Merge pull request #2511 from vishpat/nuage_ha

Changes for Nuage HA

filter_plugins/oo_filters.py

@@ -7,6 +7,7 @@ Custom filters for use in openshift-ansible
 
 from ansible import errors
 from collections import Mapping
+from distutils.util import strtobool
 from distutils.version import LooseVersion
 from operator import itemgetter
 import OpenSSL.crypto
@@ -858,6 +859,35 @@ class FilterModule(object):
             # netloc wasn't parsed, assume url was missing scheme and path
             return parse_result.path
 
+    @staticmethod
+    def oo_openshift_loadbalancer_frontends(api_port, servers_hostvars, use_nuage=False, nuage_rest_port=None):
+        loadbalancer_frontends = [{'name': 'atomic-openshift-api',
+                                   'mode': 'tcp',
+                                   'options': ['tcplog'],
+                                   'binds': ["*:{0}".format(api_port)],
+                                   'default_backend': 'atomic-openshift-api'}]
+        if bool(strtobool(str(use_nuage))) and nuage_rest_port is not None:
+            loadbalancer_frontends.append({'name': 'nuage-monitor',
+                                           'mode': 'tcp',
+                                           'options': ['tcplog'],
+                                           'binds': ["*:{0}".format(nuage_rest_port)],
+                                           'default_backend': 'nuage-monitor'})
+        return loadbalancer_frontends
+
+    @staticmethod
+    def oo_openshift_loadbalancer_backends(api_port, servers_hostvars, use_nuage=False, nuage_rest_port=None):
+        loadbalancer_backends = [{'name': 'atomic-openshift-api',
+                                  'mode': 'tcp',
+                                  'option': 'tcplog',
+                                  'balance': 'source',
+                                  'servers': FilterModule.oo_haproxy_backend_masters(servers_hostvars, api_port)}]
+        if bool(strtobool(str(use_nuage))) and nuage_rest_port is not None:
+            loadbalancer_backends.append({'name': 'nuage-monitor',
+                                          'mode': 'tcp',
+                                          'option': 'tcplog',
+                                          'balance': 'source',
+                                          'servers': FilterModule.oo_haproxy_backend_masters(servers_hostvars, nuage_rest_port)})
+        return loadbalancer_backends
 
     def filters(self):
         """ returns a mapping of filters to methods """
@@ -891,4 +921,6 @@ class FilterModule(object):
             "oo_merge_dicts": self.oo_merge_dicts,
             "oo_hostname_from_url": self.oo_hostname_from_url,
             "oo_merge_hostvars": self.oo_merge_hostvars,
+            "oo_openshift_loadbalancer_frontends": self.oo_openshift_loadbalancer_frontends,
+            "oo_openshift_loadbalancer_backends": self.oo_openshift_loadbalancer_backends
         }

playbooks/common/openshift-loadbalancer/config.yml

@@ -1,5 +1,16 @@
 ---
 - name: Configure load balancers
   hosts: oo_lb_to_config
+  vars:
+    openshift_loadbalancer_frontends: "{{ (openshift_master_api_port | default(8843)
+                                           | oo_openshift_loadbalancer_frontends(hostvars | oo_select_keys(groups['oo_masters']),
+                                                                                 openshift_use_nuage | default(false),
+                                                                                 nuage_mon_rest_server_port | default(none)))
+                                           + openshift_loadbalancer_additional_frontends | default([]) }}"
+    openshift_loadbalancer_backends: "{{ (openshift_master_api_port | default(8843)
+                                          | oo_openshift_loadbalancer_backends(hostvars | oo_select_keys(groups['oo_masters']),
+                                                                               openshift_use_nuage | default(false),
+                                                                               nuage_mon_rest_server_port | default(none)))
+                                          + openshift_loadbalancer_additional_backends | default([]) }}"
   roles:
   - role: openshift_loadbalancer

roles/nuage_common/defaults/main.yaml

@@ -8,3 +8,6 @@ nuage_ca_serial: "{{ nuage_ca_dir }}/nuageMonCA.serial.txt"
 
 nuage_master_mon_dir: /usr/share/nuage-openshift-monitor
 nuage_node_plugin_dir: /usr/share/vsp-openshift
+
+nuage_mon_rest_server_port: "{{ nuage_openshift_monitor_rest_server_port | default('9443') }}"
+

roles/nuage_master/meta/main.yml

@@ -13,8 +13,10 @@ galaxy_info:
   - cloud
   - system
 dependencies:
-- role: nuage_ca
-- role: os_firewall
-  os_firewall_allow:
-  - service: openshift-monitor 
-    port: "{{ nuage_mon_rest_server_port }}/tcp"
+  - role: nuage_ca
+  - role: nuage_common
+  - role: openshift_etcd_client_certificates
+  - role: os_firewall
+    os_firewall_allow:
+    - service: openshift-monitor 
+      port: "{{ nuage_mon_rest_server_port }}/tcp"

roles/nuage_master/templates/nuage-openshift-monitor.j2

@@ -30,4 +30,11 @@ logLevel: {{ nuage_mon_log_level }}
 # Parameters related to the nuage monitor REST server
 nuageMonServer:
     URL: {{ nuage_mon_rest_server_url }}
-    certificateDirectory: {{ cert_output_dir }} 
+    certificateDirectory: {{ cert_output_dir }}
+# etcd config required for HA
+etcdClientConfig:
+    ca: {{ openshift_master_config_dir }}/{{ "ca.crt" if (openshift.master.embedded_etcd | bool) else "master.etcd-ca.crt" }}
+    certFile: {{ openshift_master_config_dir }}/master.etcd-client.crt
+    keyFile: {{ openshift_master_config_dir }}/master.etcd-client.key
+    urls:
+        - {{ openshift.master.etcd_urls }}

roles/nuage_master/vars/main.yaml

@@ -8,7 +8,6 @@ cert_output_dir: /usr/share/nuage-openshift-monitor
 kube_config: /usr/share/nuage-openshift-monitor/nuage.kubeconfig
 kubemon_yaml: /usr/share/nuage-openshift-monitor/nuage-openshift-monitor.yaml 
 master_config_yaml: "{{ openshift_master_config_dir }}/master-config.yaml"
-nuage_mon_rest_server_port: "{{ nuage_openshift_monitor_rest_server_port | default('9443') }}"
 nuage_mon_rest_server_url: "0.0.0.0:{{ nuage_mon_rest_server_port }}"
 nuage_mon_rest_server_logdir: "{{ nuage_openshift_monitor_log_dir | default('/var/log/nuage-openshift-monitor') }}"
 nuage_mon_log_level: "{{ nuage_openshift_monitor_log_level | default('3') }}"

roles/openshift_facts/library/openshift_facts.py

@@ -1609,7 +1609,6 @@ class OpenShiftFacts(object):
                    'docker',
                    'etcd',
                    'hosted',
-                   'loadbalancer',
                    'master',
                    'node']
 
@@ -1865,13 +1864,6 @@ class OpenShiftFacts(object):
                 router=dict()
             )
 
-        if 'loadbalancer' in roles:
-            loadbalancer = dict(frontend_port='8443',
-                                default_maxconn='20000',
-                                global_maxconn='20000',
-                                limit_nofile='100000')
-            defaults['loadbalancer'] = loadbalancer
-
         return defaults
 
     def guess_host_provider(self):

roles/openshift_loadbalancer/README.md

@@ -1,27 +1,68 @@
 OpenShift HAProxy Loadbalancer
 ==============================
 
-TODO
+OpenShift HaProxy Loadbalancer Configuration
 
 Requirements
 ------------
 
-TODO
+This role is intended to be applied to the [lb] host group which is
+separate from OpenShift infrastructure components.
+
+This role is not re-entrant. All haproxy configuration lives in a single file.
 
 Role Variables
 --------------
 
-TODO
+From this role:
+
+| Name                                   | Default value |                                                       |
+|----------------------------------------|---------------|-------------------------------------------------------|
+| openshift_loadbalancer_limit_nofile    | 100000        | Limit number of open files.                           |
+| openshift_loadbalancer_global_maxconn  | 20000         | Maximum per-process number of concurrent connections. |
+| openshift_loadbalancer_default_maxconn | 20000         | Maximum per-process number of concurrent connections. |
+| openshift_loadbalancer_frontends       | none          | List of frontends. See example below.                 |
+| openshift_loadbalancer_backends        | none          | List of backends. See example below.                  |
 
 Dependencies
 ------------
 
-TODO
+* openshift_facts
+* os_firewall
+* openshift_repos
 
 Example Playbook
 ----------------
 
-TODO
+```
+- name: Configure loadbalancer hosts
+  hosts: lb
+  roles:
+  - role: openshift_loadbalancer
+    openshift_loadbalancer_frontends:
+    - name: atomic-openshift-api
+      mode: tcp
+      options:
+      - tcplog
+      binds:
+      - "*:8443"
+      default_backend: atomic-openshift-api
+    openshift_loadbalancer_backends:
+    - name: atomic-openshift-api
+      mode: tcp
+      option: tcplog
+      balance: source
+      servers:
+      - name: master1
+        address: "192.168.122.221:8443"
+	opts: check
+      - name: master2
+        address: "192.168.122.222:8443"
+	opts: check
+      - name: master3
+        address: "192.168.122.223:8443"
+	opts: check
+```
 
 License
 -------

roles/openshift_loadbalancer/meta/main.yml

@@ -10,11 +10,11 @@ galaxy_info:
     versions:
     - 7
 dependencies:
-- role: openshift_loadbalancer_facts
+- role: openshift_facts
 - role: os_firewall
   os_firewall_allow:
   - service: haproxy stats
     port: "9000/tcp"
   - service: haproxy balance
-    port: "{{ openshift.loadbalancer.frontend_port }}/tcp"
+    port: "{{ openshift_master_api_port | default(8443) }}/tcp"
 - role: openshift_repos

roles/openshift_loadbalancer/tasks/main.yml

@@ -7,15 +7,13 @@
   file:
     path: /etc/systemd/system/haproxy.service.d
     state: directory
-  when: "'limit_nofile' in openshift.loadbalancer"
 
 - name: Configure the nofile limits for haproxy
   ini_file:
     dest: /etc/systemd/system/haproxy.service.d/limits.conf
     section: Service
     option: LimitNOFILE
-    value: "{{ openshift.loadbalancer.limit_nofile }}"
-  when: "'limit_nofile' in openshift.loadbalancer"
+    value: "{{ openshift_loadbalancer_limit_nofile | default(100000) }}"
   notify: restart haproxy
   register: nofile_limit_result
 

roles/openshift_loadbalancer/templates/haproxy.cfg.j2

@@ -3,7 +3,7 @@
 global
     chroot      /var/lib/haproxy
     pidfile     /var/run/haproxy.pid
-    maxconn     {{ openshift.loadbalancer.global_maxconn }}
+    maxconn     {{ openshift_loadbalancer_global_maxconn | default(20000) }}
     user        haproxy
     group       haproxy
     daemon
@@ -32,14 +32,14 @@ defaults
     timeout server          300s
     timeout http-keep-alive 10s
     timeout check           10s
-    maxconn                 {{ openshift.loadbalancer.default_maxconn }}
+    maxconn                 {{ openshift_loadbalancer_default_maxconn | default(20000) }}
 
 listen stats :9000
     mode http
     stats enable
     stats uri /
 
-{% for frontend in openshift.loadbalancer.frontends %}
+{% for frontend in openshift_loadbalancer_frontends %}
 frontend  {{ frontend.name }}
 {% for bind in frontend.binds %}
     bind {{ bind }}
@@ -60,7 +60,7 @@ frontend  {{ frontend.name }}
 {% endif %}
 {% endfor %}
 
-{% for backend in openshift.loadbalancer.backends %}
+{% for backend in openshift_loadbalancer_backends %}
 backend {{ backend.name }}
     balance {{ backend.balance }}
 {% if 'mode' in backend %}

roles/openshift_loadbalancer_facts/README.md

@@ -1,34 +0,0 @@
-OpenShift HAProxy Loadbalancer Facts
-====================================
-
-TODO
-
-Requirements
-------------
-
-TODO
-
-Role Variables
---------------
-
-TODO
-
-Dependencies
-------------
-
-TODO
-
-Example Playbook
-----------------
-
-TODO
-
-License
--------
-
-Apache License, Version 2.0
-
-Author Information
-------------------
-
-Andrew Butcher (abutcher@redhat.com)

roles/openshift_loadbalancer_facts/meta/main.yml

@@ -1,13 +0,0 @@
----
-galaxy_info:
-  author: Andrew Butcher
-  description: OpenShift loadbalancer facts
-  company: Red Hat, Inc.
-  license: Apache License, Version 2.0
-  min_ansible_version: 1.9
-  platforms:
-  - name: EL
-    versions:
-    - 7
-dependencies:
-- role: openshift_facts

roles/openshift_loadbalancer_facts/tasks/main.yml

@@ -1,30 +0,0 @@
----
-- name: Set haproxy frontend port
-  openshift_facts:
-    role: loadbalancer
-    local_facts:
-      frontend_port: "{{ openshift_master_api_port | default(None) }}"
-
-- name: Set loadbalancer facts
-  openshift_facts:
-    role: loadbalancer
-    local_facts:
-      limit_nofile: "{{ openshift_loadbalancer_limit_nofile | default(None) }}"
-      default_maxconn: "{{ openshift_loadbalancer_default_maxconn | default(None) }}"
-      global_maxconn: "{{ openshift_loadbalancer_global_maxconn | default(None) }}"
-      frontends:
-      - name: atomic-openshift-api
-        mode: tcp
-        options:
-        - tcplog
-        binds:
-        - "*:{{ openshift.loadbalancer.frontend_port }}"
-        default_backend: atomic-openshift-api
-      backends:
-      - name: atomic-openshift-api
-        mode: tcp
-        option: tcplog
-        balance: source
-        servers: "{{ hostvars
-                     | oo_select_keys(groups['oo_masters'])
-                     | oo_haproxy_backend_masters(openshift.loadbalancer.frontend_port) }}"