Initial HA master

- Ability to specify multiple masters
  - configures the CA only a single time on the first master
  - creates and distributes additional certs for additional master hosts

- Depending on the status of openshift_master_cluster_defer_ha (defaults to
  False) one of two actions are taken when multiple masters are defined
  1. If openshift_master_cluster_defer_ha is true
     a. Certs/configs for all masters are deployed
     b. openshift-master service is only started and enabled on the master
     c. HA configuration is expected to be handled by the user manually after
        the completion of the playbook run.
  2. If oepnshift_master_cluster_defer_ha is false or undefined
     a. Certs/configs for all masters are deployed
     b. a Pacemaker/RHEL HA cluster is configured
        i. VIPs are configured based on the values of
           openshift_master_cluster_vip and
           openshift_master_cluster_plublic_vip
        ii. The openshift-master service is configured as an active/passive
            cluster service

inventory/byo/hosts.example

@@ -38,14 +38,30 @@ deployment_type=enterprise
 # Allow all auth
 #openshift_master_identity_providers=[{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}]
 
+# master cluster ha variables using pacemaker or RHEL HA
+#openshift_master_cluster_password=openshift_cluster
+#openshift_master_cluster_vip=192.168.133.25
+#openshift_master_cluster_public_vip=192.168.133.25
+#openshift_master_cluster_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
+
+# master cluster ha variables when using a different HA solution
+# For installation the value of openshift_master_cluster_hostname must resolve
+# to the first master defined in the inventory.
+# The HA solution must be manually configured after installation and must ensure
+# that openshift-master is running on a single master host.
+#openshift_master_cluster_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_defer_ha=True
+
 # host group for masters
 [masters]
-ose3-master-ansible.test.example.com
+ose3-master[1:3]-ansible.test.example.com
 
 [etcd]
 ose3-etcd[1:3]-ansible.test.example.com
 
 # host group for nodes
 [nodes]
-ose3-master-ansible.test.example.com openshift_scheduleable=False
+ose3-master[1:3]-ansible.test.example.com openshift_scheduleable=False
 ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}"

playbooks/common/openshift-master/config.yml

@@ -27,6 +27,9 @@
           api_url: "{{ openshift_master_api_url | default(None) }}"
           api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}"
           public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
+          cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+          cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+          cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
           console_path: "{{ openshift_master_console_path | default(None) }}"
           console_port: "{{ openshift_master_console_port | default(None) }}"
           console_url: "{{ openshift_master_console_url | default(None) }}"
@@ -152,16 +155,26 @@
   roles:
   - openshift_master_certificates
   post_tasks:
+  - name: Remove generated etcd client certs when using external etcd
+    file:
+      path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+      state: absent
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+    with_nested:
+    - masters_needing_certs
+    - - master.etcd-client.crt
+      - master.etcd-client.key
+
   - name: Create a tarball of the master certs
     command: >
-      tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz
-        -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} .
+      tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
+        -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
     args:
-      creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
     with_items: masters_needing_certs
   - name: Retrieve the master cert tarball from the master
     fetch:
-      src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
       dest: "{{ sync_tmpdir }}/"
       flat: yes
       fail_on_missing: yes
@@ -172,6 +185,7 @@
   hosts: oo_masters_to_config
   vars:
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -192,9 +206,14 @@
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Deploy OpenShift examples
+- name: Additional master configuration
   hosts: oo_first_master
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
   roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool
   - openshift_examples
 
 # Additional instance config for online deployments

roles/openshift_facts/library/openshift_facts.py

@@ -367,9 +367,11 @@ def set_url_facts_if_unset(facts):
         console_path = facts['master']['console_path']
         etcd_use_ssl = facts['master']['etcd_use_ssl']
         etcd_hosts = facts['master']['etcd_hosts']
-        etcd_port = facts['master']['etcd_port'],
+        etcd_port = facts['master']['etcd_port']
         hostname = facts['common']['hostname']
         public_hostname = facts['common']['public_hostname']
+        cluster_hostname = facts['master'].get('cluster_hostname')
+        cluster_public_hostname = facts['master'].get('cluster_public_hostname')
 
         if 'etcd_urls' not in facts['master']:
             etcd_urls = []
@@ -384,24 +386,51 @@ def set_url_facts_if_unset(facts):
                                         etcd_port)]
             facts['master']['etcd_urls'] = etcd_urls
         if 'api_url' not in facts['master']:
-            facts['master']['api_url'] = format_url(api_use_ssl, hostname,
+            api_hostname = cluster_hostname if cluster_hostname else hostname
+            facts['master']['api_url'] = format_url(api_use_ssl, api_hostname,
                                                     api_port)
         if 'public_api_url' not in facts['master']:
+            api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
             facts['master']['public_api_url'] = format_url(api_use_ssl,
-                                                           public_hostname,
+                                                           api_public_hostname,
                                                            api_port)
         if 'console_url' not in facts['master']:
+            console_hostname = cluster_hostname if cluster_hostname else hostname
             facts['master']['console_url'] = format_url(console_use_ssl,
-                                                        hostname,
+                                                        console_hostname,
                                                         console_port,
                                                         console_path)
         if 'public_console_url' not in facts['master']:
+            console_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
             facts['master']['public_console_url'] = format_url(console_use_ssl,
-                                                               public_hostname,
+                                                               console_public_hostname,
                                                                console_port,
                                                                console_path)
     return facts
 
+def set_aggregate_facts(facts):
+    """ Set aggregate facts
+
+        Args:
+            facts (dict): existing facts
+        Returns:
+            dict: the facts dict updated with aggregated facts
+    """
+    all_hostnames = set()
+    if 'common' in facts:
+        all_hostnames.add(facts['common']['hostname'])
+        all_hostnames.add(facts['common']['public_hostname'])
+
+        if 'master' in facts:
+            if 'cluster_hostname' in facts['master']:
+                all_hostnames.add(facts['master']['cluster_hostname'])
+            if 'cluster_public_hostname' in facts['master']:
+                all_hostnames.add(facts['master']['cluster_public_hostname'])
+
+        facts['common']['all_hostnames'] = list(all_hostnames)
+
+    return facts
+
 def set_sdn_facts_if_unset(facts):
     """ Set sdn facts if not already present in facts dict
 
@@ -675,6 +704,7 @@ class OpenShiftFacts(object):
         facts = set_identity_providers_if_unset(facts)
         facts = set_registry_url_if_unset(facts)
         facts = set_sdn_facts_if_unset(facts)
+        facts = set_aggregate_facts(facts)
         return dict(openshift=facts)
 
     def get_defaults(self, roles):
@@ -713,7 +743,7 @@ class OpenShiftFacts(object):
                           session_name='ssn', session_secrets_file='',
                           access_token_max_seconds=86400,
                           auth_token_max_seconds=500,
-                          oauth_grant_method='auto')
+                          oauth_grant_method='auto', cluster_defer_ha=False)
             defaults['master'] = master
 
         if 'node' in roles:

roles/openshift_master/defaults/main.yml

@@ -15,6 +15,12 @@ os_firewall_allow:
   port: 24224/tcp
 - service: Fluentd td-agent udp
   port: 24224/udp
+- service: pcsd
+  port: 2224/tcp
+- service: Corosync UDP
+  port: 5404/udp
+- service: Corosync UDP
+  port: 5405/udp
 os_firewall_deny:
 - service: OpenShift api http
   port: 8080/tcp

roles/openshift_master/handlers/main.yml

@@ -1,3 +1,4 @@
 ---
 - name: restart openshift-master
   service: name=openshift-master state=restarted
+  when: not openshift_master_ha

roles/openshift_master/tasks/main.yml

@@ -8,6 +8,10 @@
     - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods
   when: openshift_master_oauth_grant_method is defined
 
+- fail:
+    msg: "openshift_master_cluster_password must be set for multi-master installations"
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined
+
 - name: Install OpenShift Master package
   yum: pkg=openshift-master state=present
   register: install_result
@@ -16,6 +20,9 @@
   openshift_facts:
     role: master
     local_facts:
+      cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+      cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+      cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
       debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level) }}"
       api_port: "{{ openshift_master_api_port | default(None) }}"
       api_url: "{{ openshift_master_api_url | default(None) }}"
@@ -114,12 +121,26 @@
 
 - name: Start and enable openshift-master
   service: name=openshift-master enabled=yes state=started
+  when: not openshift_master_ha
   register: start_result
 
 - name: pause to prevent service restart from interfering with bootstrapping
   pause: seconds=30
   when: start_result | changed
 
+- name: Install cluster packagese
+  yum: pkg=pcs state=present
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool
+  register: install_result
+
+- name: Start and enable cluster service
+  service: name=pcsd enabled=yes state=started
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool
+
+- name: Set the cluster user password
+  shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster
+  when: install_result | changed
+
 - name: Create the OpenShift client config dir(s)
   file:
     path: "~{{ item }}/.kube"

roles/openshift_master_ca/tasks/main.yml

@@ -14,7 +14,7 @@
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }}
+      --hostnames={{ openshift.common.all_hostnames | join(',') }}
       --master={{ openshift.master.api_url }}
       --public-master={{ openshift.master.public_api_url }}
       --cert-dir={{ openshift_master_config_dir }} --overwrite=false

roles/openshift_master_certificates/tasks/main.yml

@@ -7,14 +7,20 @@
   with_items: masters_needing_certs
 
 - file:
-    src: "{{ openshift_master_ca_cert }}"
-    dest: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/ca.crt"
-  with_items: masters_needing_certs
+    src: "{{ openshift_master_config_dir }}/{{ item.1 }}"
+    dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+    state: hard
+  with_nested:
+  - masters_needing_certs
+  - - ca.crt
+    - ca.key
+    - ca.serial.txt
+
 
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ item.openshift.common.hostname }},{{ item.openshift.common.public_hostname }}
+      --hostnames={{ item.openshift.common.all_hostnames | join(',') }}
       --master={{ item.openshift.master.api_url }}
       --public-master={{ item.openshift.master.public_api_url }}
       --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}
@@ -22,3 +28,5 @@
   args:
     creates: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/master.server.crt"
   with_items: masters_needing_certs
+
+

roles/openshift_master_certificates/vars/main.yml

@@ -1,6 +1,3 @@
 ---
 openshift_generated_configs_dir: /etc/openshift/generated-configs
 openshift_master_config_dir: /etc/openshift/master
-openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
-openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
-openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"

roles/openshift_master_cluster/README.md

@@ -0,0 +1,34 @@
+OpenShift Master Cluster
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)

roles/openshift_master_cluster/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }

roles/openshift_master_cluster/tasks/configure.yml

@@ -0,0 +1,43 @@
+---
+- fail:
+    msg: This role requires that openshift_master_cluster_vip is set
+  when: openshift_master_cluster_vip is not defined or not openshift_master_cluster_vip
+- fail:
+    msg: This role requires that openshift_master_cluster_public_vip is set
+  when: openshift_master_cluster_public_vip is not defined or not openshift_master_cluster_public_vip
+
+- name: Authenticate to the cluster
+  command: pcs cluster auth -u hacluster -p {{ openshift_master_cluster_password }} {{ omc_cluster_hosts }}
+
+- name: Create the cluster
+  command: pcs cluster setup --name openshift_master {{ omc_cluster_hosts }}
+
+- name: Start the cluster
+  command: pcs cluster start --all
+
+- name: Enable the cluster on all nodes
+  command: pcs cluster enable --all
+
+- name: Set default resource stickiness
+  command: pcs resource defaults resource-stickiness=100
+
+- name: Add the cluster VIP resource
+  command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_vip }} --group openshift-master
+
+- name: Add the cluster public VIP resource
+  command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_public_vip }} --group openshift-master
+  when: openshift_master_cluster_public_vip != openshift_master_cluster_vip
+
+- name: Add the cluster openshift-master service resource
+  command: pcs resource create master systemd:openshift-master --group openshift-master
+
+- name: Disable stonith
+  command: pcs property set stonith-enabled=false
+
+# TODO: handle case where api port is not 8443
+- name: Wait for the clustered master service to be available
+  wait_for:
+    host: "{{ openshift_master_cluster_vip }}"
+    port: 8443
+    state: started
+    timeout: 300

roles/openshift_master_cluster/tasks/configure_deferred.yml

@@ -0,0 +1,8 @@
+---
+- debug: msg="Deferring config"
+
+- name: Start and enable openshift-master
+  service:
+    name: openshift-master
+    state: started
+    enabled: yes

roles/openshift_master_cluster/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Test if cluster is already configured
+  command: pcs status
+  register: pcs_status
+  changed_when: false
+  failed_when: false
+  when: not openshift.master.cluster_defer_ha | bool
+
+- include: configure.yml
+  when: "pcs_status | failed and 'Error: cluster is not currently running on this node' in pcs_status.stderr"
+
+- include: configure_deferred.yml
+  when: openshift.master.cluster_defer_ha | bool

roles/openshift_node_certificates/tasks/main.yml

@@ -25,7 +25,7 @@
   command: >
     {{ openshift.common.admin_binary }} create-server-cert
       --cert=server.crt --key=server.key --overwrite=true
-      --hostnames={{ [item.openshift.common.hostname, item.openshift.common.public_hostname]|unique|join(",") }}
+      --hostnames={{ openshift.common.all_hostnames |join(",") }}
       --signer-cert={{ openshift_master_ca_cert }}
       --signer-key={{ openshift_master_ca_key }}
       --signer-serial={{ openshift_master_ca_serial }}