Playbook updates for clustered etcd

- Add support to bin/cluster for specifying etcd hosts
  - defaults to 0, if no etcd hosts are selected, then configures embedded
    etcd
- Updates for the byo inventory file for etcd and master as node by default
- Consolidation of cluster logic more centrally into common playbook
- Added etcd config support to playbooks
- Restructured byo playbooks to leverage the common openshift-cluster playbook
- Added support to common master playbook to generate and apply external etcd
  client certs from the etcd ca
- start of refactor for better handling of master certs in a multi-master
  environment.
  - added the openshift_master_ca and openshift_master_certificates roles to
    manage master certs instead of generating them in the openshift_master
    role
- added etcd host groups to the cluster update playbooks
- aded better handling of host groups when they are either not present or are
  empty.
- Update AWS readme

README_AWS.md

@@ -20,10 +20,11 @@ Create a credentials file
 ```
 Note: You must source this file before running any Ansible commands.
 
+Alternatively, you could configure credentials in either ~/.boto or ~/.aws/credentials, see the [boto docs](http://docs.pythonboto.org/en/latest/boto_config_tut.html) for the format.
 
 (Optional) Setup your $HOME/.ssh/config file
 -------------------------------------------
-In case of a cluster creation, or any other case where you don't know the machine hostname in advance, you can use '.ssh/config'
+In case of a cluster creation, or any other case where you don't know the machine hostname in advance, you can use `.ssh/config`
 to setup a private key file to allow ansible to connect to the created hosts.
 
 To do so, add the the following entry to your $HOME/.ssh/config file and make it point to the private key file which allows you to login on AWS.
@@ -62,10 +63,16 @@ Node specific defaults:
 If needed, these values can be changed by setting environment variables on your system.
 
 - export ec2_instance_type='m3.large'
-- export ec2_ami='ami-307b3658'
+- export ec2_image='ami-307b3658'
 - export ec2_region='us-east-1'
 - export ec2_keypair='libra'
 - export ec2_security_groups="['public']"
+- export ec2_vpc_subnet='my_vpc_subnet'
+- export ec2_assign_public_ip='true'
+- export os_etcd_root_vol_size='20'
+- export os_etcd_root_vol_type='standard'
+- export os_etcd_vol_size='20'
+- export os_etcd_vol_type='standard'
 - export os_master_root_vol_size='20'
 - export os_master_root_vol_type='standard'
 - export os_node_root_vol_size='15'
@@ -114,3 +121,12 @@ Terminating a cluster
 ```
   bin/cluster terminate aws <cluster-id>
 ```
+
+Specifying a deployment type
+---------------------------
+The --deployment-type flag can be passed to bin/cluster to specify the deployment type
+1. To launch an online cluster (requires access to private repositories and amis):
+```
+  bin/cluster create aws --deployment-type=online <cluster-id>
+```
+Note: If no deployment type is specified, then the default is origin.

bin/cluster

@@ -51,6 +51,7 @@ class Cluster(object):
 
         env['num_masters'] = args.masters
         env['num_nodes'] = args.nodes
+        env['num_etcd'] = args.etcd
 
         return self.action(args, inventory, env, playbook)
 
@@ -261,6 +262,8 @@ if __name__ == '__main__':
                                help='number of masters to create in cluster')
     create_parser.add_argument('-n', '--nodes', default=2, type=int,
                                help='number of nodes to create in cluster')
+    create_parser.add_argument('-e', '--etcd', default=0, type=int,
+                               help='number of external etcd hosts to create in cluster')
     create_parser.set_defaults(func=cluster.create)
 
     config_parser = action_parser.add_parser('config',

filter_plugins/oo_filters.py

@@ -175,9 +175,9 @@ class FilterModule(object):
         '''
         if not issubclass(type(data), dict):
             raise errors.AnsibleFilterError("|failed expects first param is a dict")
-        if host_type not in ['master', 'node']:
-            raise errors.AnsibleFilterError("|failed expects either master or node"
-                                            " host type")
+        if host_type not in ['master', 'node', 'etcd']:
+            raise errors.AnsibleFilterError("|failed expects etcd, master or node"
+                                            " as the host type")
 
         root_vol = data[host_type]['root']
         root_vol['device_name'] = '/dev/sda1'

inventory/byo/hosts

@@ -4,6 +4,7 @@
 [OSEv3:children]
 masters
 nodes
+etcd
 
 # Set variables common for all OSEv3 hosts
 [OSEv3:vars]
@@ -33,7 +34,10 @@ openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl':
 [masters]
 ose3-master-ansible.test.example.com
 
+[etcd]
+#ose3-master-ansible.test.example.com
+
 # host group for nodes
 [nodes]
-#ose3-master-ansible.test.example.com openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
+ose3-master-ansible.test.example.com openshift_scheduleable=False
 ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}"

playbooks/aws/openshift-cluster/config.yml

@@ -1,37 +1,22 @@
 ---
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
+- hosts: localhost
   gather_facts: no
   vars_files:
   - vars.yml
   tasks:
-  - name: Evaluate oo_masters_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_masters_to_config
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | default([])
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_nodes_to_config
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-node"] | default([])
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ groups['tag_env-host-type_' ~ cluster_id ~ '-openshift-master'][0] }}"
-      groups: oo_first_master
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    when: "'tag_env-host-type_{{ cluster_id }}-openshift-master' in groups"
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
 
 - include: ../../common/openshift-cluster/config.yml
   vars:
+    g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
+    g_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
     openshift_cluster_id: "{{ cluster_id }}"
     openshift_debug_level: 4
     openshift_deployment_type: "{{ deployment_type }}"
-    openshift_first_master: "{{ groups.oo_first_master.0 }}"
     openshift_hostname: "{{ ec2_private_ip_address }}"
     openshift_public_hostname: "{{ ec2_ip_address }}"

playbooks/aws/openshift-cluster/launch.yml

@@ -11,6 +11,13 @@
       msg: Deployment type not supported for aws provider yet
     when: deployment_type == 'enterprise'
 
+  - include: ../../common/openshift-cluster/set_etcd_launch_facts_tasks.yml
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ etcd_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+
   - include: ../../common/openshift-cluster/set_master_launch_facts_tasks.yml
   - include: tasks/launch_instances.yml
     vars:
@@ -25,9 +32,10 @@
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
 
-  - set_fact:
-      a_master: "{{ master_names[0] }}"
-  - add_host: name={{ a_master }} groups=service_master
+  - add_host:
+      name: "{{ master_names.0 }}"
+      groups: service_master
+    when: master_names is defined and master_names.0 is defined
 
 - include: update.yml
 

playbooks/aws/openshift-cluster/tasks/launch_instances.yml

@@ -53,6 +53,15 @@
     latest_ami: "{{ ami_result.results | oo_ami_selector(ec2_image_name) }}"
     user_data: "{{ lookup('template', '../templates/user_data.j2') }}"
     volume_defs:
+      etcd:
+        root:
+          volume_size: "{{ lookup('env', 'os_etcd_root_vol_size') | default(25, true) }}"
+          device_type: "{{ lookup('env', 'os_etcd_root_vol_type') | default('gp2', true) }}"
+          iops: "{{ lookup('env', 'os_etcd_root_vol_iops') | default(500, true) }}"
+        etcd:
+          volume_size: "{{ lookup('env', 'os_etcd_vol_size') | default(32, true) }}"
+          device_type: "{{ lookup('env', 'os_etcd_vol_type') | default('gp2', true) }}"
+          iops: "{{ lookup('env', 'os_etcd_vol_iops') | default(500, true) }}"
       master:
         root:
           volume_size: "{{ lookup('env', 'os_master_root_vol_size') | default(25, true) }}"

playbooks/aws/openshift-cluster/update.yml

@@ -11,7 +11,9 @@
       groups: oo_hosts_to_update
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-node"]) | default([])
+    with_items: (groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | default([]))
+                | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-node"] | default([]))
+                | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-etcd"] | default([]))
 
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 

playbooks/byo/config.yml

@@ -1,12 +1,2 @@
 ---
-- name: Run the openshift-master config playbook
-  include: openshift-master/config.yml
-  when: groups.masters is defined and groups.masters
-
-- name: Run the openshift-etcd playbook
-  include: openshift-etcd/config.yml
-  when: groups.etcd is defined and groups.etcd
-
-- name: Run the openshift-node config playbook
-  include: openshift-node/config.yml
-  when: groups.nodes is defined and groups.nodes and groups.masters is defined and groups.masters
+- include: openshift-cluster/config.yml

playbooks/byo/openshift-cluster/config.yml

@@ -0,0 +1,9 @@
+---
+- include: ../../common/openshift-cluster/config.yml
+  vars:
+    g_etcd_group: "{{ 'etcd' }}"
+    g_masters_group: "{{ 'masters' }}"
+    g_nodes_group: "{{ 'nodes' }}"
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_debug_level: 4
+    openshift_deployment_type: "{{ deployment_type }}"

playbooks/byo/openshift-master/config.yml

@@ -1,15 +0,0 @@
----
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - add_host:
-      name: "{{ item }}"
-      groups: oo_masters_to_config
-    with_items: groups['masters']
-
-- include: ../../common/openshift-master/config.yml
-  vars:
-    openshift_cluster_id: "{{ cluster_id | default('default') }}"
-    openshift_debug_level: 4
-    openshift_deployment_type: "{{ deployment_type }}"

playbooks/byo/openshift-master/filter_plugins

@@ -1 +0,0 @@
-../../../filter_plugins

playbooks/byo/openshift-master/lookup_plugins

@@ -1 +0,0 @@
-../../../lookup_plugins

playbooks/byo/openshift-master/roles

@@ -1 +0,0 @@
-../../../roles

playbooks/byo/openshift-node/config.yml

@@ -1,23 +0,0 @@
----
-- name: Populate oo_nodes_to_config and oo_first_master host groups
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_nodes_to_config
-    with_items: groups.nodes
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ item }}"
-      groups: oo_first_master
-    with_items: groups.masters.0
-
-
-- include: ../../common/openshift-node/config.yml
-  vars:
-    openshift_first_master: "{{ groups.masters.0 }}"
-    openshift_cluster_id: "{{ cluster_id | default('default') }}"
-    openshift_debug_level: 4
-    openshift_deployment_type: "{{ deployment_type }}"

playbooks/common/openshift-cluster/config.yml

@@ -1,4 +1,65 @@
 ---
+- name: Populate config host groups
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - fail:
+      msg: This playbook rquires g_etcd_group to be set
+    when: g_etcd_group is not defined
+
+  - fail:
+      msg: This playbook rquires g_masters_group to be set
+    when: g_masters_group is not defined
+
+  - fail:
+      msg: This playbook rquires g_nodes_group to be set
+    when: g_nodes_group is not defined
+
+  - name: Evaluate oo_etcd_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_etcd_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_etcd_group] | default([])
+
+  - name: Evaluate oo_masters_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_masters_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_masters_group] | default([])
+
+  - name: Evaluate oo_nodes_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_nodes_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_nodes_group] | default([])
+
+  - name: Evaluate oo_first_etcd
+    add_host:
+      name: "{{ groups[g_etcd_group][0] }}"
+      groups: oo_first_etcd
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    when: g_etcd_group in groups and (groups[g_etcd_group] | length) > 0
+
+  - name: Evaluate oo_first_master
+    add_host:
+      name: "{{ groups[g_masters_group][0] }}"
+      groups: oo_first_master
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    when: g_masters_group in groups and (groups[g_masters_group] | length) > 0
+
+- include: ../openshift-etcd/config.yml
+
 - include: ../openshift-master/config.yml
 
 - include: ../openshift-node/config.yml
+  vars:
+    osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
+    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"

playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml

@@ -5,7 +5,9 @@
   set_fact:
     scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}"
   register: master_names_output
-  with_sequence: start=1 end={{ num_masters }}
+  with_sequence: count={{ num_masters }}
 
 - set_fact:
-    master_names: "{{ master_names_output.results | oo_collect('ansible_facts') | oo_collect('scratch_name') }}"
+    master_names: "{{ master_names_output.results | default([])
+                      | oo_collect('ansible_facts')
+                      | oo_collect('scratch_name') }}"

playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml

@@ -5,7 +5,9 @@
   set_fact:
     scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}"
   register: node_names_output
-  with_sequence: start=1 end={{ num_nodes }}
+  with_sequence: count={{ num_nodes }}
 
 - set_fact:
-    node_names: "{{ node_names_output.results | oo_collect('ansible_facts') | oo_collect('scratch_name') }}"
+    node_names: "{{ node_names_output.results | default([])
+                    | oo_collect('ansible_facts')
+                    | oo_collect('scratch_name') }}"

playbooks/common/openshift-master/config.yml

@@ -1,19 +1,214 @@
 ---
+- name: Set master facts and determine if external etcd certs need to be generated
+  hosts: oo_masters_to_config
+  pre_tasks:
+  - set_fact:
+      openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}"
+      openshift_master_etcd_hosts: "{{ hostvars
+                                       | oo_select_keys(groups['oo_etcd_to_config']
+                                                        | default([]))
+                                       | oo_collect('openshift.common.hostname')
+                                       | default(none, true) }}"
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: "{{ item.role }}"
+      local_facts: "{{ item.local_facts }}"
+    with_items:
+      - role: common
+        local_facts:
+          hostname: "{{ openshift_hostname | default(None) }}"
+          public_hostname: "{{ openshift_public_hostname | default(None) }}"
+          deployment_type: "{{ openshift_deployment_type }}"
+      - role: master
+        local_facts:
+          api_port: "{{ openshift_master_api_port | default(None) }}"
+          api_url: "{{ openshift_master_api_url | default(None) }}"
+          api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}"
+          public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
+          console_path: "{{ openshift_master_console_path | default(None) }}"
+          console_port: "{{ openshift_master_console_port | default(None) }}"
+          console_url: "{{ openshift_master_console_url | default(None) }}"
+          console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
+          public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
+  - name: Check status of external etcd certificatees
+    stat:
+      path: "/etc/openshift/master/{{ item }}"
+    with_items:
+    - master.etcd-client.crt
+    - master.etcd-ca.crt
+    register: g_external_etcd_cert_stat_result
+  - set_fact:
+      etcd_client_certs_missing: "{{ g_external_etcd_cert_stat_result.results
+                                    | map(attribute='stat.exists')
+                                    | list | intersect([false])}}"
+      etcd_cert_subdir: openshift-master-{{ openshift.common.hostname }}
+      etcd_cert_config_dir: /etc/openshift/master
+      etcd_cert_prefix: master.etcd-
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_master_mktemp
+    changed_when: False
+
+- name: Configure etcd certificates
+  hosts: oo_first_etcd
+  vars:
+    etcd_generated_certs_dir: /etc/etcd/generated_certs
+    etcd_needing_client_certs: "{{ hostvars
+                                   | oo_select_keys(groups['oo_masters_to_config'])
+                                   | oo_filter_list(filter_attr='etcd_client_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  roles:
+  - etcd_certificates
+  post_tasks:
+  - name: Create a tarball of the etcd certs
+    command: >
+      tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz
+        -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} .
+    args:
+      creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+    with_items: etcd_needing_client_certs
+  - name: Retrieve the etcd cert tarballs
+    fetch:
+      src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: etcd_needing_client_certs
+
+- name: Copy the external etcd certs to the masters
+  hosts: oo_masters_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: /etc/openshift/master
+      state: directory
+    when: etcd_client_certs_missing is defined and etcd_client_certs_missing
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz"
+      dest: "{{ etcd_cert_config_dir }}"
+    when: etcd_client_certs_missing is defined and etcd_client_certs_missing
+  - file:
+      path: "{{ etcd_cert_config_dir }}/{{ item }}"
+      owner: root
+      group: root
+      mode: 0600
+    with_items:
+    - master.etcd-client.crt
+    - master.etcd-client.key
+    - master.etcd-ca.crt
+    when: etcd_client_certs_missing is defined and etcd_client_certs_missing
+
+- name: Determine if master certificates need to be generated
+  hosts: oo_masters_to_config
+  tasks:
+  - set_fact:
+      openshift_master_certs_no_etcd:
+      - admin.crt
+      - master.kubelet-client.crt
+      - master.server.crt
+      - openshift-master.crt
+      - openshift-registry.crt
+      - openshift-router.crt
+      - etcd.server.crt
+      openshift_master_certs_etcd:
+      - master.etcd-client.crt
+  - set_fact:
+      openshift_master_certs: "{{ (openshift_master_certs_no_etcd | union(openshift_master_certs_etcd)) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else openshift_master_certs_no_etcd }}"
+
+  - name: Check status of master certificates
+    stat:
+      path: "/etc/openshift/master/{{ item }}"
+    with_items: openshift_master_certs
+    register: g_master_cert_stat_result
+  - set_fact:
+      master_certs_missing: "{{ g_master_cert_stat_result.results
+                                | map(attribute='stat.exists')
+                                | list | intersect([false])}}"
+      master_cert_subdir: master-{{ openshift.common.hostname }}
+      master_cert_config_dir: /etc/openshift/master
+
+- name: Configure master certificates
+  hosts: oo_first_master
+  vars:
+    master_generated_certs_dir: /etc/openshift/generated-configs
+    masters_needing_certs: "{{ hostvars
+                               | oo_select_keys(groups['oo_masters_to_config'] | difference(groups['oo_first_master']))
+                               | oo_filter_list(filter_attr='master_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  roles:
+  - openshift_master_certificates
+  post_tasks:
+  - name: Create a tarball of the master certs
+    command: >
+      tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz
+        -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} .
+    args:
+      creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+    with_items: masters_needing_certs
+  - name: Retrieve the master cert tarball from the master
+    fetch:
+      src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: masters_needing_certs
+
 - name: Configure master instances
   hosts: oo_masters_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  pre_tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: /etc/openshift/master
+      state: directory
+    when: master_certs_missing and 'oo_first_master' not in group_names
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
+      dest: "{{ master_cert_config_dir }}"
+    when: master_certs_missing and 'oo_first_master' not in group_names
   roles:
   - openshift_master
-  - openshift_examples
   - role: fluentd_master
     when: openshift.common.use_fluentd | bool
-  tasks:
+  post_tasks:
   - name: Create group for deployment type
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
+- name: Deploy OpenShift examples
+  hosts: oo_first_master
+  roles:
+  - openshift_examples
+
 # Additional instance config for online deployments
 - name: Additional instance config
   hosts: oo_masters_deployment_type_online
   roles:
   - pods
   - os_env_extras
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - file: name={{ g_master_mktemp.stdout }} state=absent
+    changed_when: False

playbooks/common/openshift-node/config.yml

@@ -18,21 +18,18 @@
           deployment_type: "{{ openshift_deployment_type }}"
       - role: node
         local_facts:
-          resources_cpu: "{{ openshift_node_resources_cpu | default(None) }}"
-          resources_memory: "{{ openshift_node_resources_memory | default(None) }}"
-          pod_cidr: "{{ openshift_node_pod_cidr | default(None) }}"
           labels: "{{ openshift_node_labels | default(None) }}"
           annotations: "{{ openshift_node_annotations | default(None) }}"
   - name: Check status of node certificates
     stat:
-      path: "{{ item }}"
+      path: "/etc/openshift/node/{{ item }}"
     with_items:
-    - "/etc/openshift/node/system:node:{{ openshift.common.hostname }}.crt"
-    - "/etc/openshift/node/system:node:{{ openshift.common.hostname }}.key"
-    - "/etc/openshift/node/system:node:{{ openshift.common.hostname }}.kubeconfig"
-    - "/etc/openshift/node/ca.crt"
-    - "/etc/openshift/node/server.key"
-    - "/etc/openshift/node/server.crt"
+    - "system:node:{{ openshift.common.hostname }}.crt"
+    - "system:node:{{ openshift.common.hostname }}.key"
+    - "system:node:{{ openshift.common.hostname }}.kubeconfig"
+    - ca.crt
+    - server.key
+    - server.crt
     register: stat_result
   - set_fact:
       certs_missing: "{{ stat_result.results | map(attribute='stat.exists')
@@ -56,10 +53,9 @@
   hosts: oo_first_master
   vars:
     nodes_needing_certs: "{{ hostvars
-                             | oo_select_keys(groups['oo_nodes_to_config'])
+                             | oo_select_keys(groups['oo_nodes_to_config']
+                                              | default([]))
                              | oo_filter_list(filter_attr='certs_missing') }}"
-    openshift_nodes: "{{ hostvars
-                         | oo_select_keys(groups['oo_nodes_to_config']) }}"
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
   roles:
   - openshift_node_certificates
@@ -86,7 +82,7 @@
   hosts: oo_nodes_to_config
   vars:
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
-    openshift_node_master_api_url: "{{ hostvars[openshift_first_master].openshift.master.api_url }}"
+    openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -110,15 +106,6 @@
     group_by: key=oo_nodes_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Delete the temporary directory on the master
-  hosts: oo_first_master
-  gather_facts: no
-  vars:
-    sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
-  tasks:
-  - file: name={{ sync_tmpdir }} state=absent
-    changed_when: False
-
 - name: Delete temporary directory on localhost
   hosts: localhost
   connection: local
@@ -143,12 +130,14 @@
                          | oo_select_keys(groups['oo_nodes_to_config'])
                          | oo_collect('openshift.common.hostname') }}"
     openshift_unscheduleable_nodes: "{{ hostvars
-                                        | oo_select_keys(groups['oo_nodes_to_config']) 
+                                        | oo_select_keys(groups['oo_nodes_to_config']
+                                                         | default([]))
                                         | oo_collect('openshift.common.hostname', {'openshift_scheduleable': False}) }}"
   pre_tasks:
   - set_fact:
       openshift_scheduleable_nodes: "{{ hostvars
-                                      | oo_select_keys(groups['oo_nodes_to_config'])
+                                      | oo_select_keys(groups['oo_nodes_to_config']
+                                                       | default([]))
                                       | oo_collect('openshift.common.hostname')
                                       | difference(openshift_unscheduleable_nodes) }}"
   roles:

playbooks/gce/openshift-cluster/config.yml

@@ -1,38 +1,24 @@
 ---
 # TODO: fix firewall related bug with GCE and origin, since GCE is overriding
 # /etc/sysconfig/iptables
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
+
+- hosts: localhost
   gather_facts: no
   vars_files:
   - vars.yml
   tasks:
-  - name: Evaluate oo_masters_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_masters_to_config
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type-{{ cluster_id }}-openshift-master"] | default([])
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_nodes_to_config
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type-{{ cluster_id }}-openshift-node"] | default([])
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ groups['tag_env-host-type-' ~ cluster_id ~ '-openshift-master'][0] }}"
-      groups: oo_first_master
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    when: "'tag_env-host-type-{{ cluster_id }}-openshift-master' in groups"
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
 
 - include: ../../common/openshift-cluster/config.yml
   vars:
+    g_etcd_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_masters_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-master' }}"
+    g_nodes_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-node' }}"
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
     openshift_cluster_id: "{{ cluster_id }}"
     openshift_debug_level: 4
     openshift_deployment_type: "{{ deployment_type }}"
-    openshift_first_master: "{{ groups.oo_first_master.0 }}"
     openshift_hostname: "{{ gce_private_ip }}"

playbooks/gce/openshift-cluster/update.yml

@@ -11,7 +11,9 @@
       groups: oo_hosts_to_update
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type-{{ cluster_id }}-openshift-master"] | union(groups["tag_env-host-type-{{ cluster_id }}-openshift-node"]) | default([])
+    with_items: (groups["tag_env-host-type-{{ cluster_id }}-openshift-master"] | default([]))
+                | union(groups["tag_env-host-type-{{ cluster_id }}-openshift-node"] | default([]))
+                | union(groups["tag_env-host-type-{{ cluster_id }}-openshift-etcd"] | default([]))
 
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 

playbooks/libvirt/openshift-cluster/config.yml

@@ -3,37 +3,22 @@
 # is localhost, so no hostname value (or public_hostname) value is getting
 # assigned
 
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
+- hosts: localhost
   gather_facts: no
   vars_files:
   - vars.yml
   tasks:
-  - name: Evaluate oo_masters_to_config
-    add_host:
-      name: "{{ item }}"
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: oo_masters_to_config
-    with_items: groups["tag_env-host-type-{{ cluster_id }}-openshift-master"] | default([])
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: oo_nodes_to_config
-    with_items: groups["tag_env-host-type-{{ cluster_id }}-openshift-node"] | default([])
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ groups['tag_env-host-type-' ~ cluster_id ~ '-openshift-master'][0] }}"
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: oo_first_master
-    when: "'tag_env-host-type-{{ cluster_id }}-openshift-master' in groups"
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
 
 - include: ../../common/openshift-cluster/config.yml
   vars:
+    g_etcd_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_masters_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-master' }}"
+    g_nodes_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-node' }}"
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
     openshift_cluster_id: "{{ cluster_id }}"
     openshift_debug_level: 4
     openshift_deployment_type: "{{ deployment_type }}"
-    openshift_first_master: "{{ groups.oo_first_master.0 }}"

playbooks/libvirt/openshift-cluster/update.yml

@@ -11,7 +11,9 @@
       groups: oo_hosts_to_update
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type-{{ cluster_id }}-openshift-master"] | union(groups["tag_env-host-type-{{ cluster_id }}-openshift-node"]) | default([])
+    with_items: (groups["tag_env-host-type-{{ cluster_id }}-openshift-master"] | default([]))
+                | union(groups["tag_env-host-type-{{ cluster_id }}-openshift-node"] | default([]))
+                | union(groups["tag_env-host-type-{{ cluster_id }}-openshift-etcd"] | default([]))
 
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 

playbooks/openstack/openshift-cluster/config.yml

@@ -1,35 +1,20 @@
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
+- hosts: localhost
   gather_facts: no
   vars_files:
   - vars.yml
   tasks:
-  - name: Evaluate oo_masters_to_config
-    add_host:
-      name: "{{ item }}"
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: oo_masters_to_config
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | default([])
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: oo_nodes_to_config
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-node"] | default([])
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ groups['tag_env-host-type_' ~ cluster_id ~ '-openshift-master'][0] }}"
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: oo_first_master
-    when: "'tag_env-host-type_{{ cluster_id }}-openshift-master' in groups"
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
 
 - include: ../../common/openshift-cluster/config.yml
   vars:
+    g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
+    g_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
     openshift_cluster_id: "{{ cluster_id }}"
     openshift_debug_level: 4
     openshift_deployment_type: "{{ deployment_type }}"
-    openshift_first_master: "{{ groups.oo_first_master.0 }}"
     openshift_hostname: "{{ ansible_default_ipv4.address }}"

playbooks/openstack/openshift-cluster/update.yml

@@ -11,7 +11,9 @@
       groups: oo_hosts_to_update
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-node"]) | default([])
+    with_items: (groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | default([]))
+                | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-node"] | default([]))
+                | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-etcd"] | default([]))
 
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 

roles/openshift_master/tasks/main.yml

@@ -26,7 +26,7 @@
       console_url: "{{ openshift_master_console_url | default(None) }}"
       console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
       public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
-      etcd_hosts: "{{ groups['etcd'] | default(None)}}"
+      etcd_hosts: "{{ openshift_master_etcd_hosts | default(None)}}"
       etcd_port: "{{ openshift_master_etcd_port | default(None) }}"
       etcd_use_ssl: "{{ openshift_master_etcd_use_ssl | default(None) }}"
       etcd_urls: "{{ openshift_master_etcd_urls | default(None) }}"
@@ -61,16 +61,6 @@
     path: "{{ openshift_master_config_dir }}"
     state: directory
 
-- name: Create the master certificates if they do not already exist
-  command: >
-    {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }}
-      --master={{ openshift.master.api_url }}
-      --public-master={{ openshift.master.public_api_url }}
-      --cert-dir={{ openshift_master_config_dir }} --overwrite=false
-  args:
-    creates: "{{ openshift_master_config_dir }}/master.server.key"
-
 - name: Create the policy file if it does not already exist
   command: >
     {{ openshift.common.admin_binary }} create-bootstrap-policy-file

roles/openshift_master_ca/README.md

@@ -0,0 +1,34 @@
+OpenShift Master CA
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)

roles/openshift_master_ca/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }

roles/openshift_master_ca/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+- name: Install the OpenShift package for admin tooling
+  yum: pkg=openshift state=present
+  register: install_result
+
+- name: Reload generated facts
+  openshift_facts:
+
+- name: Create openshift_master_config_dir if it doesn't exist
+  file:
+    path: "{{ openshift_master_config_dir }}"
+    state: directory
+
+- name: Create the master certificates if they do not already exist
+  command: >
+    {{ openshift.common.admin_binary }} create-master-certs
+      --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }}
+      --master={{ openshift.master.api_url }}
+      --public-master={{ openshift.master.public_api_url }}
+      --cert-dir={{ openshift_master_config_dir }} --overwrite=false
+  args:
+    creates: "{{ openshift_master_config_dir }}/master.server.key"

roles/openshift_master_ca/vars/main.yml

@@ -0,0 +1,5 @@
+---
+openshift_master_config_dir: /etc/openshift/master
+openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
+openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
+openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"

roles/openshift_master_certificates/README.md

@@ -0,0 +1,34 @@
+OpenShift Master Certificates
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)

roles/openshift_master_certificates/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_master_ca }

roles/openshift_master_certificates/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- name: Ensure the generated_configs directory present
+  file:
+    path: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}"
+    state: directory
+    mode: 0700
+  with_items: masters_needing_certs
+
+- file:
+    src: "{{ openshift_master_ca_cert }}"
+    dest: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/ca.crt"
+  with_items: masters_needing_certs
+
+- name: Create the master certificates if they do not already exist
+  command: >
+    {{ openshift.common.admin_binary }} create-master-certs
+      --hostnames={{ item.openshift.common.hostname }},{{ item.openshift.common.public_hostname }}
+      --master={{ item.openshift.master.api_url }}
+      --public-master={{ item.openshift.master.public_api_url }}
+      --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}
+      --overwrite=false
+  args:
+    creates: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/master.server.crt"
+  with_items: masters_needing_certs

roles/openshift_master_certificates/vars/main.yml

@@ -0,0 +1,6 @@
+---
+openshift_generated_configs_dir: /etc/openshift/generated-configs
+openshift_master_config_dir: /etc/openshift/master
+openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
+openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
+openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"

roles/openshift_node/tasks/main.yml

@@ -1,5 +1,11 @@
 ---
 # TODO: allow for overriding default ports where possible
+- fail:
+    msg: This role requres that osn_cluster_dns_domain is set
+  when: osn_cluster_dns_domain is not defined or not osn_cluster_dns_domain
+- fail:
+    msg: This role requres that osn_cluster_dns_ip is set
+  when: osn_cluster_dns_ip is not defined or not osn_cluster_dns_ip
 
 - name: Install OpenShift Node package
   yum: pkg=openshift-node state=present

roles/openshift_node/templates/node.yaml.v1.j2

@@ -1,7 +1,7 @@
 allowDisabledDocker: false
 apiVersion: v1
-dnsDomain: {{ hostvars[openshift_first_master].openshift.dns.domain }}
-dnsIP: {{ hostvars[openshift_first_master].openshift.dns.ip }}
+dnsDomain: {{ osn_cluster_dns_domain }}
+dnsIP: {{ osn_cluster_dns_ip }}
 dockerConfig:
   execHandlerName: ""
 imageConfig:

roles/openshift_node_certificates/tasks/main.yml

@@ -3,6 +3,7 @@
   file:
     path: "{{ openshift_generated_configs_dir }}"
     state: directory
+  when: nodes_needing_certs | length > 0
 
 - name: Generate the node client config
   command: >