Add etcd role that builds out basic etcd cluster

- Add initial etcd role
- Add etcd playbook to create etcd client certs
- Hookup master to etcd

playbooks/byo/config.yml

@@ -3,6 +3,10 @@
   include: openshift-master/config.yml
   when: groups.masters is defined and groups.masters
 
+- name: Run the openshift-etcd playbook
+  include: openshift-etcd/config.yml
+  when: groups.etcd is defined and groups.etcd
+
 - name: Run the openshift-node config playbook
   include: openshift-node/config.yml
   when: groups.nodes is defined and groups.nodes and groups.masters is defined and groups.masters

playbooks/byo/etcd/config.yml

@@ -0,0 +1,7 @@
+## deploys a simple etcd cluster, this cluster does not provide client side ssl
+## and cannot be used directly for openshift. This should only be used for testing.
+---
+- name: Configure etcd
+  hosts: etcd
+  roles:
+  - etcd

playbooks/byo/etcd/filter_plugins

@@ -0,0 +1 @@
+../../../filter_plugins/

playbooks/byo/etcd/roles

@@ -0,0 +1 @@
+../../../roles/

playbooks/byo/openshift-etcd/config.yml

@@ -0,0 +1,20 @@
+---
+- name: Populate oo_etcd_hosts_to_config and oo_first_master host groups
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - name: Evaluate oo_etcd_hosts_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_etcd_hosts_to_config
+    with_items: groups.etcd
+  - name: Evaluate oo_first_master
+    add_host:
+      name: "{{ item }}"
+      groups: oo_first_master
+    with_items: groups.masters.0
+
+
+- include: ../../common/openshift-etcd/config.yml
+  vars:
+    openshift_first_master: "{{ groups.masters.0 }}"

playbooks/byo/openshift-etcd/filter_plugins

@@ -0,0 +1 @@
+../../../filter_plugins

playbooks/byo/openshift-etcd/roles

@@ -0,0 +1 @@
+../../../roles

playbooks/common/openshift-etcd/config.yml

@@ -0,0 +1,106 @@
+---
+- name: Gather and set facts for etcd hosts
+  hosts: oo_etcd_hosts_to_config
+  roles:
+  - openshift_facts
+  tasks:
+  - openshift_facts:
+      role: common
+      local_facts:
+        hostname: "{{ openshift_hostname | default(None) }}"
+  - name: Check for etcd certificates
+    stat:
+      path: "{{ item }}"
+    with_items:
+    - "/etc/etcd/ca.crt"
+    - "/etc/etcd/client.crt"
+    - "/etc/etcd/client.key"
+    - "/etc/etcd/peer-ca.crt"
+    - "/etc/etcd/peer.crt"
+    - "/etc/etcd/peer.key"
+    register: g_etcd_certs_stat
+  - set_fact:
+      etcd_certs_missing: "{{ g_etcd_certs_stat.results | map(attribute='stat.exists')
+                              | list | intersect([false])}}"
+      etcd_subdir: etcd-{{ openshift.common.hostname }}
+      etcd_dir: /etc/openshift/generated-configs/etcd-{{ openshift.common.hostname }}
+      etcd_cert_dir: /etc/etcd
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_etcd_mktemp
+    changed_when: False
+
+- name: Create etcd certs
+  hosts: oo_first_master
+  vars:
+    etcd_hosts_needing_certs: "{{ hostvars
+                             | oo_select_keys(groups['oo_etcd_hosts_to_config'])
+                             | oo_filter_list(filter_attr='etcd_certs_missing') }}"
+    etcd_hosts: "{{ hostvars
+                         | oo_select_keys(groups['oo_etcd_hosts_to_config']) }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}"
+  roles:
+  - openshift_etcd_certs
+  post_tasks:
+  - name: Create a tarball of the etcd certs
+    command: >
+      tar -czvf {{ item.etcd_dir }}.tgz
+        -C {{ item.etcd_dir }} .
+    args:
+      creates: "{{ item.etcd_dir }}.tgz"
+    with_items: etcd_hosts_needing_certs
+
+  - name: Retrieve the etcd cert tarballs from the master
+    fetch:
+      src: "{{ item.etcd_dir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: etcd_hosts_needing_certs
+
+- name: Deploy etcd
+  hosts: oo_etcd_hosts_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}"
+    etcd_url_scheme: https
+  pre_tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: "{{ etcd_cert_dir }}"
+      state: directory
+  - name: Unarchive the tarball on the node
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ etcd_subdir }}.tgz"
+      dest: "{{ etcd_cert_dir }}"
+    when: etcd_certs_missing
+  - file: path=/etc/etcd/client.crt mode=0600 owner=etcd group=etcd
+  - file: path=/etc/etcd/client.key mode=0600 owner=etcd group=etcd
+  - file: path=/etc/etcd/ca.crt mode=0644 owner=etcd group=etcd
+  roles:
+  - etcd
+
+- name: Delete the temporary directory on the master
+  hosts: oo_first_master
+  gather_facts: no
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}"
+  tasks:
+  - file: name={{ sync_tmpdir }} state=absent
+    changed_when: False
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - file: name={{ g_etcd_mktemp.stdout }} state=absent
+    changed_when: False

playbooks/common/openshift-etcd/filter_plugins

@@ -0,0 +1 @@
+../../../filter_plugins

playbooks/common/openshift-etcd/roles

@@ -0,0 +1 @@
+../../../roles/

roles/etcd/README.md

@@ -0,0 +1,39 @@
+Role Name
+=========
+
+Configures an etcd cluster for an arbitrary number of hosts
+
+Requirements
+------------
+
+This role assumes it's being deployed on a RHEL/Fedora based host with package
+named 'etcd' available via yum.
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+None
+
+Example Playbook
+----------------
+
+    - hosts: etcd
+      roles:
+         - { etcd }
+
+License
+-------
+
+MIT
+
+Author Information
+------------------
+
+Scott Dodson <sdodson@redhat.com>
+Adapted from https://github.com/retr0h/ansible-etcd for use on RHEL/Fedora. We
+should at some point submit a PR to merge this with that module.

roles/etcd/defaults/main.yaml

@@ -0,0 +1,28 @@
+---
+etcd_interface: eth0
+etcd_client_port: 2379
+etcd_peer_port: 2380
+etcd_peers_group: etcd
+etcd_url_scheme: http
+etcd_peer_url_scheme: http
+etcd_ca_file: /etc/etcd/ca.crt
+etcd_cert_file: /etc/etcd/client.crt
+etcd_key_file: /etc/etcd/client.key
+etcd_peer_ca_file: /etc/etcd/ca.crt
+etcd_peer_cert_file: /etc/etcd/peer.crt
+etcd_peer_key_file: /etc/etcd/peer.key
+
+etcd_initial_cluster_state: new
+etcd_initial_cluster_token: etcd-cluster-1
+
+etcd_initial_advertise_peer_urls: "{{ etcd_peer_url_scheme }}://{{ hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_peer_port }}"
+etcd_listen_peer_urls: "{{ etcd_peer_url_scheme }}://{{ hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_peer_port }}"
+etcd_advertise_client_urls: "{{ etcd_url_scheme }}://{{ hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_client_port }}"
+etcd_listen_client_urls: "{{ etcd_url_scheme }}://{{ hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_client_port }}"
+
+etcd_data_dir: /var/lib/etcd/
+os_firewall_allow:
+- service: etcd
+  port: "{{etcd_client_port}}/tcp"
+- service: etcd peering
+  port: "{{ etcd_peer_port }}/tcp"

roles/etcd/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: restart etcd
+  service: name=etcd state=restarted

roles/etcd/meta/main.yml

@@ -0,0 +1,17 @@
+---
+# This module is based on https://github.com/retr0h/ansible-etcd with most
+# changes centered around installing from a pre-existing rpm
+# TODO: Extend https://github.com/retr0h/ansible-etcd rather than forking
+galaxy_info:
+  author: Scott Dodson
+  description: etcd management
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.2
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system

roles/etcd/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: Install etcd
+  yum: pkg=etcd state=present disable_gpg_check=yes
+
+- name: Write etcd global config file
+  template:
+    src: etcd.conf.j2
+    dest: /etc/etcd/etcd.conf
+  notify:
+    - restart etcd
+
+- name: Enable etcd
+  service:
+    name: etcd
+    state: started
+    enabled: yes

roles/etcd/templates/etcd.conf.j2

@@ -0,0 +1,46 @@
+{% macro initial_cluster() -%}
+{% for host in groups[etcd_peers_group] -%}
+{% if loop.last -%}
+{{ host }}={{ etcd_peer_url_scheme }}://{{ hostvars[host]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_peer_port }}
+{%- else -%}
+{{ host }}={{ etcd_peer_url_scheme }}://{{ hostvars[host]['ansible_' + etcd_interface]['ipv4']['address'] }}:{{ etcd_peer_port }},
+{%- endif -%}
+{% endfor -%}
+{% endmacro -%}
+
+ETCD_NAME={{ inventory_hostname }}
+ETCD_DATA_DIR={{ etcd_data_dir }}
+#ETCD_SNAPSHOT_COUNTER="10000"
+#ETCD_HEARTBEAT_INTERVAL="100"
+#ETCD_ELECTION_TIMEOUT="1000"
+ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }}
+ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }}
+#ETCD_MAX_SNAPSHOTS="5"
+#ETCD_MAX_WALS="5"
+#ETCD_CORS=""
+#
+#[cluster]
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }}
+ETCD_INITIAL_CLUSTER={{ initial_cluster() }}
+ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }}
+ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }}
+ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }}
+#ETCD_DISCOVERY=""
+#ETCD_DISCOVERY_SRV=""
+#ETCD_DISCOVERY_FALLBACK="proxy"
+#ETCD_DISCOVERY_PROXY=""
+#
+#[proxy]
+#ETCD_PROXY="off"
+#
+#[security]
+{% if etcd_url_scheme == 'https' -%}
+ETCD_CA_FILE={{ etcd_ca_file }}
+ETCD_CERT_FILE={{ etcd_cert_file }}
+ETCD_KEY_FILE={{ etcd_key_file }}
+{% endif -%}
+{% if etcd_peer_url_scheme == 'https' -%}
+ETCD_PEER_CA_FILE={{ etcd_peer_ca_file }}
+ETCD_PEER_CERT_FILE={{ etcd_peer_cert_file }}
+ETCD_PEER_KEY_FILE={{ etcd_peer_key_file }}
+{% endif -%}

roles/openshift_etcd_certs/README.md

@@ -0,0 +1,34 @@
+OpenShift etcd certs
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Scott Dodson (sdodson@redhat.com)

roles/openshift_etcd_certs/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Scott Dodson
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }

roles/openshift_etcd_certs/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+- name: Create openshift_generated_configs_dir if it doesn't exist
+  file:
+    path: "{{ openshift_generated_configs_dir }}"
+    state: directory
+
+- name: Create openshift_generated_configs_dir for each etcd host
+  file:
+    path: "{{ openshift_generated_configs_dir }}/etcd-{{ item.openshift.common.hostname}}"
+    state: directory
+  with_items: etcd_hosts_needing_certs
+
+- name: Generate the etcd client side certs
+  delegate_to: "{{ openshift_first_master }}"
+  command: >
+    {{ openshift.common.admin_binary }} create-server-cert
+      --cert=client.crt --key=client.key --overwrite=true
+      --hostnames={{ [item.openshift.common.hostname, item.openshift.common.public_hostname, item.openshift.common.ip]|unique|join(",") }}
+      --signer-cert={{ openshift_master_ca_cert }}
+      --signer-key={{ openshift_master_ca_key }}
+      --signer-serial={{ openshift_master_ca_serial }}
+  args:
+    chdir: "{{ openshift_generated_configs_dir }}/etcd-{{ item.openshift.common.hostname }}"
+    creates: "{{ openshift_generated_configs_dir }}/etcd-{{ item.openshift.common.hostname }}/client.crt"
+  with_items: etcd_hosts_needing_certs
+
+- name: Copy CA cert
+  delegate_to: "{{ openshift_first_master }}"
+  command: "cp {{ openshift_master_ca_cert }} ."
+  args:
+    chdir: "{{ openshift_generated_configs_dir }}/etcd-{{ item.openshift.common.hostname }}"
+    creates: "{{ openshift_generated_configs_dir }}/etcd-{{ item.openshift.common.hostname }}/ca.crt"
+  with_items: etcd_hosts_needing_certs

roles/openshift_etcd_certs/vars/main.yml

@@ -0,0 +1,8 @@
+---
+openshift_node_config_dir: /etc/openshift/node
+openshift_master_config_dir: /etc/openshift/master
+openshift_generated_configs_dir: /etc/openshift/generated-configs
+openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
+openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
+openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"
+openshift_kube_api_version: v1beta3

roles/openshift_facts/library/openshift_facts.py

@@ -366,13 +366,24 @@ def set_url_facts_if_unset(facts):
         console_port = facts['master']['console_port']
         console_path = facts['master']['console_path']
         etcd_use_ssl = facts['master']['etcd_use_ssl']
+        etcd_hosts = facts['master']['etcd_hosts']
         etcd_port = facts['master']['etcd_port'],
         hostname = facts['common']['hostname']
         public_hostname = facts['common']['public_hostname']
 
         if 'etcd_urls' not in facts['master']:
-            facts['master']['etcd_urls'] = [format_url(etcd_use_ssl, hostname,
-                                                       etcd_port)]
+            etcd_urls = []
+            if etcd_hosts != '':
+                etcd_port = 2379
+                facts['master']['etcd_port'] = etcd_port
+                facts['master']['embedded_etcd'] = False
+                for host in etcd_hosts:
+                    etcd_urls.append(format_url(etcd_use_ssl, host,
+                                                etcd_port))
+            else:
+                etcd_urls = [format_url(etcd_use_ssl, hostname,
+                                        etcd_port)]
+            facts['master']['etcd_urls'] = etcd_urls
         if 'api_url' not in facts['master']:
             facts['master']['api_url'] = format_url(api_use_ssl, hostname,
                                                     api_port)
@@ -695,7 +706,7 @@ class OpenShiftFacts(object):
         if 'master' in roles:
             master = dict(api_use_ssl=True, api_port='8443',
                           console_use_ssl=True, console_path='/console',
-                          console_port='8443', etcd_use_ssl=True,
+                          console_port='8443', etcd_use_ssl=True, etcd_hosts='',
                           etcd_port='4001', portal_net='172.30.0.0/16',
                           embedded_etcd=True, embedded_kube=True,
                           embedded_dns=True, dns_port='53',

roles/openshift_master/tasks/main.yml

@@ -31,6 +31,7 @@
       console_url: "{{ openshift_master_console_url | default(None) }}"
       console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
       public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
+      etcd_hosts: "{{ groups['etcd'] | default(None)}}"
       etcd_port: "{{ openshift_master_etcd_port | default(None) }}"
       etcd_use_ssl: "{{ openshift_master_etcd_use_ssl | default(None) }}"
       etcd_urls: "{{ openshift_master_etcd_urls | default(None) }}"