Fix etcd scaleup playbook

* Added static pod and rpm based install to scaleup
* Standardized cluster-health check usage
* etcdctl '-C' option is deprecated, use '--endpoints'

playbooks/openshift-etcd/private/remove-etcdv2-data.yml

@@ -6,5 +6,3 @@
   - import_role:
       name: etcd
       tasks_from: remove-etcd-v2-data.yml
-    vars:
-      etcd_peer: "{{ openshift.common.hostname }}"

playbooks/openshift-etcd/private/scaleup.yml

@@ -19,24 +19,31 @@
   hosts: oo_new_etcd_to_config
   serial: 1
   any_errors_fatal: true
-  pre_tasks:
+  tasks:
   - import_role:
       name: etcd
       tasks_from: add_new_member.yml
+    vars:
+      etcd_peer: "{{ hostvars[etcd_ca_host].etcd_ip }}"
+
   - import_role:
       name: etcd
       tasks_from: server_certificates.yml
     vars:
       etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}"
       etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}"
-  tasks:
+
   - import_role:
       name: os_firewall
     when: etcd_add_check.rc == 0
 
+  # Setup etcd as a static pod if collocated with a master
   - import_role:
       name: etcd
-    when: etcd_add_check.rc == 0
+      tasks_from: static.yml
+    when:
+    - etcd_add_check.rc == 0
+    - inventory_hostname in groups['oo_masters']
     vars:
       etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}"
       etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
@@ -44,24 +51,27 @@
       etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}"
       etcd_ca_setup: False
 
-  # etcd_hostname fact is set in add_new_member.yml called above.
-  - name: Verify cluster is stable
-    command: >
-      {{ r_etcd_common_etcdctl_command }}
-                       --cert-file {{ etcd_peer_cert_file }}
-                       --key-file {{ etcd_peer_key_file }}
-                       --ca-file {{ etcd_peer_ca_file }}
-                       -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }}
-                       cluster-health
-    register: scaleup_health
-    retries: 3
-    delay: 30
-    until: scaleup_health.rc == 0
+  - import_role:
+      name: etcd
+      tasks_from: rpm.yml
+    when:
+    - etcd_add_check.rc == 0
+    - not inventory_hostname in groups['oo_masters']
+    vars:
+      etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}"
+      etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+      etcd_initial_cluster_state: "existing"
+      etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}"
+      etcd_ca_setup: False
+
+  - import_role:
+      name: etcd
+      tasks_from: verify_cluster_health.yml
 
 - name: Update master etcd client urls
   hosts: oo_masters_to_config
   serial: 1
-  pre_tasks:
+  tasks:
   - set_fact:
       openshift_master_etcd_hosts: "{{ hostvars
                                        | lib_utils_oo_select_keys(groups['oo_etcd_to_config'] | union(groups['oo_new_etcd_to_config'] | default([]) ))
@@ -78,7 +88,6 @@
     vars:
       l_use_ssl: "{{ openshift_master_etcd_use_ssl | default(True) | bool}}"
 
-  post_tasks:
   - import_role:
       name: openshift_control_plane
       tasks_from: update_etcd_client_urls.yml

playbooks/openshift-etcd/private/server_certificates.yml

@@ -2,7 +2,7 @@
 - name: Create etcd server certificates for etcd hosts
   hosts: oo_etcd_to_config
   any_errors_fatal: true
-  post_tasks:
+  tasks:
     - import_role:
         name: etcd
         tasks_from: server_certificates.yml

playbooks/openshift-etcd/private/upgrade_rpm_members.yml

@@ -10,7 +10,6 @@
       tasks_from: upgrade_rpm.yml
     vars:
       r_etcd_upgrade_version: "{{ etcd_upgrade_version }}"
-      etcd_peer: "{{ openshift.common.hostname }}"
     when:
     - etcd_rpm_version.stdout | default('99') is version(etcd_upgrade_version, '<')
     - ansible_distribution == 'RedHat'

playbooks/openshift-etcd/private/upgrade_static.yml

@@ -6,6 +6,4 @@
   - import_role:
       name: etcd
       tasks_from: upgrade_static.yml
-    vars:
-      etcd_peer: "{{ openshift.common.hostname }}"
     when: inventory_hostname in groups['oo_masters']

roles/etcd/defaults/main.yaml

@@ -83,9 +83,8 @@ etcd_listen_peer_urls: "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_
 etcd_advertise_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}"
 etcd_listen_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}"
 
-# required role variable
-#etcd_peer: 127.0.0.1
-etcdctlv2: "{{ r_etcd_common_etcdctl_command }} --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://{{ etcd_peer }}:{{ etcd_client_port }}"
+etcd_peer: "{{ openshift.common.hostname }}"
+etcdctlv2: "{{ r_etcd_common_etcdctl_command }} --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoints {{ etcd_peer_url_scheme }}://{{ etcd_peer }}:{{ etcd_client_port }}"
 
 etcd_service: etcd
 # Location of the service file is fixed and not meant to be changed

roles/etcd/etcdctl.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-# Sets up handy aliases for etcd, need etcdctl2 and etcdctl3 because
-# command flags are different between the two. Should work on stand
-# alone etcd hosts and master + etcd hosts too because we use the peer keys.
-etcdctl2() {
- /usr/bin/etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://`hostname`:2379 ${@}
-}
-
-etcdctl3() {
- ETCDCTL_API=3 /usr/bin/etcdctl --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt --endpoints https://`hostname`:2379 ${@}
-}

roles/etcd/tasks/add_new_member.yml

@@ -3,13 +3,7 @@
 - import_tasks: set_facts.yml
 
 - name: Add new etcd members to cluster
-  command: >
-    {{ r_etcd_common_etcdctl_command }}
-                     --cert-file {{ etcd_peer_cert_file }}
-                     --key-file {{ etcd_peer_key_file }}
-                     --ca-file {{ etcd_peer_ca_file }}
-                     -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_ip }}:{{ etcd_client_port }}
-                     member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}
+  command: "{{ etcdctlv2 }} member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}"
   delegate_to: "{{ etcd_ca_host }}"
   failed_when:
   - etcd_add_check.rc == 1

roles/etcd/tasks/remove-etcd-v2-data.yml

@@ -1,10 +1,5 @@
 ---
-- name: Verify cluster is healthy pre-upgrade
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: cluster_health
-  retries: 30
-  delay: 6
-  until: cluster_health.rc == 0
+- import_tasks: verify_cluster_health.yml
 
 - name: Check migrated status
   command: "{{ etcdctlv2 }} get /kubernetes.io"

roles/etcd/tasks/static.yml

@@ -84,7 +84,7 @@
       - "{{ etcd_peer_key_file }}"
       - "--ca-file"
       - "{{ etcd_peer_ca_file }}"
-      - "-C"
+      - "--endpoints"
       - "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}"
       - "cluster-health"
   with_items:

roles/etcd/tasks/upgrade_rpm.yml

@@ -8,12 +8,7 @@
 # RHEL 7.3.3 with etcd-3.1.0-2.el7
 # RHEL 7.3.2 with etcd-3.0.15-1.el7
 
-- name: Verify cluster is healthy pre-upgrade
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: cluster_health
-  retries: 30
-  delay: 6
-  until: cluster_health.rc == 0
+- import_tasks: verify_cluster_health.yml
 
 - set_fact:
     l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}"
@@ -30,9 +25,4 @@
 - name: restart etcd
   command: "{{ l_etcd_restart_command }}"
 
-- name: Verify cluster is healthy
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: etcdctl
-  until: etcdctl.rc == 0
-  retries: 3
-  delay: 10
+- import_tasks: verify_cluster_health.yml

roles/etcd/tasks/upgrade_static.yml

@@ -3,12 +3,7 @@
 
 # INPUT r_etcd_upgrade_version
 
-- name: Verify cluster is healthy pre-upgrade
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: cluster_health
-  retries: 30
-  delay: 6
-  until: cluster_health.rc == 0
+- import_tasks: verify_cluster_health.yml
 
 - name: Check for old etcd service files
   stat:
@@ -54,9 +49,4 @@
 - set_fact:
     r_etcd_common_etcd_runtime: static_pod
 
-- name: Verify cluster is healthy
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: etcdctl
-  until: etcdctl.rc == 0 and 'stopped' not in etcdctl.stderr
-  retries: 30
-  delay: 10
+- import_tasks: verify_cluster_health.yml

roles/etcd/tasks/verify_cluster_health.yml

@@ -0,0 +1,9 @@
+---
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: cluster_health
+  retries: 30
+  delay: 6
+  until:
+  - cluster_health.rc == 0
+  - ('stopped' not in cluster_health.stderr)

roles/etcd/templates/etcdctl.sh.j2

@@ -4,7 +4,7 @@
 # alone etcd hosts and master + etcd hosts too because we use the peer keys.
 etcdctl2() {
 
- cmd="ETCDCTL_API=2 etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://`hostname`:2379 ${@}"
+ cmd="ETCDCTL_API=2 etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoints https://`hostname`:2379 ${@}"
  if [[ -f /usr/local/bin/master-exec ]]; then
    /usr/local/bin/master-exec etcd etcd /bin/sh -c "$cmd"
  else