Convert etcd to static pods on upgrade

This commit converts any etcd hosts that are colocated with
masters to static pods.  Additionally, we stop, disable and mask
any existing etcd services.

This commit also ensures that rpm-standalone etcd hosts are
upgraded.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1571054

playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml

@@ -2,7 +2,6 @@
 ###############################################################################
 # Upgrade Control Plane
 ###############################################################################
-
 - name: Backup and upgrade etcd
   import_playbook: ../../../openshift-etcd/private/upgrade_main.yml
 

playbooks/openshift-etcd/private/upgrade_image_members.yml

@@ -1,7 +1,6 @@
 ---
 # INPUT etcd_upgrade_version
 # INPUT etcd_container_version
-# INPUT openshift_is_containerized
 - name: Upgrade containerized hosts to {{ etcd_upgrade_version }}
   hosts: oo_etcd_hosts_to_upgrade
   serial: 1
@@ -14,4 +13,4 @@
       etcd_peer: "{{ openshift.common.hostname }}"
     when:
     - etcd_container_version | default('99') is version_compare(etcd_upgrade_version,'<')
-    - openshift_is_containerized | bool
+    - openshift_is_atomic | bool

playbooks/openshift-etcd/private/upgrade_main.yml

@@ -51,9 +51,17 @@
       name: etcd
       tasks_from: drop_etcdctl.yml
 
+# We only want to upgrade with the old method if the host is not part of
+# master host.
 - name: Perform etcd upgrade
   import_playbook: upgrade_step.yml
-  when: openshift_etcd_upgrade | default(true) | bool
+  when:
+  - openshift_etcd_upgrade | default(true) | bool
+  - inventory_hostname not in groups['oo_masters']
+
+# Upgrade / convert etcd to static pods
+- name: Upgrade etcd static pods
+  import_playbook: upgrade_static.yml
 
 - name: Backup etcd
   import_playbook: upgrade_backup.yml

playbooks/openshift-etcd/private/upgrade_static.yml

@@ -0,0 +1,11 @@
+---
+- name: Upgrade etcd static pods
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - import_role:
+      name: etcd
+      tasks_from: upgrade_static.yml
+    vars:
+      etcd_peer: "{{ openshift.common.hostname }}"
+    when: inventory_hostname in groups['oo_masters']

playbooks/openshift-etcd/private/upgrade_step.yml

@@ -8,42 +8,6 @@
 
 - import_playbook: upgrade_rpm_members.yml
   vars:
-    etcd_upgrade_version: '2.1'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '2.2'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '2.2.5'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '2.3'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '2.3.7'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '3.0'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '3.0.15'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '3.1'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '3.1.3'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
     etcd_upgrade_version: '3.2'
 
 - import_playbook: upgrade_image_members.yml

roles/etcd/tasks/upgrade/upgrade_image.yml

@@ -25,22 +25,6 @@
 - name: restart etcd
   command: /usr/local/bin/master-restart etcd
 
-## TODO: probably should just move this into the backup playbooks, also this
-## will fail on atomic host. We need to revisit how to do etcd backups there as
-## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1)
-- name: Detecting Atomic Host Operating System
-  stat:
-    path: /run/ostree-booted
-  register: l_ostree_booted
-
-- name: Upgrade etcd for etcdctl when not atomic
-  package:
-    name: etcd
-    state: latest
-  when: not l_ostree_booted.stat.exists | bool
-  register: result
-  until: result is succeeded
-
 - name: Verify cluster is healthy
   command: "{{ etcdctlv2 }} cluster-health"
   register: etcdctl

roles/etcd/tasks/upgrade/upgrade_static.yml

@@ -1,27 +0,0 @@
----
-# PREREQ Node service is ready to run static pods
-
-# INPUT r_etcd_upgrade_version
-- name: Verify cluster is healthy pre-upgrade
-  command: "{{ etcdctlv2 }} cluster-health"
-
-- name: Stop etcd
-  systemd:
-    name: "{{ etcd_service }}"
-    state: stopped
-  when: not openshift.node.bootstrapped
-
-- name: Remove old service file
-  file:
-    path: "{{ etcd_service }}"
-    state: absent
-
-- name: Configure static definition
-  import_tasks: ../static.yml
-
-- name: Verify cluster is healthy
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: etcdctl
-  until: etcdctl.rc == 0
-  retries: 3
-  delay: 10

roles/etcd/tasks/upgrade_image.yml

@@ -1,6 +1,2 @@
 ---
-- include_tasks: upgrade/upgrade_static.yml
-  when: openshift_master_bootstrap_enabled | default(False) | bool
-
 - include_tasks: upgrade/upgrade_image.yml
-  when: not openshift_master_bootstrap_enabled | default(False) | bool

roles/etcd/tasks/upgrade_rpm.yml

@@ -1,6 +1,2 @@
 ---
-- include_tasks: upgrade/upgrade_static.yml
-  when: openshift_master_bootstrap_enabled | default(False) | bool
-
 - include_tasks: upgrade/upgrade_rpm.yml
-  when: not openshift_master_bootstrap_enabled | default(False) | bool

roles/etcd/tasks/upgrade_static.yml

@@ -0,0 +1,45 @@
+---
+# PREREQ Node service is ready to run static pods
+
+# INPUT r_etcd_upgrade_version
+- name: Verify cluster is healthy pre-upgrade
+  command: "{{ etcdctlv2 }} cluster-health"
+
+- name: Remove old etcd service files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - "/etc/systemd/system/etcd.service"
+  - "/etc/systemd/system/etcd_container.service"
+
+# We removed the ability to detect what was previously 'containerized'
+# Need to stop and disable this service, but might not be present.
+- name: Stop, disable old etcd services
+  systemd:
+    name: "{{ item }}"
+    state: stopped
+    enabled: no
+  failed_when: False
+  with_items:
+  - etcd
+  - etcd_container
+
+- name: Mask old etcd services
+  command: "systemctl mask {{ etcd_service }}"
+  with_items:
+  - etcd
+  - etcd_container
+
+- name: Reload systemd daemon
+  command: "systemctl daemon-reload"
+
+- name: Configure static definition
+  import_tasks: static.yml
+
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 30
+  delay: 10