Merge pull request #8206 from mgugino-upstream-stage/etcd-to-pod-upgrade

Convert etcd to static pods on upgrade

playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml

@@ -2,7 +2,6 @@
 ###############################################################################
 # Upgrade Control Plane
 ###############################################################################
-
 - name: Backup and upgrade etcd
   import_playbook: ../../../openshift-etcd/private/upgrade_main.yml
 

playbooks/openshift-etcd/private/upgrade_image_members.yml

@@ -1,7 +1,6 @@
 ---
 # INPUT etcd_upgrade_version
 # INPUT etcd_container_version
-# INPUT openshift_is_containerized
 - name: Upgrade containerized hosts to {{ etcd_upgrade_version }}
   hosts: oo_etcd_hosts_to_upgrade
   serial: 1
@@ -14,4 +13,4 @@
       etcd_peer: "{{ openshift.common.hostname }}"
     when:
     - etcd_container_version | default('99') is version_compare(etcd_upgrade_version,'<')
-    - openshift_is_containerized | bool
+    - openshift_is_atomic | bool

playbooks/openshift-etcd/private/upgrade_main.yml

@@ -51,9 +51,17 @@
       name: etcd
       tasks_from: drop_etcdctl.yml
 
+# We only want to upgrade with the old method if the host is not part of
+# master host.
 - name: Perform etcd upgrade
   import_playbook: upgrade_step.yml
-  when: openshift_etcd_upgrade | default(true) | bool
+  when:
+  - openshift_etcd_upgrade | default(true) | bool
+  - inventory_hostname not in groups['oo_masters']
+
+# Upgrade / convert etcd to static pods
+- name: Upgrade etcd static pods
+  import_playbook: upgrade_static.yml
 
 - name: Backup etcd
   import_playbook: upgrade_backup.yml

playbooks/openshift-etcd/private/upgrade_static.yml

@@ -0,0 +1,11 @@
+---
+- name: Upgrade etcd static pods
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - import_role:
+      name: etcd
+      tasks_from: upgrade_static.yml
+    vars:
+      etcd_peer: "{{ openshift.common.hostname }}"
+    when: inventory_hostname in groups['oo_masters']

playbooks/openshift-etcd/private/upgrade_step.yml

@@ -8,42 +8,6 @@
 
 - import_playbook: upgrade_rpm_members.yml
   vars:
-    etcd_upgrade_version: '2.1'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '2.2'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '2.2.5'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '2.3'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '2.3.7'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '3.0'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '3.0.15'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
-    etcd_upgrade_version: '3.1'
-
-- import_playbook: upgrade_image_members.yml
-  vars:
-    etcd_upgrade_version: '3.1.3'
-
-- import_playbook: upgrade_rpm_members.yml
-  vars:
     etcd_upgrade_version: '3.2'
 
 - import_playbook: upgrade_image_members.yml

roles/etcd/tasks/upgrade/upgrade_image.yml

@@ -25,22 +25,6 @@
 - name: restart etcd
   command: /usr/local/bin/master-restart etcd
 
-## TODO: probably should just move this into the backup playbooks, also this
-## will fail on atomic host. We need to revisit how to do etcd backups there as
-## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1)
-- name: Detecting Atomic Host Operating System
-  stat:
-    path: /run/ostree-booted
-  register: l_ostree_booted
-
-- name: Upgrade etcd for etcdctl when not atomic
-  package:
-    name: etcd
-    state: latest
-  when: not l_ostree_booted.stat.exists | bool
-  register: result
-  until: result is succeeded
-
 - name: Verify cluster is healthy
   command: "{{ etcdctlv2 }} cluster-health"
   register: etcdctl

roles/etcd/tasks/upgrade/upgrade_static.yml

@@ -1,27 +0,0 @@
----
-# PREREQ Node service is ready to run static pods
-
-# INPUT r_etcd_upgrade_version
-- name: Verify cluster is healthy pre-upgrade
-  command: "{{ etcdctlv2 }} cluster-health"
-
-- name: Stop etcd
-  systemd:
-    name: "{{ etcd_service }}"
-    state: stopped
-  when: not openshift.node.bootstrapped
-
-- name: Remove old service file
-  file:
-    path: "{{ etcd_service }}"
-    state: absent
-
-- name: Configure static definition
-  import_tasks: ../static.yml
-
-- name: Verify cluster is healthy
-  command: "{{ etcdctlv2 }} cluster-health"
-  register: etcdctl
-  until: etcdctl.rc == 0
-  retries: 3
-  delay: 10

roles/etcd/tasks/upgrade_image.yml

@@ -1,6 +1,2 @@
 ---
-- include_tasks: upgrade/upgrade_static.yml
-  when: openshift_master_bootstrap_enabled | default(False) | bool
-
 - include_tasks: upgrade/upgrade_image.yml
-  when: not openshift_master_bootstrap_enabled | default(False) | bool

roles/etcd/tasks/upgrade_rpm.yml

@@ -1,6 +1,2 @@
 ---
-- include_tasks: upgrade/upgrade_static.yml
-  when: openshift_master_bootstrap_enabled | default(False) | bool
-
 - include_tasks: upgrade/upgrade_rpm.yml
-  when: not openshift_master_bootstrap_enabled | default(False) | bool

roles/etcd/tasks/upgrade_static.yml

@@ -0,0 +1,45 @@
+---
+# PREREQ Node service is ready to run static pods
+
+# INPUT r_etcd_upgrade_version
+- name: Verify cluster is healthy pre-upgrade
+  command: "{{ etcdctlv2 }} cluster-health"
+
+- name: Remove old etcd service files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - "/etc/systemd/system/etcd.service"
+  - "/etc/systemd/system/etcd_container.service"
+
+# We removed the ability to detect what was previously 'containerized'
+# Need to stop and disable this service, but might not be present.
+- name: Stop, disable old etcd services
+  systemd:
+    name: "{{ item }}"
+    state: stopped
+    enabled: no
+  failed_when: False
+  with_items:
+  - etcd
+  - etcd_container
+
+- name: Mask old etcd services
+  command: "systemctl mask {{ etcd_service }}"
+  with_items:
+  - etcd
+  - etcd_container
+
+- name: Reload systemd daemon
+  command: "systemctl daemon-reload"
+
+- name: Configure static definition
+  import_tasks: static.yml
+
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 30
+  delay: 10