Make etcd containerized upgrade stepwise

playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml

@@ -2,13 +2,14 @@
 - name: Verify cluster is healthy pre-upgrade
   command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
 
-- name: Update systemd unit
-  openshift_facts:
-    role: etcd
+- name: Get current image
+  shell: grep 'ExecStart=' /etc/systemd/system/etcd_container.service | awk '{print $NF}'
+  register: current_image
 
-- name: Set etcd3 image name
+- name: Set new_etcd_image
   set_fact:
-    new_etcd_image: "{{ openshift.etcd.etcd_image | regex_replace('/etcd$','/etcd3') }}"
+    new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd3:' ~ upgrade_version ) if upgrade_version | version_compare('3.0','>=')
+                        else current_image.stdout.split(':')[0] ~ ':' ~ upgrade_version }}"
 
 - name: Pull new etcd image
   command: "docker pull {{ new_etcd_image }}"
@@ -16,15 +17,22 @@
 - name: Update to latest etcd image
   replace:
     dest: /etc/systemd/system/etcd_container.service
-    regexp: "{{ openshift.etcd.etcd_image }}$"
+    regexp: "{{ current_image.stdout }}$"
     replace: "{{ new_etcd_image }}"
 
-- name: Update etcd package not excluding etcd3
+- name: Restart etcd_container
   systemd:
     name: etcd_container
     daemon_reload: yes
     state: restarted
 
+## TODO: probably should just move this into the backup playbooks, also this
+## will fail on atomic host. We need to revisit how to do etcd backups there as
+## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1)
+- name: Upgrade etcdctl to 3.0 if necessary and able
+  action: "{{ ansible_pkg_mgr }} name=etcd ensure=latest"
+  when: openshift.common.is_containerized | bool and not openshift.common.is_atomic | bool and upgrade_version | version_compare('3.0','>=')
+
 - name: Verify cluster is healthy
   command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
   register: etcdctl
@@ -36,4 +44,4 @@
   openshift_facts:
     role: etcd
     local_facts:
-    - etcd_image: "{{ new_etcd_image }}"
+      etcd_image: "{{ new_etcd_image }}"

playbooks/common/openshift-cluster/upgrades/etcd/main.yml

@@ -33,11 +33,16 @@
 - name: Determine etcd version
   hosts: etcd_hosts_to_upgrade
   tasks:
-  - name: Record etcd version
+  - name: Record RPM based etcd version
     command: rpm -qa --qf '%{version}' etcd\*
     register: etcd_installed_version
     failed_when: false
     when: not openshift.common.is_containerized | bool
+  - name: Record containerized etcd version
+    command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\*
+    register: etcd_installed_version
+    failed_when: false
+    when: openshift.common.is_containerized | bool
 
 # I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
 # through hosts, then loop through tasks only when appropriate
@@ -50,7 +55,7 @@
   - include: rhel_tasks.yml
     when: etcd_installed_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
 
-- name: Upgrade to 2.2
+- name: Upgrade RPM hosts to 2.2
   hosts: etcd_hosts_to_upgrade
   serial: 1
   vars:
@@ -59,7 +64,16 @@
   - include: rhel_tasks.yml
     when: etcd_installed_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
 
-- name: Upgrade to 2.3
+- name: Upgrade containerized hosts to 2.2.5
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: 2.2.5
+  tasks:
+  - include: containerized_tasks.yml
+    when: etcd_installed_version.stdout | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool
+
+- name: Upgrade RPM hosts to 2.3
   hosts: etcd_hosts_to_upgrade
   serial: 1
   vars:
@@ -68,7 +82,16 @@
   - include: rhel_tasks.yml
     when: etcd_installed_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
 
-- name: Upgrade to 3.0
+- name: Upgrade containerized hosts to 2.3.7
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: 2.3.7
+  tasks:
+  - include: containerized_tasks.yml
+    when: etcd_installed_version.stdout | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool
+
+- name: Upgrade RPM hosts to 3.0
   hosts: etcd_hosts_to_upgrade
   serial: 1
   vars:
@@ -77,19 +100,21 @@
   - include: rhel_tasks.yml
     when: etcd_installed_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
 
-- name: Upgrade fedora to latest
+- name: Upgrade containerized hosts to etcd3 image
   hosts: etcd_hosts_to_upgrade
   serial: 1
+  vars:
+    upgrade_version: 3.0.3
   tasks:
-  - include: fedora_tasks.yml
-    when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool
+  - include: containerized_tasks.yml
+    when: etcd_installed_version.stdout | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool
 
-- name: Upgrade containerized hosts to etcd3 image
+- name: Upgrade fedora to latest
   hosts: etcd_hosts_to_upgrade
   serial: 1
   tasks:
-  - include: containerized_tasks.yml
-    when: openshift.common.is_containerized | bool
+  - include: fedora_tasks.yml
+    when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool
 
 - name: Backup etcd
   include: backup.yml