Upgrade cri-o

When cri-o is upgraded we need to clean up reminants of pods from
previous versions

playbooks/openshift-master/private/upgrade.yml

@@ -67,6 +67,8 @@
     failed_when:
     - l_pb_upgrade_control_plane_pre_upgrade_storage.rc != 0
     - openshift_upgrade_pre_storage_migration_fatal | default(true) | bool
+    retries: 2
+    delay: 30
 
 # Set openshift_master_facts separately. In order to reconcile
 # admission_config's, we currently must run openshift_master_facts and
@@ -189,6 +191,8 @@
     failed_when:
     - l_pb_upgrade_control_plane_post_upgrade_storage.rc != 0
     - openshift_upgrade_post_storage_migration_fatal | default(false) | bool
+    retries: 2
+    delay: 30
 
   - set_fact:
       reconcile_complete: True

roles/openshift_node/files/clean-up-crio-pods.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+for c in $(runc list -q); do
+        output=$(runc state $c | grep io.kubernetes.cri-o.ContainerType)
+        if [[ "$output" =~ "container" ]]; then
+                runc delete -f $c
+        fi
+        for m in $(mount | grep $c | awk '{print $3}'); do
+                umount -R $m
+        done
+done
+for c in $(runc list -q); do
+        output=$(runc state $c | grep io.kubernetes.cri-o.ContainerType)
+        if [[ "$output" =~ "sandbox" ]]; then
+                runc delete -f $c
+        fi
+        for m in $(mount | grep $c | awk '{print $3}'); do
+                umount -R $m
+        done
+done
+mount | grep overlay | awk '{print $3}' | xargs umount | true
+umount -R /var/lib/containers/storage/overlay
+umount -R /var/lib/containers/storage
+rm -rf /var/run/containers/storage/*
+rm -rf /var/lib/containers/storage/*

roles/openshift_node/tasks/upgrade.yml

@@ -22,19 +22,24 @@
 
 - name: Ensure cri-o is updated
   package:
-    name: cri-o
+    name: "{{ crio_pkgs | join (',') }}"
     state: latest
   when:
   - openshift_use_crio | default(False)
   register: crio_update
+  vars:
+    crio_pkgs:
+    - "cri-o"
+    - "cri-tools"
 
-- name: Restart cri-o
-  systemd:
-    name: cri-o
-    state: restarted
-  when:
-  - openshift_use_crio | default(False)
-  - crio_update is changed
+- name: Remove CRI-O default configuration files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - "/etc/cni/net.d/200-loopback.conf"
+  - "/etc/cni/net.d/100-crio-bridge.conf"
+  when: crio_update | changed
 
 - name: install pre-pulled rpms.
   import_tasks: upgrade/rpm_upgrade_install.yml

roles/openshift_node/tasks/upgrade/restart.yml

@@ -30,6 +30,12 @@
   retries: 3
   delay: 30
 
+- name: Restart cri-o
+  service:
+    name: cri-o
+    state: started
+  when: openshift_use_crio | default(False)
+
 - name: Start node service
   service:
     name: "{{ openshift_service_type }}-node"

roles/openshift_node/tasks/upgrade/stop_services.yml

@@ -8,14 +8,6 @@
   - "{{ openshift_service_type }}-node"
   failed_when: false
 
-- name: Ensure static containerized services stopped before Docker restart
-  command: /usr/local/bin/master-restart "{{ item }}"
-  with_items:
-  - api
-  - controllers
-  - etcd
-  failed_when: false
-
 - service:
     name: docker
     state: stopped
@@ -26,3 +18,16 @@
   when:
   - l_docker_upgrade is defined
   - l_docker_upgrade | bool
+
+- name: Stop crio
+  service:
+    name: cri-o
+    state: stopped
+  when: openshift_use_crio | default(False)
+
+# TODO: Need to determine if this is needed long term or just 3.9 to 3.10
+# Upgrading cri-o, at least from 1.9 to 1.10, requires that all
+# pods be stopped
+- name: Clean up cri-o pods
+  script: clean-up-crio-pods.sh
+  when: openshift_use_crio | default(False)

roles/openshift_node/tasks/upgrade_pre.yml

@@ -35,11 +35,15 @@
   - l_docker_upgrade | bool
 
 - name: Stage cri-o updates
-  command: "{{ ansible_pkg_mgr }} install -y --downloadonly cri-o"
+  command: "{{ ansible_pkg_mgr }} install -y --downloadonly {{ crio_pkgs | join(' ') }}"
   register: result
   until: result is succeeded
   when:
   - openshift_use_crio | default(False)
+  vars:
+    crio_pkgs:
+    - "cri-o"
+    - "cri-tools"
 
 - import_tasks: upgrade/rpm_upgrade.yml
   when: not openshift_is_atomic | bool