Add playbook to migrate node imageConfig.format

There does not currently exist a way to migrate a node's
config imageConfig.format during transition between registries.

This commit adds a playbook that can be run on an as-needed
basis to update configmaps for each group in openshift_node_groups

playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_control_plane.yml

@@ -1,129 +1,12 @@
 ---
-#
-# Control Plane Upgrade Playbook
-#
-# Upgrades masters and Docker (only on standalone etcd hosts)
-#
-# This upgrade does not include:
-# - node service running on masters
-# - docker running on masters
-# - node service running on dedicated nodes
-#
-# You can run the upgrade_nodes.yml playbook after this to upgrade these components separately.
-#
-- import_playbook: ../init.yml
-  vars:
-    l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
-    l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
-    l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
-    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
+# This file has been refactored for release 3.11 to ensure that new registry
+# credentials are placed on all nodes and node imageConfig.format strings
+# are updated to ensure new images can be pull.
+# This file can be skipped if:
+#   1) the above steps are not necessary due to having already run upgrade_control_plane_part1
+#   2) or you were already using a private oreg_url.
+#   3) you are running origin.
 
-- name: Configure the upgrade target for the common upgrade tasks 3.11
-  hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config
-  tasks:
-  - set_fact:
-      openshift_upgrade_target: '3.11'
-      openshift_upgrade_min: '3.10'
-      openshift_release: '3.11'
+- import_playbook: upgrade_control_plane_part1.yml
 
-- import_playbook: ../pre/config.yml
-  # These vars a meant to exclude oo_nodes from plays that would otherwise include
-  # them by default.
-  vars:
-    l_openshift_version_set_hosts: "oo_etcd_to_config:oo_masters_to_config:!oo_first_master"
-    l_upgrade_repo_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
-    l_upgrade_no_proxy_hosts: "oo_masters_to_config"
-    l_upgrade_health_check_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
-    l_upgrade_verify_targets_hosts: "oo_masters_to_config"
-    l_upgrade_docker_target_hosts: "oo_masters_to_config:oo_etcd_to_config"
-    l_upgrade_excluder_hosts: "oo_masters_to_config"
-    openshift_protect_installed_version: False
-
-# Need to run sanity checks after version has been run.
-- import_playbook: ../../../../init/sanity_checks.yml
-  vars:
-    # oo_lb_to_config might not be present; Can't use !oo_nodes because masters are nodes.
-    l_sanity_check_hosts: "{{ groups['oo_etcd_to_config'] | union(groups['oo_masters_to_config']) | union(groups['oo_lb_to_config'] | default([]) ) }}"
-
-# Some change makes critical outage on current cluster.
-- name: Confirm upgrade will not make critical changes
-  hosts: oo_first_master
-  tasks:
-  - name: Confirm Reconcile Security Context Constraints will not change current SCCs
-    command: >
-      {{ openshift_client_binary }} adm policy --config={{ openshift.common.config_base }}/master/admin.kubeconfig reconcile-sccs --additive-only=true -o name
-    register: check_reconcile_scc_result
-    when: openshift_reconcile_sccs_reject_change | default(true) | bool
-    until: check_reconcile_scc_result.rc == 0
-    retries: 3
-
-  - fail:
-      msg: >
-        Changes to bootstrapped SCCs have been detected. Please review the changes by running
-        "{{ openshift_client_binary }} adm policy --config={{ openshift.common.config_base }}/master/admin.kubeconfig reconcile-sccs --additive-only=true"
-        After reviewing the changes please apply those changes by adding the '--confirm' flag.
-        Do not modify the default SCCs. Customizing the default SCCs will cause this check to fail when upgrading.
-        If you require non standard SCCs please refer to https://docs.okd.io/latest/admin_guide/manage_scc.html
-    when:
-    - openshift_reconcile_sccs_reject_change | default(true) | bool
-    - check_reconcile_scc_result.stdout != '' or check_reconcile_scc_result.rc != 0
-
-# TODO: need to verify settings about the bootstrap configs
-# 1. Does network policy match the master config
-
-- name: Ensure metrics-server is installed before upgrading the controller-manager
-  hosts: oo_first_master
-  roles:
-  - role: metrics_server
-    # a default is set on the actual variable in the role, so no fancy logic is needed here
-    when: openshift_metrics_server_install | default(true) | bool
-
-
-- name: Configure components that must be available prior to upgrade
-  hosts: oo_first_master
-  roles:
-  - role: openshift_sdn
-    when: openshift_use_openshift_sdn | default(True) | bool
-
-- import_playbook: ../upgrade_control_plane.yml
-  vars:
-    openshift_release: '3.11'
-
-- name: Update master nodes
-  hosts: oo_masters
-  serial: 1
-  tasks:
-  - import_role:
-      name: openshift_node
-      tasks_from: upgrade_pre.yml
-  - import_role:
-      name: openshift_node
-      tasks_from: upgrade.yml
-  - import_role:
-      name: openshift_storage_glusterfs
-      tasks_from: check_cluster_health.yml
-    when: >
-          ('glusterfs' in groups and inventory_hostname in groups['glusterfs'])
-          or ('glusterfs_registry' in groups and inventory_hostname in groups['glusterfs_registry'])
-
-
-- import_playbook: ../post_control_plane.yml
-
-- hosts: oo_masters
-  tasks:
-  - import_role:
-      name: openshift_web_console
-      tasks_from: remove_old_asset_config.yml
-
-# This is a one time migration. No need to save it in the 3.11.
-# https://bugzilla.redhat.com/show_bug.cgi?id=1565736
-- hosts: oo_first_master
-  tasks:
-  - import_role:
-      name: openshift_hosted
-      tasks_from: registry_service_account.yml
-    when: openshift_hosted_manage_registry | default(True) | bool
-  - import_role:
-      name: openshift_hosted
-      tasks_from: remove_legacy_env_variables.yml
-    when: openshift_hosted_manage_registry | default(True) | bool
+- import_playbook: upgrade_control_plane_part2.yml

playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_control_plane_part1.yml

@@ -0,0 +1,7 @@
+---
+# If you have already run this playbook, you can run upgrade_control_plane_part2.yml
+# instead to skip this.
+- import_playbook: ../../../../init/main.yml
+  vars:
+    openshift_protect_installed_version: False
+- import_playbook: ../../../../openshift-node/private/registry_auth.yml

playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_control_plane_part2.yml

@@ -0,0 +1,131 @@
+---
+#
+# Control Plane Upgrade Playbook
+#
+# Upgrades masters and Docker (only on standalone etcd hosts)
+#
+# This upgrade does not include:
+# - node service running on masters
+# - docker running on masters
+# - node service running on dedicated nodes
+#
+# You can run the upgrade_nodes.yml playbook after this to upgrade these components separately.
+#
+- import_playbook: ../init.yml
+  vars:
+    l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
+
+- name: Configure the upgrade target for the common upgrade tasks 3.11
+  hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config
+  tasks:
+  - set_fact:
+      openshift_upgrade_target: '3.11'
+      openshift_upgrade_min: '3.10'
+      openshift_release: '3.11'
+
+- import_playbook: ../pre/config.yml
+  # These vars a meant to exclude oo_nodes from plays that would otherwise include
+  # them by default.
+  vars:
+    l_openshift_version_set_hosts: "oo_etcd_to_config:oo_masters_to_config:!oo_first_master"
+    l_upgrade_repo_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_no_proxy_hosts: "oo_masters_to_config"
+    l_upgrade_health_check_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_verify_targets_hosts: "oo_masters_to_config"
+    l_upgrade_docker_target_hosts: "oo_masters_to_config:oo_etcd_to_config"
+    l_upgrade_excluder_hosts: "oo_masters_to_config"
+    openshift_protect_installed_version: False
+
+# Need to run sanity checks after version has been run.
+- import_playbook: ../../../../init/sanity_checks.yml
+  vars:
+    # oo_lb_to_config might not be present; Can't use !oo_nodes because masters are nodes.
+    l_sanity_check_hosts: "{{ groups['oo_etcd_to_config'] | union(groups['oo_masters_to_config']) | union(groups['oo_lb_to_config'] | default([]) ) }}"
+
+# Some change makes critical outage on current cluster.
+- name: Confirm upgrade will not make critical changes
+  hosts: oo_first_master
+  tasks:
+  - name: Confirm Reconcile Security Context Constraints will not change current SCCs
+    command: >
+      {{ openshift_client_binary }} adm policy --config={{ openshift.common.config_base }}/master/admin.kubeconfig reconcile-sccs --additive-only=true -o name
+    register: check_reconcile_scc_result
+    when: openshift_reconcile_sccs_reject_change | default(true) | bool
+    until: check_reconcile_scc_result.rc == 0
+    retries: 3
+
+  - fail:
+      msg: >
+        Changes to bootstrapped SCCs have been detected. Please review the changes by running
+        "{{ openshift_client_binary }} adm policy --config={{ openshift.common.config_base }}/master/admin.kubeconfig reconcile-sccs --additive-only=true"
+        After reviewing the changes please apply those changes by adding the '--confirm' flag.
+        Do not modify the default SCCs. Customizing the default SCCs will cause this check to fail when upgrading.
+        If you require non standard SCCs please refer to https://docs.okd.io/latest/admin_guide/manage_scc.html
+    when:
+    - openshift_reconcile_sccs_reject_change | default(true) | bool
+    - check_reconcile_scc_result.stdout != '' or check_reconcile_scc_result.rc != 0
+
+# TODO: need to verify settings about the bootstrap configs
+# 1. Does network policy match the master config
+
+- name: Ensure metrics-server is installed before upgrading the controller-manager
+  hosts: oo_first_master
+  roles:
+  - role: metrics_server
+    # a default is set on the actual variable in the role, so no fancy logic is needed here
+    when: openshift_metrics_server_install | default(true) | bool
+
+
+- name: Configure components that must be available prior to upgrade
+  hosts: oo_first_master
+  roles:
+  - role: openshift_sdn
+    when: openshift_use_openshift_sdn | default(True) | bool
+
+- import_playbook: ../upgrade_control_plane.yml
+  vars:
+    openshift_release: '3.11'
+
+- name: Update master nodes
+  hosts: oo_masters
+  serial: 1
+  tasks:
+  - import_role:
+      name: openshift_node
+      tasks_from: upgrade_pre.yml
+  - import_role:
+      name: openshift_node
+      tasks_from: upgrade.yml
+  - import_role:
+      name: openshift_control_plane
+      tasks_from: verify_api_server.yml
+  - import_role:
+      name: openshift_storage_glusterfs
+      tasks_from: check_cluster_health.yml
+    when: >
+          ('glusterfs' in groups and inventory_hostname in groups['glusterfs'])
+          or ('glusterfs_registry' in groups and inventory_hostname in groups['glusterfs_registry'])
+
+- import_playbook: ../post_control_plane.yml
+
+- hosts: oo_masters
+  tasks:
+  - import_role:
+      name: openshift_web_console
+      tasks_from: remove_old_asset_config.yml
+
+# This is a one time migration. No need to save it in the 3.11.
+# https://bugzilla.redhat.com/show_bug.cgi?id=1565736
+- hosts: oo_first_master
+  tasks:
+  - import_role:
+      name: openshift_hosted
+      tasks_from: registry_service_account.yml
+    when: openshift_hosted_manage_registry | default(True) | bool
+  - import_role:
+      name: openshift_hosted
+      tasks_from: remove_legacy_env_variables.yml
+    when: openshift_hosted_manage_registry | default(True) | bool

playbooks/openshift-master/private/upgrade.yml

@@ -16,20 +16,9 @@
       get_mime: false
     register: service_signer_cert_stat
     changed_when: false
-  - name: verify api server
-    command: >
-      curl --silent --tlsv1.2
-      --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt
-      {{ openshift.master.api_url }}/healthz/ready
-    args:
-      # Disables the following warning:
-      # Consider using get_url or uri module rather than running curl
-      warn: no
-    register: api_available_output
-    until: api_available_output.stdout == 'ok'
-    retries: 120
-    delay: 1
-    changed_when: false
+  - import_role:
+      name: openshift_control_plane
+      tasks_from: verify_api_server.yml
 
 - import_playbook: create_service_signer_cert.yml
 

playbooks/openshift-node/imageconfig.yml

@@ -0,0 +1,9 @@
+---
+- import_playbook: ../init/main.yml
+  vars:
+    l_init_fact_hosts: "oo_masters_to_config"
+    l_openshift_version_determine_hosts: "all:!all"
+    l_openshift_version_set_hosts: "all:!all"
+    skip_sanity_checks: True
+
+- import_playbook: private/imageconfig.yml

playbooks/openshift-node/private/imageconfig.yml

@@ -0,0 +1,11 @@
+---
+- import_playbook: registry_auth.yml
+  vars:
+    l_reg_auth_restart_hosts: "all:!all"
+
+- name: Update node imageConfig.format
+  hosts: oo_first_master
+  tasks:
+  - import_role:
+      name: openshift_node_group
+      tasks_from: migrate_imageconfig.yml

playbooks/openshift-node/private/registry_auth.yml

@@ -15,8 +15,11 @@
       tasks_from: node_system_container_install.yml
     when: openshift_is_atomic
 
+# l_reg_auth_restart_hosts is passed in via imageconfig.yml to prevent
+# the nodes from restarting because the sync pod will be restarting them
+# anyway.
 - name: Restart nodes
-  hosts: oo_nodes_to_config
+  hosts: "{{ l_reg_auth_restart_hosts | default('oo_nodes_to_config') }}"
   serial: "{{ openshift_restart_nodes_serial | default(1) }}"
   roles:
   - lib_openshift

roles/openshift_control_plane/tasks/verify_api_server.yml

@@ -0,0 +1,16 @@
+---
+# Ensure localhost master is responding.
+- name: verify API server
+  command: >
+    curl --silent --tlsv1.2 --max-time 2
+    --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt
+    {{ openshift.master.api_url }}/healthz/ready
+  args:
+    # Disables the following warning:
+    # Consider using get_url or uri module rather than running curl
+    warn: no
+  register: l_api_available_output
+  until: l_api_available_output.stdout == 'ok'
+  retries: 120
+  delay: 1
+  changed_when: false

roles/openshift_facts/defaults/main.yml

@@ -35,6 +35,7 @@ openshift_image_default: "{{ l_os_registry_url | regex_replace('${component}' |
 openshift_cli_image: "{{ (system_images_registry == 'docker') | ternary(openshift_image_default, (openshift_image_default.split('/')|length==2) | ternary(system_images_registry + '/' + openshift_image_default, openshift_image_default)) }}"
 system_openshift_cli_image: "{{ (system_images_registry == 'docker') | ternary('docker:' + openshift_cli_image, openshift_cli_image) }}"
 osn_image: "{{ l_os_registry_url | regex_replace('${component}' | regex_escape, 'node') }}"
+osn_pod_image: "{{ l_os_registry_url | regex_replace('${component}' | regex_escape, 'pod') }}"
 osm_image: "{{ l_osm_registry_url | regex_replace('${component}' | regex_escape, 'control-plane') }}"
 
 repoquery_cmd: "{{ (ansible_pkg_mgr == 'dnf') | ternary('dnf repoquery --latest-limit 1 -d 0', 'repoquery --plugins') }}"

roles/openshift_node/tasks/upgrade/stop_services.yml

@@ -7,16 +7,17 @@
   - "{{ openshift_service_type }}-node"
   failed_when: false
 
-- service:
+- name: stop docker to kill static pods
+  service:
     name: docker
     state: stopped
   register: l_openshift_node_upgrade_docker_stop_result
   until: not (l_openshift_node_upgrade_docker_stop_result is failed)
   retries: 3
   delay: 30
-  when:
-  - l_docker_upgrade is defined
-  - l_docker_upgrade | bool
+  when: >
+        inventory_hostname in groups['oo_masters_to_config']
+        or (l_docker_upgrade is defined and l_docker_upgrade | bool)
 
 - name: Stop crio
   service:

roles/openshift_node/tasks/upgrade_pre.yml

@@ -7,6 +7,22 @@
 
 - import_tasks: registry_auth.yml
 
+- name: Check that pod image is present
+  command: "{{ openshift_container_cli }} images -q {{ osn_pod_image }}"
+  register: pod_image
+
+# This task runs async to save time while other downloads proceed
+- name: pre-pull pod image
+  docker_image:
+    name: "{{ osn_pod_image }}"
+  environment:
+    NO_PROXY: "{{ openshift.common.no_proxy | default('') }}"
+  when: pod_image.stdout_lines == []
+  # 10 minutes to pull the image
+  async: 600
+  poll: 0
+  register: pod_image_prepull
+
 - name: update package meta data to speed install later.
   command: "{{ ansible_pkg_mgr }} makecache"
   register: result
@@ -35,3 +51,13 @@
 
 - import_tasks: upgrade/rpm_upgrade.yml
   when: not openshift_is_atomic | bool
+
+- name: Check status of node pod image pre-pull
+  async_status:
+    jid: "{{ pod_image_prepull.ansible_job_id }}"
+  register: job_result
+  until: job_result.finished
+  when: pod_image.stdout_lines == []
+  retries: 20
+  delay: 30
+  failed_when: false

roles/openshift_node_group/tasks/migrate_imageconfig.yml

@@ -0,0 +1,15 @@
+---
+# This file is used to migrate imageConfig.format when changing component
+# registries.
+
+- name: patch existing node config maps
+  include_tasks: create_config.yml
+  vars:
+    l_openshift_node_group_name: "{{ node_group.name }}"
+    l_openshift_node_group_edits:
+    - key: "imageConfig.format"
+      value: "{{ l_os_non_standard_reg_url }}"
+    l_openshift_node_group_labels: "{{ node_group.labels }}"
+  with_items: "{{ openshift_node_groups }}"
+  loop_control:
+    loop_var: node_group