Add wait_for_pods to upgrade for hosted components

Sometimes upgrading routers and registries can take
a while.  This will cause openshift-ansible to determine
that registry needs new certs to be deployed and
deploying those certs will fail because dc rollout is
already in progress.

This commit adds wait_for_pods to registry and routers
before checking certs.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1552164

playbooks/common/openshift-cluster/upgrades/post_control_plane.yml

@@ -10,16 +10,12 @@
     - openshift_web_console_install | default(true) | bool
     - openshift_upgrade_target is version_compare('3.9','>=')
 
+# upgrade registry and router pods; we defer waiting for these pods
+# until after the next play to hopefully save some time polling.
+- import_playbook: ../../../openshift-hosted/private/upgrade.yml
+
 - name: Upgrade default router and default registry
   hosts: oo_first_master
-  pre_tasks:
-  - import_role:
-      name: openshift_hosted
-      tasks_from: upgrade_routers.yml
-  - import_role:
-      name: openshift_hosted
-      tasks_from: upgrade_registry.yml
-
   roles:
   - lib_utils
   - openshift_manageiq
@@ -40,28 +36,8 @@
   - role: openshift_hosted_templates
     openshift_hosted_templates_import_command: replace
 
-  post_tasks:
-  # Do not perform these tasks when the registry is insecure.  The default registry is insecure in openshift_hosted/defaults/main.yml
-  - when: not (openshift_docker_hosted_registry_insecure | default(False))
-    block:
-    # we need to migrate customers to the new pattern of pushing to the registry via dns
-    # Step 1: verify the certificates have the docker registry service name
-    - name: shell command to determine if the docker-registry.default.svc is found in the registry certificate
-      shell: >
-        echo -n | openssl s_client -showcerts -servername docker-registry.default.svc -connect docker-registry.default.svc:5000  | openssl x509 -text |  grep -A1 'X509v3 Subject Alternative Name:' | grep -Pq 'DNS:docker-registry\.default\.svc(,|$)'
-      register: cert_output
-      changed_when: false
-      failed_when:
-      - cert_output.rc not in [0, 1]
-
-    # Step 2: Set a fact to be used to determine if we should run the redeploy of registry certs
-    - name: set a fact to include the registry certs playbook if needed
-      set_fact:
-        openshift_hosted_rollout_certs_and_registry: "{{ cert_output.rc != 0  }}"
-
-# Run the redeploy certs based upon the certificates. Defaults to False for insecure registries
-- when: (hostvars[groups.oo_first_master.0].openshift_hosted_rollout_certs_and_registry | default(False)) | bool
-  import_playbook: ../../../openshift-hosted/private/redeploy-registry-certificates.yml
+# Poll for registry and router pods, redeploy registry certs if needed.
+- import_playbook: ../../../openshift-hosted/private/upgrade_poll_and_check_certs.yml
 
 # Check for warnings to be printed at the end of the upgrade:
 - name: Clean up and display warnings

playbooks/openshift-hosted/private/openshift_hosted_wait_for_pods.yml

@@ -5,6 +5,10 @@
 - name: Poll for hosted pod deployments
   hosts: oo_first_master
   tasks:
+  # Need to set_fact variable for registry.
+  - import_role:
+      name: openshift_hosted
+      tasks_from: set_fact_workaround.yml
   - import_role:
       name: openshift_hosted
       tasks_from: wait_for_pod.yml

playbooks/openshift-hosted/private/upgrade.yml

@@ -0,0 +1,14 @@
+---
+# This play is called during upgrade_control_plane via post_control_plane.yml
+- name: Upgrade default router and default registry
+  hosts: oo_first_master
+  roles:
+  - lib_utils
+  - openshift_facts
+  tasks:
+  - import_role:
+      name: openshift_hosted
+      tasks_from: upgrade_routers.yml
+  - import_role:
+      name: openshift_hosted
+      tasks_from: upgrade_registry.yml

playbooks/openshift-hosted/private/upgrade_poll_and_check_certs.yml

@@ -0,0 +1,32 @@
+---
+# This playbook is called during upgrade_control_plane via post_control_plane.yml
+
+# Need to poll to ensure the pods are deployed.
+- import_playbook: openshift_hosted_wait_for_pods.yml
+
+- name: Upgrade default router and default registry
+  hosts: oo_first_master
+  roles:
+  - lib_utils
+  - openshift_facts
+  tasks:
+  # Do not perform these tasks when the registry is insecure.  The default registry is insecure in openshift_hosted/defaults/main.yml
+  - when: not (openshift_docker_hosted_registry_insecure | default(False))
+    block:
+    # we need to migrate customers to the new pattern of pushing to the registry via dns
+    # Step 1: verify the certificates have the docker registry service name
+    - name: shell command to determine if the docker-registry.default.svc is found in the registry certificate
+      shell: >
+        echo -n | openssl s_client -showcerts -servername docker-registry.default.svc -connect docker-registry.default.svc:5000  | openssl x509 -text |  grep -A1 'X509v3 Subject Alternative Name:' | grep -Pq 'DNS:docker-registry\.default\.svc(,|$)'
+      register: cert_output
+      changed_when: false
+      failed_when:
+      - cert_output.rc not in [0, 1]
+
+    # Step 2: Set a fact to be used to determine if we should run the redeploy of registry certs
+    - name: set a fact to include the registry certs playbook if needed
+      set_fact:
+        openshift_hosted_rollout_certs_and_registry: "{{ cert_output.rc != 0  }}"
+
+- when: (hostvars[groups.oo_first_master.0].openshift_hosted_rollout_certs_and_registry | default(False)) | bool
+  import_playbook: redeploy-registry-certificates.yml

roles/openshift_hosted/tasks/registry.yml

@@ -125,11 +125,3 @@
     volume_mounts: "{{ openshift_hosted_registry_volumes }}"
     edits: "{{ openshift_hosted_registry_edits }}"
     force: "{{ True|bool in openshift_hosted_registry_force }}"
-
-# TODO(michaelgugino) remove this set fact.  It is currently necessary due to
-# custom module not properly templating variables.
-- name: setup registry list
-  set_fact:
-    r_openshift_hosted_registry_list:
-    - name: "{{ openshift_hosted_registry_name }}"
-      namespace: "{{ openshift_hosted_registry_namespace }}"

roles/openshift_hosted/tasks/set_fact_workaround.yml

@@ -0,0 +1,8 @@
+---
+# TODO(michaelgugino) remove this set fact.  It is currently necessary due to
+# custom module not properly templating variables.
+- name: setup registry list
+  set_fact:
+    r_openshift_hosted_registry_list:
+    - name: "{{ openshift_hosted_registry_name }}"
+      namespace: "{{ openshift_hosted_registry_namespace }}"