Merge pull request #11808 from mtnbikenc/node-ready

Add check to ensure nodes are Ready after scaleup/upgrade

roles/openshift_node/tasks/apply_machine_config.yml

@@ -62,7 +62,7 @@
   vars:
     podman_flags: "--privileged --rm --entrypoint=/usr/bin/machine-config-daemon -ti {{ l_mcd_image }}"
     podman_mounts: "-v /:/rootfs -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd"
-    mcd_command: "start --node-name {{ ansible_hostname }} --once-from {{ temp_dir.path }}/worker_ignition_config.json --skip-reboot"
+    mcd_command: "start --node-name {{ ansible_nodename | lower }} --once-from {{ temp_dir.path }}/worker_ignition_config.json --skip-reboot"
 
 - name: Remove temp directory
   file:
@@ -72,3 +72,16 @@
 - name: Reboot the host and wait for it to come back
   reboot:
   #  reboot_timeout: 600  # default, 10 minutes
+
+- name: Wait for nodes to report ready
+  command: >
+    oc get node {{ ansible_nodename | lower }}
+    --config={{ openshift_node_kubeconfig_path }}
+    --output=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  delegate_to: localhost
+  run_once: true
+  register: oc_get
+  until:
+  - oc_get.stdout == "True"
+  retries: 36
+  delay: 5

roles/openshift_node/tasks/config.yml

@@ -95,7 +95,7 @@
     vars:
       podman_flags: "--privileged --rm --entrypoint=/usr/bin/machine-config-daemon -ti {{ release_image_mcd.stdout }}"
       podman_mounts: "-v /:/rootfs -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd"
-      mcd_command: "start --node-name {{ ansible_hostname }} --once-from {{ temp_dir.path }}/bootstrap.ign --skip-reboot"
+      mcd_command: "start --node-name {{ ansible_nodename | lower }} --once-from {{ temp_dir.path }}/bootstrap.ign --skip-reboot"
 
   - name: Remove temp directory
     file:
@@ -109,3 +109,17 @@
   rescue:
   - fail:
       msg: "Ignition apply failed"
+
+- name: Wait for nodes to report ready
+  command: >
+    oc get node {{ hostvars[item].ansible_nodename | lower }}
+    --config={{ openshift_node_kubeconfig_path }}
+    --output=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+  loop: "{{ ansible_play_batch }}"
+  delegate_to: localhost
+  run_once: true
+  register: oc_get
+  until:
+  - oc_get.stdout == "True"
+  retries: 36
+  delay: 5

roles/openshift_node/tasks/upgrade.yml

@@ -7,18 +7,16 @@
 
 - name: Cordon node prior to upgrade
   command: >
-    oc adm cordon {{ item | lower }}
+    oc adm cordon {{ ansible_nodename | lower }}
     --config={{ openshift_node_kubeconfig_path }}
   delegate_to: localhost
-  with_items: "{{ ansible_play_batch }}"
 
 - name: Drain node prior to upgrade
   command: >
-    oc adm drain {{ item | lower }}
+    oc adm drain {{ ansible_nodename | lower }}
     --config={{ openshift_node_kubeconfig_path }}
     --force --delete-local-data --ignore-daemonsets
   delegate_to: localhost
-  with_items: "{{ ansible_play_batch }}"
 
 # Run the openshift_node_pre_upgrade_hook if defined
 - block:
@@ -42,10 +40,9 @@
 
 - name: Uncordon node after upgrade
   command: >
-    oc adm uncordon {{ item | lower }}
+    oc adm uncordon {{ ansible_nodename | lower }}
     --config={{ openshift_node_kubeconfig_path }}
   delegate_to: localhost
-  with_items: "{{ ansible_play_batch }}"
 
 # Run the openshift_node_post_upgrade_hook if defined
 - block: