Merge pull request #2981 from dgoodwin/upgrade-wait-for-node

Wait for nodes to be ready before proceeding with upgrade.

playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml

@@ -87,6 +87,19 @@
   - name: Restart rpm node service
     service: name="{{ openshift.common.service_type }}-node" state=restarted
     when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool
+
+  - name: Wait for node to be ready
+    command: >
+      {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.common.hostname | lower }} --no-headers
+    register: node_output
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: inventory_hostname in groups.oo_nodes_to_upgrade
+    until: "{{ node_output.stdout.split()[1].startswith('Ready')}}"
+    # Give the node two minutes to come back online. Note that we pre-pull images now
+    # so containerized services should restart quickly as well.
+    retries: 24
+    delay: 5
+
   - name: Set node schedulability
     command: >
       {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true