retry package operations

When a package install/update fails due to network blips or other spotty
availability, retry it. If the failure is a real failure (e.g. package
is really not there) it still fails after 3 tries (Ansible default).

playbooks/adhoc/uninstall.yml

@@ -126,10 +126,14 @@
         - origin-sdn-ovs
         - tuned-profiles-openshift-node
         - tuned-profiles-origin-node
+        register: result
+        until: result | success
 
       - name: Remove flannel package
         package: name=flannel state=absent
         when: openshift_use_flannel | default(false) | bool
+        register: result
+        until: result | success
       when: not is_atomic | bool
 
     - shell: systemctl reset-failed
@@ -382,6 +386,8 @@
     - origin-master
     - pacemaker
     - pcs
+    register: result
+    until: result | success
 
   - shell: systemctl reset-failed
     changed_when: False
@@ -497,6 +503,8 @@
     with_items:
     - etcd
     - etcd3
+    register: result
+    until: result | success
 
   - shell: systemctl reset-failed
     changed_when: False
@@ -554,6 +562,8 @@
     when: not is_atomic | bool and openshift_remove_all | default(True) | bool
     with_items:
     - haproxy
+    register: result
+    until: result | success
 
   - shell: systemctl reset-failed
     changed_when: False

playbooks/common/openshift-cluster/upgrades/docker/tasks/upgrade.yml

@@ -41,6 +41,8 @@
 
 - name: Upgrade Docker
   package: name=docker{{ '-' + docker_version }} state=present
+  register: result
+  until: result | success
 
 - include: restart.yml
   when: not skip_docker_restart | default(False) | bool

playbooks/init/facts.yml

@@ -84,6 +84,8 @@
       - "{{ 'python3-dbus' if ansible_distribution == 'Fedora' else 'dbus-python' }}"
       - "{{ 'python3-PyYAML' if ansible_distribution == 'Fedora' else 'PyYAML' }}"
       - yum-utils
+      register: result
+      until: result | success
 
     - name: Ensure various deps for running system containers are installed
       package:
@@ -100,6 +102,8 @@
         or (openshift_use_openvswitch_system_container | default(False)) | bool
         or (openshift_use_node_system_container | default(False)) | bool
         or (openshift_use_master_system_container | default(False)) | bool
+      register: result
+      until: result | success
 
   - name: Gather Cluster facts and set is_containerized if needed
     openshift_facts:

playbooks/openshift-node/private/network_manager.yml

@@ -9,6 +9,8 @@
     package:
       name: 'NetworkManager'
       state: present
+    register: result
+    until: result | success
 
   - name: configure NetworkManager
     lineinfile:

roles/cockpit/tasks/main.yml

@@ -11,6 +11,8 @@
     - cockpit-docker
     - "{{ cockpit_plugins }}"
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Enable cockpit-ws
   systemd:

roles/contiv/tasks/download_bins.yml

@@ -8,6 +8,8 @@
   yum:
     name: bzip2
     state: installed
+  register: result
+  until: result | success
 
 - name: Download Bins | Download Contiv tar file
   get_url:

roles/contiv/tasks/pkgMgrInstallers/centos-install.yml

@@ -3,6 +3,8 @@
   yum:
     pkg=net-tools
     state=latest
+  register: result
+  until: result | success
 
 - name: PkgMgr RHEL/CentOS | Get openstack ocata rpm
   get_url:
@@ -20,6 +22,8 @@
   yum: name=/tmp/rdo-release-ocata-2.noarch.rpm state=present
   tags:
     - ovs_install
+  register: result
+  until: result | success
 
 - name: PkgMgr RHEL/CentOS | Install ovs
   yum:
@@ -31,3 +35,5 @@
     no_proxy: "{{ no_proxy|default('') }}"
   tags:
     - ovs_install
+  register: result
+  until: result | success

roles/contiv_facts/tasks/fedora-install.yml

@@ -3,6 +3,8 @@
   yum:
     name: dnf
     state: installed
+  register: result
+  until: result | success
 
 - name: Update repo cache
   command: dnf update -y

roles/docker/tasks/package_docker.yml

@@ -37,6 +37,8 @@
 - name: Install Docker
   package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present
   when: not openshift.common.is_atomic | bool and not curr_docker_version | skipped and not curr_docker_version.stdout != ''
+  register: result
+  until: result | success
 
 - block:
   # Extend the default Docker service unit file when using iptables-services

roles/docker/tasks/systemcontainer_crio.yml

@@ -29,6 +29,8 @@
     name: container-selinux
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check we are not using node as a Docker container with CRI-O
   fail: msg='Cannot use CRI-O with node configured as a Docker container'
@@ -42,6 +44,8 @@
     name: atomic
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # At the time of writing the atomic command requires runc for it's own use. This
 # task is here in the even that the atomic package ever removes the dependency.
@@ -50,6 +54,8 @@
     name: runc
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 
 - name: Check that overlay is in the kernel

roles/docker/tasks/systemcontainer_docker.yml

@@ -34,6 +34,8 @@
     name: container-selinux
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # Used to pull and install the system container
 - name: Ensure atomic is installed
@@ -41,6 +43,8 @@
     name: atomic
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # At the time of writing the atomic command requires runc for it's own use. This
 # task is here in the even that the atomic package ever removes the dependency.
@@ -49,11 +53,15 @@
     name: runc
     state: present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # Make sure Docker is installed so we are able to use the client
 - name: Install Docker so we can use the client
   package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 # Make sure docker is disabled. Errors are ignored.
 - name: Disable Docker

roles/etcd/tasks/auxiliary/drop_etcdctl.yml

@@ -2,6 +2,8 @@
 - name: Install etcd for etcdctl
   package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Configure etcd profile.d aliases
   template:

roles/etcd/tasks/backup/backup.yml

@@ -43,6 +43,8 @@
   when:
   - r_etcd_common_embedded_etcd | bool
   - not l_ostree_booted.stat.exists | bool
+  register: result
+  until: result | success
 
 - name: Check selinux label of '{{ etcd_data_dir }}'
   command: >

roles/etcd/tasks/certificates/deploy_ca.yml

@@ -6,6 +6,8 @@
   when: not etcd_is_atomic | bool
   delegate_to: "{{ etcd_ca_host }}"
   run_once: true
+  register: result
+  until: result | success
 
 - file:
     path: "{{ item }}"

roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml

@@ -4,6 +4,8 @@
     name: "etcd{{ '-' + etcd_version if etcd_version is defined else '' }}"
     state: present
   when: not etcd_is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Check status of etcd certificates
   stat:

roles/etcd/tasks/main.yml

@@ -12,6 +12,8 @@
 - name: Install etcd
   package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present
   when: not etcd_is_containerized | bool
+  register: result
+  until: result | success
 
 - include_tasks: drop_etcdctl.yml
   when:

roles/etcd/tasks/upgrade/upgrade_image.yml

@@ -44,6 +44,8 @@
     name: etcd
     state: latest
   when: not l_ostree_booted.stat.exists | bool
+  register: result
+  until: result | success
 
 - name: Verify cluster is healthy
   command: "{{ etcdctlv2 }} cluster-health"

roles/etcd/tasks/upgrade/upgrade_rpm.yml

@@ -18,6 +18,8 @@
   package:
     name: "{{ l_etcd_target_package }}"
     state: latest
+  register: result
+  until: result | success
 
 - lineinfile:
     destfile: "{{ etcd_conf_file }}"

roles/flannel/tasks/main.yml

@@ -3,6 +3,8 @@
   become: yes
   package: name=flannel state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Set flannel etcd options
   become: yes

roles/nickhammond.logrotate/tasks/main.yml

@@ -2,6 +2,8 @@
 - name: nickhammond.logrotate | Install logrotate
   package: name=logrotate state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: nickhammond.logrotate | Setup logrotate.d scripts
   template:

roles/nuage_ca/tasks/main.yaml

@@ -2,6 +2,8 @@
 - name: Install openssl
   package: name=openssl state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Create CA directory
   file: path="{{ nuage_ca_dir }}" state=directory

roles/openshift_ca/tasks/main.yml

@@ -13,6 +13,7 @@
     state: present
   when: not openshift.common.is_containerized | bool
   register: install_result
+  until: install_result | success
   delegate_to: "{{ openshift_ca_host }}"
   run_once: true
 

roles/openshift_cli/tasks/main.yml

@@ -8,6 +8,8 @@
 - name: Install clients
   package: name={{ openshift.common.service_type }}-clients state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - block:
   - name: Pull CLI Image
@@ -47,3 +49,5 @@
 - name: Install bash completion for oc tools
   package: name=bash-completion state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success

roles/openshift_clock/tasks/main.yaml

@@ -9,6 +9,8 @@
   when:
     - openshift_clock_enabled | bool
     - chrony_installed.rc != 0
+  register: result
+  until: result | success
 
 - name: Start and enable ntpd/chronyd
   command: timedatectl set-ntp true

roles/openshift_excluder/tasks/install.yml

@@ -13,6 +13,8 @@
     when:
     - r_openshift_excluder_enable_docker_excluder | bool
     - ansible_pkg_mgr == "yum"
+    register: result
+    until: result | success
 
 
   # For DNF we do not need the "*" and if we add it, it causes an error because
@@ -26,6 +28,8 @@
     when:
     - r_openshift_excluder_enable_docker_excluder | bool
     - ansible_pkg_mgr == "dnf"
+    register: result
+    until: result | success
 
   - name: Install openshift excluder - yum
     package:
@@ -34,6 +38,8 @@
     when:
     - r_openshift_excluder_enable_openshift_excluder | bool
     - ansible_pkg_mgr == "yum"
+    register: result
+    until: result | success
 
   # For DNF we do not need the "*" and if we add it, it causes an error because
   # it's not a valid pkg_spec
@@ -46,6 +52,8 @@
     when:
     - r_openshift_excluder_enable_openshift_excluder | bool
     - ansible_pkg_mgr == "dnf"
+    register: result
+    until: result | success
 
   - set_fact:
       r_openshift_excluder_install_ran: True

roles/openshift_expand_partition/tasks/main.yml

@@ -2,6 +2,8 @@
 - name: Ensure growpart is installed
   package: name=cloud-utils-growpart state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Determine if growpart is installed
   command: "rpm -q cloud-utils-growpart"

roles/openshift_loadbalancer/tasks/main.yml

@@ -5,6 +5,8 @@
 - name: Install haproxy
   package: name=haproxy state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Pull haproxy image
   command: >

roles/openshift_master/tasks/main.yml

@@ -39,6 +39,8 @@
     state: present
   when:
   - not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Create r_openshift_master_data_dir
   file:
@@ -88,6 +90,8 @@
   - item.kind == 'HTPasswdPasswordIdentityProvider'
   - not openshift.common.is_atomic | bool
   with_items: "{{ openshift.master.identity_providers }}"
+  register: result
+  until: result | success
 
 - name: Ensure htpasswd directory exists
   file:
@@ -306,6 +310,7 @@
   - openshift.master.cluster_method == 'pacemaker'
   - not openshift.common.is_containerized | bool
   register: l_install_result
+  until: l_install_result | success
 
 - name: Start and enable cluster service
   systemd:

roles/openshift_master/tasks/upgrade/rpm_upgrade.yml

@@ -18,3 +18,5 @@
       - "{{ openshift.common.service_type }}-sdn-ovs{{ openshift_pkg_version }}"
       - "{{ openshift.common.service_type }}-clients{{ openshift_pkg_version }}"
       - "tuned-profiles-{{ openshift.common.service_type }}-node{{ openshift_pkg_version }}"
+  register: result
+  until: result | success

roles/openshift_nfs/tasks/setup.yml

@@ -5,6 +5,8 @@
 
 - name: Install nfs-utils
   package: name=nfs-utils state=present
+  register: result
+  until: result | success
 
 - name: Configure NFS
   lineinfile:

roles/openshift_node/tasks/bootstrap.yml

@@ -4,6 +4,8 @@
     name: "{{ item }}"
     state: present
   with_items: "{{ r_openshift_node_image_prep_packages }}"
+  register: result
+  until: result | success
 
 - name: create the directory for node
   file:

roles/openshift_node/tasks/dnsmasq.yml

@@ -13,6 +13,8 @@
 - name: Install dnsmasq
   package: name=dnsmasq state=installed
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: ensure origin/node directory exists
   file:

roles/openshift_node/tasks/dnsmasq/no-network-manager.yml

@@ -7,5 +7,7 @@
     name: NetworkManager
     state: present
   notify: restart NetworkManager
+  register: result
+  until: result | success
 
 - include_tasks: network-manager.yml

roles/openshift_node/tasks/docker/upgrade.yml

@@ -36,5 +36,7 @@
 
 - name: Upgrade Docker
   package: name=docker{{ '-' + docker_version }} state=present
+  register: result
+  until: result | success
 
 # starting docker happens back in ../main.yml where it calls ../restart.yml

roles/openshift_node/tasks/install.yml

@@ -5,6 +5,8 @@
     package:
       name: "{{ openshift.common.service_type }}-node{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
+    register: result
+    until: result | success
 
   - name: Install sdn-ovs package
     package:
@@ -12,11 +14,15 @@
       state: present
     when:
     - openshift_node_use_openshift_sdn | bool
+    register: result
+    until: result | success
 
   - name: Install conntrack-tools package
     package:
       name: "conntrack-tools"
       state: present
+    register: result
+    until: result | success
 
 - when:
   - openshift.common.is_containerized | bool

roles/openshift_node/tasks/storage_plugins/ceph.yml

@@ -2,3 +2,5 @@
 - name: Install Ceph storage plugin dependencies
   package: name=ceph-common state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success

roles/openshift_node/tasks/storage_plugins/glusterfs.yml

@@ -2,6 +2,8 @@
 - name: Install GlusterFS storage plugin dependencies
   package: name=glusterfs-fuse state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check for existence of fusefs sebooleans
   command: getsebool {{ item }}

roles/openshift_node/tasks/storage_plugins/iscsi.yml

@@ -2,3 +2,5 @@
 - name: Install iSCSI storage plugin dependencies
   package: name=iscsi-initiator-utils state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success

roles/openshift_node/tasks/storage_plugins/nfs.yml

@@ -2,6 +2,8 @@
 - name: Install NFS storage plugin dependencies
   package: name=nfs-utils state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check for existence of nfs sebooleans
   command: getsebool {{ item }}

roles/openshift_node/tasks/upgrade.yml

@@ -107,6 +107,8 @@
     name: openvswitch
     state: latest
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Update oreg value
   yedit:

roles/openshift_node/tasks/upgrade/rpm_upgrade.yml

@@ -8,10 +8,14 @@
 # We verified latest rpm available is suitable, so just yum update.
 - name: Upgrade packages
   package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present"
+  register: result
+  until: result | success
 
 - name: Ensure python-yaml present for config upgrade
   package: name=PyYAML state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Install Node service file
   template:

roles/openshift_openstack/tasks/node-packages.yml

@@ -6,6 +6,8 @@
     name: "{{ item }}"
     state: latest
   with_items: "{{ openshift_openstack_required_packages }}"
+  register: result
+  until: result | success
 
 - name: Install debug packages (optional)
   yum:
@@ -13,3 +15,5 @@
     state: latest
   with_items: "{{ openshift_openstack_debug_packages }}"
   when: openshift_openstack_install_debug_packages|bool
+  register: result
+  until: result | success

roles/openshift_repos/tasks/main.yaml

@@ -9,6 +9,8 @@
   # TODO: This needs to be removed and placed into a role
   - name: Ensure libselinux-python is installed
     package: name=libselinux-python state=present
+    register: result
+    until: result | success
 
   - name: Remove openshift_additional.repo file
     file:

roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml

@@ -4,6 +4,8 @@
   when:
   - not openshift.common.is_atomic | bool
   - not glusterfs_heketi_is_native | bool
+  register: result
+  until: result | success
 
 - name: Verify heketi-cli is installed
   shell: "command -v {{ glusterfs_heketi_cli }} >/dev/null 2>&1 || { echo >&2 'ERROR: Make sure heketi-cli is available, then re-run the installer'; exit 1; }"

roles/openshift_storage_nfs/tasks/main.yml

@@ -4,6 +4,8 @@
 
 - name: Install nfs-utils
   package: name=nfs-utils state=present
+  register: result
+  until: result | success
 
 - name: Configure NFS
   lineinfile:

roles/openshift_storage_nfs_lvm/tasks/nfs.yml

@@ -2,6 +2,8 @@
 - name: Install NFS server
   package: name=nfs-utils state=present
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Start rpcbind
   systemd:

roles/os_firewall/tasks/firewalld.yml

@@ -8,6 +8,8 @@
   package:
     name: firewalld
     state: present
+  register: result
+  until: result | success
 
 - name: Ensure iptables services are not enabled
   systemd:

roles/os_firewall/tasks/iptables.yml

@@ -22,6 +22,8 @@
     - iptables
     - iptables-services
   when: not r_os_firewall_is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Start and enable iptables service
   systemd:

roles/os_update_latest/tasks/main.yml

@@ -1,3 +1,5 @@
 ---
 - name: Update all packages
   package: name=* state=latest
+  register: result
+  until: result | success

roles/rhel_subscribe/tasks/main.yml

@@ -36,6 +36,8 @@
   yum:
     name: subscription-manager
     state: present
+  register: result
+  until: result | success
 
 - name: RedHat subscriptions
   redhat_subscription: