Merge pull request #5948 from mtnbikenc/checkpoint-times

Automatic merge from submit-queue.

Add execution times to checkpoint status

This PR adds execution times to the checkpoint status report at the end of an installer playbook run.

- Checkpoint plays now use `all` host group to avoid using an undefined host group during the Initialization phase as well as maintaining the need to collect inventory vars for host group conditionals
- `run_once: true` is used to limit the execution to only one host
- `installer_phase_*` data is now stored as a dict so multiple data points can be stored for that phase
- Documentation updated


```
INSTALLER STATUS ***************************************************************
Initialization             : Complete (0:02:14)
Health Check               : Complete (0:01:10)
etcd Install               : Complete (0:02:01)
Master Install             : Complete (0:11:43)
Master Additional Install  : Complete (0:00:54)
Node Install               : Complete (0:14:11)
Hosted Install             : Complete (0:03:28)
```

playbooks/common/openshift-checks/install.yml

@@ -1,13 +1,15 @@
 ---
 - name: Health Check Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Health Check 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_health: "In Progress"
-      aggregate: false
+        installer_phase_health:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: OpenShift Health Checks
   hosts: oo_all_hosts
@@ -37,11 +39,13 @@
       - docker_image_availability
 
 - name: Health Check Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Health Check 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_health: "Complete"
-      aggregate: false
+        installer_phase_health:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-cluster/openshift_hosted.yml

@@ -1,13 +1,15 @@
 ---
 - name: Hosted Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Hosted install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_hosted: "In Progress"
-      aggregate: false
+        installer_phase_hosted:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: create_persistent_volumes.yml
 
@@ -30,11 +32,13 @@
   - openshift_crio_enable_docker_gc | default(False) | bool
 
 - name: Hosted Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Hosted install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_hosted: "Complete"
-      aggregate: false
+        installer_phase_hosted:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-cluster/openshift_logging.yml

@@ -1,13 +1,15 @@
 ---
 - name: Logging Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Logging install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_logging: "In Progress"
-      aggregate: false
+        installer_phase_logging:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: OpenShift Aggregated Logging
   hosts: oo_first_master
@@ -23,11 +25,13 @@
         tasks_from: update_master_config
 
 - name: Logging Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Logging install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_logging: "Complete"
-      aggregate: false
+        installer_phase_logging:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-cluster/openshift_metrics.yml

@@ -1,13 +1,15 @@
 ---
 - name: Metrics Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Metrics install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_metrics: "In Progress"
-      aggregate: false
+        installer_phase_metrics:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: OpenShift Metrics
   hosts: oo_first_master
@@ -24,11 +26,13 @@
       tasks_from: update_master_config.yaml
 
 - name: Metrics Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Metrics install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_metrics: "Complete"
-      aggregate: false
+        installer_phase_metrics:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-cluster/openshift_prometheus.yml

@@ -1,13 +1,15 @@
 ---
 - name: Prometheus Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Prometheus install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_prometheus: "In Progress"
-      aggregate: false
+        installer_phase_prometheus:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Create Hosted Resources - openshift_prometheus
   hosts: oo_first_master
@@ -15,11 +17,13 @@
   - role: openshift_prometheus
 
 - name: Prometheus Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Prometheus install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_prometheus: "Complete"
-      aggregate: false
+        installer_phase_prometheus:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-cluster/service_catalog.yml

@@ -1,13 +1,15 @@
 ---
 - name: Service Catalog Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Service Catalog install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_servicecatalog: "In Progress"
-      aggregate: false
+        installer_phase_servicecatalog:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Service Catalog
   hosts: oo_first_master
@@ -19,11 +21,13 @@
     first_master: "{{ groups.oo_first_master[0] }}"
 
 - name: Service Catalog Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Service Catalog install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_servicecatalog: "Complete"
-      aggregate: false
+        installer_phase_servicecatalog:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-cluster/std_include.yml

@@ -1,15 +1,17 @@
 ---
 - name: Initialization Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   roles:
   - installer_checkpoint
   tasks:
   - name: Set install initialization 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "In Progress"
-      aggregate: false
+        installer_phase_initialize:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: evaluate_groups.yml
   tags:
@@ -36,11 +38,13 @@
   - always
 
 - name: Initialization Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set install initialization 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "Complete"
-      aggregate: false
+        installer_phase_initialize:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-etcd/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: etcd Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set etcd install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_etcd: "In Progress"
-      aggregate: false
+        installer_phase_etcd:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: ca.yml
 
@@ -26,11 +28,13 @@
   - role: nickhammond.logrotate
 
 - name: etcd Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set etcd install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_etcd: "Complete"
-      aggregate: false
+        installer_phase_etcd:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-glusterfs/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: GlusterFS Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set GlusterFS install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_glusterfs: "In Progress"
-      aggregate: false
+        installer_phase_glusterfs:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Open firewall ports for GlusterFS nodes
   hosts: glusterfs
@@ -46,11 +48,13 @@
     when: groups.oo_glusterfs_to_config | default([]) | count > 0
 
 - name: GlusterFS Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set GlusterFS install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_glusterfs: "Complete"
-      aggregate: false
+        installer_phase_glusterfs:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-loadbalancer/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: Load Balancer Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set load balancer install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_loadbalancer: "In Progress"
-      aggregate: false
+        installer_phase_loadbalancer:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Configure firewall and docker for load balancers
   hosts: oo_lb_to_config:!oo_masters_to_config:!oo_nodes_to_config
@@ -37,11 +39,13 @@
   - role: tuned
 
 - name: Load Balancer Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set load balancer install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_loadbalancer: "Complete"
-      aggregate: false
+        installer_phase_loadbalancer:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-management/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: Management Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Management install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_management: "In Progress"
-      aggregate: false
+        installer_phase_management:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Setup CFME
   hosts: oo_first_master
@@ -25,11 +27,13 @@
       template_dir: "{{ hostvars[groups.masters.0].r_openshift_management_mktemp.stdout }}"
 
 - name: Management Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Management install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_management: "Complete"
-      aggregate: false
+        installer_phase_management:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-master/additional_config.yml

@@ -1,13 +1,15 @@
 ---
 - name: Master Additional Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master Additional install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master_additional: "In Progress"
-      aggregate: false
+        installer_phase_master_additional:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Additional master configuration
   hosts: oo_first_master
@@ -36,11 +38,13 @@
     when: openshift_use_flannel | default(false) | bool
 
 - name: Master Additional Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master Additional install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master_additional: "Complete"
-      aggregate: false
+        installer_phase_master_additional:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-master/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: Master Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master: "In Progress"
-      aggregate: false
+        installer_phase_master:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: certificates.yml
 
@@ -238,11 +240,13 @@
     r_openshift_excluder_service_type: "{{ openshift.common.service_type }}"
 
 - name: Master Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master: "Complete"
-      aggregate: false
+        installer_phase_master:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-nfs/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: NFS Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set NFS install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_nfs: "In Progress"
-      aggregate: false
+        installer_phase_nfs:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Configure nfs
   hosts: oo_nfs_to_config
@@ -16,11 +18,13 @@
   - role: openshift_storage_nfs
 
 - name: NFS Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set NFS install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_nfs: "Complete"
-      aggregate: false
+        installer_phase_nfs:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

playbooks/common/openshift-node/config.yml

@@ -1,13 +1,15 @@
 ---
 - name: Node Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Node install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_node: "In Progress"
-      aggregate: false
+        installer_phase_node:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: certificates.yml
 
@@ -24,11 +26,13 @@
 - include: enable_excluders.yml
 
 - name: Node Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Node install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_node: "Complete"
-      aggregate: false
+        installer_phase_node:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"

roles/installer_checkpoint/README.md

@@ -92,31 +92,34 @@ phase/component and then a final play for setting `installer_hase_initialize` to
 # common/openshift-cluster/std_include.yml
 ---
 - name: Initialization Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   roles:
   - installer_checkpoint
   tasks:
   - name: Set install initialization 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "In Progress"
-      aggregate: false
+        installer_phase_initialize:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 #...
 # Various plays here
 #...
 
 - name: Initialization Checkpoint End
-  hosts: localhost
-  connection: local
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set install initialization 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "Complete"
-      aggregate: false
+        installer_phase_initialize:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 ``` 
 
 Each phase or component of the installer will follow a similar pattern, with the
@@ -139,37 +142,25 @@ localhost            : ok=24   changed=0    unreachable=0    failed=0
 
 
 INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : Complete
-Master Additional Install  : Complete
-Node Install               : Complete
-GlusterFS Install          : Not Started
-Hosted Install             : Complete
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Service Catalog Install    : Not Started
+Initialization             : Complete (0:02:14)
+Health Check               : Complete (0:01:10)
+etcd Install               : Complete (0:02:01)
+Master Install             : Complete (0:11:43)
+Master Additional Install  : Complete (0:00:54)
+Node Install               : Complete (0:14:11)
+Hosted Install             : Complete (0:03:28)
 ```
 
 Example display if a failure occurs during execution:
 
 ```
 INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : In Progress
-     This phase can be restarted by running: playbooks/byo/openshift-master/config.yml
-Master Additional Install  : Not Started
-Node Install               : Not Started
-GlusterFS Install          : Not Started
-Hosted Install             : Not Started
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Service Catalog Install    : Not Started
+Initialization             : Complete (0:02:14)
+Health Check               : Complete (0:01:10)
+etcd Install               : Complete (0:02:58)
+Master Install             : Complete (0:09:20)
+Master Additional Install  : In Progress (0:20:04)
+    This phase can be restarted by running: playbooks/byo/openshift-master/additional_config.yml
 ```
 
 [set_stats]: http://docs.ansible.com/ansible/latest/set_stats_module.html

roles/installer_checkpoint/callback_plugins/installer_checkpoint.py

@@ -1,60 +1,10 @@
 """Ansible callback plugin to print a summary completion status of installation
 phases.
 """
+from datetime import datetime
 from ansible.plugins.callback import CallbackBase
 from ansible import constants as C
 
-DOCUMENTATION = '''
-
-'''
-
-EXAMPLES = '''
----------------------------------------------
-Example display of a successful playbook run:
-
-PLAY RECAP *********************************************************************
-master01.example.com : ok=158  changed=16   unreachable=0    failed=0
-node01.example.com   : ok=469  changed=74   unreachable=0    failed=0
-node02.example.com   : ok=157  changed=17   unreachable=0    failed=0
-localhost            : ok=24   changed=0    unreachable=0    failed=0
-
-
-INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : Complete
-Master Additional Install  : Complete
-Node Install               : Complete
-GlusterFS Install          : Not Started
-Hosted Install             : Complete
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Prometheus Install         : Not Started
-Service Catalog Install    : Not Started
-
------------------------------------------------------
-Example display if a failure occurs during execution:
-
-INSTALLER STATUS ***************************************************************
-Initialization             : Complete
-etcd Install               : Complete
-NFS Install                : Not Started
-Load balancer Install      : Not Started
-Master Install             : In Progress
-     This phase can be restarted by running: playbooks/byo/openshift-master/config.yml
-Master Additional Install  : Not Started
-Node Install               : Not Started
-GlusterFS Install          : Not Started
-Hosted Install             : Not Started
-Metrics Install            : Not Started
-Logging Install            : Not Started
-Prometheus Install         : Not Started
-Service Catalog Install    : Not Started
-
-'''
-
 
 class CallbackModule(CallbackBase):
     """This callback summarizes installation phase status."""
@@ -163,9 +113,10 @@ class CallbackModule(CallbackBase):
                 phase_title = phase_attributes[phase]['title']
                 padding = max_column - len(phase_title) + 2
                 if phase in stats.custom['_run']:
-                    phase_status = stats.custom['_run'][phase]
+                    phase_status = stats.custom['_run'][phase]['status']
+                    phase_time = phase_time_delta(stats.custom['_run'][phase])
                     self._display.display(
-                        '{}{}: {}'.format(phase_title, ' ' * padding, phase_status),
+                        '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time),
                         color=self.phase_color(phase_status))
                     if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
                         self._display.display(
@@ -192,3 +143,17 @@ class CallbackModule(CallbackBase):
             phase_color = C.COLOR_WARN
 
         return phase_color
+
+
+def phase_time_delta(phase):
+    """ Calculate the difference between phase start and end times """
+    time_format = '%Y%m%d%H%M%SZ'
+    phase_start = datetime.strptime(phase['start'], time_format)
+    if 'end' not in phase:
+        # The phase failed so set the end time to now
+        phase_end = datetime.now()
+    else:
+        phase_end = datetime.strptime(phase['end'], time_format)
+    delta = str(phase_end - phase_start).split(".")[0]  # Trim microseconds
+
+    return delta