Merge pull request #8289 from mtnbikenc/fix-checkpointing

Fix and simplify Installer Checkpoint

playbooks/init/main.yml

@@ -13,6 +13,8 @@
     set_stats:
       data:
         installer_phase_initialize:
+          title: "Initialization"
+          playbook: ""
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-autoheal/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_autoheal:
+          title: "Auto-heal Install"
+          playbook: "playbooks/openshift-autoheal/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-checks/private/install.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_health:
+          title: "Health Check"
+          playbook: "playbooks/openshift-checks/pre-install.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-descheduler/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_descheduler:
+          title: "Descheduler Install"
+          playbook: "playbooks/openshift-descheduler/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-etcd/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_etcd:
+          title: "etcd Install"
+          playbook: "playbooks/openshift-etcd/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-glusterfs/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_glusterfs:
+          title: "GlusterFS Install"
+          playbook: "playbooks/openshift-glusterfs/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-grafana/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_grafana:
+          title: "Grafana Install"
+          playbook: "playbooks/openshift-grafana/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-hosted/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_hosted:
+          title: "Hosted Install"
+          playbook: "playbooks/openshift-hosted/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-loadbalancer/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_loadbalancer:
+          title: "Load Balancer Install"
+          playbook: "playbooks/openshift-loadbalancer/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-logging/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_logging:
+          title: "Logging Install"
+          playbook: "playbooks/openshift-logging/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-management/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_management:
+          title: "Management Install"
+          playbook: "playbooks/openshift-management/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-master/private/additional_config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_master_additional:
+          title: "Master Additional Install"
+          playbook: "playbooks/openshift-master/additional_config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-master/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_master:
+          title: "Master Install"
+          playbook: "playbooks/openshift-master/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-metrics/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_metrics:
+          title: "Metrics Install"
+          playbook: "playbooks/openshift-metrics/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-monitor-availability/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_monitor_availability:
+          title: "Monitor Availability"
+          playbook: "playbooks/openshift-monitor-availability/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-monitoring/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_cluster_monitoring_operator:
+          title: "Cluster Monitoring Operator"
+          playbook: "playbooks/openshift-monitoring/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-nfs/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_nfs:
+          title: "NFS Install"
+          playbook: "playbooks/openshift-nfs/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-node-problem-detector/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_node_problem_detector:
+          title: "Node Problem Detector Install"
+          playbook: "playbooks/openshift-node-problem-detector/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-node/private/bootstrap.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_node:
+          title: "Node Preparation"
+          playbook: "(no entry point playbook)"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-prometheus/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_prometheus:
+          title: "Prometheus Install"
+          playbook: "playbooks/openshift-prometheus/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-service-catalog/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_servicecatalog:
+          title: "Service Catalog Install"
+          playbook: "playbooks/openshift-service-catalog/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

playbooks/openshift-web-console/private/config.yml

@@ -8,6 +8,8 @@
     set_stats:
       data:
         installer_phase_web_console:
+          title: "Web Console Install"
+          playbook: "playbooks/openshift-web-console/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 

roles/installer_checkpoint/README.md

@@ -28,52 +28,6 @@ The installer_checkpoint.py callback plugin extends the Ansible
 display the status of each phase which was run.  The INSTALLER STATUS report is
 displayed immediately following the PLAY RECAP.
 
-Phases of cluster installation are mapped to the steps in the
-[common/openshift-cluster/config.yml][openshift_cluster_config] playbook.
-
-To correctly display the order of the installer phases, the `installer_phases`
-variable defines the phase or component order.
-
-```python
-        # Set the order of the installer phases
-        installer_phases = [
-            'installer_phase_initialize',
-            'installer_phase_etcd',
-            'installer_phase_nfs',
-            'installer_phase_loadbalancer',
-            'installer_phase_master',
-            'installer_phase_master_additional',
-            'installer_phase_node',
-            'installer_phase_glusterfs',
-            'installer_phase_hosted',
-            'installer_phase_metrics',
-            'installer_phase_logging',
-            'installer_phase_servicecatalog',
-        ]
-```
-
-Additional attributes, such as display title and component playbook, of each
-phase are stored in the `phase_attributes` variable.
-
-```python
-        # Define the attributes of the installer phases
-        phase_attributes = {
-            'installer_phase_initialize': {
-                'title': 'Initialization',
-                'playbook': ''
-            },
-            'installer_phase_etcd': {
-                'title': 'etcd Install',
-                'playbook': 'playbooks/openshift-etcd/config.yml'
-            },
-            'installer_phase_nfs': {
-                'title': 'NFS Install',
-                'playbook': 'playbooks/openshift-nfs/config.yml'
-            },
-            #...
-        }
-```
-
 Usage
 -----
 
@@ -82,26 +36,27 @@ added to the beginning of the main playbook for the component to set the phase
 status to "In Progress".  Additionally, a play must be added after the last play
 for that component to set the phase status to "Complete".  
 
-The following example shows the first play of the 'installer phase' loading the
-`installer_checkpoint` role, as well as the `set_stats` task for setting
-`installer_phase_initialize` to "In Progress".  Various plays are run for the
-phase/component and then a final play for setting `installer_hase_initialize` to
-"Complete".
+The following example shows the first play of the etcd install using the
+`set_stats` module for setting the required checkpoint data points.
+
+* `title` - Name of the component phase
+* `playbook` - Entry point playbook used to run only this component
+* `status` - "In Progress" or "Complete"
 
 ```yaml
-# init/main.yml
+# playbooks/openshift-etcd/private/config.yml
 ---
-- name: Initialization Checkpoint Start
+- name: etcd Install Checkpoint Start
   hosts: all
   gather_facts: false
-  roles:
-  - installer_checkpoint
   tasks:
-  - name: Set install initialization 'In Progress'
+  - name: Set etcd install 'In Progress'
     run_once: true
     set_stats:
       data:
-        installer_phase_initialize:
+        installer_phase_etcd:
+          title: "etcd Install"
+          playbook: "playbooks/openshift-etcd/config.yml"
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
@@ -109,25 +64,19 @@ phase/component and then a final play for setting `installer_hase_initialize` to
 # Various plays here
 #...
 
-- name: Initialization Checkpoint End
+- name: etcd Install Checkpoint End
   hosts: all
   gather_facts: false
   tasks:
-  - name: Set install initialization 'Complete'
+  - name: Set etcd install 'Complete'
     run_once: true
     set_stats:
       data:
-        installer_phase_initialize:
+        installer_phase_etcd:
           status: "Complete"
           end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 ``` 
 
-Each phase or component of the installer will follow a similar pattern, with the
-exception that the `installer_checkpoint` role does not need to be called since
-it was already loaded by the play in `init/main.yml`.  It is important to
-place the 'In Progress' and 'Complete' plays as the first and last plays of the
-phase or component.
- 
 Examples
 --------
 
@@ -164,4 +113,3 @@ Master Additional Install  : In Progress (0:20:04)
 ```
 
 [set_stats]: http://docs.ansible.com/ansible/latest/set_stats_module.html
-[openshift_cluster_config]: https://github.com/openshift/openshift-ansible/blob/master/playbooks/common/openshift-cluster/config.yml

roles/installer_checkpoint/callback_plugins/installer_checkpoint.py

@@ -19,120 +19,36 @@ class CallbackModule(CallbackBase):
 
     def v2_playbook_on_stats(self, stats):
 
-        # Set the order of the installer phases
-        installer_phases = [
-            'installer_phase_initialize',
-            'installer_phase_health',
-            'installer_phase_etcd',
-            'installer_phase_nfs',
-            'installer_phase_loadbalancer',
-            'installer_phase_master',
-            'installer_phase_master_additional',
-            'installer_phase_node',
-            'installer_phase_glusterfs',
-            'installer_phase_hosted',
-            'installer_phase_web_console',
-            'installer_phase_metrics',
-            'installer_phase_logging',
-            'installer_phase_prometheus',
-            'installer_phase_servicecatalog',
-            'installer_phase_management',
-        ]
-
-        # Define the attributes of the installer phases
-        phase_attributes = {
-            'installer_phase_initialize': {
-                'title': 'Initialization',
-                'playbook': ''
-            },
-            'installer_phase_health': {
-                'title': 'Health Check',
-                'playbook': 'playbooks/openshift-checks/pre-install.yml'
-            },
-            'installer_phase_etcd': {
-                'title': 'etcd Install',
-                'playbook': 'playbooks/openshift-etcd/config.yml'
-            },
-            'installer_phase_nfs': {
-                'title': 'NFS Install',
-                'playbook': 'playbooks/openshift-nfs/config.yml'
-            },
-            'installer_phase_loadbalancer': {
-                'title': 'Load balancer Install',
-                'playbook': 'playbooks/openshift-loadbalancer/config.yml'
-            },
-            'installer_phase_master': {
-                'title': 'Master Install',
-                'playbook': 'playbooks/openshift-master/config.yml'
-            },
-            'installer_phase_master_additional': {
-                'title': 'Master Additional Install',
-                'playbook': 'playbooks/openshift-master/additional_config.yml'
-            },
-            'installer_phase_node': {
-                'title': 'Node Install',
-                'playbook': 'playbooks/openshift-node/config.yml'
-            },
-            'installer_phase_glusterfs': {
-                'title': 'GlusterFS Install',
-                'playbook': 'playbooks/openshift-glusterfs/config.yml'
-            },
-            'installer_phase_hosted': {
-                'title': 'Hosted Install',
-                'playbook': 'playbooks/openshift-hosted/config.yml'
-            },
-            'installer_phase_web_console': {
-                'title': 'Web Console Install',
-                'playbook': 'playbooks/openshift-web-console/config.yml'
-            },
-            'installer_phase_metrics': {
-                'title': 'Metrics Install',
-                'playbook': 'playbooks/openshift-metrics/config.yml'
-            },
-            'installer_phase_logging': {
-                'title': 'Logging Install',
-                'playbook': 'playbooks/openshift-logging/config.yml'
-            },
-            'installer_phase_prometheus': {
-                'title': 'Prometheus Install',
-                'playbook': 'playbooks/openshift-prometheus/config.yml'
-            },
-            'installer_phase_servicecatalog': {
-                'title': 'Service Catalog Install',
-                'playbook': 'playbooks/openshift-service-catalog/config.yml'
-            },
-            'installer_phase_management': {
-                'title': 'Management Install',
-                'playbook': 'playbooks/openshift-management/config.yml'
-            },
-        }
+        phases = stats.custom['_run']
 
         # Find the longest phase title
         max_column = 0
-        for phase in phase_attributes:
-            max_column = max(max_column, len(phase_attributes[phase]['title']))
-
-        if '_run' in stats.custom:
-            self._display.banner('INSTALLER STATUS')
-            for phase in installer_phases:
-                phase_title = phase_attributes[phase]['title']
-                padding = max_column - len(phase_title) + 2
-                if phase in stats.custom['_run']:
-                    phase_status = stats.custom['_run'][phase]['status']
-                    phase_time = phase_time_delta(stats.custom['_run'][phase])
-                    self._display.display(
-                        '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time),
-                        color=self.phase_color(phase_status))
-                    if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
-                        self._display.display(
-                            '\tThis phase can be restarted by running: {}'.format(
-                                phase_attributes[phase]['playbook']))
-                    if 'message' in stats.custom['_run'][phase]:
-                        self._display.display(
-                            '\t{}'.format(
-                                stats.custom['_run'][phase]['message']))
-
-        self._display.display("", screen_only=True)
+        for phase in phases:
+            max_column = max(max_column, len(phases[phase]['title']))
+
+        # Sort the phases by start time
+        ordered_phases = sorted(phases, key=lambda x: (phases[x]['start']))
+
+        self._display.banner('INSTALLER STATUS')
+        # Display status information for each phase
+        for phase in ordered_phases:
+            phase_title = phases[phase]['title']
+            padding = max_column - len(phase_title) + 2
+            phase_status = phases[phase]['status']
+            phase_time = phase_time_delta(phases[phase])
+            self._display.display(
+                '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time),
+                color=self.phase_color(phase_status))
+            # If the phase is not complete, tell the user what playbook to rerun
+            if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
+                self._display.display(
+                    '\tThis phase can be restarted by running: {}'.format(
+                        phases[phase]['playbook']))
+            # Display any extra messages stored during the phase
+            if 'message' in phases[phase]:
+                self._display.display(
+                    '\t{}'.format(
+                        phases[phase]['message']))
 
     def phase_color(self, status):
         """ Return color code for installer phase"""