Merge pull request #5527 from mtnbikenc/installer-checkpoint

Automatic merge from submit-queue.

Installer checkpointing

This pull request adds an Ansible callback plugin to allow for 'checkpointing' the status of the OpenShift-Ansible installer.  The installer is divided up into phases and each phase will start with a status of 'Not Started'.  As the phase is performed the status will be updated to 'In Progress' and finally 'Complete'.  Should the install fail or complete successfully, the callback plugin will report the status of all phases after the PLAY RECAP.  The INSTALLER STATUS is also written to the Ansible log file.

https://trello.com/c/VNEvT2Fo

Changes:
- Added installer_checkpoint role and callback plugin
- Added checkpoint 'Start' and 'End' plays to each installation phase

Additional items related to enabling proper checkpointing:
- Removed openshift_set_hostname and related task (related to 3.0)
- Added openshift-hosted entry point playbook
- Moved openshift metrics and logging out of openshift_hosted playbook
- Moved API Aggregation play to master install


Example playbook output:
```
<- snip ->
PLAY RECAP ***********************************************************************************************************************************
ec2-34-207-180-218.compute-1.amazonaws.com : ok=158  changed=16   unreachable=0    failed=0   
ec2-52-87-169-12.compute-1.amazonaws.com : ok=469  changed=74   unreachable=0    failed=0   
ec2-52-90-209-20.compute-1.amazonaws.com : ok=157  changed=17   unreachable=0    failed=0   
localhost                  : ok=24   changed=0    unreachable=0    failed=0   


INSTALLER STATUS *****************************************************************************************************************************
Initialization             : Complete
etcd Install               : Complete
NFS Install                : Not Started
Load balancer Install      : Not Started
Master Install             : Complete
Master Additional Install  : Complete
Node Install               : Complete
GlusterFS Install          : Not Started
Hosted Install             : Complete
Metrics Install            : Not Started
Logging Install            : Not Started
Service Catalog Install    : Not Started
```

inventory/byo/hosts.origin.example

@@ -638,7 +638,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # the CIDRs reserved for external IPs, nodes, pods, or services.
 #openshift_master_ingress_ip_network_cidr=172.46.0.0/16
 
-# Configure number of bits to allocate to each host’s subnet e.g. 9
+# Configure number of bits to allocate to each host's subnet e.g. 9
 # would mean a /23 network on the host.
 # When upgrading or scaling up the following must match whats in your master config!
 #  Inventory: master yaml field
@@ -707,11 +707,6 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # interface other than the default network interface.
 #openshift_set_node_ip=True
 
-# Force setting of system hostname when configuring OpenShift
-# This works around issues related to installations that do not have valid dns
-# entries for the interfaces attached to the host.
-#openshift_set_hostname=True
-
 # Configure dnsIP in the node config
 #openshift_dns_ip=172.30.0.1
 

inventory/byo/hosts.ose.example

@@ -646,7 +646,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # the CIDRs reserved for external IPs, nodes, pods, or services.
 #openshift_master_ingress_ip_network_cidr=172.46.0.0/16
 
-# Configure number of bits to allocate to each host’s subnet e.g. 9
+# Configure number of bits to allocate to each host's subnet e.g. 9
 # would mean a /23 network on the host.
 # When upgrading or scaling up the following must match whats in your master config!
 #  Inventory: master yaml field
@@ -715,11 +715,6 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # interface other than the default network interface.
 #openshift_set_node_ip=True
 
-# Force setting of system hostname when configuring OpenShift
-# This works around issues related to installations that do not have valid dns
-# entries for the interfaces attached to the host.
-#openshift_set_hostname=True
-
 # Configure dnsIP in the node config
 #openshift_dns_ip=172.30.0.1
 

playbooks/byo/openshift-cluster/openshift-hosted.yml

@@ -0,0 +1,6 @@
+---
+- include: initialize_groups.yml
+
+- include: ../../common/openshift-cluster/std_include.yml
+
+- include: ../../common/openshift-cluster/openshift_hosted.yml

playbooks/common/openshift-cluster/config.yml

@@ -22,57 +22,33 @@
   tags:
   - always
 
-- name: Set hostname
-  hosts: oo_masters_to_config:oo_nodes_to_config
-  tasks:
-  # TODO: switch back to hostname module once we depend on ansible-2.4
-  # https://github.com/ansible/ansible/pull/25906
-  - name: Set hostname
-    command: "hostnamectl set-hostname {{ openshift.common.hostname }}"
-    when: openshift_set_hostname | default(false,true) | bool
-
 - include: ../openshift-etcd/config.yml
 
 - include: ../openshift-nfs/config.yml
-  tags:
-  - nfs
+  when: groups.oo_nfs_to_config | default([]) | count > 0
 
 - include: ../openshift-loadbalancer/config.yml
-  tags:
-  - loadbalancer
+  when: groups.oo_lb_to_config | default([]) | count > 0
 
 - include: ../openshift-master/config.yml
 
 - include: ../openshift-master/additional_config.yml
 
 - include: ../openshift-node/config.yml
-  tags:
-  - node
 
 - include: ../openshift-glusterfs/config.yml
-  tags:
-  - glusterfs
+  when: groups.oo_glusterfs_to_config | default([]) | count > 0
 
 - include: openshift_hosted.yml
-  tags:
-  - hosted
 
-- name: Configure API Aggregation on masters
-  hosts: oo_masters
-  serial: 1
-  tasks:
-  - block:
-    - include_role:
-        name: openshift_service_catalog
-        tasks_from: wire_aggregator
-      vars:
-        first_master: "{{ groups.oo_first_master[0] }}"
+- include: openshift_metrics.yml
+  when: openshift_metrics_install_metrics | default(false) | bool
+
+- include: openshift_logging.yml
+  when: openshift_logging_install_logging | default(false) | bool
 
 - include: service_catalog.yml
-  when:
-  - openshift_enable_service_catalog | default(false) | bool
-  tags:
-  - servicecatalog
+  when: openshift_enable_service_catalog | default(false) | bool
 
 - name: Print deprecated variable warning message if necessary
   hosts: oo_first_master

playbooks/common/openshift-cluster/openshift_hosted.yml

@@ -1,4 +1,15 @@
 ---
+- name: Hosted Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Hosted install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_hosted: "In Progress"
+      aggregate: false
+
 - include: create_persistent_volumes.yml
 
 - include: openshift_default_storage_class.yml
@@ -9,10 +20,17 @@
 
 - include: openshift_hosted_registry.yml
 
-- include: openshift_metrics.yml
-
-- include: openshift_logging.yml
-
 - include: cockpit-ui.yml
 
 - include: openshift_prometheus.yml
+
+- name: Hosted Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Hosted install 'Complete'
+    set_stats:
+      data:
+        installer_phase_hosted: "Complete"
+      aggregate: false

playbooks/common/openshift-cluster/openshift_logging.yml

@@ -1,4 +1,15 @@
 ---
+- name: Logging Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Logging install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_logging: "In Progress"
+      aggregate: false
+
 - name: OpenShift Aggregated Logging
   hosts: oo_first_master
   roles:
@@ -11,4 +22,14 @@
     - include_role:
         name: openshift_logging
         tasks_from: update_master_config
-    when: openshift_logging_install_logging | default(false) | bool
+
+- name: Logging Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Logging install 'Complete'
+    set_stats:
+      data:
+        installer_phase_logging: "Complete"
+      aggregate: false

playbooks/common/openshift-cluster/openshift_metrics.yml

@@ -1,10 +1,19 @@
 ---
+- name: Metrics Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Metrics install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_metrics: "In Progress"
+      aggregate: false
+
 - name: OpenShift Metrics
   hosts: oo_first_master
   roles:
   - role: openshift_metrics
-    when: openshift_metrics_install_metrics | default(false) | bool
-
 
 - name: OpenShift Metrics
   hosts: oo_masters:!oo_first_master
@@ -14,4 +23,14 @@
     include_role:
       name: openshift_metrics
       tasks_from: update_master_config.yaml
-    when: openshift_metrics_install_metrics | default(false) | bool
+
+- name: Metrics Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Metrics install 'Complete'
+    set_stats:
+      data:
+        installer_phase_metrics: "Complete"
+      aggregate: false

playbooks/common/openshift-cluster/service_catalog.yml

@@ -1,4 +1,15 @@
 ---
+- name: Service Catalog Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Service Catalog install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_servicecatalog: "In Progress"
+      aggregate: false
+
 - name: Service Catalog
   hosts: oo_first_master
   roles:
@@ -7,3 +18,14 @@
   - template_service_broker
   vars:
     first_master: "{{ groups.oo_first_master[0] }}"
+
+- name: Service Catalog Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Service Catalog install 'Complete'
+    set_stats:
+      data:
+        installer_phase_servicecatalog: "Complete"
+      aggregate: false

playbooks/common/openshift-cluster/std_include.yml

@@ -1,4 +1,17 @@
 ---
+- name: Initialization Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles:
+  - installer_checkpoint
+  tasks:
+  - name: Set install initialization 'In Progress'
+    set_stats:
+      data:
+        installer_phase_initialize: "In Progress"
+      aggregate: false
+
 - include: evaluate_groups.yml
   tags:
   - always
@@ -22,3 +35,14 @@
 - include: initialize_openshift_version.yml
   tags:
   - always
+
+- name: Initialization Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set install initialization 'Complete'
+    set_stats:
+      data:
+        installer_phase_initialize: "Complete"
+      aggregate: false

playbooks/common/openshift-etcd/config.yml

@@ -1,4 +1,15 @@
 ---
+- name: etcd Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set etcd install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_etcd: "In Progress"
+      aggregate: false
+
 - name: Configure etcd
   hosts: oo_etcd_to_config
   any_errors_fatal: true
@@ -10,3 +21,14 @@
     etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
     r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   - role: nickhammond.logrotate
+
+- name: etcd Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set etcd install 'Complete'
+    set_stats:
+      data:
+        installer_phase_etcd: "Complete"
+      aggregate: false

playbooks/common/openshift-glusterfs/config.yml

@@ -1,4 +1,15 @@
 ---
+- name: GlusterFS Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set GlusterFS install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_glusterfs: "In Progress"
+      aggregate: false
+
 - name: Open firewall ports for GlusterFS nodes
   hosts: glusterfs
   tasks:
@@ -24,3 +35,14 @@
     include_role:
       name: openshift_storage_glusterfs
     when: groups.oo_glusterfs_to_config | default([]) | count > 0
+
+- name: GlusterFS Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set GlusterFS install 'Complete'
+    set_stats:
+      data:
+        installer_phase_glusterfs: "Complete"
+      aggregate: false

playbooks/common/openshift-loadbalancer/config.yml

@@ -1,4 +1,15 @@
 ---
+- name: Load Balancer Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set load balancer install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_loadbalancer: "In Progress"
+      aggregate: false
+
 - name: Configure load balancers
   hosts: oo_lb_to_config
   vars:
@@ -16,3 +27,14 @@
   roles:
   - role: os_firewall
   - role: openshift_loadbalancer
+
+- name: Load Balancer Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set load balancer install 'Complete'
+    set_stats:
+      data:
+        installer_phase_loadbalancer: "Complete"
+      aggregate: false

playbooks/common/openshift-master/additional_config.yml

@@ -1,4 +1,15 @@
 ---
+- name: Master Additional Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Master Additional install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_master_additional: "In Progress"
+      aggregate: false
+
 - name: Additional master configuration
   hosts: oo_first_master
   vars:
@@ -24,3 +35,14 @@
     - openshift.common.deployment_subtype != 'registry'
   - role: flannel_register
     when: openshift_use_flannel | default(false) | bool
+
+- name: Master Additional Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Master Additional install 'Complete'
+    set_stats:
+      data:
+        installer_phase_master_additional: "Complete"
+      aggregate: false

playbooks/common/openshift-master/config.yml

@@ -1,4 +1,15 @@
 ---
+- name: Master Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Master install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_master: "In Progress"
+      aggregate: false
+
 - name: Disable excluders
   hosts: oo_masters_to_config
   gather_facts: no
@@ -231,6 +242,12 @@
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
+- name: Configure API Aggregation on masters
+  hosts: oo_masters
+  serial: 1
+  tasks:
+  - include: tasks/wire_aggregator.yml
+
 - name: Re-enable excluder if it was previously enabled
   hosts: oo_masters_to_config
   gather_facts: no
@@ -238,3 +255,14 @@
   - role: openshift_excluder
     r_openshift_excluder_action: enable
     r_openshift_excluder_service_type: "{{ openshift.common.service_type }}"
+
+- name: Master Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Master install 'Complete'
+    set_stats:
+      data:
+        installer_phase_master: "Complete"
+      aggregate: false

roles/openshift_service_catalog/tasks/wire_aggregator.yml

@@ -9,23 +9,23 @@
     path: /etc/origin/master/front-proxy-ca.crt
   register: first_proxy_ca_crt
   changed_when: false
-  delegate_to: "{{ first_master }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
 
 - name: Check for First Master Aggregator Signer key
   stat:
     path: /etc/origin/master/front-proxy-ca.crt
   register: first_proxy_ca_key
   changed_when: false
-  delegate_to: "{{ first_master }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
 
 # TODO: this currently has a bug where hostnames are required
 - name: Creating First Master Aggregator signer certs
   command: >
-    {{ hostvars[first_master].openshift.common.client_binary }} adm ca create-signer-cert
+    {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm ca create-signer-cert
     --cert=/etc/origin/master/front-proxy-ca.crt
     --key=/etc/origin/master/front-proxy-ca.key
     --serial=/etc/origin/master/ca.serial.txt
-  delegate_to: "{{ first_master }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
   when:
   - not first_proxy_ca_crt.stat.exists
   - not first_proxy_ca_key.stat.exists
@@ -50,7 +50,7 @@
   with_items:
   - front-proxy-ca.crt
   - front-proxy-ca.key
-  delegate_to: "{{ first_master }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
   when:
   - not proxy_ca_key.stat.exists
   - not proxy_ca_crt.stat.exists
@@ -74,7 +74,7 @@
   stat:
     path: /etc/origin/master/aggregator-front-proxy.kubeconfig
   register: first_front_proxy_kubeconfig
-  delegate_to: "{{ first_master }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
   run_once: true
 
 # create-api-client-config generates a ca.crt file which will
@@ -84,14 +84,14 @@
 - block:
   - name: Create first master api-client config for Aggregator
     command: >
-      {{ hostvars[first_master].openshift.common.client_binary }} adm create-api-client-config
+      {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm create-api-client-config
       --certificate-authority=/etc/origin/master/front-proxy-ca.crt
       --signer-cert=/etc/origin/master/front-proxy-ca.crt
       --signer-key=/etc/origin/master/front-proxy-ca.key
       --user aggregator-front-proxy
       --client-dir={{ certtemp.stdout }}
       --signer-serial=/etc/origin/master/ca.serial.txt
-    delegate_to: "{{ first_master }}"
+    delegate_to: "{{ groups.oo_first_master.0 }}"
     run_once: true
   - name: Copy first master api-client config for Aggregator
     copy:
@@ -102,7 +102,7 @@
     - aggregator-front-proxy.crt
     - aggregator-front-proxy.key
     - aggregator-front-proxy.kubeconfig
-    delegate_to: "{{ first_master }}"
+    delegate_to: "{{ groups.oo_first_master.0 }}"
     run_once: true
   when:
   - not first_front_proxy_kubeconfig.stat.exists
@@ -117,7 +117,7 @@
     src: "/etc/origin/master/{{ item }}"
     dest: "{{ certtemp.stdout }}/{{ item }}"
     flat: yes
-  delegate_to: "{{ first_master }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
   with_items:
   - aggregator-front-proxy.crt
   - aggregator-front-proxy.key

playbooks/common/openshift-nfs/config.yml

@@ -1,6 +1,28 @@
 ---
+- name: NFS Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set NFS install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_nfs: "In Progress"
+      aggregate: false
+
 - name: Configure nfs
   hosts: oo_nfs_to_config
   roles:
   - role: os_firewall
   - role: openshift_storage_nfs
+
+- name: NFS Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set NFS install 'Complete'
+    set_stats:
+      data:
+        installer_phase_nfs: "Complete"
+      aggregate: false

playbooks/common/openshift-node/config.yml

@@ -1,4 +1,15 @@
 ---
+- name: Node Install Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Node install 'In Progress'
+    set_stats:
+      data:
+        installer_phase_node: "In Progress"
+      aggregate: false
+
 - name: Disable excluders
   hosts: oo_nodes_to_config
   gather_facts: no
@@ -99,3 +110,14 @@
   - role: openshift_excluder
     r_openshift_excluder_action: enable
     r_openshift_excluder_service_type: "{{ openshift.common.service_type }}"
+
+- name: Node Install Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set Node install 'Complete'
+    set_stats:
+      data:
+        installer_phase_node: "Complete"
+      aggregate: false

roles/installer_checkpoint/README.md

@@ -0,0 +1,177 @@
+OpenShift-Ansible Installer Checkpoint
+======================================
+
+A complete OpenShift cluster installation is comprised of many different
+components which can take 30 minutes to several hours to complete.  If the
+installation should fail, it could be confusing to understand at which component
+the failure occurred.  Additionally, it may be desired to re-run only the
+component which failed instead of starting over from the beginning.  Components
+which came after the failed component would also need to be run individually.
+
+Design
+------
+
+The Installer Checkpoint implements an Ansible callback plugin to allow
+displaying and logging of the installer status at the end of a playbook run.
+
+To ensure the callback plugin is loaded, regardless of ansible.cfg file
+configuration, the plugin has been placed inside the installer_checkpoint role
+which must be called early in playbook execution. The `std_include.yml` playbook
+is run first for all entry point playbooks, therefore, the initialization of the
+checkpoint plugin has been placed at the beginning of that file.
+
+Playbooks use the [set_stats][set_stats] Ansible module to set a custom stats
+variable indicating the status of the phase being executed.
+
+The installer_checkpoint.py callback plugin extends the Ansible
+`v2_playbook_on_stats` method, which is called at the end of a playbook run, to
+display the status of each phase which was run.  The INSTALLER STATUS report is
+displayed immediately following the PLAY RECAP.
+
+Phases of cluster installation are mapped to the steps in the
+[common/openshift-cluster/config.yml][openshift_cluster_config] playbook.
+
+To correctly display the order of the installer phases, the `installer_phases`
+variable defines the phase or component order.
+
+```python
+        # Set the order of the installer phases
+        installer_phases = [
+            'installer_phase_initialize',
+            'installer_phase_etcd',
+            'installer_phase_nfs',
+            'installer_phase_loadbalancer',
+            'installer_phase_master',
+            'installer_phase_master_additional',
+            'installer_phase_node',
+            'installer_phase_glusterfs',
+            'installer_phase_hosted',
+            'installer_phase_metrics',
+            'installer_phase_logging',
+            'installer_phase_servicecatalog',
+        ]
+```
+
+Additional attributes, such as display title and component playbook, of each
+phase are stored in the `phase_attributes` variable.
+
+```python
+        # Define the attributes of the installer phases
+        phase_attributes = {
+            'installer_phase_initialize': {
+                'title': 'Initialization',
+                'playbook': ''
+            },
+            'installer_phase_etcd': {
+                'title': 'etcd Install',
+                'playbook': 'playbooks/byo/openshift-etcd/config.yml'
+            },
+            'installer_phase_nfs': {
+                'title': 'NFS Install',
+                'playbook': 'playbooks/byo/openshift-nfs/config.yml'
+            },
+            #...
+        }
+```
+
+Usage
+-----
+
+In order to indicate the beginning of a component installation, a play must be
+added to the beginning of the main playbook for the component to set the phase
+status to "In Progress".  Additionally, a play must be added after the last play
+for that component to set the phase status to "Complete".  
+
+The following example shows the first play of the 'installer phase' loading the
+`installer_checkpoint` role, as well as the `set_stats` task for setting
+`installer_phase_initialize` to "In Progress".  Various plays are run for the
+phase/component and then a final play for setting `installer_hase_initialize` to
+"Complete".
+
+```yaml
+# common/openshift-cluster/std_include.yml
+---
+- name: Initialization Checkpoint Start
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles:
+  - installer_checkpoint
+  tasks:
+  - name: Set install initialization 'In Progress'
+    set_stats:
+      data:
+        installer_phase_initialize: "In Progress"
+      aggregate: false
+
+#...
+# Various plays here
+#...
+
+- name: Initialization Checkpoint End
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+  - name: Set install initialization 'Complete'
+    set_stats:
+      data:
+        installer_phase_initialize: "Complete"
+      aggregate: false
+``` 
+
+Each phase or component of the installer will follow a similar pattern, with the
+exception that the `installer_checkpoint` role does not need to be called since
+it was already loaded by the play in `std_include.yml`.  It is important to
+place the 'In Progress' and 'Complete' plays as the first and last plays of the
+phase or component.
+ 
+Examples
+--------
+
+Example display of a successful playbook run:
+
+```
+PLAY RECAP *********************************************************************
+master01.example.com : ok=158  changed=16   unreachable=0    failed=0
+node01.example.com   : ok=469  changed=74   unreachable=0    failed=0
+node02.example.com   : ok=157  changed=17   unreachable=0    failed=0
+localhost            : ok=24   changed=0    unreachable=0    failed=0
+
+
+INSTALLER STATUS ***************************************************************
+Initialization             : Complete
+etcd Install               : Complete
+NFS Install                : Not Started
+Load balancer Install      : Not Started
+Master Install             : Complete
+Master Additional Install  : Complete
+Node Install               : Complete
+GlusterFS Install          : Not Started
+Hosted Install             : Complete
+Metrics Install            : Not Started
+Logging Install            : Not Started
+Service Catalog Install    : Not Started
+```
+
+Example display if a failure occurs during execution:
+
+```
+INSTALLER STATUS ***************************************************************
+Initialization             : Complete
+etcd Install               : Complete
+NFS Install                : Not Started
+Load balancer Install      : Not Started
+Master Install             : In Progress
+     This phase can be restarted by running: playbooks/byo/openshift-master/config.yml
+Master Additional Install  : Not Started
+Node Install               : Not Started
+GlusterFS Install          : Not Started
+Hosted Install             : Not Started
+Metrics Install            : Not Started
+Logging Install            : Not Started
+Service Catalog Install    : Not Started
+```
+
+[set_stats]: http://docs.ansible.com/ansible/latest/set_stats_module.html
+[openshift_cluster_config]: https://github.com/openshift/openshift-ansible/blob/master/playbooks/common/openshift-cluster/config.yml

roles/installer_checkpoint/callback_plugins/installer_checkpoint.py

@@ -0,0 +1,182 @@
+"""Ansible callback plugin to print a summary completion status of installation
+phases.
+"""
+from ansible.plugins.callback import CallbackBase
+from ansible import constants as C
+
+DOCUMENTATION = '''
+
+'''
+
+EXAMPLES = '''
+---------------------------------------------
+Example display of a successful playbook run:
+
+PLAY RECAP *********************************************************************
+master01.example.com : ok=158  changed=16   unreachable=0    failed=0
+node01.example.com   : ok=469  changed=74   unreachable=0    failed=0
+node02.example.com   : ok=157  changed=17   unreachable=0    failed=0
+localhost            : ok=24   changed=0    unreachable=0    failed=0
+
+
+INSTALLER STATUS ***************************************************************
+Initialization             : Complete
+etcd Install               : Complete
+NFS Install                : Not Started
+Load balancer Install      : Not Started
+Master Install             : Complete
+Master Additional Install  : Complete
+Node Install               : Complete
+GlusterFS Install          : Not Started
+Hosted Install             : Complete
+Metrics Install            : Not Started
+Logging Install            : Not Started
+Service Catalog Install    : Not Started
+
+-----------------------------------------------------
+Example display if a failure occurs during execution:
+
+INSTALLER STATUS ***************************************************************
+Initialization             : Complete
+etcd Install               : Complete
+NFS Install                : Not Started
+Load balancer Install      : Not Started
+Master Install             : In Progress
+     This phase can be restarted by running: playbooks/byo/openshift-master/config.yml
+Master Additional Install  : Not Started
+Node Install               : Not Started
+GlusterFS Install          : Not Started
+Hosted Install             : Not Started
+Metrics Install            : Not Started
+Logging Install            : Not Started
+Service Catalog Install    : Not Started
+
+'''
+
+
+class CallbackModule(CallbackBase):
+    """This callback summarizes installation phase status."""
+
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'aggregate'
+    CALLBACK_NAME = 'installer_checkpoint'
+    CALLBACK_NEEDS_WHITELIST = False
+
+    def __init__(self):
+        super(CallbackModule, self).__init__()
+
+    def v2_playbook_on_stats(self, stats):
+
+        # Set the order of the installer phases
+        installer_phases = [
+            'installer_phase_initialize',
+            'installer_phase_etcd',
+            'installer_phase_nfs',
+            'installer_phase_loadbalancer',
+            'installer_phase_master',
+            'installer_phase_master_additional',
+            'installer_phase_node',
+            'installer_phase_glusterfs',
+            'installer_phase_hosted',
+            'installer_phase_metrics',
+            'installer_phase_logging',
+            'installer_phase_servicecatalog',
+        ]
+
+        # Define the attributes of the installer phases
+        phase_attributes = {
+            'installer_phase_initialize': {
+                'title': 'Initialization',
+                'playbook': ''
+            },
+            'installer_phase_etcd': {
+                'title': 'etcd Install',
+                'playbook': 'playbooks/byo/openshift-etcd/config.yml'
+            },
+            'installer_phase_nfs': {
+                'title': 'NFS Install',
+                'playbook': 'playbooks/byo/openshift-nfs/config.yml'
+            },
+            'installer_phase_loadbalancer': {
+                'title': 'Load balancer Install',
+                'playbook': 'playbooks/byo/openshift-loadbalancer/config.yml'
+            },
+            'installer_phase_master': {
+                'title': 'Master Install',
+                'playbook': 'playbooks/byo/openshift-master/config.yml'
+            },
+            'installer_phase_master_additional': {
+                'title': 'Master Additional Install',
+                'playbook': 'playbooks/byo/openshift-master/additional_config.yml'
+            },
+            'installer_phase_node': {
+                'title': 'Node Install',
+                'playbook': 'playbooks/byo/openshift-node/config.yml'
+            },
+            'installer_phase_glusterfs': {
+                'title': 'GlusterFS Install',
+                'playbook': 'playbooks/byo/openshift-glusterfs/config.yml'
+            },
+            'installer_phase_hosted': {
+                'title': 'Hosted Install',
+                'playbook': 'playbooks/byo/openshift-cluster/openshift-hosted.yml'
+            },
+            'installer_phase_metrics': {
+                'title': 'Metrics Install',
+                'playbook': 'playbooks/byo/openshift-cluster/openshift-metrics.yml'
+            },
+            'installer_phase_logging': {
+                'title': 'Logging Install',
+                'playbook': 'playbooks/byo/openshift-cluster/openshift-logging.yml'
+            },
+            'installer_phase_servicecatalog': {
+                'title': 'Service Catalog Install',
+                'playbook': 'playbooks/byo/openshift-cluster/service-catalog.yml'
+            },
+        }
+
+        # Find the longest phase title
+        max_column = 0
+        for phase in phase_attributes:
+            max_column = max(max_column, len(phase_attributes[phase]['title']))
+
+        if '_run' in stats.custom:
+            self._display.banner('INSTALLER STATUS')
+            for phase in installer_phases:
+                phase_title = phase_attributes[phase]['title']
+                padding = max_column - len(phase_title) + 2
+                if phase in stats.custom['_run']:
+                    phase_status = stats.custom['_run'][phase]
+                    self._display.display(
+                        '{}{}: {}'.format(phase_title, ' ' * padding, phase_status),
+                        color=self.phase_color(phase_status))
+                    if phase_status == 'In Progress' and phase != 'installer_phase_initialize':
+                        self._display.display(
+                            '\tThis phase can be restarted by running: {}'.format(
+                                phase_attributes[phase]['playbook']))
+                else:
+                    # Phase was not found in custom stats
+                    self._display.display(
+                        '{}{}: {}'.format(phase_title, ' ' * padding, 'Not Started'),
+                        color=C.COLOR_SKIP)
+
+        self._display.display("", screen_only=True)
+
+    def phase_color(self, status):
+        """ Return color code for installer phase"""
+        valid_status = [
+            'In Progress',
+            'Complete',
+        ]
+
+        if status not in valid_status:
+            self._display.warning('Invalid phase status defined: {}'.format(status))
+
+        if status == 'Complete':
+            phase_color = C.COLOR_OK
+        elif status == 'In Progress':
+            phase_color = C.COLOR_ERROR
+        else:
+            phase_color = C.COLOR_WARN
+
+        return phase_color

roles/openshift_metrics/tasks/main.yaml

@@ -45,11 +45,11 @@
 
 - include: install_metrics.yaml
   when:
-    - openshift_metrics_install_metrics | default(false) | bool
+    - openshift_metrics_install_metrics | bool
 
 - include: uninstall_metrics.yaml
   when:
-    - openshift_metrics_uninstall_metrics | default(false) | bool
+    - openshift_metrics_uninstall_metrics | bool
 
 - include: uninstall_hosa.yaml
   when: not openshift_metrics_install_hawkular_agent | bool