Add cert expiry check to upgrades

This commit extends cert expiry check to fail
if certs are within warning timeline as well
as utilizing the check during upgrades.

playbooks/common/openshift-cluster/upgrades/init.yml

@@ -9,6 +9,13 @@
 - import_playbook: ../../../init/base_packages.yml
 - import_playbook: ../../../init/cluster_facts.yml
 
+- name: Inspect cluster certificates
+  hosts: "{{ l_upgrade_cert_check_hosts }}"
+  tasks:
+  - import_role:
+      name: openshift_certificate_expiry
+      tasks_from: main.yml
+
 - name: Ensure essential node configmaps are present
   hosts: oo_first_master
   tasks:

playbooks/common/openshift-cluster/upgrades/upgrade_scale_group.yml

@@ -16,6 +16,8 @@
 
 - name: initialize upgrade bits
   import_playbook: init.yml
+  vars:
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
 
 - name: unschedule nodes
   hosts: oo_sg_current_nodes

playbooks/common/openshift-cluster/upgrades/v3_10/upgrade_control_plane.yml

@@ -16,6 +16,7 @@
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config:oo_nodes_to_config"
     l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
 
 - name: Configure the upgrade target for the common upgrade tasks 3.10
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config

playbooks/common/openshift-cluster/upgrades/v3_10/upgrade_nodes.yml

@@ -5,6 +5,8 @@
 # Upgrades nodes only, but requires the control plane to have already been upgraded.
 #
 - import_playbook: ../init.yml
+  vars:
+    l_upgrade_cert_check_hosts: "oo_nodes_to_config"
 
 - name: Configure the upgrade target for the common upgrade tasks
   hosts: oo_all_hosts

playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_control_plane.yml

@@ -16,6 +16,7 @@
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
 
 - name: Configure the upgrade target for the common upgrade tasks 3.11
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config

playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_nodes.yml

@@ -5,6 +5,8 @@
 # Upgrades nodes only, but requires the control plane to have already been upgraded.
 #
 - import_playbook: ../init.yml
+  vars:
+    l_upgrade_cert_check_hosts: "oo_nodes_to_config"
 
 - name: Configure the upgrade target for the common upgrade tasks
   hosts: oo_all_hosts

roles/lib_utils/library/openshift_cert_expiry.py

@@ -779,7 +779,7 @@ an OpenShift Container Platform cluster
     ######################################################################
 
     res = tabulate_summary(ocp_certs, kubeconfigs, etcd_certs, router_certs, registry_certs)
-
+    warn_certs = bool(res['expired'] + res['warning'])
     msg = "Checked {count} total certificates. Expired/Warning/OK: {exp}/{warn}/{ok}. Warning window: {window} days".format(
         count=res['total'],
         exp=res['expired'],
@@ -822,6 +822,7 @@ an OpenShift Container Platform cluster
     # error we noticed earlier
     module.exit_json(
         check_results=check_results,
+        warn_certs=warn_certs,
         summary=res,
         msg=msg,
         rc=0,

roles/openshift_certificate_expiry/defaults/main.yml

@@ -1,8 +1,9 @@
 ---
 openshift_certificate_expiry_config_base: "/etc/origin"
-openshift_certificate_expiry_warning_days: 30
+openshift_certificate_expiry_warning_days: 365
 openshift_certificate_expiry_show_all: no
 openshift_certificate_expiry_generate_html_report: no
 openshift_certificate_expiry_html_report_path: "/tmp/cert-expiry-report.html"
 openshift_certificate_expiry_save_json_results: no
 openshift_certificate_expiry_json_results_path: "/tmp/cert-expiry-report.json"
+openshift_certificate_expiry_fail_on_warn: True

roles/openshift_certificate_expiry/tasks/main.yml

@@ -12,14 +12,10 @@
     src: cert-expiry-table.html.j2
     dest: "{{ openshift_certificate_expiry_html_report_path }}"
   delegate_to: localhost
-  when: openshift_certificate_expiry_generate_html_report|bool
-
-- name: Generate the result JSON string
-  run_once: yes
-  set_fact:
-    # oo_cert_expiry_results_to_json is a custom filter in role lib_utils
-    json_result_string: "{{ hostvars|oo_cert_expiry_results_to_json(play_hosts) }}"
-  when: openshift_certificate_expiry_save_json_results|bool
+  when: >
+        openshift_certificate_expiry_generate_html_report | bool
+        or (openshift_certificate_expiry_fail_on_warn | bool and
+            check_results.warn_certs | bool)
 
 - name: Generate results JSON file
   run_once: yes
@@ -27,4 +23,20 @@
     src: save_json_results.j2
     dest: "{{ openshift_certificate_expiry_json_results_path }}"
   delegate_to: localhost
-  when: openshift_certificate_expiry_save_json_results|bool
+  when: >
+        openshift_certificate_expiry_save_json_results | bool
+        or (openshift_certificate_expiry_fail_on_warn | bool and
+            check_results.warn_certs | bool)
+  vars:
+    json_result_string: "{{ hostvars|oo_cert_expiry_results_to_json(play_hosts) }}"
+
+- name: Fail when certs are near or already expired
+  fail:
+    msg: >
+      Cluster certificates found to be expired or within
+      {{ openshift_certificate_expiry_warning_days|int }} days of expiring.
+      You may view the report at {{ openshift_certificate_expiry_html_report_path }}
+      or {{ openshift_certificate_expiry_json_results_path }}.
+  when:
+  - openshift_certificate_expiry_fail_on_warn | bool
+  - check_results.warn_certs | bool

roles/openshift_certificate_expiry/templates/cert-expiry-table.html.j2

@@ -65,7 +65,7 @@
     </nav>
 
     {# Each host has a header and table to itself #}
-    {% for host in play_hosts %}
+    {% for host in ansible_play_hosts %}
       <h1>{{ host }}</h1>
 
       <p>