Browse Source

Add cert expiry check to upgrades

This commit extends cert expiry check to fail
if certs are within warning timeline as well
as utilizing the check during upgrades.
Michael Gugino 6 years ago
parent
commit
1c0e23bf1e

+ 7 - 0
playbooks/common/openshift-cluster/upgrades/init.yml

@@ -9,6 +9,13 @@
 - import_playbook: ../../../init/base_packages.yml
 - import_playbook: ../../../init/base_packages.yml
 - import_playbook: ../../../init/cluster_facts.yml
 - import_playbook: ../../../init/cluster_facts.yml
 
 
+- name: Inspect cluster certificates
+  hosts: "{{ l_upgrade_cert_check_hosts }}"
+  tasks:
+  - import_role:
+      name: openshift_certificate_expiry
+      tasks_from: main.yml
+
 - name: Ensure essential node configmaps are present
 - name: Ensure essential node configmaps are present
   hosts: oo_first_master
   hosts: oo_first_master
   tasks:
   tasks:

+ 2 - 0
playbooks/common/openshift-cluster/upgrades/upgrade_scale_group.yml

@@ -16,6 +16,8 @@
 
 
 - name: initialize upgrade bits
 - name: initialize upgrade bits
   import_playbook: init.yml
   import_playbook: init.yml
+  vars:
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
 
 
 - name: unschedule nodes
 - name: unschedule nodes
   hosts: oo_sg_current_nodes
   hosts: oo_sg_current_nodes

+ 1 - 0
playbooks/common/openshift-cluster/upgrades/v3_10/upgrade_control_plane.yml

@@ -16,6 +16,7 @@
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config:oo_nodes_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config:oo_nodes_to_config"
     l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
 
 
 - name: Configure the upgrade target for the common upgrade tasks 3.10
 - name: Configure the upgrade target for the common upgrade tasks 3.10
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config

+ 2 - 0
playbooks/common/openshift-cluster/upgrades/v3_10/upgrade_nodes.yml

@@ -5,6 +5,8 @@
 # Upgrades nodes only, but requires the control plane to have already been upgraded.
 # Upgrades nodes only, but requires the control plane to have already been upgraded.
 #
 #
 - import_playbook: ../init.yml
 - import_playbook: ../init.yml
+  vars:
+    l_upgrade_cert_check_hosts: "oo_nodes_to_config"
 
 
 - name: Configure the upgrade target for the common upgrade tasks
 - name: Configure the upgrade target for the common upgrade tasks
   hosts: oo_all_hosts
   hosts: oo_all_hosts

+ 1 - 0
playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_control_plane.yml

@@ -16,6 +16,7 @@
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_upgrade_cert_check_hosts: "oo_masters_to_config:oo_etcd_to_config"
 
 
 - name: Configure the upgrade target for the common upgrade tasks 3.11
 - name: Configure the upgrade target for the common upgrade tasks 3.11
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config

+ 2 - 0
playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_nodes.yml

@@ -5,6 +5,8 @@
 # Upgrades nodes only, but requires the control plane to have already been upgraded.
 # Upgrades nodes only, but requires the control plane to have already been upgraded.
 #
 #
 - import_playbook: ../init.yml
 - import_playbook: ../init.yml
+  vars:
+    l_upgrade_cert_check_hosts: "oo_nodes_to_config"
 
 
 - name: Configure the upgrade target for the common upgrade tasks
 - name: Configure the upgrade target for the common upgrade tasks
   hosts: oo_all_hosts
   hosts: oo_all_hosts

+ 2 - 1
roles/lib_utils/library/openshift_cert_expiry.py

@@ -779,7 +779,7 @@ an OpenShift Container Platform cluster
     ######################################################################
     ######################################################################
 
 
     res = tabulate_summary(ocp_certs, kubeconfigs, etcd_certs, router_certs, registry_certs)
     res = tabulate_summary(ocp_certs, kubeconfigs, etcd_certs, router_certs, registry_certs)
-
+    warn_certs = bool(res['expired'] + res['warning'])
     msg = "Checked {count} total certificates. Expired/Warning/OK: {exp}/{warn}/{ok}. Warning window: {window} days".format(
     msg = "Checked {count} total certificates. Expired/Warning/OK: {exp}/{warn}/{ok}. Warning window: {window} days".format(
         count=res['total'],
         count=res['total'],
         exp=res['expired'],
         exp=res['expired'],
@@ -822,6 +822,7 @@ an OpenShift Container Platform cluster
     # error we noticed earlier
     # error we noticed earlier
     module.exit_json(
     module.exit_json(
         check_results=check_results,
         check_results=check_results,
+        warn_certs=warn_certs,
         summary=res,
         summary=res,
         msg=msg,
         msg=msg,
         rc=0,
         rc=0,

+ 2 - 1
roles/openshift_certificate_expiry/defaults/main.yml

@@ -1,8 +1,9 @@
 ---
 ---
 openshift_certificate_expiry_config_base: "/etc/origin"
 openshift_certificate_expiry_config_base: "/etc/origin"
-openshift_certificate_expiry_warning_days: 30
+openshift_certificate_expiry_warning_days: 365
 openshift_certificate_expiry_show_all: no
 openshift_certificate_expiry_show_all: no
 openshift_certificate_expiry_generate_html_report: no
 openshift_certificate_expiry_generate_html_report: no
 openshift_certificate_expiry_html_report_path: "/tmp/cert-expiry-report.html"
 openshift_certificate_expiry_html_report_path: "/tmp/cert-expiry-report.html"
 openshift_certificate_expiry_save_json_results: no
 openshift_certificate_expiry_save_json_results: no
 openshift_certificate_expiry_json_results_path: "/tmp/cert-expiry-report.json"
 openshift_certificate_expiry_json_results_path: "/tmp/cert-expiry-report.json"
+openshift_certificate_expiry_fail_on_warn: True

+ 21 - 9
roles/openshift_certificate_expiry/tasks/main.yml

@@ -12,14 +12,10 @@
     src: cert-expiry-table.html.j2
     src: cert-expiry-table.html.j2
     dest: "{{ openshift_certificate_expiry_html_report_path }}"
     dest: "{{ openshift_certificate_expiry_html_report_path }}"
   delegate_to: localhost
   delegate_to: localhost
-  when: openshift_certificate_expiry_generate_html_report|bool
-
-- name: Generate the result JSON string
-  run_once: yes
-  set_fact:
-    # oo_cert_expiry_results_to_json is a custom filter in role lib_utils
-    json_result_string: "{{ hostvars|oo_cert_expiry_results_to_json(play_hosts) }}"
-  when: openshift_certificate_expiry_save_json_results|bool
+  when: >
+        openshift_certificate_expiry_generate_html_report | bool
+        or (openshift_certificate_expiry_fail_on_warn | bool and
+            check_results.warn_certs | bool)
 
 
 - name: Generate results JSON file
 - name: Generate results JSON file
   run_once: yes
   run_once: yes
@@ -27,4 +23,20 @@
     src: save_json_results.j2
     src: save_json_results.j2
     dest: "{{ openshift_certificate_expiry_json_results_path }}"
     dest: "{{ openshift_certificate_expiry_json_results_path }}"
   delegate_to: localhost
   delegate_to: localhost
-  when: openshift_certificate_expiry_save_json_results|bool
+  when: >
+        openshift_certificate_expiry_save_json_results | bool
+        or (openshift_certificate_expiry_fail_on_warn | bool and
+            check_results.warn_certs | bool)
+  vars:
+    json_result_string: "{{ hostvars|oo_cert_expiry_results_to_json(play_hosts) }}"
+
+- name: Fail when certs are near or already expired
+  fail:
+    msg: >
+      Cluster certificates found to be expired or within
+      {{ openshift_certificate_expiry_warning_days|int }} days of expiring.
+      You may view the report at {{ openshift_certificate_expiry_html_report_path }}
+      or {{ openshift_certificate_expiry_json_results_path }}.
+  when:
+  - openshift_certificate_expiry_fail_on_warn | bool
+  - check_results.warn_certs | bool

+ 1 - 1
roles/openshift_certificate_expiry/templates/cert-expiry-table.html.j2

@@ -65,7 +65,7 @@
     </nav>
     </nav>
 
 
     {# Each host has a header and table to itself #}
     {# Each host has a header and table to itself #}
-    {% for host in play_hosts %}
+    {% for host in ansible_play_hosts %}
       <h1>{{ host }}</h1>
       <h1>{{ host }}</h1>
 
 
       <p>
       <p>