Redeploy etcd certificates during upgrade when etcd hostname not present in etcd serving cert SAN.

playbooks/common/openshift-cluster/upgrades/pre/verify_cluster.yml

@@ -92,3 +92,25 @@
         state: started
         enabled: yes
       with_items: "{{ master_services }}"
+
+# Until openshift-ansible is determining which host is the CA host we
+# must (unfortunately) ensure that the first host in the etcd group is
+# the etcd CA host.
+# https://bugzilla.redhat.com/show_bug.cgi?id=1469358
+- name: Verify we can proceed on first etcd
+  hosts: oo_first_etcd
+  gather_facts: no
+  tasks:
+  - name: Ensure CA exists on first etcd
+    stat:
+      path: /etc/etcd/generated_certs
+    register: __etcd_ca_stat
+
+  - fail:
+      msg: >
+        In order to correct an etcd certificate signing problem
+        upgrading may require re-generating etcd certificates. Please
+        ensure that the /etc/etcd/generated_certs directory exists on
+        the first host defined in your [etcd] group.
+    when:
+    - not __etcd_ca_stat.stat.exists | bool

playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml

@@ -2,6 +2,30 @@
 ###############################################################################
 # Upgrade Masters
 ###############################################################################
+
+# Prior to 3.6, openshift-ansible created etcd serving certificates
+# without a SubjectAlternativeName entry for the system hostname. The
+# SAN list in Go 1.8 is now (correctly) authoritative and since
+# openshift-ansible configures masters to talk to etcd hostnames
+# rather than IP addresses, we must correct etcd certificates.
+#
+# This play examines the etcd serving certificate SANs on each etcd
+# host and records whether or not the system hostname is missing.
+- name: Examine etcd serving certificate SAN
+  hosts: oo_etcd_to_config
+  tasks:
+  - slurp:
+      src: /etc/etcd/server.crt
+    register: etcd_serving_cert
+  - set_fact:
+      __etcd_cert_lacks_hostname: "{{ (openshift.common.hostname not in (etcd_serving_cert.content | b64decode | lib_utils_oo_parse_certificate_san)) | bool }}"
+
+# Redeploy etcd certificates when hostnames were missing from etcd
+# serving certificate SANs.
+- import_playbook: ../../../openshift-etcd/redeploy-certificates.yml
+  when:
+  - true in hostvars | lib_utils_oo_select_keys(groups['oo_etcd_to_config']) | lib_utils_oo_collect('__etcd_cert_lacks_hostname') | default([false])
+
 - name: Backup and upgrade etcd
   import_playbook: ../../../openshift-etcd/private/upgrade_main.yml
 

roles/lib_utils/filter_plugins/oo_filters.py

@@ -340,6 +340,58 @@ def lib_utils_oo_parse_named_certificates(certificates, named_certs_dir, interna
     return certificates
 
 
+def lib_utils_oo_parse_certificate_san(certificate):
+    """ Parses SubjectAlternativeNames from a PEM certificate.
+
+        Ex: certificate = '''-----BEGIN CERTIFICATE-----
+                MIIEcjCCAlqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADAhMR8wHQYDVQQDDBZldGNk
+                LXNpZ25lckAxNTE2ODIwNTg1MB4XDTE4MDEyNDE5MDMzM1oXDTIzMDEyMzE5MDMz
+                M1owHzEdMBsGA1UEAwwUbWFzdGVyMS5hYnV0Y2hlci5jb20wggEiMA0GCSqGSIb3
+                DQEBAQUAA4IBDwAwggEKAoIBAQD4wBdWXNI3TF1M0b0bEIGyJPvdqKeGwF5XlxWg
+                NoA1Ain/Xz0N1SW5pXW2CDo9HX+ay8DyhzR532yrBa+RO3ivNCmfnexTQinfSLWG
+                mBEdiu7HO3puR/GNm74JNyXoEKlMAIRiTGq9HPoTo7tNV5MLodgYirpHrkSutOww
+                DfFSrNjH/ehqxwQtrIOnTAHigdTOrKVdoYxqXblDEMONTPLI5LMvm4/BqnAVaOyb
+                9RUzND6lxU/ei3FbUS5IoeASOHx0l1ifxae3OeSNAimm/RIRo9rieFNUFh45TzID
+                elsdGrLB75LH/gnRVV1xxVbwPN6xW1mEwOceRMuhIArJQ2G5AgMBAAGjgbYwgbMw
+                UQYDVR0jBEowSIAUXTqN88vCI6E7wONls3QJ4/63unOhJaQjMCExHzAdBgNVBAMM
+                FmV0Y2Qtc2lnbmVyQDE1MTY4MjA1ODWCCQDMaopfom6OljAMBgNVHRMBAf8EAjAA
+                MBMGA1UdJQQMMAoGCCsGAQUFBwMBMAsGA1UdDwQEAwIFoDAdBgNVHQ4EFgQU7l05
+                OYeY3HppL6/0VJSirudj8t0wDwYDVR0RBAgwBocEwKh6ujANBgkqhkiG9w0BAQsF
+                AAOCAgEAFU8sicE5EeQsUPnFEqDvoJd1cVE+8aCBqkW0++4GsVw2A/JOJ3OBJL6r
+                BV3b1u8/e8xBNi8hPi42Q+LWBITZZ/COFyhwEAK94hcr7eZLCV2xfUdMJziP4Qkh
+                /WRN7vXHTtJ6NP/d6A22SPbtnMSt9Y6G8y9qa5HBrqIqmkYbLzDw/SdZbDbuGhRk
+                xUwg2ahXNblVoE5P6rxPONgXliA94telZ1/61iyrVaiGQb1/GUP/DRfvvR4dOCrA
+                lMosW6fm37Wdi/8iYW+aDPWGS+yVK/sjSnHNjxqvrzkfGk+COa5riT9hJ7wZY0Hb
+                YiJS74SZgZt/nnr5PI2zFRUiZLECqCkZnC/sz29i+irLabnq7Cif9Mv+TUcXWvry
+                TdJuaaYdTSMRSUkDd/c9Ife8tOr1i1xhFzDNKNkZjTVRk1MBquSXndVCDKucdfGi
+                YoWm+NDFrayw8yxK/KTHo3Db3lu1eIXTHxriodFx898b//hysHr4hs4/tsEFUTZi
+                705L2ScIFLfnyaPby5GK/3sBIXtuhOFM3QV3JoYKlJB5T6wJioVoUmSLc+UxZMeE
+                t9gGVQbVxtLvNHUdW7uKQ5pd76nIJqApQf8wg2Pja8oo56fRZX2XLt8nm9cswcC4
+                Y1mDMvtfxglQATwMTuoKGdREuu1mbdb8QqdyQmZuMa72q+ax2kQ=
+                -----END CERTIFICATE-----'''
+
+            returns ['192.168.122.186']
+    """
+
+    if not HAS_OPENSSL:
+        raise errors.AnsibleFilterError("|missing OpenSSL python bindings")
+
+    names = []
+
+    try:
+        lcert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, certificate)
+        for i in range(lcert.get_extension_count()):
+            if lcert.get_extension(i).get_short_name() == 'subjectAltName':
+                sanstr = str(lcert.get_extension(i))
+                sanstr = sanstr.replace('DNS:', '')
+                sanstr = sanstr.replace('IP Address:', '')
+                names = sanstr.split(', ')
+    except Exception:
+        raise errors.AnsibleFilterError("|failed to parse certificate")
+
+    return names
+
+
 def lib_utils_oo_generate_secret(num_bytes):
     """ generate a session secret """
 
@@ -612,6 +664,7 @@ class FilterModule(object):
             "lib_utils_oo_dict_to_keqv_list": lib_utils_oo_dict_to_keqv_list,
             "lib_utils_oo_list_to_dict": lib_utils_oo_list_to_dict,
             "lib_utils_oo_parse_named_certificates": lib_utils_oo_parse_named_certificates,
+            "lib_utils_oo_parse_certificate_san": lib_utils_oo_parse_certificate_san,
             "lib_utils_oo_generate_secret": lib_utils_oo_generate_secret,
             "lib_utils_oo_pods_match_component": lib_utils_oo_pods_match_component,
             "lib_utils_oo_image_tag_to_rpm_version": lib_utils_oo_image_tag_to_rpm_version,