HACK GCP: create and remove etcd discovery entries via a script

roles/openshift_gcp/tasks/deprovision.yml

@@ -68,18 +68,6 @@
     type: A
     state: absent
 
-- name: Remove etcd discovery record
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-      - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
-    managed_zone: "{{ managed_zone }}"
-    type: SRV
-    state: absent
-
 - name: Remove etcd records for masters
   gcp_dns_resource_record_set:
     auth_kind: serviceaccount
@@ -143,3 +131,11 @@
     project: "{{ openshift_gcp_project }}"
     name: "{{ openshift_gcp_network_name }}"
     state: absent
+
+- name: Templatize DNS script
+  template: src=remove.j2.sh dest=/tmp/remove.sh mode=u+rx
+
+- name: Run DNS cleanup script
+  command: /tmp/remove.sh
+  args:
+    chdir: "{{ files_dir }}"

roles/openshift_gcp/tasks/main.yml

@@ -238,6 +238,7 @@
     name: "{{ entry_name }}"
     managed_zone: "{{ managed_zone }}"
     type: A
+    ttl: 600
     target: "{{ master_ip }}"
     state: present
   with_indexed_items: "{{ master_instances }}"
@@ -245,16 +246,10 @@
     entry_name: "{{ openshift_gcp_prefix }}etcd-{{ item.0 }}.{{ public_hosted_zone }}."
     master_ip: "{{ item.1.networkInterfaces[0].networkIP }}"
 
-- name: Create etcd discovery entry
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-      - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
-    managed_zone: "{{ managed_zone }}"
-    type: SRV
-    ttl: 600
-    target: "{{ etcd_discovery_targets }}"
-    state: present
+- name: Templatize DNS script
+  template: src=additional_settings.j2.sh dest=/tmp/additional_settings.sh mode=u+rx
+
+- name: Run addition provision GCP script
+  command: /tmp/additional_settings.sh
+  args:
+    chdir: "{{ files_dir }}"

roles/openshift_gcp/templates/additional_settings.j2.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
+# configure DNS
+(
+# Retry DNS changes until they succeed since this may be a shared resource
+while true; do
+    dns="${TMPDIR:-/tmp}/dns.yaml"
+    rm -f $dns
+
+    # DNS records for etcd discovery
+    ETCD_DNS_NAME="_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
+    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "${ETCD_DNS_NAME}" 2>/dev/null | grep -q "${ETCD_DNS_NAME}"; then
+        if [[ ! -f $dns ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+        fi
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "${ETCD_DNS_NAME}" --type SRV {% for etcd in etcd_discovery_targets %}'{{ etcd }}' {% endfor %}
+
+    else
+        echo "DNS record for '${ETCD_DNS_NAME}' already exists"
+    fi
+
+    # Commit all DNS changes, retrying if preconditions are not met
+    if [[ -f $dns ]]; then
+        if ! out="$( gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns execute -z "${dns_zone}" 2>&1 )"; then
+            rc=$?
+            if [[ "${out}" == *"HTTPError 412: Precondition not met"* ]]; then
+                continue
+            fi
+            exit $rc
+        fi
+    fi
+    break
+done
+) &
+
+for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/remove.j2.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# DNS
+(
+dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
+if gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" &>/dev/null; then
+    # Retry DNS changes until they succeed since this may be a shared resource
+    while true; do
+        dns="${TMPDIR:-/tmp}/dns.yaml"
+        rm -f "${dns}"
+
+        # export all dns records that match into a zone format, and turn each line into a set of args for
+        # record-sets transaction.
+        gcloud dns record-sets export --project "{{ openshift_gcp_project }}" -z "${dns_zone}" --zone-file-format "${dns}"
+
+        # Remove etcd discovery record
+        ETCD_DNS_NAME="_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
+        grep -F -e "${ETCD_DNS_NAME}" "${dns}" | awk '{ print "--name", $1, "--ttl", $2, "--type", $4, "\x27"$5" "$6" "$7" "$8"\x27"; }'  >> "${dns}.input" || true
+
+        if [ -s "${dns}.input" ]; then
+            rm -f "${dns}"
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+            cat "${dns}.input" | xargs -L1 gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file="${dns}" remove -z "${dns_zone}"
+
+            # Commit all DNS changes, retrying if preconditions are not met
+            if ! out="$( gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns execute -z "${dns_zone}" 2>&1 )"; then
+                rc=$?
+                if [[ "${out}" == *"HTTPError 412: Precondition not met"* ]]; then
+                    continue
+                fi
+                exit $rc
+            fi
+        fi
+        rm "${dns}.input"
+        break
+    done
+fi
+) &
+
+for i in `jobs -p`; do wait $i; done