Add playbooks to setup cluster via bootkube and scale up nodes

inventory/dynamic/gcp/hosts.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Use a playbook to calculate the inventory dynamically from
 # the provided cluster variables.
 src="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-if ! out="$( ansible-playbook --inventory-file "${src}/none" ${src}/../../../playbooks/gcp/openshift-cluster/inventory.yml 2>&1 )"; then
+if ! out="$( ansible-playbook --inventory-file "${src}/none" ${src}/../../../test/gcp/inventory.yml 2>&1 )"; then
   echo "error: Inventory configuration failed" 1>&2
   echo "$out" 1>&2
   echo "{}"

playbooks/bootkube.yml

@@ -0,0 +1,102 @@
+---
+# Generate config using openshift-installer, set Base Domain to testing.tt
+# Add bootstrap host in [bootstrap] group and set ignition_file
+# Add master host to [masters] group
+# Add worker hosts in [workers] group
+# Make sure bootstrap has <clusterid>-api.<dns base> name
+# Make sure masters have <clusterid>-etcd-<index>.<dns base> name
+
+# FIXME: use dnsmasq to fake DNS entries
+
+- import_playbook: init/main.yml
+  vars:
+    l_install_base_packages: True
+    l_repo_hosts: "all:!all"
+
+# TODO: proper firewalld setup
+# 49500 on bootstrap; 2379, 6443, 10250 on masters, 10250 on workers
+
+- import_playbook: container-runtime/private/setup_storage.yml
+
+- import_playbook: container-runtime/private/config.yml
+
+- name: install nodes
+  hosts: nodes
+  tasks:
+  - import_role:
+      name: openshift_node40
+      tasks_from: install.yml
+
+- name: setup AWS creds
+  hosts: masters:bootstrap:workers
+  tasks:
+  - import_role:
+      name: openshift_node40
+      tasks_from: aws.yml
+
+- name: Config bootstrap node
+  hosts: bootstrap
+  tasks:
+  - import_role:
+      name: openshift_node40
+      tasks_from: aws.yml
+  - import_role:
+      name: openshift_node40
+      tasks_from: config.yml
+  - import_role:
+      name: openshift_node40
+      tasks_from: systemd.yml
+
+- name: Start masters
+  hosts: masters
+  tasks:
+  # TODO Read this from master's ignition file
+  - set_fact:
+      openshift_bootstrap_endpoint: "https://{{ bootstrap }}:49500/config/master?etcd_index={{ index }}"
+    vars:
+      bootstrap: "{{ hostvars[groups['bootstrap'][0]]['ansible_host'] }}"
+      index: "{{ groups['masters'].index(inventory_hostname) }}"
+  - name: Wait for bootstrap endpoint to show up
+    uri:
+      url: "{{ openshift_bootstrap_endpoint }}"
+      validate_certs: false
+    delay: 10
+    retries: 60
+    register: result
+    until:
+    - "'status' in result"
+    - result.status == 200
+  - import_role:
+      name: openshift_node40
+      tasks_from: config.yml
+  - name: Make sure etcd user exists
+    user:
+      name: etcd
+  - import_role:
+      name: openshift_node40
+      tasks_from: systemd.yml
+
+- name: Start workers
+  hosts: workers
+  tasks:
+  # TODO Read this from master's ignition file
+  - set_fact:
+      openshift_bootstrap_endpoint: "https://{{ bootstrap }}:49500/config/worker"
+    vars:
+      bootstrap: "{{ hostvars[groups['bootstrap'][0]]['ansible_host'] }}"
+  - name: Wait for bootstrap endpoint to show up
+    uri:
+      url: "{{ openshift_bootstrap_endpoint }}"
+      validate_certs: false
+    delay: 10
+    retries: 60
+    register: result
+    until:
+    - "'status' in result"
+    - result.status == 200
+  - import_role:
+      name: openshift_node40
+      tasks_from: config.yml
+  - import_role:
+      name: openshift_node40
+      tasks_from: systemd.yml

playbooks/gcp/OWNERS

@@ -1,14 +0,0 @@
-# approval == this is a good idea /approve
-approvers:
-  - smarterclayton
-  - michaelgugino
-  - mtnbikenc
-  - sdodson
-  - vrutkovs
-# review == this code is good /lgtm
-reviewers:
-  - smarterclayton
-  - michaelgugino
-  - mtnbikenc
-  - sdodson
-  - vrutkovs

playbooks/gcp/openshift-cluster/build_base_image.yml

@@ -1,186 +0,0 @@
----
-# This playbook ensures that a base image is up to date with all of the required settings
-- name: Verify prerequisites for image build
-  hosts: localhost
-  connection: local
-  gather_facts: no
-  tasks:
-  - name: Require openshift_gcp_root_image
-    fail:
-      msg: "A root OS image name or family is required for base image building.  Please ensure `openshift_gcp_root_image` is defined."
-    when: openshift_gcp_root_image is undefined
-
-- name: Provision ssh key
-  hosts: localhost
-  connection: local
-  gather_facts: no
-  tasks:
-  - name: Set up core host GCP configuration
-    import_role:
-      name: openshift_gcp
-      tasks_from: provision_ssh_keys.yml
-
-- name: Launch image build instance
-  hosts: localhost
-  connection: local
-  gather_facts: no
-  tasks:
-  - name: Create the image instance disk
-    gce_pd:
-      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
-      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-      project_id: "{{ openshift_gcp_project }}"
-      zone: "{{ openshift_gcp_zone }}"
-      name: "{{ openshift_gcp_prefix }}build-image-instance"
-      disk_type: pd-ssd
-      image: "{{ openshift_gcp_root_image }}"
-      size_gb: 10
-      state: present
-
-  - name: Launch the image build instance
-    gce:
-      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
-      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-      project_id: "{{ openshift_gcp_project }}"
-      zone: "{{ openshift_gcp_zone }}"
-      machine_type: n1-standard-1
-      instance_names: "{{ openshift_gcp_prefix }}build-image-instance"
-      state: present
-      tags:
-      - build-image-instance
-      disk_auto_delete: false
-      disks:
-      - "{{ openshift_gcp_prefix }}build-image-instance"
-    register: gce
-
-  - add_host:
-      hostname: "{{ item.public_ip }}"
-      groupname: build_instance_ips
-    with_items: "{{ gce.instance_data }}"
-
-  - name: Wait for instance to respond to SSH
-    wait_for:
-      delay: 1
-      host: "{{ item.public_ip }}"
-      port: 22
-      state: started
-      timeout: 120
-    with_items: "{{ gce.instance_data }}"
-
-- name: Prepare instance content sources
-  pre_tasks:
-  - set_fact:
-      allow_rhel_subscriptions: "{{ rhsub_skip | default('no', True) | lower in ['no', 'false'] }}"
-  - set_fact:
-      using_rhel_subscriptions: "{{ (deployment_type in ['enterprise', 'atomic-enterprise', 'openshift-enterprise'] or ansible_distribution == 'RedHat') and allow_rhel_subscriptions }}"
-  hosts: build_instance_ips
-  roles:
-  - role: rhel_subscribe
-    when: using_rhel_subscriptions
-  - role: openshift_repos
-    vars:
-      openshift_additional_repos: []
-  post_tasks:
-  - name: Add custom repositories
-    include_role:
-      name: openshift_gcp
-      tasks_from: add_custom_repositories.yml
-  - name: Add the Google Cloud repo
-    yum_repository:
-      name: google-cloud
-      description: Google Cloud Compute
-      baseurl: https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64
-      gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-      gpgcheck: yes
-      repo_gpgcheck: yes
-      state: present
-    when: ansible_os_family == "RedHat"
-  - name: Add the jdetiber-qemu-user-static copr repo
-    yum_repository:
-      name: jdetiber-qemu-user-static
-      description: QEMU user static COPR
-      baseurl: https://copr-be.cloud.fedoraproject.org/results/jdetiber/qemu-user-static/epel-7-$basearch/
-      gpgkey: https://copr-be.cloud.fedoraproject.org/results/jdetiber/qemu-user-static/pubkey.gpg
-      gpgcheck: yes
-      repo_gpgcheck: no
-      state: present
-    when: ansible_os_family == "RedHat"
-  - name: Accept GPG keys for the repos
-    command: yum -q makecache -y --disablerepo='*' --enablerepo='google-cloud,jdetiber-qemu-user-static'
-  - name: Install qemu-user-static
-    package:
-      name: qemu-user-static
-      state: present
-  - name: Disable yum-cron service (installed by Google Cloud by default)
-    systemd:
-      name: yum-cron
-      state: stopped
-      enabled: no
-  - name: Start and enable systemd-binfmt service
-    systemd:
-      name: systemd-binfmt
-      state: started
-      enabled: yes
-
-- name: Build image
-  hosts: build_instance_ips
-  pre_tasks:
-  - name: Set up core host GCP configuration
-    include_role:
-      name: openshift_gcp
-      tasks_from: configure_gcp_base_image.yml
-  roles:
-  - role: os_update_latest
-  post_tasks:
-  - name: Disable all repos on RHEL
-    command: subscription-manager repos --disable="*"
-    when: using_rhel_subscriptions
-  - name: Enable repos for packages on RHEL
-    command: subscription-manager repos --enable="rhel-7-server-rpms" --enable="rhel-7-server-extras-rpms"
-    when: using_rhel_subscriptions
-  - name: Install common image prerequisites
-    package:
-      name: "{{ pkg_list | join(',') }}"
-      state: latest
-    vars:
-      pkg_list:
-      # required by Ansible
-      - PyYAML
-      - google-compute-engine
-      - google-compute-engine-init
-      - google-config
-      - wget
-      - git
-      - net-tools
-      - bind-utils
-      - iptables-services
-      - bridge-utils
-      - bash-completion
-  - name: Clean yum metadata
-    command: yum clean all
-    args:
-      warn: no
-    when: ansible_os_family == "RedHat"
-
-- name: Commit image
-  hosts: localhost
-  connection: local
-  tasks:
-  - name: Terminate the image build instance
-    gce:
-      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
-      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-      project_id: "{{ openshift_gcp_project }}"
-      zone: "{{ openshift_gcp_zone }}"
-      instance_names: "{{ openshift_gcp_prefix }}build-image-instance"
-      state: absent
-  - name: Save the new image
-    command: gcloud --project "{{ openshift_gcp_project}}" compute images create "{{ openshift_gcp_base_image_name | default(openshift_gcp_base_image + '-' + lookup('pipe','date +%Y%m%d-%H%M%S')) }}" --source-disk "{{ openshift_gcp_prefix }}build-image-instance" --source-disk-zone "{{ openshift_gcp_zone }}" --family "{{ openshift_gcp_base_image }}"
-  - name: Remove the image instance disk
-    gce_pd:
-      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
-      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-      project_id: "{{ openshift_gcp_project }}"
-      zone: "{{ openshift_gcp_zone }}"
-      name: "{{ openshift_gcp_prefix }}build-image-instance"
-      state: absent

playbooks/gcp/openshift-cluster/install.yml

@@ -1,36 +0,0 @@
-# This playbook installs onto a provisioned cluster
----
-- hosts: localhost
-  connection: local
-  tasks:
-  - name: place all scale groups into Ansible groups
-    include_role:
-      name: openshift_gcp
-      tasks_from: setup_scale_group_facts.yml
-
-- name: run the init
-  import_playbook: ../../init/main.yml
-
-- import_playbook: ../../openshift-checks/private/install.yml
-
-- name: ensure master nodes are ready for bootstrapping
-  import_playbook: ../../openshift-node/private/bootstrap.yml
-
-- name: configure the control plane
-  import_playbook: ../../common/private/control_plane.yml
-
-- name: run the GCP specific post steps
-  import_playbook: install_gcp.yml
-
-- name: install components
-  import_playbook: ../../common/private/components.yml
-
-- name: Copy the kubeconfig, used by CI to determine when the containers are ready
-  hosts: oo_first_master
-  gather_facts: no
-  tasks:
-  - name: Retrieve cluster configuration
-    fetch:
-      src: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
-      dest: "/tmp/"
-      flat: yes

playbooks/gcp/openshift-cluster/install_gcp.yml

@@ -1,12 +0,0 @@
----
-- hosts: masters
-  gather_facts: no
-  tasks:
-  - name: create master health check service
-    include_role:
-      name: openshift_gcp
-      tasks_from: configure_master_healthcheck.yml
-  - name: configure master bootstrap distribution
-    include_role:
-      name: openshift_gcp
-      tasks_from: configure_master_bootstrap.yml

playbooks/gcp/openshift-cluster/openshift_node_group.yml

@@ -1,23 +0,0 @@
-# This playbook installs onto a provisioned cluster
----
-- hosts: localhost
-  connection: local
-  tasks:
-  - name: place all scale groups into Ansible groups
-    include_role:
-      name: openshift_gcp
-      tasks_from: setup_scale_group_facts.yml
-    vars:
-      all_nodes: true
-
-- import_playbook: ../../init/main.yml
-  vars:
-    l_init_fact_hosts: "oo_masters_to_config"
-    l_openshift_version_set_hosts: "all:!all"
-    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
-
-- name: Setup node-group configmaps
-  hosts: oo_first_master
-  tasks:
-  - import_role:
-      name: openshift_node_group

playbooks/gcp/openshift-cluster/publish_image.yml

@@ -1,9 +0,0 @@
----
-- name: Publish the most recent image
-  hosts: localhost
-  connection: local
-  gather_facts: no
-  tasks:
-  - import_role:
-      name: openshift_gcp
-      tasks_from: publish_image.yml

playbooks/gcp/openshift-cluster/roles

@@ -1 +0,0 @@
-../../../roles

requirements.txt

@@ -1,6 +1,6 @@
 # Versions are pinned to prevent pypi releases arbitrarily breaking
 # tests with new APIs/semantics. We want to update versions deliberately.
-ansible==2.6.5
+ansible==2.7.1
 boto==2.44.0
 click==6.7
 pyOpenSSL==17.5.0

roles/lib_utils/action_plugins/parse_ignition.py

@@ -2,12 +2,12 @@
 
 import base64
 import os
-
-from ansible.plugins.action import ActionBase
-from ansible import errors
+import six
 from six.moves import urllib
+from ansible.plugins.action import ActionBase
 
 
+# pylint: disable=too-many-function-args
 def get_files(files_dict, systemd_dict, dir_list, data):
     """parse data to populate file_dict"""
     files = data.get('storage', []).get('files', [])
@@ -28,7 +28,10 @@ def get_files(files_dict, systemd_dict, dir_list, data):
     # get the systemd units files we're here
     systemd_units = data.get('systemd', []).get('units', [])
     for item in systemd_units:
-        contents = item['contents'].decode('unicode-escape')
+        contents = item['contents']
+        if six.PY2:
+            # pylint: disable=redefined-variable-type
+            contents = contents.decode('unicode-escape')
         mode = "0644"
         inode = {"contents": contents, "mode": mode}
         name = item['name']
@@ -39,6 +42,7 @@ def get_files(files_dict, systemd_dict, dir_list, data):
         systemd_dict[name] = enabled
 
 
+# pylint: disable=too-few-public-methods
 class ActionModule(ActionBase):
     """ActionModule for parse_ignition.py"""
 

roles/lib_utils/test/test_parse_ignition.py

@@ -7,9 +7,10 @@ import sys
 
 MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'action_plugins'))
 sys.path.insert(0, MODULE_PATH)
+ASSET_PATH = os.path.realpath(os.path.join(__file__, os.pardir, 'test_data'))
 
 # pylint: disable=import-error,wrong-import-position,missing-docstring
-import parse_ignition # noqa: E402
+import parse_ignition  # noqa: E402
 
 
 def read_ign(path):
@@ -25,7 +26,7 @@ def write_out_files(files_dict):
 
 
 def test_parse_json():
-    ign_data = read_ign('test_data/example.ign.json')
+    ign_data = read_ign(os.path.join(ASSET_PATH, 'example.ign.json'))
     files_dict = {}
     systemd_dict = {}
     dir_list = set()
@@ -36,7 +37,7 @@ def test_parse_json():
 
 
 def test_parse_json_encoded_files():
-    ign_data = read_ign('test_data/bootstrap.ign.json')
+    ign_data = read_ign(os.path.join(ASSET_PATH, 'bootstrap.ign.json'))
     files_dict = {}
     systemd_dict = {}
     dir_list = set()
@@ -44,11 +45,11 @@ def test_parse_json_encoded_files():
     result['files_dict'] = files_dict
     result['systemd_dict'] = systemd_dict
     parse_ignition.get_files(files_dict, systemd_dict, dir_list, ign_data)
-    #print(files_dict['/opt/tectonic/manifests/cluster-config.yaml']['contents'])
+    # print(files_dict['/opt/tectonic/manifests/cluster-config.yaml']['contents'])
 
 
 def parse_json2():
-    ign_data = read_ign('test_data/bs.ign.json')
+    ign_data = read_ign(os.path.join(ASSET_PATH, 'bs.ign.json'))
     files_dict = {}
     systemd_dict = {}
     dir_list = set()

roles/openshift_gcp/defaults/main.yml

@@ -62,3 +62,84 @@ openshift_gcp_user_data_file: ''
 openshift_gcp_multizone: False
 
 provision_custom_repositories: []
+
+mcd_port: 49500
+
+openshift_gcp_firewall_rules:
+  - rule: icmp
+    allowed:
+      - ip_protocol: 'icmp'
+  - rule: ssh-external
+    allowed:
+      - ip_protocol: 'tcp'
+        ports:
+          - '22'
+  - rule: ssh-internal
+    allowed:
+      - ip_protocol: 'tcp'
+        ports:
+          - '22'
+    source_tags:
+      - ssh-bastion
+  - rule: master-internal
+    allowed:
+      - ip_protocol: 'tcp'
+        ports:
+          - '2224'
+          - '2379'
+          - '2380'
+          - '4001'
+          - "{{ openshift_gcp_kubernetes_api_port }}"
+          - "{{ internal_console_port }}"
+          - '8053'
+          - '8444'
+          - "{{ openshift_gcp_master_healthcheck_port }}"
+          - '10250'
+          - '10255'
+          - '24224'
+          - "{{ mcd_port }}"
+      - ip_protocol: 'udp'
+        ports:
+          - '4789'
+          - '5404'
+          - '5405'
+          - '10255'
+          - '24224'
+    source_tags:
+      - ocp
+    target_tags:
+      - ocp-master
+      - ocp-bootstrap
+  - rule: master-external
+    allowed:
+      - ip_protocol: 'tcp'
+        ports:
+          - '80'
+          - '443'
+          - '1936'
+          - "{{ openshift_gcp_master_healthcheck_port }}"
+          - "{{ openshift_gcp_kubernetes_api_port }}"
+          - "{{ openshift_master_api_port }}"
+          - "{{ mcd_port }}"
+          - "{{ openshift_node_port_range }}"
+      - ip_protocol: 'udp'
+        ports:
+          - "{{ openshift_node_port_range }}"
+    target_tags:
+      - ocp-master
+      - ocp-bootstrap
+  - rule: node-internal
+    allowed:
+      - ip_protocol: 'tcp'
+        ports:
+          - '10250'
+          - '10255'
+          - '9000-10000'
+      - ip_protocol: 'udp'
+        ports:
+          - '4789'
+          - '10255'
+    source_tags:
+      - ocp
+    target_tags:
+      - ocp-node

roles/openshift_gcp/tasks/add_custom_repositories.yml

@@ -1,20 +0,0 @@
----
-- name: Copy custom repository secrets
-  copy:
-    src: "{{ files_dir }}/{{ item.1.sslclientcert }}"
-    dest: /var/lib/yum/custom_secret_{{ item.0 }}_cert
-  when: item.1.sslclientcert | default(false)
-  with_indexed_items: "{{ provision_custom_repositories }}"
-- name: Copy custom repository secrets
-  copy:
-    src: "{{ files_dir }}/{{ item.1.sslclientkey }}"
-    dest: /var/lib/yum/custom_secret_{{ item.0 }}_key
-  when: item.1.sslclientkey | default(false)
-  with_indexed_items: "{{ provision_custom_repositories }}"
-
-- name: Create any custom repos that are defined
-  template:
-    src: yum_repo.j2
-    dest: /etc/yum.repos.d/provision_custom_repositories.repo
-  when: provision_custom_repositories | length > 0
-  notify: refresh cache

roles/openshift_gcp/tasks/configure_gcp_base_image.yml

@@ -1,10 +0,0 @@
-# GCE instances are starting with xfs AND barrier=1, which is only for extfs.
----
-- name: Remove barrier=1 from XFS fstab entries
-  command: sed -i -e 's/xfs\(.*\)barrier=1/xfs\1/g; s/, / /g' /etc/fstab
-
-- name: Ensure the root filesystem has XFS group quota turned on
-  command: sed -i -e 's/linux16 \(.*\)$/linux16 \1 rootflags=gquota/g' /boot/grub2/grub.cfg
-
-- name: Ensure the root partition grows on startup
-  copy: src=partition.conf dest=/etc/systemd/system/google-instance-setup.service.d/

roles/openshift_gcp/tasks/configure_master_bootstrap.yml

@@ -1,40 +0,0 @@
-#
-# These tasks configure the instance to periodically update the project metadata with the
-# latest bootstrap kubeconfig from the project metadata. This keeps the project metadata
-# in sync with the cluster's configuration. We then invoke a CSR approve on any nodes that
-# are waiting to join the cluster.
-#
----
-- name: Copy unit service
-  copy:
-    src: openshift-bootstrap-update.timer
-    dest: /etc/systemd/system/openshift-bootstrap-update.timer
-    owner: root
-    group: root
-    mode: 0664
-
-- name: Copy unit timer
-  copy:
-    src: openshift-bootstrap-update.service
-    dest: /etc/systemd/system/openshift-bootstrap-update.service
-    owner: root
-    group: root
-    mode: 0664
-
-- name: Create bootstrap update script
-  template: src=openshift-bootstrap-update.j2 dest=/usr/bin/openshift-bootstrap-update mode=u+rx
-
-- name: Start bootstrap update timer
-  systemd:
-    name: "openshift-bootstrap-update.timer"
-    state: started
-
-- name: Approve node certificates when bootstrapping
-  oc_csr_approve:
-    oc_bin: "{{ hostvars[groups.masters.0]['first_master_client_binary'] }}"
-    oc_conf: "{{ hostvars[groups.masters.0].openshift.common.config_base }}/master/admin.kubeconfig"
-    node_list: "{{ groups['all'] | map('extract', hostvars) | selectattr('gce_metadata.bootstrap', 'match', 'true') | map(attribute='gce_name') | list }}"
-  register: gcp_csr_approve
-  retries: 30
-  until: gcp_csr_approve is succeeded
-  when: groups['all'] | map('extract', hostvars) | selectattr('gce_metadata.bootstrap', 'match', 'true') | map(attribute='gce_name') | list | length > 0

roles/openshift_gcp/tasks/configure_master_healthcheck.yml

@@ -24,3 +24,9 @@
     name: haproxy
     state: started
     enabled: yes
+
+- name: allow haproxy to connect to any port
+  seboolean:
+    name: haproxy_connect_any
+    state: yes
+    persistent: yes

roles/openshift_gcp/tasks/deprovision.yml

@@ -0,0 +1,79 @@
+---
+- name: Fetch instance group managers
+  gcp_compute_instance_group_manager_facts:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    zone: "{{ openshift_gcp_zone }}"
+    filters:
+    - "name : {{ openshift_gcp_prefix }}ig*"
+  register: instance_group_managers
+
+- name: Fetch instance templates
+  gcp_compute_instance_template_facts:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    filters:
+    - "name : {{ openshift_gcp_prefix }}instance-template*"
+  register: instance_templates
+
+- name: Remove GCP Instance Groups
+  gcp_compute_instance_group_manager:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    zone: "{{ openshift_gcp_zone }}"
+    name: "{{ item[0].name }}"
+    base_instance_name: "{{ item[0].name }}"
+    instance_template: "{{ item[1] }}"
+    state: absent
+  with_nested:
+  - "{{ instance_group_managers['items'] }}"
+  - "{{ instance_templates['items'] }}"
+
+- name: Remove GCP instance templates
+  gcp_compute_instance_template:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    name: "{{ item.name }}"
+    state: absent
+  with_items: "{{ instance_templates['items'] }}"
+
+- name: Remove GCP firewall
+  gcp_compute_firewall:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    name: "{{ openshift_gcp_prefix }}{{ item.rule }}"
+    state: absent
+  with_items: "{{ openshift_gcp_firewall_rules }}"
+
+- name: Remove GCP network
+  gcp_compute_network:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    name: "{{ openshift_gcp_network_name }}"
+    state: absent
+
+- name: Templatize DNS script
+  template: src=remove.j2.sh dest=/tmp/remove.sh mode=u+rx
+
+- name: Run DNS cleanup script
+  command: /tmp/remove.sh
+  args:
+    chdir: "{{ files_dir }}"

roles/openshift_gcp/tasks/dynamic_inventory.yml

@@ -1,5 +1,12 @@
 ---
 - name: Extract PEM from service account file
-  copy: content="{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).private_key }}" dest=/tmp/gce.pem mode=0600
+  copy:
+    content: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).private_key }}"
+    dest: /tmp/gce.pem
+    mode: 0600
+
 - name: Templatize environment script
-  template: src=inventory.j2.sh dest=/tmp/inventory.sh mode=u+rx
+  template:
+    src: inventory.j2.sh
+    dest: /tmp/inventory.sh
+    mode: u+rx

roles/openshift_gcp/tasks/main.yml

@@ -1,45 +1,176 @@
-#
-# This role relies on gcloud invoked via templated bash in order to
-# provide a high performance deployment option. The next logical step
-# is to transition to a deployment manager template which is then instantiated.
-# TODO: use a formal set of role parameters consistent with openshift_aws
-#
 ---
-- name: Templatize DNS script
-  template: src=dns.j2.sh dest=/tmp/openshift_gcp_provision_dns.sh mode=u+rx
-- name: Templatize provision script
-  template: src=provision.j2.sh dest=/tmp/openshift_gcp_provision.sh mode=u+rx
-- name: Templatize de-provision script
-  template: src=remove.j2.sh dest=/tmp/openshift_gcp_provision_remove.sh mode=u+rx
-  when:
-  - state | default('present') == 'absent'
+- name: Create GCP network
+  gcp_compute_network:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    name: "{{ openshift_gcp_network_name }}"
+    state: present
+  register: network
 
-- name: Provision GCP DNS domain
-  command: /tmp/openshift_gcp_provision_dns.sh
-  args:
-    chdir: "{{ files_dir }}"
-  register: dns_provision
+- name: Create GCP firewall
+  gcp_compute_firewall:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    name: "{{ openshift_gcp_prefix }}{{ item.rule }}"
+    allowed: "{{ item.allowed }}"
+    network: "{{ network.selfLink }}"
+    target_tags: "{{ item.target_tags | default(omit) }}"
+    source_tags: "{{ item.source_tags | default(omit) }}"
+    state: present
+  with_items: "{{ openshift_gcp_firewall_rules }}"
+
+- import_tasks: provision_ssh_keys.yml
+
+- name: Find GCP image
+  gcp_compute_image_facts:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    filters:
+    - "family = {{ openshift_gcp_image }}"
+  register: gcp_node_image
+
+- name: Provision GCP instance templates
+  gcp_compute_instance_template:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    name: "{{ openshift_gcp_prefix }}instance-template-{{ item.name }}"
+    properties:
+      machine_type: "{{ item.machine_type }}"
+      network_interfaces:
+      - network: "{{ network }}"
+        access_configs:
+        - name: "{{ openshift_gcp_prefix }}instance-template-{{ item.name }}-config"
+          type: 'ONE_TO_ONE_NAT'
+      disks:
+      - auto_delete: true
+        boot: true
+        initialize_params:
+          disk_size_gb: "{{ item.boot_disk_size }}"
+          source_image: "{{ gcp_node_image['items'][0].selfLink }}"
+      metadata:
+        "cluster-id": "{{ openshift_gcp_prefix + openshift_gcp_clusterid }}"
+        "node-group": "{{ item.name }}"
+      tags:
+        items:
+        - "ocp"
+        - "{{ openshift_gcp_prefix }}ocp"
+        - "{{ item.tags }}"
+    state: present
+  with_items: "{{ openshift_gcp_node_group_config }}"
+  register: instance_template
+
+- name: Create GCP Instance Groups
+  gcp_compute_instance_group_manager:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    zone: "{{ openshift_gcp_zone }}"
+    name: "{{ openshift_gcp_prefix }}ig-{{ item.item.suffix }}"
+    base_instance_name: "{{ openshift_gcp_prefix }}ig-{{ item.item.suffix }}"
+    instance_template: "{{ item }}"
+    target_size: "{{ item.item.scale | int}}"
+    named_ports:
+    - name: "{{ openshift_gcp_prefix }}port-kube-api"
+      port: "{{ openshift_gcp_kubernetes_api_port }}"
+    - name: "{{ openshift_gcp_prefix }}port-openshift-api"
+      port: "{{ openshift_master_api_port }}"
+    state: present
+  with_items: "{{ instance_template.results }}"
+  register: instance_groups
+
+- name: Get master instance group
+  gcp_compute_instance_group_facts:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    zone: "{{ openshift_gcp_zone }}"
+    filters:
+    - name = "{{ openshift_gcp_prefix }}ig-m"
+  register: master_instance_group
+
+- set_fact:
+    master_instance_group: "{{ master_instance_group['items'][0] }}"
+
+- name: Wait for master instance group to start all instances
+  gcp_compute_instance_group_manager_facts:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    zone: "{{ openshift_gcp_zone }}"
+    filters: "name = {{ master_instance_group['name'] }}"
+  register: master_group_result
+  # Wait for 3 minutes
+  retries: 36
+  delay: 5
+  until:
+  - "master_group_result['items'][0]['currentActions']['none'] == master_group_result['items'][0]['targetSize']"
+
+- name: Collect a list of instances
+  gcp_compute_instance_facts:
+    auth_kind: serviceaccount
+    scopes:
+    - https://www.googleapis.com/auth/compute
+    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+    project: "{{ openshift_gcp_project }}"
+    zone: "{{ openshift_gcp_zone }}"
+  register: all_instances
+
+- name: Filter instances to fetch bootstrap
+  set_fact:
+    bootstrap_instance: "{{ item }}"
+  with_items:
+  - "{{ all_instances['items'] }}"
   when:
-  - state | default('present') == 'present'
+  - "'tags' in item"
+  - "'items' in item['tags']"
+  - "cluster_tag in item['tags']['items']"
+  - "'ocp-bootstrap' in item['tags']['items']"
+  vars:
+    cluster_tag: "{{ openshift_gcp_prefix }}ocp"
 
-- name: Ensure that DNS resolves to the hosted zone
-  assert:
-    that:
-    - "lookup('dig', public_hosted_zone, 'qtype=NS', wantlist=True) | sort | join(',') == dns_provision.stdout"
-    msg: "The DNS domain {{ public_hosted_zone }} defined in 'public_hosted_zone' must have NS records pointing to the Google nameservers: '{{ dns_provision.stdout }}' instead of '{{ lookup('dig', public_hosted_zone, 'qtype=NS') }}'."
+- name: Filter instances to fetch masters
+  set_fact:
+    master_instances: "{{ master_instances | default([]) }} + [ {{ item }} ]"
+  with_items:
+  - "{{ all_instances['items'] }}"
   when:
-  - state | default('present') == 'present'
+  - "'tags' in item"
+  - "'items' in item['tags']"
+  - "cluster_tag in item['tags']['items']"
+  - "'ocp-master' in item['tags']['items']"
+  vars:
+    cluster_tag: "{{ openshift_gcp_prefix }}ocp"
 
-- import_tasks: provision_ssh_keys.yml
+- set_fact:
+    etcd_discovery_targets: "{{ etcd_discovery_targets | default('') }} '0 0 2380 {{ entry_name }}'"
+    master_external_ips: "{{ master_external_ips | default('') }} '{{ master_ip }}'"
+  with_indexed_items: "{{ master_instances }}"
+  vars:
+    entry_name: "{{ openshift_gcp_prefix }}etcd-{{ item.0 }}.{{ public_hosted_zone }}."
+    master_ip: "{{ item.1.networkInterfaces[0].accessConfigs[0].natIP }}"
 
-- name: Provision GCP resources
-  command: /tmp/openshift_gcp_provision.sh
+- name: Templatize DNS script
+  template: src=additional_settings.j2.sh dest=/tmp/additional_settings.sh mode=u+rx
+
+- name: Run addition provision GCP script
+  command: /tmp/additional_settings.sh
   args:
     chdir: "{{ files_dir }}"
-  when:
-  - state | default('present') == 'present'
-
-- name: De-provision GCP resources
-  command: /tmp/openshift_gcp_provision_remove.sh
-  when:
-  - state | default('present') == 'absent'

roles/openshift_gcp/tasks/provision_ssh_keys.yml

@@ -1,6 +1,9 @@
 ---
 - name: Templatize SSH key provision script
-  template: src=provision_ssh.j2.sh dest=/tmp/openshift_gcp_provision_ssh.sh mode=u+rx
+  template:
+    src: provision_ssh.j2.sh
+    dest: /tmp/openshift_gcp_provision_ssh.sh
+    mode: u+rx
 
 - name: Provision GCP SSH key resources
   command: /tmp/openshift_gcp_provision_ssh.sh

roles/openshift_gcp/tasks/publish_image.yml

@@ -1,32 +0,0 @@
----
-- name: Require openshift_gcp_image
-  fail:
-    msg: "A source image name or family is required for image publishing.  Please ensure `openshift_gcp_image` is defined."
-  when: openshift_gcp_image is undefined
-
-- name: Require openshift_gcp_target_image
-  fail:
-    msg: "A target image name or family is required for image publishing.  Please ensure `openshift_gcp_target_image` is defined."
-  when: openshift_gcp_target_image is undefined
-
-- block:
-  - name: Retrieve images in the {{ openshift_gcp_target_image }} family
-    command: >
-      gcloud --project "{{ openshift_gcp_project }}" compute images list
-        "--filter=family={{ openshift_gcp_target_image }}"
-        --format=json --sort-by ~creationTimestamp
-    register: images
-  - name: Prune oldest images
-    command: >
-      gcloud --project "{{ openshift_gcp_project }}" compute images delete "{{ item['name'] }}"
-    with_items: "{{ (images.stdout | default('[]') | from_json )[( openshift_gcp_keep_images | int ):] }}"
-  when: openshift_gcp_keep_images is defined
-
-- name: Copy the latest image in the family {{ openshift_gcp_image }} to {{ openshift_gcp_target_image }}
-  command: >
-    gcloud --project "{{ openshift_gcp_target_project | default(openshift_gcp_project) }}"
-      beta compute images create
-      "{{ openshift_gcp_target_image_name | default(openshift_gcp_target_image + '-' + lookup('pipe','date +%Y%m%d-%H%M%S')) }}"
-      --family "{{ openshift_gcp_target_image }}"
-      --source-image-family "{{ openshift_gcp_image }}"
-      --source-image-project "{{ openshift_gcp_project }}"

roles/openshift_gcp/tasks/setup_scale_group_facts.yml

@@ -1,38 +1,19 @@
 ---
-- name: Set var to exclude bootstrapped nodes
-  set_fact:
-    bootstrapped_nodes: "{{ all_nodes | default(false) | ternary([], groups['tag_ocp-bootstrap']) | default([]) }}"
-
-- name: Add node instances to node group
+- name: Add bootstrap instances
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups: nodes, new_nodes
-    openshift_node_group_name: "{{ openshift_gcp_node_group_mapping['compute'] }}"
-  with_items: "{{ groups['tag_ocp-node'] | default([]) | difference(bootstrapped_nodes) }}"
-
-- name: Add bootstrap node instances as nodes
-  add_host:
-    name: "{{ item }}"
-    groups: nodes, new_nodes
+    groups: bootstrap
+    ignition_file: "{{ openshift_gcp_bootstrap_ignition_file }}"
   with_items: "{{ groups['tag_ocp-bootstrap'] | default([]) }}"
-  when: all_nodes | default(False)
-
-- name: Add non-bootstrapping master node instances to node group
-  add_host:
-    name: "{{ hostvars[item].gce_name }}"
-    groups: nodes
-  with_items: "{{ groups['tag_ocp-master'] | default([]) | difference(bootstrapped_nodes) }}"
 
-- name: Add infra node instances to node group
+- name: Add master instances
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups: nodes, new_nodes
-    openshift_node_group_name: "{{ openshift_gcp_node_group_mapping['infra'] }}"
-  with_items: "{{ groups['tag_ocp-infra-node'] | default([]) | difference(bootstrapped_nodes) }}"
+    groups: masters
+  with_items: "{{ groups['tag_ocp-master'] | default([]) }}"
 
-- name: Add masters to requisite groups
+- name: Add worker instances
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups: masters, etcd
-    openshift_node_group_name: "{{ openshift_gcp_node_group_mapping['masters'] }}"
-  with_items: "{{ groups['tag_ocp-master'] }}"
+    groups: workers
+  with_items: "{{ groups['tag_ocp-worker'] | default([]) }}"

roles/openshift_gcp/templates/additional_settings.j2.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
+# configure DNS
+(
+# Retry DNS changes until they succeed since this may be a shared resource
+while true; do
+    dns="${TMPDIR:-/tmp}/dns.yaml"
+    rm -f $dns
+
+    # DNS records for etcd servers
+    {% for master in master_instances %}
+      MASTER_DNS_NAME="{{ openshift_gcp_prefix }}etcd-{{ loop.index-1 }}.{{ public_hosted_zone }}."
+      IP="{{ master.networkInterfaces[0].networkIP }}"
+      if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ openshift_master_cluster_hostname }}" 2>/dev/null | grep -q "${MASTER_DNS_NAME}"; then
+          if [[ ! -f $dns ]]; then
+              gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+          fi
+          gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "${MASTER_DNS_NAME}" --type A "$IP"
+      else
+          echo "DNS record for '${MASTER_DNS_NAME}' already exists"
+      fi
+    {% endfor %}
+
+    # DNS records for etcd discovery
+    ETCD_DNS_NAME="_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
+    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "${ETCD_DNS_NAME}" 2>/dev/null | grep -q "${ETCD_DNS_NAME}"; then
+        if [[ ! -f $dns ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+        fi
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "${ETCD_DNS_NAME}" --type SRV {{ etcd_discovery_targets }}
+    else
+        echo "DNS record for '${ETCD_DNS_NAME}' already exists"
+    fi
+
+    # Roundrobin masters and bootstrap
+    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ openshift_master_cluster_public_hostname }}" 2>/dev/null | grep -q "{{ openshift_master_cluster_public_hostname }}"; then
+        if [[ ! -f $dns ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+        fi
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ openshift_master_cluster_public_hostname }}" --type A {{ bootstrap_instance.networkInterfaces[0].accessConfigs[0].natIP }} {{ master_external_ips }}
+    else
+        echo "DNS record for '{{ openshift_master_cluster_public_hostname }}' already exists"
+    fi
+
+    # Commit all DNS changes, retrying if preconditions are not met
+    if [[ -f $dns ]]; then
+        if ! out="$( gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns execute -z "${dns_zone}" 2>&1 )"; then
+            rc=$?
+            if [[ "${out}" == *"HTTPError 412: Precondition not met"* ]]; then
+                continue
+            fi
+            exit $rc
+        fi
+    fi
+    break
+done
+) &
+
+# Add groups to target pools
+# Add bootstrap
+# gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}ig-b" --target-pools "{{ openshift_gcp_prefix }}master-lb-pool" --zone "{{ openshift_gcp_zone }}"
+
+# # Add masters
+# gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}ig-m" --target-pools "{{ openshift_gcp_prefix }}master-lb-pool" --zone "{{ openshift_gcp_zone }}"
+
+# wait until all node groups are stable
+{% for node_group in openshift_gcp_node_group_config %}
+{% if node_group.wait_for_stable | default(False) %}
+# wait for stable {{ node_group.name }}
+( gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed wait-until-stable "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --timeout=600 ) &
+{% else %}
+# not waiting for {{ node_group.name }} due to bootstrapping
+{% endif %}
+{% endfor %}
+
+for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/dns.j2.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
-
-# Check the DNS managed zone in Google Cloud DNS, create it if it doesn't exist
-if ! gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" dns managed-zones create "${dns_zone}" --dns-name "{{ public_hosted_zone }}" --description "{{ public_hosted_zone }} domain" >/dev/null
-fi
-
-# Always output the expected nameservers as a comma delimited list
-gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" --format='value(nameServers)' | tr ';' ','

roles/openshift_gcp/templates/master_healthcheck.j2

@@ -60,9 +60,9 @@ defaults
 #---------------------------------------------------------------------
 # main frontend which proxys to the backends
 #---------------------------------------------------------------------
-frontend  http-proxy *:8080
+frontend  http-proxy *:{{ openshift_gcp_master_healthcheck_port }}
     acl          url_healthz  path_beg  -i /healthz
     use_backend  ocp          if url_healthz
 
 backend ocp
-    server       ocp localhost:{{ internal_console_port }} ssl verify none
+    server       ocp localhost:{{ openshift_gcp_kubernetes_api_port }} ssl verify none

roles/openshift_gcp/templates/openshift-bootstrap-update.j2

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-oc serviceaccounts create-kubeconfig -n openshift-infra node-bootstrapper > /root/bootstrap.kubeconfig
-gcloud compute project-info --project '{{ openshift_gcp_project }}' add-metadata --metadata-from-file '{{ openshift_gcp_prefix + openshift_gcp_clusterid | default("default") }}-bootstrap-config=/root/bootstrap.kubeconfig'
-rm -f /root/bootstrap.kubeconfig

roles/openshift_gcp/templates/provision.j2.sh

@@ -1,304 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-metadata=""
-if [[ -n "{{ openshift_gcp_startup_script_file }}" ]]; then
-    if [[ ! -f "{{ openshift_gcp_startup_script_file }}" ]]; then
-        echo "Startup script file missing at {{ openshift_gcp_startup_script_file }} from=$(pwd)"
-        exit 1
-    fi
-    metadata+="--metadata-from-file=startup-script={{ openshift_gcp_startup_script_file }}"
-fi
-if [[ -n "{{ openshift_gcp_user_data_file }}" ]]; then
-    if [[ ! -f "{{ openshift_gcp_user_data_file }}" ]]; then
-        echo "User data file missing at {{ openshift_gcp_user_data_file }}"
-        exit 1
-    fi
-    if [[ -n "${metadata}" ]]; then
-        metadata+=","
-    else
-        metadata="--metadata-from-file="
-    fi
-    metadata+="user-data={{ openshift_gcp_user_data_file }}"
-fi
-
-# Select image or image family
-image="{{ openshift_gcp_image }}"
-if ! gcloud --project "{{ openshift_gcp_project }}" compute images describe "${image}" &>/dev/null; then
-    if ! gcloud --project "{{ openshift_gcp_project }}" compute images describe-from-family "${image}" &>/dev/null; then
-        echo "No compute image or image-family found, create an image named '{{ openshift_gcp_image }}' to continue'"
-        exit 1
-    fi
-    image="family/${image}"
-fi
-
-### PROVISION THE INFRASTRUCTURE ###
-
-dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
-
-# Check the DNS managed zone in Google Cloud DNS, create it if it doesn't exist and exit after printing NS servers
-if ! gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" &>/dev/null; then
-    echo "DNS zone '${dns_zone}' doesn't exist. Must be configured prior to running this script"
-    exit 1
-fi
-
-# Create network
-if ! gcloud --project "{{ openshift_gcp_project }}" compute networks describe "{{ openshift_gcp_network_name }}" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute networks create "{{ openshift_gcp_network_name }}" --mode "auto"
-else
-    echo "Network '{{ openshift_gcp_network_name }}' already exists"
-fi
-
-# Firewall rules in a form:
-# ['name']='parameters for "gcloud compute firewall-rules create"'
-# For all possible parameters see: gcloud compute firewall-rules create --help
-range=""
-if [[ -n "{{ openshift_node_port_range }}" ]]; then
-    range=",tcp:{{ openshift_node_port_range }},udp:{{ openshift_node_port_range }}"
-fi
-declare -A FW_RULES=(
-  ['icmp']='--allow icmp'
-  ['ssh-external']='--allow tcp:22'
-  ['ssh-internal']='--allow tcp:22 --source-tags bastion'
-  ['master-internal']="--allow tcp:2224,tcp:2379,tcp:2380,tcp:4001,udp:4789,udp:5404,udp:5405,tcp:8053,udp:8053,tcp:8444,tcp:10250,tcp:10255,udp:10255,tcp:24224,udp:24224 --source-tags ocp --target-tags ocp-master"
-  ['master-external']="--allow tcp:80,tcp:443,tcp:1936,tcp:8080,tcp:8443${range} --target-tags ocp-master"
-  ['node-internal']="--allow udp:4789,tcp:10250,tcp:10255,udp:10255,tcp:9000-10000 --source-tags ocp --target-tags ocp-node,ocp-infra-node"
-  ['infra-node-internal']="--allow tcp:5000 --source-tags ocp --target-tags ocp-infra-node"
-  ['infra-node-external']="--allow tcp:80,tcp:443,tcp:1936${range} --target-tags ocp-infra-node"
-)
-for rule in "${!FW_RULES[@]}"; do
-    ( if ! gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules describe "{{ openshift_gcp_prefix }}$rule" &>/dev/null; then
-        gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules create "{{ openshift_gcp_prefix }}$rule" --network "{{ openshift_gcp_network_name }}" ${FW_RULES[$rule]}
-    else
-        echo "Firewall rule '{{ openshift_gcp_prefix }}${rule}' already exists"
-    fi ) &
-done
-
-
-# Master IP
-( if ! gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute addresses create "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global
-else
-    echo "IP '{{ openshift_gcp_prefix }}master-ssl-lb-ip' already exists"
-fi ) &
-
-# Internal master IP
-( if ! gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute addresses create "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}"
-else
-    echo "IP '{{ openshift_gcp_prefix }}master-network-lb-ip' already exists"
-fi ) &
-
-# Router IP
-( if ! gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute addresses create "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}"
-else
-    echo "IP '{{ openshift_gcp_prefix }}router-network-lb-ip' already exists"
-fi ) &
-
-
-{% for node_group in openshift_gcp_node_group_config %}
-# configure {{ node_group.name }}
-(
-    if ! gcloud --project "{{ openshift_gcp_project }}" compute instance-templates describe "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" &>/dev/null; then
-        gcloud --project "{{ openshift_gcp_project }}" compute instance-templates create "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" \
-                --machine-type "{{ node_group.machine_type }}" --network "{{ openshift_gcp_network_name }}" \
-                --tags "{{ openshift_gcp_prefix }}ocp,ocp,ocp-bootstrap,{{ node_group.tags }}" \
-                --boot-disk-size "{{ node_group.boot_disk_size }}" --boot-disk-type "pd-ssd" \
-                --scopes "logging-write,monitoring-write,useraccounts-ro,service-control,service-management,storage-ro,compute-rw" \
-                --image "{{ node_group.image | default('${image}') }}" ${metadata}  \
-                --metadata "bootstrap={{ node_group.bootstrap | default(False) | bool | to_json }},cluster-id={{ openshift_gcp_prefix + openshift_gcp_clusterid }},node-group={{ node_group.name }}"
-    else
-        echo "Instance template '{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}' already exists"
-    fi
-
-    # Create instance group
-    if ! gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed describe "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" &>/dev/null; then
-        gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed create "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" \
-                --zone "{{ openshift_gcp_zone }}" --template "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" --size "{{ node_group.scale }}"
-    else
-        echo "Instance group '{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}' already exists"
-    fi
-) &
-{% endfor %}
-
-for i in `jobs -p`; do wait $i; done
-
-
-# Configure the master external LB rules
-(
-# Master health check
-if ! gcloud --project "{{ openshift_gcp_project }}" compute health-checks describe "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute health-checks create https "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" --port "{{ internal_console_port }}" --request-path "/healthz"
-else
-    echo "Health check '{{ openshift_gcp_prefix }}master-ssl-lb-health-check' already exists"
-fi
-
-gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-named-ports "{{ openshift_gcp_prefix }}ig-m" \
-        --zone "{{ openshift_gcp_zone }}" --named-ports "{{ openshift_gcp_prefix }}port-name-master:{{ internal_console_port }}"
-
-# Master backend service
-if ! gcloud --project "{{ openshift_gcp_project }}" compute backend-services describe "{{ openshift_gcp_prefix }}master-ssl-lb-backend" --global &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute backend-services create "{{ openshift_gcp_prefix }}master-ssl-lb-backend" --health-checks "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" --port-name "{{ openshift_gcp_prefix }}port-name-master" --protocol "TCP" --global --timeout="{{ openshift_gcp_master_lb_timeout }}"
-    gcloud --project "{{ openshift_gcp_project }}" compute backend-services add-backend "{{ openshift_gcp_prefix }}master-ssl-lb-backend" --instance-group "{{ openshift_gcp_prefix }}ig-m" --global --instance-group-zone "{{ openshift_gcp_zone }}"
-else
-    echo "Backend service '{{ openshift_gcp_prefix }}master-ssl-lb-backend' already exists"
-fi
-
-# Master tcp proxy target
-if ! gcloud --project "{{ openshift_gcp_project }}" compute target-tcp-proxies describe "{{ openshift_gcp_prefix }}master-ssl-lb-target" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute target-tcp-proxies create "{{ openshift_gcp_prefix }}master-ssl-lb-target" --backend-service "{{ openshift_gcp_prefix }}master-ssl-lb-backend"
-else
-    echo "Proxy target '{{ openshift_gcp_prefix }}master-ssl-lb-target' already exists"
-fi
-
-# Master forwarding rule
-if ! gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules describe "{{ openshift_gcp_prefix }}master-ssl-lb-rule" --global &>/dev/null; then
-    IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global --format='value(address)')
-    gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules create "{{ openshift_gcp_prefix }}master-ssl-lb-rule" --address "$IP" --global --ports "{{ console_port }}" --target-tcp-proxy "{{ openshift_gcp_prefix }}master-ssl-lb-target"
-else
-    echo "Forwarding rule '{{ openshift_gcp_prefix }}master-ssl-lb-rule' already exists"
-fi
-) &
-
-
-# Configure the master internal LB rules
-(
-# Internal master health check
-if ! gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks describe "{{ openshift_gcp_prefix }}master-network-lb-health-check" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks create "{{ openshift_gcp_prefix }}master-network-lb-health-check" --port "8080" --request-path "/healthz"
-else
-    echo "Health check '{{ openshift_gcp_prefix }}master-network-lb-health-check' already exists"
-fi
-
-# Internal master target pool
-if ! gcloud --project "{{ openshift_gcp_project }}" compute target-pools describe "{{ openshift_gcp_prefix }}master-network-lb-pool" --region "{{ openshift_gcp_region }}" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute target-pools create "{{ openshift_gcp_prefix }}master-network-lb-pool" --http-health-check "{{ openshift_gcp_prefix }}master-network-lb-health-check" --region "{{ openshift_gcp_region }}"
-else
-    echo "Target pool '{{ openshift_gcp_prefix }}master-network-lb-pool' already exists"
-fi
-
-# Internal master forwarding rule
-if ! gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules describe "{{ openshift_gcp_prefix }}master-network-lb-rule" --region "{{ openshift_gcp_region }}" &>/dev/null; then
-    IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
-    gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules create "{{ openshift_gcp_prefix }}master-network-lb-rule" --address "$IP" --region "{{ openshift_gcp_region }}" --target-pool "{{ openshift_gcp_prefix }}master-network-lb-pool"
-else
-    echo "Forwarding rule '{{ openshift_gcp_prefix }}master-network-lb-rule' already exists"
-fi
-) &
-
-
-# Configure the infra node rules
-(
-# Router health check
-if ! gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks describe "{{ openshift_gcp_prefix }}router-network-lb-health-check" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks create "{{ openshift_gcp_prefix }}router-network-lb-health-check" --port "1936" --request-path "/healthz"
-else
-    echo "Health check '{{ openshift_gcp_prefix }}router-network-lb-health-check' already exists"
-fi
-
-# Router target pool
-if ! gcloud --project "{{ openshift_gcp_project }}" compute target-pools describe "{{ openshift_gcp_prefix }}router-network-lb-pool" --region "{{ openshift_gcp_region }}" &>/dev/null; then
-    gcloud --project "{{ openshift_gcp_project }}" compute target-pools create "{{ openshift_gcp_prefix }}router-network-lb-pool" --http-health-check "{{ openshift_gcp_prefix }}router-network-lb-health-check" --region "{{ openshift_gcp_region }}"
-else
-    echo "Target pool '{{ openshift_gcp_prefix }}router-network-lb-pool' already exists"
-fi
-
-# Router forwarding rule
-if ! gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules describe "{{ openshift_gcp_prefix }}router-network-lb-rule" --region "{{ openshift_gcp_region }}" &>/dev/null; then
-    IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
-    gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules create "{{ openshift_gcp_prefix }}router-network-lb-rule" --address "$IP" --region "{{ openshift_gcp_region }}" --target-pool "{{ openshift_gcp_prefix }}router-network-lb-pool"
-else
-    echo "Forwarding rule '{{ openshift_gcp_prefix }}router-network-lb-rule' already exists"
-fi
-) &
-
-for i in `jobs -p`; do wait $i; done
-
-# set the target pools
-(
-if [[ "ig-m" == "{{ openshift_gcp_infra_network_instance_group }}" ]]; then
-    gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}ig-m" --target-pools "{{ openshift_gcp_prefix }}master-network-lb-pool,{{ openshift_gcp_prefix }}router-network-lb-pool" --zone "{{ openshift_gcp_zone }}"
-else
-    gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}ig-m" --target-pools "{{ openshift_gcp_prefix }}master-network-lb-pool" --zone "{{ openshift_gcp_zone }}"
-    gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}{{ openshift_gcp_infra_network_instance_group }}" --target-pools "{{ openshift_gcp_prefix }}router-network-lb-pool" --zone "{{ openshift_gcp_zone }}"
-fi
-) &
-
-# configure DNS
-(
-# Retry DNS changes until they succeed since this may be a shared resource
-while true; do
-    dns="${TMPDIR:-/tmp}/dns.yaml"
-    rm -f $dns
-
-    # DNS record for master lb
-    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ openshift_master_cluster_public_hostname }}" 2>/dev/null | grep -q "{{ openshift_master_cluster_public_hostname }}"; then
-        IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global --format='value(address)')
-        if [[ ! -f $dns ]]; then
-            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
-        fi
-        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ openshift_master_cluster_public_hostname }}." --type A "$IP"
-    else
-        echo "DNS record for '{{ openshift_master_cluster_public_hostname }}' already exists"
-    fi
-
-    # DNS record for internal master lb
-    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ openshift_master_cluster_hostname }}" 2>/dev/null | grep -q "{{ openshift_master_cluster_hostname }}"; then
-        IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
-        if [[ ! -f $dns ]]; then
-            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
-        fi
-        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ openshift_master_cluster_hostname }}." --type A "$IP"
-    else
-        echo "DNS record for '{{ openshift_master_cluster_hostname }}' already exists"
-    fi
-
-    # DNS record for router lb
-    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ wildcard_zone }}" 2>/dev/null | grep -q "{{ wildcard_zone }}"; then
-        IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
-        if [[ ! -f $dns ]]; then
-            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
-        fi
-        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ wildcard_zone }}." --type A "$IP"
-        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "*.{{ wildcard_zone }}." --type CNAME "{{ wildcard_zone }}."
-    else
-        echo "DNS record for '{{ wildcard_zone }}' already exists"
-    fi
-
-    # Commit all DNS changes, retrying if preconditions are not met
-    if [[ -f $dns ]]; then
-        if ! out="$( gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns execute -z "${dns_zone}" 2>&1 )"; then
-            rc=$?
-            if [[ "${out}" == *"HTTPError 412: Precondition not met"* ]]; then
-                continue
-            fi
-            exit $rc
-        fi
-    fi
-    break
-done
-) &
-
-# Create bucket for registry
-(
-if ! gsutil ls -p "{{ openshift_gcp_project }}" "gs://{{ openshift_gcp_registry_bucket_name }}" &>/dev/null; then
-    gsutil mb -p "{{ openshift_gcp_project }}" -l "{{ openshift_gcp_region }}" "gs://{{ openshift_gcp_registry_bucket_name }}"
-else
-    echo "Bucket '{{ openshift_gcp_registry_bucket_name }}' already exists"
-fi
-) &
-
-# wait until all node groups are stable
-{% for node_group in openshift_gcp_node_group_config %}
-{% if node_group.wait_for_stable | default(False) or not (node_group.bootstrap | default(False)) %}
-# wait for stable {{ node_group.name }}
-( gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed wait-until-stable "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --timeout=600 ) &
-{% else %}
-# not waiting for {{ node_group.name }} due to bootstrapping
-{% endif %}
-{% endfor %}
-
-
-for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/remove.j2.sh

@@ -1,78 +1,6 @@
 #!/bin/bash
 
-set -euo pipefail
-
-function teardown_cmd() {
-    a=( $@ )
-    local name=$1
-    a=( "${a[@]:1}" )
-    local flag=0
-    local found=
-    for i in ${a[@]}; do
-        if [[ "$i" == "--"* ]]; then
-            found=true
-            break
-        fi
-        flag=$((flag+1))
-    done
-    if [[ -z "${found}" ]]; then
-      flag=$((flag+1))
-    fi
-    if gcloud --project "{{ openshift_gcp_project }}" ${a[@]::$flag} describe "${name}" ${a[@]:$flag} &>/dev/null; then
-        gcloud --project "{{ openshift_gcp_project }}" ${a[@]::$flag} delete -q "${name}" ${a[@]:$flag}
-    fi
-}
-
-function teardown() {
-    for i in `seq 1 20`; do
-        if teardown_cmd $@; then
-            break
-        fi
-        sleep 0.5
-    done
-}
-
-# Preemptively spin down the instances
-{% for node_group in openshift_gcp_node_group_config %}
-# scale down {{ node_group.name }}
-(
-    # performs a delete and scale down as one operation to ensure maximum parallelism
-    if ! instances=$( gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed list-instances "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --format='value[terminator=","](instance)' 2>/dev/null ); then
-        exit 0
-    fi
-    instances="${instances%?}"
-    if [[ -z "${instances}" ]]; then
-        echo "warning: No instances in {{ node_group.name }}" 1>&2
-        exit 0
-    fi
-    if ! gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed delete-instances "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --instances "${instances}"; then
-        echo "warning: Unable to scale down the node group {{ node_group.name }}" 1>&2
-        exit 0
-    fi
-) &
-{% endfor %}
-
-# Bucket for registry
-(
-if gsutil ls -p "{{ openshift_gcp_project }}" "gs://{{ openshift_gcp_registry_bucket_name }}" &>/dev/null; then
-    gsutil -m rm -r "gs://{{ openshift_gcp_registry_bucket_name }}"
-fi
-) &
-
-# Project metadata prefixed with {{ openshift_gcp_prefix }}
-(
-    for key in $( gcloud --project "{{ openshift_gcp_project }}" compute project-info describe --flatten=commonInstanceMetadata.items[] '--format=value(commonInstanceMetadata.items.key)' ); do
-        if [[ "${key}" == "{{ openshift_gcp_prefix }}"* ]]; then
-            gcloud --project "{{ openshift_gcp_project }}" compute project-info remove-metadata "--keys=${key}"
-        fi
-    done
-) &
-
-# Instances and disks used for image building
-(
-    teardown "{{ openshift_gcp_prefix }}build-image-instance" compute instances --zone "{{ openshift_gcp_zone }}"
-    teardown "{{ openshift_gcp_prefix }}build-image-instance" compute disks --zone "{{ openshift_gcp_zone }}"
-) &
+set -euxo pipefail
 
 # DNS
 (
@@ -86,9 +14,34 @@ if gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${
         # export all dns records that match into a zone format, and turn each line into a set of args for
         # record-sets transaction.
         gcloud dns record-sets export --project "{{ openshift_gcp_project }}" -z "${dns_zone}" --zone-file-format "${dns}"
-        if grep -F -e '{{ openshift_master_cluster_hostname }}' -e '{{ openshift_master_cluster_public_hostname }}' -e '{{ wildcard_zone }}' "${dns}" | \
-                awk '{ print "--name", $1, "--ttl", $2, "--type", $4, $5; }' > "${dns}.input"
-        then
+
+        # Fetch API record to get a list of masters + bootstrap node
+        bootstrap_and_masters=""
+        public_ip_output=($(grep -F -e '{{ openshift_master_cluster_public_hostname }}.' "${dns}" | awk '{ print $5 }')) || public_ip_output=""
+
+        for index in "${!public_ip_output[@]}"; do
+            bootstrap_and_masters="${bootstrap_and_masters} ${public_ip_output[${index}]}"
+            if [ ${index} -eq 0 ]; then
+                # First record is bootstrap
+                continue
+            fi
+            # etcd server name
+            MASTER_DNS_NAME="{{ openshift_gcp_prefix }}etcd-$((index-1)).{{ public_hosted_zone }}."
+            # Add a extra space here so that it won't match etcd discovery record
+            grep -F -e "${MASTER_DNS_NAME} " "${dns}" | awk '{ print "--name", $1, "--ttl", $2, "--type", $4, $5; }' >> "${dns}.input" || true
+        done
+
+        # Remove API record
+        if [ ! -z "${public_ip_output}" ]; then
+            args=`grep -F -e '{{ openshift_master_cluster_public_hostname }}.' "${dns}" | awk '{ print "--name", $1, "--ttl", $2, "--type", $4; }' | head -n1`
+            echo "${args}${bootstrap_and_masters}" >> "${dns}.input"
+        fi
+
+        # Remove etcd discovery record
+        ETCD_DNS_NAME="_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
+        grep -F -e "${ETCD_DNS_NAME}" "${dns}" | awk '{ print "--name", $1, "--ttl", $2, "--type", $4, "\x27"$5" "$6" "$7" "$8"\x27"; }'  >> "${dns}.input" || true
+
+        if [ -s "${dns}.input" ]; then
             rm -f "${dns}"
             gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
             cat "${dns}.input" | xargs -L1 gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file="${dns}" remove -z "${dns_zone}"
@@ -108,72 +61,4 @@ if gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${
 fi
 ) &
 
-(
-# Router network rules
-teardown "{{ openshift_gcp_prefix }}router-network-lb-rule" compute forwarding-rules --region "{{ openshift_gcp_region }}"
-teardown "{{ openshift_gcp_prefix }}router-network-lb-pool" compute target-pools --region "{{ openshift_gcp_region }}"
-teardown "{{ openshift_gcp_prefix }}router-network-lb-health-check" compute http-health-checks
-teardown "{{ openshift_gcp_prefix }}router-network-lb-ip" compute addresses --region "{{ openshift_gcp_region }}"
-
-# Internal master network rules
-teardown "{{ openshift_gcp_prefix }}master-network-lb-rule" compute forwarding-rules --region "{{ openshift_gcp_region }}"
-teardown "{{ openshift_gcp_prefix }}master-network-lb-pool" compute target-pools --region "{{ openshift_gcp_region }}"
-teardown "{{ openshift_gcp_prefix }}master-network-lb-health-check" compute http-health-checks
-teardown "{{ openshift_gcp_prefix }}master-network-lb-ip" compute addresses --region "{{ openshift_gcp_region }}"
-) &
-
-(
-# Master SSL network rules
-teardown "{{ openshift_gcp_prefix }}master-ssl-lb-rule" compute forwarding-rules --global
-teardown "{{ openshift_gcp_prefix }}master-ssl-lb-target" compute target-tcp-proxies
-teardown "{{ openshift_gcp_prefix }}master-ssl-lb-ip" compute addresses --global
-teardown "{{ openshift_gcp_prefix }}master-ssl-lb-backend" compute backend-services --global
-teardown "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" compute health-checks
-) &
-
-# Firewall rules
-(
-    if ! firewalls=$( gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules list --filter="network~projects/{{ openshift_gcp_project }}/global/networks/{{ openshift_gcp_network_name }}" --format="value[terminator=' '](name)" 2>/dev/null ); then
-        exit 0
-    fi
-    firewalls="${firewalls%?}"
-    if [[ -z "${firewalls}" ]]; then
-        exit 0
-    fi
-    gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules delete -q ${firewalls}
-) &
-
-for i in `jobs -p`; do wait $i; done
-
-{% for node_group in openshift_gcp_node_group_config %}
-# teardown {{ node_group.name }} - any load balancers referencing these groups must be removed
-(
-    teardown "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" compute instance-groups managed --zone "{{ openshift_gcp_zone }}"
-    teardown "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" compute instance-templates
-) &
-{% endfor %}
-
 for i in `jobs -p`; do wait $i; done
-
-# Images specifically located under this cluster prefix family
-for name in $( gcloud --project "{{ openshift_gcp_project }}" compute images list "--filter=family={{ openshift_gcp_prefix }}images" '--format=value(name)' ); do
-    ( gcloud --project "{{ openshift_gcp_project }}" compute images delete "${name}" ) &
-done
-
-# Disks
-(
-    if ! disks=$( gcloud --project "{{ openshift_gcp_project }}" compute disks list --filter="users~projects/{{ openshift_gcp_project }}/zones/{{ openshift_gcp_zone }}/instances/{{ openshift_gcp_prefix }}.*" --format="value[terminator=' '](name)" 2>/dev/null ); then
-        exit 0
-    fi
-    disks="${disks%?}"
-    if [[ -z "${disks}" ]]; then
-        echo "warning: No disks in use by {{ openshift_gcp_prefix }}" 1>&2
-        exit 0
-    fi
-    gcloud --project "{{ openshift_gcp_project }}" compute disks delete -q "${disks}"
-) &
-
-# Network
-( teardown "{{ openshift_gcp_network_name }}" compute networks ) &
-
-for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/yum_repo.j2

@@ -1,20 +0,0 @@
-{% for repo in provision_custom_repositories %}
-[{{ repo.id | default(repo.name) }}]
-name={{ repo.name | default(repo.id) }}
-baseurl={{ repo.baseurl }}
-{% set enable_repo = repo.enabled | default(1) %}
-enabled={{ 1 if ( enable_repo == 1 or enable_repo == True ) else 0 }}
-{% set enable_gpg_check = repo.gpgcheck | default(1) %}
-gpgcheck={{ 1 if ( enable_gpg_check == 1 or enable_gpg_check == True ) else 0 }}
-{% if 'sslclientcert' in repo %}
-sslclientcert={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_cert" if repo.sslclientcert }}
-{% endif %}
-{% if 'sslclientkey' in repo %}
-sslclientkey={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_key" if repo.sslclientkey }}
-{% endif %}
-{% for key, value in repo.iteritems() %}
-{% if key not in ['id', 'name', 'baseurl', 'enabled', 'gpgcheck', 'sslclientkey', 'sslclientcert'] and value is defined %}
-{{ key }}={{ value }}
-{% endif %}
-{% endfor %}
-{% endfor %}

roles/openshift_node40/tasks/aws.yml

@@ -1,3 +1,4 @@
+---
 - name: Configure AWS Cloud Provider Settings
   lineinfile:
     dest: /etc/kubernetes/kubelet-env

playbooks/gcp/openshift-cluster/build_image.yml

@@ -24,10 +24,6 @@
   connection: local
   gather_facts: no
   tasks:
-  - name: Set facts
-    set_fact:
-      openshift_master_unsupported_embedded_etcd: True
-
   - name: Create the image instance disk
     gce_pd:
       service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
@@ -79,16 +75,12 @@
 
 - name: Add custom repositories
   hosts: nodes
-  handlers:
-  - import_tasks: ../../roles/openshift_repos/handlers/main.yml
-  tasks:
-  - include_role:
-      name: openshift_gcp
-      tasks_from: add_custom_repositories.yml
+  roles:
+  - role: openshift_repos
 
-# This is the part that installs all of the software and configs for the instance
-# to become a node.
-- import_playbook: ../../openshift-node/private/image_prep.yml
+- import_playbook: ../../playbooks/init/base_packages.yml
+  vars:
+    l_base_packages_hosts: nodes
 
 # Add additional GCP specific behavior
 - hosts: nodes

playbooks/gcp/openshift-cluster/deprovision.yml

@@ -6,5 +6,4 @@
   tasks:
   - include_role:
       name: openshift_gcp
-    vars:
-      state: absent
+      tasks_from: deprovision.yml

test/gcp/install.yml

@@ -0,0 +1,145 @@
+# This playbook installs onto a provisioned cluster
+#TODO: split into parts: nodes.yml, bootstrap.yml, masters.yml, workers.yml, bootkube/post_setup.yml
+---
+- hosts: localhost
+  connection: local
+  tasks:
+  - name: place all scale groups into Ansible groups
+    include_role:
+      name: openshift_gcp
+      tasks_from: setup_scale_group_facts.yml
+
+- name: run the init
+  import_playbook: ../../playbooks/init/main.yml
+  vars:
+    l_init_fact_hosts: "bootstrap:masters:workers"
+    l_openshift_version_set_hosts: "bootstrap:masters:workers"
+    l_install_base_packages: True
+    l_repo_hosts: "all:!all"
+
+- name: Install nodes
+  hosts: bootstrap:masters:workers
+  roles:
+  - role: container_runtime
+  tasks:
+  - import_role:
+      name: container_runtime
+      tasks_from: docker_storage_setup_overlay.yml
+  - import_role:
+      name: container_runtime
+      tasks_from: extra_storage_setup.yml
+  - import_role:
+      name: container_runtime
+      tasks_from: package_crio.yml
+  - name: FIXME pause_image
+    ini_file:
+      dest: "/etc/crio/crio.conf"
+      section: crio.image
+      option: pause_image
+      value: '"docker.io/openshift/origin-pod:v4.0"'
+  - name: FIXME restart crio
+    service:
+      name: crio
+      state: restarted
+  - import_role:
+      name: openshift_node40
+      tasks_from: install.yml
+
+- name: Config bootstrap node
+  hosts: bootstrap
+  tasks:
+  - import_role:
+      name: openshift_node40
+      tasks_from: config.yml
+  - import_role:
+      name: openshift_node40
+      tasks_from: systemd.yml
+    vars:
+      excluded_services:
+      - progress.service
+
+- name: Start masters
+  hosts: masters
+  tasks:
+  # This is required for openshift_node40/config.yml
+  - set_fact:
+      openshift_bootstrap_endpoint: "https://{{ openshift_master_cluster_hostname }}:{{ mcd_port }}/config/master"
+  - name: Wait for bootstrap endpoint to show up
+    uri:
+      url: "{{ openshift_bootstrap_endpoint }}"
+      validate_certs: false
+    delay: 10
+    retries: 60
+    register: result
+    until:
+    - "'status' in result"
+    - result.status == 200
+  - import_role:
+      name: openshift_node40
+      tasks_from: config.yml
+  - name: Make sure etcd user exists
+    user:
+      name: etcd
+  - import_role:
+      name: openshift_node40
+      tasks_from: systemd.yml
+
+- name: Start workers
+  hosts: workers
+  tasks:
+  # This is required for openshift_node40/config.yml
+  - set_fact:
+      openshift_bootstrap_endpoint: "https://{{ openshift_master_cluster_hostname }}:{{ mcd_port }}/config/worker"
+  - name: Wait for bootstrap endpoint to show up
+    uri:
+      url: "{{ openshift_bootstrap_endpoint }}"
+      validate_certs: false
+    delay: 10
+    retries: 60
+    register: result
+    until:
+    - "'status' in result"
+    - result.status == 200
+  - import_role:
+      name: openshift_node40
+      tasks_from: config.yml
+  - import_role:
+      name: openshift_node40
+      tasks_from: systemd.yml
+
+- name: Wait for nodes to become ready
+  hosts: bootstrap
+  tasks:
+  - name: Wait for temporary control plane to show up
+    #TODO: Rework with k8s module
+    oc_obj:
+      state: list
+      kind: pod
+      namespace: kube-system
+      kubeconfig: /opt/tectonic/auth/kubeconfig
+    register: control_plane_pods
+    retries: 60
+    delay: 10
+    until:
+    - "'results' in control_plane_pods and 'results' in control_plane_pods.results"
+    - control_plane_pods.results.results[0]['items'] | length > 0
+  - name: Wait for master nodes to show up
+    #TODO: Rework with k8s module
+    oc_obj:
+      state: list
+      kind: node
+      selector: "node-role.kubernetes.io/master"
+      kubeconfig: /opt/tectonic/auth/kubeconfig
+    register: master_nodes
+    retries: 60
+    delay: 10
+    until:
+    - "'results' in master_nodes and 'results' in master_nodes.results"
+    - master_nodes.results.results[0]['items'] | length > 0
+  - name: Wait for bootkube service to finish
+    service_facts: {}
+    #10 mins to complete temp plane
+    retries: 120
+    delay: 5
+    until: ansible_facts.services['bootkube.service'].state == 'stopped'
+    ignore_errors: true

test/gcp/roles

@@ -0,0 +1 @@
+../../roles