Revert "Devel 4.0: CI test"

images/installer/Dockerfile

@@ -10,13 +10,13 @@ COPY images/installer/origin-extra-root /
 # install ansible and deps
 RUN INSTALL_PKGS="python-lxml python-dns pyOpenSSL python2-cryptography openssl python2-passlib httpd-tools openssh-clients origin-clients iproute patch" \
  && yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS \
- && EPEL_PKGS="ansible python2-boto python2-crypto which python2-pip.noarch python2-scandir python2-packaging azure-cli-2.0.46" \
+ && EPEL_PKGS="ansible-2.6.5 python2-boto python2-boto3 python2-crypto which python2-pip.noarch python2-scandir python2-packaging azure-cli-2.0.46" \
  && yum install -y epel-release \
  && yum install -y --setopt=tsflags=nodocs $EPEL_PKGS \
  && if [ "$(uname -m)" == "x86_64" ]; then yum install -y https://sdodson.fedorapeople.org/google-cloud-sdk-183.0.0-3.el7.x86_64.rpm ; fi \
  && yum install -y java-1.8.0-openjdk-headless \
  && rpm -V $INSTALL_PKGS $EPEL_PKGS $EPEL_TESTING_PKGS \
- && pip install 'apache-libcloud~=2.2.1' 'SecretStorage<3' 'ansible[azure]' 'google-auth' 'boto3==1.4.6' \
+ && pip install 'apache-libcloud~=2.2.1' 'SecretStorage<3' 'ansible[azure]' \
  && yum clean all
 
 LABEL name="openshift/origin-ansible" \

images/installer/origin-extra-root/etc/yum.repos.d/centos-ansible26.repo

@@ -0,0 +1,6 @@
+
+[centos-ansible26-testing]
+name=CentOS Ansible 2.6 testing repo
+baseurl=https://cbs.centos.org/repos/configmanagement7-ansible-26-testing/x86_64/os/
+enabled=1
+gpgcheck=0

images/installer/root/usr/local/bin/entrypoint-provider

@@ -45,7 +45,7 @@ if [[ -f "${FILES}/ssh-privatekey" ]]; then
   else
     keyfile="${HOME}/.ssh/id_rsa"
   fi
-  mkdir -p "${HOME}/.ssh"
+  mkdir "${HOME}/.ssh"
   rm -f "${keyfile}"
   cat "${FILES}/ssh-privatekey" > "${keyfile}"
   chmod 0600 "${keyfile}"

inventory/dynamic/gcp/ansible.cfg

@@ -28,8 +28,6 @@ inventory_ignore_extensions = secrets.py, .pyc, .cfg, .crt
 # work around privilege escalation timeouts in ansible:
 timeout = 30
 
-stdout_callback = yaml
-
 # Uncomment to use the provided example inventory
 inventory = hosts.sh
 

inventory/dynamic/gcp/group_vars/all/00_defaults.yml

@@ -20,9 +20,6 @@ openshift_master_cluster_hostname: "internal-openshift-master.{{ public_hosted_z
 openshift_master_cluster_public_hostname: "openshift-master.{{ public_hosted_zone }}"
 openshift_master_default_subdomain: "{{ wildcard_zone }}"
 
-mcd_port: 49500
-mcd_endpoint: "{{ openshift_master_cluster_public_hostname }}:{{ mcd_port }}"
-
 # Cloud specific settings
 openshift_cloudprovider_kind: gce
 openshift_hosted_registry_storage_provider: gcs
@@ -34,7 +31,7 @@ openshift_master_identity_providers:
 openshift_node_port_range: 30000-32000
 openshift_node_open_ports: [{"service":"Router stats port", "port":"1936/tcp"}, {"service":"Allowed open host ports", "port":"9000-10000/tcp"}, {"service":"Allowed open host ports", "port":"9000-10000/udp"}]
 os_sdn_network_plugin_name: redhat/openshift-ovs-networkpolicy
-openshift_node_sdn_mtu: 1500
+openshift_node_sdn_mtu: 1410
 osm_cluster_network_cidr: 172.16.0.0/16
 osm_host_subnet_length: 9
 openshift_portal_net: 172.30.0.0/16

inventory/dynamic/gcp/hosts.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Use a playbook to calculate the inventory dynamically from
 # the provided cluster variables.
 src="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-if ! out="$( ansible-playbook --inventory-file "${src}/none" ${src}/../../../test/gcp/inventory.yml 2>&1 )"; then
+if ! out="$( ansible-playbook --inventory-file "${src}/none" ${src}/../../../playbooks/gcp/openshift-cluster/inventory.yml 2>&1 )"; then
   echo "error: Inventory configuration failed" 1>&2
   echo "$out" 1>&2
   echo "{}"

playbooks/bootkube.yml

@@ -1,92 +0,0 @@
----
-# Generate config using openshift-installer, set Base Domain to testing.tt
-# Add bootstrap host in [bootstrap] group and set ignition_file
-# Add master host to [masters] group
-# Add worker hosts in [workers] group
-# Make sure bootstrap has <clusterid>-api.<dns base> name
-# Make sure masters have <clusterid>-etcd-<index>.<dns base> name
-
-# FIXME: use dnsmasq to fake DNS entries
-
-- import_playbook: init/main.yml
-  vars:
-    l_install_base_packages: True
-    l_repo_hosts: "all:!all"
-
-# TODO: proper firewalld setup
-# 49500 on bootstrap; 2379, 6443, 10250 on masters, 10250 on workers
-
-- import_playbook: container-runtime/private/setup_storage.yml
-
-- import_playbook: container-runtime/private/config.yml
-
-- name: install nodes
-  hosts: nodes
-  tasks:
-  - import_role:
-      name: openshift_node40
-      tasks_from: install.yml
-
-- name: Config bootstrap node
-  hosts: bootstrap
-  tasks:
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml
-
-- name: Start masters
-  hosts: masters
-  tasks:
-  # TODO Read this from master's ignition file
-  - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ bootstrap }}:49500/config/master?etcd_index={{ index }}"
-    vars:
-      bootstrap: "{{ hostvars[groups['bootstrap'][0]]['ansible_host'] }}"
-      index: "{{ groups['masters'].index(inventory_hostname) }}"
-  - name: Wait for bootstrap endpoint to show up
-    uri:
-      url: "{{ openshift_bootstrap_endpoint }}"
-      validate_certs: false
-    delay: 10
-    retries: 60
-    register: result
-    until:
-    - "'status' in result"
-    - result.status == 200
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - name: Make sure etcd user exists
-    user:
-      name: etcd
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml
-
-- name: Start workers
-  hosts: workers
-  tasks:
-  # TODO Read this from master's ignition file
-  - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ bootstrap }}:49500/config/worker"
-    vars:
-      bootstrap: "{{ hostvars[groups['bootstrap'][0]]['ansible_host'] }}"
-  - name: Wait for bootstrap endpoint to show up
-    uri:
-      url: "{{ openshift_bootstrap_endpoint }}"
-      validate_certs: false
-    delay: 10
-    retries: 60
-    register: result
-    until:
-    - "'status' in result"
-    - result.status == 200
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml

playbooks/bootkube_node.yml

@@ -1,29 +0,0 @@
----
-# l_scale_up_hosts may be passed in via various scaleup plays.
-
-- import_playbook: init/main.yml
-  vars:
-    l_install_base_packages: True
-    l_repo_hosts: "all:!all"
-
-# This is required for container runtime for crio, only needs to run once.
-- name: Configure os_firewall
-  hosts: "{{ l_scale_up_hosts | default(l_default_firewall_hosts) }}"
-  vars:
-    l_default_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config:oo_nfs_to_config:oo_nodes_to_config"
-  roles:
-  - role: os_firewall
-
-- import_playbook: container-runtime/private/setup_storage.yml
-
-- import_playbook: container-runtime/private/config.yml
-
-- name: install nodes
-  hosts: bootkube_nodes
-  tasks:
-  - import_role:
-      name: openshift_node40
-      tasks_from: install.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: bootkube_config.yml

playbooks/deploy_cluster_40.yml

@@ -1,174 +0,0 @@
----
-- name: run the init
-  import_playbook: init/main.yml
-  vars:
-    l_init_fact_hosts: "nodes"
-    l_openshift_version_set_hosts: "nodes"
-    l_install_base_packages: True
-    l_repo_hosts: "all:!all"
-
-# TODO(michaelgugino): break up the rest of this file into reusable chunks.
-- name: Install nodes
-  hosts: nodes
-  roles:
-  - role: container_runtime
-  tasks:
-  - import_role:
-      name: container_runtime
-      tasks_from: docker_storage_setup_overlay.yml
-  - import_role:
-      name: container_runtime
-      tasks_from: extra_storage_setup.yml
-  - import_role:
-      name: container_runtime
-      tasks_from: package_crio.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: install.yml
-
-- name: Config bootstrap node
-  hosts: bootstrap
-  tasks:
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml
-    vars:
-      excluded_services:
-      - progress.service
-
-- name: Start masters
-  hosts: masters
-  tasks:
-  # This is required for openshift_node40/config.yml
-  - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ mcd_endpoint }}/config/master"
-  - name: Wait for bootstrap endpoint to show up
-    uri:
-      url: "{{ openshift_bootstrap_endpoint }}"
-      validate_certs: false
-    delay: 10
-    retries: 60
-    register: result
-    until:
-    - "'status' in result"
-    - result.status == 200
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - name: Make sure etcd user exists
-    user:
-      name: etcd
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml
-
-- name: Start workers
-  hosts: workers
-  tasks:
-  # This is required for openshift_node40/config.yml
-  - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ mcd_endpoint }}/config/worker"
-  - name: Wait for bootstrap endpoint to show up
-    uri:
-      url: "{{ openshift_bootstrap_endpoint }}"
-      validate_certs: false
-    delay: 10
-    retries: 60
-    register: result
-    until:
-    - "'status' in result"
-    - result.status == 200
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml
-
-- name: Wait for nodes to become ready
-  hosts: bootstrap
-  tasks:
-  - name: Wait for temporary control plane to show up
-    oc_obj:
-      state: list
-      kind: pod
-      namespace: kube-system
-      kubeconfig: /opt/openshift/auth/kubeconfig
-    register: control_plane_pods
-    retries: 60
-    delay: 10
-    until:
-    - "'results' in control_plane_pods and 'results' in control_plane_pods.results"
-    - control_plane_pods.results.results[0]['items'] | length > 0
-  - name: Wait for master nodes to show up
-    oc_obj:
-      state: list
-      kind: node
-      selector: "node-role.kubernetes.io/master"
-      kubeconfig: /opt/openshift/auth/kubeconfig
-    register: master_nodes
-    retries: 60
-    delay: 10
-    until:
-    - "'results' in master_nodes and 'results' in master_nodes.results"
-    - master_nodes.results.results[0]['items'] | length > 0
-  - name: Wait for bootkube service to finish
-    service_facts: {}
-    #10 mins to complete temp plane
-    retries: 120
-    delay: 5
-    until: "'bootkube.service' not in ansible_facts.services"
-    ignore_errors: true
-  - name: Fetch kubeconfig for test container
-    fetch:
-      src: /opt/openshift/auth/kubeconfig
-      dest: /tmp/artifacts/installer/auth/kubeconfig
-      flat: yes
-
-  - name: Wait for core operators to appear and complete
-    oc_obj:
-      state: list
-      kind: ClusterOperator
-      name: "{{ item }}"
-      kubeconfig: /opt/openshift/auth/kubeconfig
-    register: operator
-    #Give each operator 5 mins to come up
-    retries: 60
-    delay: 5
-    until:
-    - "'results' in operator"
-    - "'results' in operator.results"
-    - operator.results.results | length > 0
-    - "'status' in operator.results.results[0]"
-    - "'conditions' in operator.results.results[0]['status']"
-    - operator.results.results[0].status.conditions | selectattr('type', 'match', '^Available$') | map(attribute='status') | join | bool == True
-    - operator.results.results[0].status.conditions | selectattr('type', 'match', '^Progressing$') | map(attribute='status') | join | bool == False
-    - operator.results.results[0].status.conditions | selectattr('type', 'match', '^Failing$') | map(attribute='status') | join | bool == False
-    with_items:
-    - machine-config-operator
-    # Fails often with 'x of y nodes are not at revision n'
-    #- openshift-cluster-kube-apiserver-operator
-    # Failing with 'ConfigObservationFailing: configmap/cluster-config-v1.kube-system: no recognized cloud provider platform found' - https://github.com/openshift/cluster-kube-controller-manager-operator/issues/100
-    #- openshift-cluster-kube-controller-manager-operator
-    # Fails often with 'x of y nodes are not at revision n'
-    #- openshift-cluster-kube-scheduler-operator
-    #- openshift-cluster-openshift-apiserver-operator
-    - openshift-cluster-openshift-controller-manager-operator
-    - openshift-ingress-operator
-    ignore_errors: true
-
-  - block:
-    - name: Output the operators status
-      oc_obj:
-        state: list
-        kind: ClusterOperator
-        selector: ""
-        kubeconfig: /opt/openshift/auth/kubeconfig
-    - fail:
-        msg: Required operators didn't complete the install
-    when: operator.failed
-
-  - pause: {}

playbooks/gcp/OWNERS

@@ -0,0 +1,14 @@
+# approval == this is a good idea /approve
+approvers:
+  - smarterclayton
+  - michaelgugino
+  - mtnbikenc
+  - sdodson
+  - vrutkovs
+# review == this code is good /lgtm
+reviewers:
+  - smarterclayton
+  - michaelgugino
+  - mtnbikenc
+  - sdodson
+  - vrutkovs

playbooks/gcp/openshift-cluster/build_base_image.yml

@@ -0,0 +1,186 @@
+---
+# This playbook ensures that a base image is up to date with all of the required settings
+- name: Verify prerequisites for image build
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  tasks:
+  - name: Require openshift_gcp_root_image
+    fail:
+      msg: "A root OS image name or family is required for base image building.  Please ensure `openshift_gcp_root_image` is defined."
+    when: openshift_gcp_root_image is undefined
+
+- name: Provision ssh key
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  tasks:
+  - name: Set up core host GCP configuration
+    import_role:
+      name: openshift_gcp
+      tasks_from: provision_ssh_keys.yml
+
+- name: Launch image build instance
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  tasks:
+  - name: Create the image instance disk
+    gce_pd:
+      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
+      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+      project_id: "{{ openshift_gcp_project }}"
+      zone: "{{ openshift_gcp_zone }}"
+      name: "{{ openshift_gcp_prefix }}build-image-instance"
+      disk_type: pd-ssd
+      image: "{{ openshift_gcp_root_image }}"
+      size_gb: 10
+      state: present
+
+  - name: Launch the image build instance
+    gce:
+      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
+      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+      project_id: "{{ openshift_gcp_project }}"
+      zone: "{{ openshift_gcp_zone }}"
+      machine_type: n1-standard-1
+      instance_names: "{{ openshift_gcp_prefix }}build-image-instance"
+      state: present
+      tags:
+      - build-image-instance
+      disk_auto_delete: false
+      disks:
+      - "{{ openshift_gcp_prefix }}build-image-instance"
+    register: gce
+
+  - add_host:
+      hostname: "{{ item.public_ip }}"
+      groupname: build_instance_ips
+    with_items: "{{ gce.instance_data }}"
+
+  - name: Wait for instance to respond to SSH
+    wait_for:
+      delay: 1
+      host: "{{ item.public_ip }}"
+      port: 22
+      state: started
+      timeout: 120
+    with_items: "{{ gce.instance_data }}"
+
+- name: Prepare instance content sources
+  pre_tasks:
+  - set_fact:
+      allow_rhel_subscriptions: "{{ rhsub_skip | default('no', True) | lower in ['no', 'false'] }}"
+  - set_fact:
+      using_rhel_subscriptions: "{{ (deployment_type in ['enterprise', 'atomic-enterprise', 'openshift-enterprise'] or ansible_distribution == 'RedHat') and allow_rhel_subscriptions }}"
+  hosts: build_instance_ips
+  roles:
+  - role: rhel_subscribe
+    when: using_rhel_subscriptions
+  - role: openshift_repos
+    vars:
+      openshift_additional_repos: []
+  post_tasks:
+  - name: Add custom repositories
+    include_role:
+      name: openshift_gcp
+      tasks_from: add_custom_repositories.yml
+  - name: Add the Google Cloud repo
+    yum_repository:
+      name: google-cloud
+      description: Google Cloud Compute
+      baseurl: https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64
+      gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+      gpgcheck: yes
+      repo_gpgcheck: yes
+      state: present
+    when: ansible_os_family == "RedHat"
+  - name: Add the jdetiber-qemu-user-static copr repo
+    yum_repository:
+      name: jdetiber-qemu-user-static
+      description: QEMU user static COPR
+      baseurl: https://copr-be.cloud.fedoraproject.org/results/jdetiber/qemu-user-static/epel-7-$basearch/
+      gpgkey: https://copr-be.cloud.fedoraproject.org/results/jdetiber/qemu-user-static/pubkey.gpg
+      gpgcheck: yes
+      repo_gpgcheck: no
+      state: present
+    when: ansible_os_family == "RedHat"
+  - name: Accept GPG keys for the repos
+    command: yum -q makecache -y --disablerepo='*' --enablerepo='google-cloud,jdetiber-qemu-user-static'
+  - name: Install qemu-user-static
+    package:
+      name: qemu-user-static
+      state: present
+  - name: Disable yum-cron service (installed by Google Cloud by default)
+    systemd:
+      name: yum-cron
+      state: stopped
+      enabled: no
+  - name: Start and enable systemd-binfmt service
+    systemd:
+      name: systemd-binfmt
+      state: started
+      enabled: yes
+
+- name: Build image
+  hosts: build_instance_ips
+  pre_tasks:
+  - name: Set up core host GCP configuration
+    include_role:
+      name: openshift_gcp
+      tasks_from: configure_gcp_base_image.yml
+  roles:
+  - role: os_update_latest
+  post_tasks:
+  - name: Disable all repos on RHEL
+    command: subscription-manager repos --disable="*"
+    when: using_rhel_subscriptions
+  - name: Enable repos for packages on RHEL
+    command: subscription-manager repos --enable="rhel-7-server-rpms" --enable="rhel-7-server-extras-rpms"
+    when: using_rhel_subscriptions
+  - name: Install common image prerequisites
+    package:
+      name: "{{ pkg_list | join(',') }}"
+      state: latest
+    vars:
+      pkg_list:
+      # required by Ansible
+      - PyYAML
+      - google-compute-engine
+      - google-compute-engine-init
+      - google-config
+      - wget
+      - git
+      - net-tools
+      - bind-utils
+      - iptables-services
+      - bridge-utils
+      - bash-completion
+  - name: Clean yum metadata
+    command: yum clean all
+    args:
+      warn: no
+    when: ansible_os_family == "RedHat"
+
+- name: Commit image
+  hosts: localhost
+  connection: local
+  tasks:
+  - name: Terminate the image build instance
+    gce:
+      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
+      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+      project_id: "{{ openshift_gcp_project }}"
+      zone: "{{ openshift_gcp_zone }}"
+      instance_names: "{{ openshift_gcp_prefix }}build-image-instance"
+      state: absent
+  - name: Save the new image
+    command: gcloud --project "{{ openshift_gcp_project}}" compute images create "{{ openshift_gcp_base_image_name | default(openshift_gcp_base_image + '-' + lookup('pipe','date +%Y%m%d-%H%M%S')) }}" --source-disk "{{ openshift_gcp_prefix }}build-image-instance" --source-disk-zone "{{ openshift_gcp_zone }}" --family "{{ openshift_gcp_base_image }}"
+  - name: Remove the image instance disk
+    gce_pd:
+      service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
+      credentials_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
+      project_id: "{{ openshift_gcp_project }}"
+      zone: "{{ openshift_gcp_zone }}"
+      name: "{{ openshift_gcp_prefix }}build-image-instance"
+      state: absent

test/gcp/build_image.yml

@@ -24,6 +24,10 @@
   connection: local
   gather_facts: no
   tasks:
+  - name: Set facts
+    set_fact:
+      openshift_master_unsupported_embedded_etcd: True
+
   - name: Create the image instance disk
     gce_pd:
       service_account_email: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
@@ -75,12 +79,16 @@
 
 - name: Add custom repositories
   hosts: nodes
-  roles:
-  - role: openshift_repos
+  handlers:
+  - import_tasks: ../../roles/openshift_repos/handlers/main.yml
+  tasks:
+  - include_role:
+      name: openshift_gcp
+      tasks_from: add_custom_repositories.yml
 
-- import_playbook: ../../playbooks/init/base_packages.yml
-  vars:
-    l_base_packages_hosts: nodes
+# This is the part that installs all of the software and configs for the instance
+# to become a node.
+- import_playbook: ../../openshift-node/private/image_prep.yml
 
 # Add additional GCP specific behavior
 - hosts: nodes
@@ -91,21 +99,6 @@
   - include_role:
       name: openshift_gcp
       tasks_from: frequent_log_rotation.yml
-  - name: Install networkmanager-glib to reset MTU
-    package:
-      name: NetworkManager-glib
-      state: present
-  - name: Set MTU
-    nmcli:
-      conn_name: "System eth0"
-      mtu: "{{ openshift_node_sdn_mtu }}"
-      type: ethernet
-      state: present
-  #Required for storage tests to mount NFS shares
-  - name: Install packages for tests
-    package:
-      name: "nfs-utils"
-      state: present
 
 - name: Commit image
   hosts: localhost

test/gcp/deprovision.yml

@@ -6,4 +6,5 @@
   tasks:
   - include_role:
       name: openshift_gcp
-      tasks_from: deprovision.yml
+    vars:
+      state: absent

playbooks/gcp/openshift-cluster/install.yml

@@ -0,0 +1,36 @@
+# This playbook installs onto a provisioned cluster
+---
+- hosts: localhost
+  connection: local
+  tasks:
+  - name: place all scale groups into Ansible groups
+    include_role:
+      name: openshift_gcp
+      tasks_from: setup_scale_group_facts.yml
+
+- name: run the init
+  import_playbook: ../../init/main.yml
+
+- import_playbook: ../../openshift-checks/private/install.yml
+
+- name: ensure master nodes are ready for bootstrapping
+  import_playbook: ../../openshift-node/private/bootstrap.yml
+
+- name: configure the control plane
+  import_playbook: ../../common/private/control_plane.yml
+
+- name: run the GCP specific post steps
+  import_playbook: install_gcp.yml
+
+- name: install components
+  import_playbook: ../../common/private/components.yml
+
+- name: Copy the kubeconfig, used by CI to determine when the containers are ready
+  hosts: oo_first_master
+  gather_facts: no
+  tasks:
+  - name: Retrieve cluster configuration
+    fetch:
+      src: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+      dest: "/tmp/"
+      flat: yes

playbooks/gcp/openshift-cluster/install_gcp.yml

@@ -0,0 +1,12 @@
+---
+- hosts: masters
+  gather_facts: no
+  tasks:
+  - name: create master health check service
+    include_role:
+      name: openshift_gcp
+      tasks_from: configure_master_healthcheck.yml
+  - name: configure master bootstrap distribution
+    include_role:
+      name: openshift_gcp
+      tasks_from: configure_master_bootstrap.yml

playbooks/gcp/openshift-cluster/openshift_node_group.yml

@@ -0,0 +1,23 @@
+# This playbook installs onto a provisioned cluster
+---
+- hosts: localhost
+  connection: local
+  tasks:
+  - name: place all scale groups into Ansible groups
+    include_role:
+      name: openshift_gcp
+      tasks_from: setup_scale_group_facts.yml
+    vars:
+      all_nodes: true
+
+- import_playbook: ../../init/main.yml
+  vars:
+    l_init_fact_hosts: "oo_masters_to_config"
+    l_openshift_version_set_hosts: "all:!all"
+    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
+
+- name: Setup node-group configmaps
+  hosts: oo_first_master
+  tasks:
+  - import_role:
+      name: openshift_node_group

playbooks/gcp/openshift-cluster/publish_image.yml

@@ -0,0 +1,9 @@
+---
+- name: Publish the most recent image
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  tasks:
+  - import_role:
+      name: openshift_gcp
+      tasks_from: publish_image.yml

playbooks/gcp/openshift-cluster/roles

@@ -0,0 +1 @@
+../../../roles

playbooks/init/base_packages.yml

@@ -36,7 +36,7 @@
       - "{{ 'python3-PyYAML' if ansible_distribution == 'Fedora' else 'PyYAML' }}"
       - libsemanage-python
       - yum-utils
-      - "{{ 'python3-docker' if ansible_distribution == 'Fedora' else 'python-docker-py' }}"
+      - "{{ 'python3-docker' if ansible_distribution == 'Fedora' else 'python-docker' }}"
       pkg_list_non_fedora:
       - 'python-ipaddress'
       pkg_list_use_non_fedora: "{{ ansible_distribution != 'Fedora' | bool }}"

playbooks/init/basic_facts.yml

@@ -25,7 +25,66 @@
   - name: set openshift_deployment_type if unset
     set_fact:
       openshift_deployment_type: "{{ deployment_type }}"
-      openshift_is_atomic: False
     when:
     - openshift_deployment_type is undefined
     - deployment_type is defined
+
+- name: Retrieve existing master configs and validate
+  hosts: oo_masters_to_config
+  gather_facts: no
+  any_errors_fatal: true
+  roles:
+  - openshift_facts
+  tasks:
+  - import_role:
+      name: openshift_control_plane
+      tasks_from: check_existing_config.yml
+
+  - when:
+    - l_existing_config_master_config is defined
+    - l_existing_config_master_config.networkConfig is defined
+    block:
+    - set_fact:
+        openshift_portal_net: "{{ l_existing_config_master_config.networkConfig.serviceNetworkCIDR }}"
+
+    - set_fact:
+        osm_cluster_network_cidr: "{{ l_existing_config_master_config.networkConfig.clusterNetworks[0].cidr }}"
+        osm_host_subnet_length: "{{ l_existing_config_master_config.networkConfig.clusterNetworks[0].hostSubnetLength }}"
+      when:
+      - l_existing_config_master_config.networkConfig.clusterNetworks is defined
+      # End block
+
+- name: Initialize special first-master variables
+  hosts: oo_first_master
+  roles:
+  - role: openshift_facts
+  tasks:
+  - when: not (osm_default_node_selector is defined)
+    block:
+    - set_fact:
+        # l_existing_config_master_config is set in openshift_control_plane/tasks/check_existing_config.yml
+        openshift_master_config_node_selector: "{{ l_existing_config_master_config.projectConfig.defaultNodeSelector }}"
+      when:
+      - l_existing_config_master_config is defined
+      - l_existing_config_master_config.projectConfig is defined
+      - l_existing_config_master_config.projectConfig.defaultNodeSelector is defined
+      - l_existing_config_master_config.projectConfig.defaultNodeSelector != ''
+
+  - set_fact:
+      # We need to setup openshift_client_binary here for special uses of delegate_to in
+      # later roles and plays.
+      first_master_client_binary: "{{  openshift_client_binary }}"
+      #Some roles may require this to be set for first master
+      openshift_client_binary: "{{ openshift_client_binary }}"
+      # we need to know if a default node selector has been manually set outside the installer
+      l_osm_default_node_selector: '{{ osm_default_node_selector | default(openshift_master_config_node_selector) | default("node-role.kubernetes.io/compute=true") }}'
+
+- name: Disable web console if required
+  hosts: oo_masters_to_config
+  gather_facts: no
+  tasks:
+  - set_fact:
+      openshift_web_console_install: False
+    when:
+    - openshift_deployment_subtype is defined
+    - openshift_deployment_subtype == 'registry' or ( osm_disabled_features is defined and 'WebConsole' in osm_disabled_features )

playbooks/init/main.yml

@@ -30,8 +30,13 @@
 - import_playbook: base_packages.yml
   when: l_install_base_packages | default(False) | bool
 
+- import_playbook: cluster_facts.yml
+
 - import_playbook: version.yml
 
+- import_playbook: sanity_checks.yml
+  when: not (skip_sanity_checks | default(False))
+
 - name: Initialization Checkpoint End
   hosts: all
   gather_facts: false

playbooks/init/version.yml

@@ -1,7 +1,23 @@
 ---
 - name: Determine openshift_version to configure on first master
-  hosts: oo_nodes_to_config
+  hosts: "{{ l_openshift_version_determine_hosts | default('oo_first_master') }}"
   tasks:
   - include_role:
       name: openshift_version
       tasks_from: first_master.yml
+
+# NOTE: We set this even on etcd hosts as they may also later run as masters,
+# and we don't want to install wrong version of docker and have to downgrade
+# later.
+- name: Set openshift_version for etcd, node, and master hosts
+  hosts: "{{ l_openshift_version_set_hosts | default(l_default_version_set_hosts) }}"
+  vars:
+    l_default_version_set_hosts: "oo_etcd_to_config:oo_nodes_to_config:oo_masters_to_config:!oo_first_master"
+    l_first_master_openshift_version: "{{ hostvars[groups.oo_first_master.0].openshift_version }}"
+    l_first_master_openshift_pkg_version: "{{ hostvars[groups.oo_first_master.0].openshift_pkg_version | default('') }}"
+    l_first_master_openshift_image_tag: "{{ hostvars[groups.oo_first_master.0].openshift_image_tag}}"
+  tasks:
+  - set_fact:
+      openshift_version: "{{ l_first_master_openshift_version }}"
+      openshift_pkg_version: "{{ l_first_master_openshift_pkg_version }}"
+      openshift_image_tag: "{{ l_first_master_openshift_image_tag }}"

playbooks/openshift-node/scaleup.yml

@@ -31,15 +31,11 @@
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config:oo_nodes_to_config"
     l_sanity_check_hosts: "{{ groups['oo_nodes_to_config'] | union(groups['oo_masters_to_config']) }}"
 
-- name: install nodes
-  hosts: oo_nodes_to_config
-  tasks:
-  - import_role:
-      name: openshift_node40
-      tasks_from: install.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: config.yml
-  - import_role:
-      name: openshift_node40
-      tasks_from: systemd.yml
+- import_playbook: ../init/version.yml
+  vars:
+    l_openshift_version_set_hosts: "oo_nodes_to_config:!oo_first_master"
+
+- import_playbook: private/bootstrap.yml
+- import_playbook: private/join.yml
+
+- import_playbook: ../openshift-glusterfs/private/add_hosts.yml

playbooks/prerequisites.yml

@@ -13,6 +13,9 @@
     l_install_base_packages: True
     l_repo_hosts: "{{ l_scale_up_hosts | default('oo_all_hosts') }}"
 
+- import_playbook: init/validate_hostnames.yml
+  when: not (skip_validate_hostnames | default(False))
+
 # This is required for container runtime for crio, only needs to run once.
 - name: Configure os_firewall
   hosts: "{{ l_scale_up_hosts | default(l_default_firewall_hosts) }}"

requirements.txt

@@ -1,6 +1,6 @@
 # Versions are pinned to prevent pypi releases arbitrarily breaking
 # tests with new APIs/semantics. We want to update versions deliberately.
-ansible==2.7.1
+ansible==2.6.5
 boto==2.44.0
 click==6.7
 pyOpenSSL==17.5.0

roles/container_runtime/tasks/crio_firewall.yml

@@ -1,11 +1,6 @@
 ---
 - when: r_crio_firewall_enabled | bool and not r_crio_use_firewalld | bool
   block:
-  - name: Make sure iptables-services is installed
-    package:
-      name: iptables-services
-      state: present
-
   - name: Add iptables allow rules
     os_firewall_manage_iptables:
       name: "{{ item.service }}"

roles/container_runtime/tasks/docker_storage_setup_overlay.yml

@@ -6,3 +6,5 @@
     owner: root
     group: root
     mode: 0664
+  when:
+  - container_runtime_docker_storage_type == 'overlay2'

roles/container_runtime/tasks/package_crio.yml

@@ -35,8 +35,8 @@
     pkg_list:
       - cri-o
       - cri-tools
+      - atomic
       - skopeo
-      - podman
 
 - name: Remove CRI-O default configuration files
   file:

roles/container_runtime/tasks/package_docker.yml

@@ -1 +1,165 @@
 ---
+- import_tasks: common/pre.yml
+
+- name: Get current installed Docker version
+  command: "{{ repoquery_installed }} --qf '%{version}' docker"
+  register: curr_docker_version
+  retries: 4
+  until: curr_docker_version is succeeded
+  changed_when: false
+
+# Some basic checks to ensure the role will complete
+- import_tasks: docker_sanity.yml
+
+# Make sure Docker is installed, but does not update a running version.
+# Docker upgrades are handled by a separate playbook.
+# Note: The curr_docker_version.stdout check can be removed when https://github.com/ansible/ansible/issues/33187 gets fixed.
+- name: Install Docker
+  package:
+    name: "{{ pkg_list | join(',') }}"
+    state: present
+  register: result
+  until: result is succeeded
+  vars:
+    pkg_list:
+    - "docker{{ '-' + docker_version if docker_version is defined else '' }}"
+    - skopeo
+
+- block:
+  # Extend the default Docker service unit file when using iptables-services
+  - name: Ensure docker.service.d directory exists
+    file:
+      path: "{{ docker_systemd_dir }}"
+      state: directory
+
+  - name: Configure Docker service unit file
+    template:
+      dest: "{{ docker_systemd_dir }}/custom.conf"
+      src: custom.conf.j2
+    notify:
+    - restart container runtime
+  when: not (os_firewall_use_firewalld | default(False)) | bool
+
+- stat:
+    path: /etc/sysconfig/docker
+    get_checksum: false
+    get_mime: false
+  register: docker_check
+
+- name: Set registry params
+  lineinfile:
+    dest: /etc/sysconfig/docker
+    regexp: '^{{ item.reg_conf_var }}=.*$'
+    line: "{{ item.reg_conf_var }}='{{ item.reg_fact_val | lib_utils_oo_prepend_strings_in_list(item.reg_flag ~ ' ') | join(' ') }}'"
+  when:
+  - item.reg_fact_val != []
+  - docker_check.stat.isreg is defined
+  - docker_check.stat.isreg
+  with_items:
+  - reg_conf_var: ADD_REGISTRY
+    reg_fact_val: "{{ l2_docker_additional_registries }}"
+    reg_flag: --add-registry
+  - reg_conf_var: BLOCK_REGISTRY
+    reg_fact_val: "{{ l2_docker_blocked_registries }}"
+    reg_flag: --block-registry
+  - reg_conf_var: INSECURE_REGISTRY
+    reg_fact_val: "{{ l2_docker_insecure_registries }}"
+    reg_flag: --insecure-registry
+  notify:
+  - restart container runtime
+
+- name: Place additional/blocked/insecure registries in /etc/containers/registries.conf
+  template:
+    dest: "{{ containers_registries_conf_path }}"
+    src: registries.conf.j2
+  when: openshift_docker_use_etc_containers | bool
+  notify:
+  - restart container runtime
+
+- name: Set Proxy Settings
+  lineinfile:
+    dest: /etc/sysconfig/docker
+    regexp: '^{{ item.reg_conf_var }}=.*$'
+    line: "{{ item.reg_conf_var }}='{{ item.reg_fact_val }}'"
+    state: "{{ 'present' if item.reg_fact_val != '' else 'absent'}}"
+  with_items:
+  - reg_conf_var: HTTP_PROXY
+    reg_fact_val: "{{ docker_http_proxy }}"
+  - reg_conf_var: HTTPS_PROXY
+    reg_fact_val: "{{ docker_https_proxy }}"
+  - reg_conf_var: NO_PROXY
+    reg_fact_val: "{{ docker_no_proxy }}"
+  notify:
+  - restart container runtime
+  when:
+  - docker_check.stat.isreg is defined
+  - docker_check.stat.isreg
+  - docker_http_proxy != '' or docker_https_proxy != ''
+
+- name: Set various Docker options
+  lineinfile:
+    dest: /etc/sysconfig/docker
+    regexp: '^OPTIONS=.*$'
+    line: "OPTIONS='\
+      {% if ansible_selinux.status | default(None) == 'enabled' and openshift_docker_selinux_enabled | default(true) | bool %} --selinux-enabled {% endif %} \
+      {% if openshift_docker_log_driver %} --log-driver {{ openshift_docker_log_driver }}{% endif %} \
+      {% if l2_docker_log_options != [] %} {{ l2_docker_log_options |  lib_utils_oo_split() | lib_utils_oo_prepend_strings_in_list('--log-opt ') | join(' ')}}{% endif %} \
+      {% if (openshift_docker_hosted_registry_insecure | bool) and openshift_docker_hosted_registry_network %} --insecure-registry={{ openshift_docker_hosted_registry_network }} {% endif %} \
+      {% if docker_options is defined %} {{ docker_options }}{% endif %} \
+      {% if openshift_docker_options %} {{ openshift_docker_options }}{% endif %} \
+      --signature-verification={{ openshift_docker_signature_verification | bool }}'"
+  when: docker_check.stat.isreg is defined and docker_check.stat.isreg
+  notify:
+  - restart container runtime
+
+- stat:
+    path: /etc/sysconfig/docker-network
+    get_checksum: false
+    get_mime: false
+  register: sysconfig_docker_network_check
+
+- name: Configure Docker Network OPTIONS
+  lineinfile:
+    dest: /etc/sysconfig/docker-network
+    regexp: '^DOCKER_NETWORK_OPTIONS=.*$'
+    line: "DOCKER_NETWORK_OPTIONS='\
+      {% if openshift.node is defined and openshift.node.sdn_mtu is defined %} --mtu={{ openshift.node.sdn_mtu }}{% endif %}'"
+  when:
+  - sysconfig_docker_network_check.stat.isreg is defined
+  - sysconfig_docker_network_check.stat.isreg
+  notify:
+  - restart container runtime
+
+# The following task is needed as the systemd module may report a change in
+# state even though docker is already running.
+- name: Detect if docker is already started
+  command: "systemctl show docker -p ActiveState"
+  changed_when: False
+  register: r_docker_already_running_result
+
+- name: Start the Docker service
+  systemd:
+    name: docker
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  register: r_docker_package_docker_start_result
+  until: not (r_docker_package_docker_start_result is failed)
+  retries: 3
+  delay: 30
+
+- set_fact:
+    docker_service_status_changed: "{{ (r_docker_package_docker_start_result is changed) and (r_docker_already_running_result.stdout != 'ActiveState=active' ) }}"
+
+- name: Check for docker_storage_path/overlay2
+  stat:
+    path: "{{ docker_storage_path }}/overlay2"
+  register: dsp_stat
+
+- name: Fixup SELinux permissions for docker
+  shell: |
+           semanage fcontext -a -e /var/lib/docker/overlay2 "{{ docker_storage_path }}/overlay2"
+           restorecon -R -v "{{ docker_storage_path }}/overlay2"
+  when: dsp_stat.stat.exists
+
+- import_tasks: common/post.yml

roles/container_runtime/templates/crio-network.j2

@@ -1 +1,9 @@
-CRIO_NETWORK_OPTIONS="--cni-config-dir=/etc/kubernetes/cni/net.d --cni-plugin-dir=/var/lib/cni/bin"
+{% if 'http_proxy' in openshift.common %}
+HTTP_PROXY={{ openshift.common.http_proxy }}
+{% endif %}
+{% if 'https_proxy' in openshift.common %}
+HTTPS_PROXY={{ openshift.common.https_proxy }}
+{% endif %}
+{% if 'no_proxy' in openshift.common %}
+NO_PROXY={{ openshift.common.no_proxy }}
+{% endif %}

roles/container_runtime/templates/crio.conf.j2

@@ -152,7 +152,7 @@ image_volumes = "mkdir"
 
 # registries is used to specify a comma separated list of registries to be used
 # when pulling an unqualified image (e.g. fedora:rawhide).
-registries = ['docker.io']
+# registries = []
 
 # The "crio.network" table contains settings pertaining to the
 # management of CNI plugins.

roles/lib_utils/action_plugins/parse_ignition.py

@@ -1,83 +0,0 @@
-"""Ansible action plugin to decode ignition payloads"""
-
-import base64
-import os
-import six
-from six.moves import urllib
-from ansible.plugins.action import ActionBase
-
-
-# pylint: disable=too-many-function-args
-def get_file_data(encoded_contents):
-    """Decode data URLs as specified in RFC 2397"""
-    # The following source is adapted from Python3 source
-    # License: https://github.com/python/cpython/blob/3.7/LICENSE
-    # retrieved from: https://github.com/python/cpython/blob/3.7/Lib/urllib/request.py
-    _, data = encoded_contents.split(":", 1)
-    mediatype, data = data.split(",", 1)
-
-    # even base64 encoded data URLs might be quoted so unquote in any case:
-    data = urllib.parse.unquote(data)
-    if mediatype.endswith(";base64"):
-        data = base64.b64decode(data).decode('utf-8')
-        mediatype = mediatype[:-7]
-    # End PSF software
-    return data
-
-
-# pylint: disable=too-many-function-args
-def get_files(files_dict, systemd_dict, dir_list, data):
-    """parse data to populate file_dict"""
-    files = data.get('storage', []).get('files', [])
-    for item in files:
-        path = item["path"]
-        dir_list.add(os.path.dirname(path))
-        # remove prefix "data:,"
-        encoded_contents = item['contents']['source']
-        contents = get_file_data(encoded_contents)
-        # convert from int to octal, padding at least to 4 places.
-        # eg, 420 becomes '0644'
-        mode = str(format(int(item["mode"]), '04o'))
-        inode = {"contents": contents, "mode": mode}
-        files_dict[path] = inode
-    # get the systemd units files we're here
-    systemd_units = data.get('systemd', []).get('units', [])
-    for item in systemd_units:
-        contents = item['contents']
-        if six.PY2:
-            # pylint: disable=redefined-variable-type
-            contents = contents.decode('unicode-escape')
-        mode = "0644"
-        inode = {"contents": contents, "mode": mode}
-        name = item['name']
-        path = '/etc/systemd/system/' + name
-        dir_list.add(os.path.dirname(path))
-        files_dict[path] = inode
-        enabled = item.get('enabled') or True
-        systemd_dict[name] = enabled
-
-
-# pylint: disable=too-few-public-methods
-class ActionModule(ActionBase):
-    """ActionModule for parse_ignition.py"""
-
-    def run(self, tmp=None, task_vars=None):
-        """Run parse_ignition action plugin"""
-        result = super(ActionModule, self).run(tmp, task_vars)
-        result["changed"] = False
-        result["failed"] = False
-        result["msg"] = "Parsed successfully"
-        files_dict = {}
-        systemd_dict = {}
-        dir_list = set()
-        result["files_dict"] = files_dict
-        result["systemd_dict"] = systemd_dict
-
-        # self.task_vars holds all in-scope variables.
-        # Ignore settting self.task_vars outside of init.
-        # pylint: disable=W0201
-        self.task_vars = task_vars or {}
-        ign_file_contents = self._task.args.get('ign_file_contents')
-        get_files(files_dict, systemd_dict, dir_list, ign_file_contents)
-        result["dir_list"] = list(dir_list)
-        return result

roles/lib_utils/test/test_parse_ignition.py

@@ -1,66 +0,0 @@
-'''
- Unit tests for wildcard
-'''
-import json
-import os
-import sys
-
-MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'action_plugins'))
-sys.path.insert(0, MODULE_PATH)
-ASSET_PATH = os.path.realpath(os.path.join(__file__, os.pardir, 'test_data'))
-
-# pylint: disable=import-error,wrong-import-position,missing-docstring
-import parse_ignition  # noqa: E402
-
-
-def read_ign(path):
-    with open(path) as ign_in:
-        data = json.loads(ign_in.read())
-    return data
-
-
-def write_out_files(files_dict):
-    for path in files_dict:
-        with open('/tmp/bsoutput/' + path.replace('/', '__'), 'w') as fpath:
-            fpath.write(files_dict[path]['contents'])
-
-
-def test_parse_json():
-    ign_data = read_ign(os.path.join(ASSET_PATH, 'example.ign.json'))
-    files_dict = {}
-    systemd_dict = {}
-    dir_list = set()
-    result = {}
-    result['files_dict'] = files_dict
-    result['systemd_dict'] = systemd_dict
-    parse_ignition.get_files(files_dict, systemd_dict, dir_list, ign_data)
-
-
-def test_parse_json_encoded_files():
-    ign_data = read_ign(os.path.join(ASSET_PATH, 'bootstrap.ign.json'))
-    files_dict = {}
-    systemd_dict = {}
-    dir_list = set()
-    result = {}
-    result['files_dict'] = files_dict
-    result['systemd_dict'] = systemd_dict
-    parse_ignition.get_files(files_dict, systemd_dict, dir_list, ign_data)
-    # print(files_dict['/opt/tectonic/manifests/cluster-config.yaml']['contents'])
-
-
-def parse_json2():
-    ign_data = read_ign(os.path.join(ASSET_PATH, 'bs.ign.json'))
-    files_dict = {}
-    systemd_dict = {}
-    dir_list = set()
-    result = {}
-    result['files_dict'] = files_dict
-    result['systemd_dict'] = systemd_dict
-    parse_ignition.get_files(files_dict, systemd_dict, dir_list, ign_data)
-    write_out_files(files_dict)
-
-
-if __name__ == '__main__':
-    test_parse_json()
-    test_parse_json_encoded_files()
-    parse_json2()

roles/openshift_facts/defaults/main.yml

@@ -45,8 +45,8 @@ osm_image: "{{ l_osm_registry_url | regex_replace('${component}' | regex_escape,
 repoquery_cmd: "{{ (ansible_pkg_mgr == 'dnf') | ternary('dnf repoquery --latest-limit 1 -d 0', 'repoquery --plugins') }}"
 repoquery_installed: "{{ (ansible_pkg_mgr == 'dnf') | ternary('dnf repoquery --latest-limit 1 -d 0 --disableexcludes=all --installed', 'repoquery --plugins --installed') }}"
 
-openshift_use_crio: True
-openshift_use_crio_only: True
+openshift_use_crio: False
+openshift_use_crio_only: False
 openshift_crio_enable_docker_gc: False
 openshift_crio_var_sock: "/var/run/crio/crio.sock"
 openshift_crio_pause_image: "{{ l_os_registry_url | regex_replace('${component}' | regex_escape, 'pod') }}"
@@ -136,7 +136,7 @@ openshift_service_type_dict:
   openshift-enterprise: atomic-openshift
 
 openshift_service_type: "{{ openshift_service_type_dict[openshift_deployment_type] }}"
-openshift_master_api_port: "6443"
+openshift_master_api_port: "8443"
 openshift_ca_host: "{{ groups.oo_first_master.0 }}"
 openshift_use_openshift_sdn: true
 os_sdn_network_plugin_name: "redhat/openshift-ovs-subnet"

roles/openshift_gcp/defaults/main.yml

@@ -31,21 +31,21 @@ openshift_gcp_registry_bucket_name: "{{ openshift_gcp_prefix }}-docker-registry"
 openshift_gcp_master_dns_ttl: 300
 
 openshift_gcp_node_group_config:
-  - name: bootstrap
-    suffix: b
-    tags: ocp-bootstrap ocp-node
+  - name: master
+    suffix: m
+    tags: ocp-master
     machine_type: n1-standard-2
     boot_disk_size: 150
     scale: 1
-  - name: master
-    suffix: m
-    tags: ocp-master ocp-node
+  - name: infra
+    suffix: i
+    tags: ocp-infra-node ocp-node
     machine_type: n1-standard-2
     boot_disk_size: 150
     scale: 1
-  - name: worker
+  - name: node
     suffix: n
-    tags: ocp-worker ocp-node
+    tags: ocp-node
     machine_type: n1-standard-2
     boot_disk_size: 150
     scale: 3
@@ -61,97 +61,7 @@ openshift_gcp_user_data_file: ''
 
 openshift_gcp_multizone: False
 
-provision_custom_repositories: []
-
-mcd_port: 49500
-openshift_gcp_kubernetes_api_port: 6443
-openshift_gcp_master_healthcheck_port: 8080
-
-openshift_gcp_firewall_rules:
-  - rule: icmp
-    allowed:
-      - ip_protocol: 'icmp'
-  - rule: ssh-external
-    allowed:
-      - ip_protocol: 'tcp'
-        ports:
-          - '22'
-  - rule: ssh-internal
-    allowed:
-      - ip_protocol: 'tcp'
-        ports:
-          - '22'
-    source_tags:
-      - ssh-bastion
-  - rule: master-internal
-    allowed:
-      - ip_protocol: 'tcp'
-        ports:
-          - '2224'
-          - '2379'
-          - '2380'
-          - '4001'
-          #kube-system/kubelet:cadvisor
-          - '4193'
-          - "{{ openshift_gcp_kubernetes_api_port }}"
-          - "{{ internal_console_port }}"
-          - '8053'
-          - '8444'
-          - "{{ openshift_gcp_master_healthcheck_port }}"
-          #cadvisor port
-          - '9100'
-          # CVO port
-          - '9099'
-          - '10250'
-          - '10255'
-          - '24224'
-          - "{{ mcd_port }}"
-      - ip_protocol: 'udp'
-        ports:
-          - '4789'
-          - '5404'
-          - '5405'
-          - '10255'
-          - '24224'
-    source_tags:
-      - ocp
-    target_tags:
-      - ocp-master
-      - ocp-bootstrap
-  - rule: master-external
-    allowed:
-      - ip_protocol: 'tcp'
-        ports:
-          - "{{ openshift_gcp_master_healthcheck_port }}"
-          - "{{ openshift_gcp_kubernetes_api_port }}"
-          - "{{ openshift_master_api_port }}"
-          - "{{ mcd_port }}"
-    target_tags:
-      - ocp-master
-      - ocp-bootstrap
-  - rule: node-internal
-    allowed:
-      - ip_protocol: 'tcp'
-        ports:
-          - '1936'
-          - '10250'
-          - '10255'
-          - '9000-10000'
-      - ip_protocol: 'udp'
-        ports:
-          - '4789'
-          - '10255'
-    source_tags:
-      - ocp
-    target_tags:
-      - ocp-worker
-  - rule: node-external
-    allowed:
-      - ip_protocol: 'tcp'
-        ports:
-          - "{{ openshift_node_port_range }}"
-      - ip_protocol: 'udp'
-        ports:
-          - "{{ openshift_node_port_range }}"
-    target_tags:
-      - ocp-worker
+openshift_gcp_node_group_mapping:
+  masters: 'node-config-master'
+  infra: 'node-config-infra'
+  compute: 'node-config-compute'

roles/openshift_gcp/tasks/add_custom_repositories.yml

@@ -0,0 +1,20 @@
+---
+- name: Copy custom repository secrets
+  copy:
+    src: "{{ files_dir }}/{{ item.1.sslclientcert }}"
+    dest: /var/lib/yum/custom_secret_{{ item.0 }}_cert
+  when: item.1.sslclientcert | default(false)
+  with_indexed_items: "{{ provision_custom_repositories }}"
+- name: Copy custom repository secrets
+  copy:
+    src: "{{ files_dir }}/{{ item.1.sslclientkey }}"
+    dest: /var/lib/yum/custom_secret_{{ item.0 }}_key
+  when: item.1.sslclientkey | default(false)
+  with_indexed_items: "{{ provision_custom_repositories }}"
+
+- name: Create any custom repos that are defined
+  template:
+    src: yum_repo.j2
+    dest: /etc/yum.repos.d/provision_custom_repositories.repo
+  when: provision_custom_repositories | length > 0
+  notify: refresh cache

roles/openshift_gcp/tasks/configure_gcp_base_image.yml

@@ -0,0 +1,10 @@
+# GCE instances are starting with xfs AND barrier=1, which is only for extfs.
+---
+- name: Remove barrier=1 from XFS fstab entries
+  command: sed -i -e 's/xfs\(.*\)barrier=1/xfs\1/g; s/, / /g' /etc/fstab
+
+- name: Ensure the root filesystem has XFS group quota turned on
+  command: sed -i -e 's/linux16 \(.*\)$/linux16 \1 rootflags=gquota/g' /boot/grub2/grub.cfg
+
+- name: Ensure the root partition grows on startup
+  copy: src=partition.conf dest=/etc/systemd/system/google-instance-setup.service.d/

roles/openshift_gcp/tasks/configure_master_bootstrap.yml

@@ -0,0 +1,40 @@
+#
+# These tasks configure the instance to periodically update the project metadata with the
+# latest bootstrap kubeconfig from the project metadata. This keeps the project metadata
+# in sync with the cluster's configuration. We then invoke a CSR approve on any nodes that
+# are waiting to join the cluster.
+#
+---
+- name: Copy unit service
+  copy:
+    src: openshift-bootstrap-update.timer
+    dest: /etc/systemd/system/openshift-bootstrap-update.timer
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Copy unit timer
+  copy:
+    src: openshift-bootstrap-update.service
+    dest: /etc/systemd/system/openshift-bootstrap-update.service
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Create bootstrap update script
+  template: src=openshift-bootstrap-update.j2 dest=/usr/bin/openshift-bootstrap-update mode=u+rx
+
+- name: Start bootstrap update timer
+  systemd:
+    name: "openshift-bootstrap-update.timer"
+    state: started
+
+- name: Approve node certificates when bootstrapping
+  oc_csr_approve:
+    oc_bin: "{{ hostvars[groups.masters.0]['first_master_client_binary'] }}"
+    oc_conf: "{{ hostvars[groups.masters.0].openshift.common.config_base }}/master/admin.kubeconfig"
+    node_list: "{{ groups['all'] | map('extract', hostvars) | selectattr('gce_metadata.bootstrap', 'match', 'true') | map(attribute='gce_name') | list }}"
+  register: gcp_csr_approve
+  retries: 30
+  until: gcp_csr_approve is succeeded
+  when: groups['all'] | map('extract', hostvars) | selectattr('gce_metadata.bootstrap', 'match', 'true') | map(attribute='gce_name') | list | length > 0

roles/openshift_gcp/tasks/configure_master_healthcheck.yml

@@ -24,9 +24,3 @@
     name: haproxy
     state: started
     enabled: yes
-
-- name: allow haproxy to connect to any port
-  seboolean:
-    name: haproxy_connect_any
-    state: yes
-    persistent: yes

roles/openshift_gcp/tasks/deprovision.yml

@@ -1,141 +0,0 @@
----
-- name: Fetch instance group managers
-  gcp_compute_instance_group_manager_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    filters:
-    - "name : {{ openshift_gcp_prefix }}ig*"
-  register: instance_group_managers
-
-- name: Fetch instance templates
-  gcp_compute_instance_template_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    filters:
-    - "name : {{ openshift_gcp_prefix }}instance-template*"
-  register: instance_templates
-
-- name: Collect a list of instances
-  gcp_compute_instance_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-  register: all_instances
-
-- name: Filter instances to fetch masters
-  set_fact:
-    master_instances: "{{ master_instances | default([]) }} + [ {{ item }} ]"
-  with_items:
-  - "{{ all_instances['items'] }}"
-  when:
-  - "'tags' in item"
-  - "'items' in item['tags']"
-  - "cluster_tag in item['tags']['items']"
-  - "'ocp-master' in item['tags']['items']"
-  vars:
-    cluster_tag: "{{ openshift_gcp_prefix }}ocp"
-
-- name: Get managed zone
-  gcp_dns_managed_zone:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
-    state: present
-  register: managed_zone
-
-- name: Remove public API hostname
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_master_cluster_public_hostname }}."
-    managed_zone: "{{ managed_zone }}"
-    type: A
-    state: absent
-
-- name: Remove etcd records for masters
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ entry_name }}"
-    managed_zone: "{{ managed_zone }}"
-    type: A
-    state: absent
-  with_indexed_items: "{{ master_instances }}"
-  when: master_instances is defined
-  vars:
-    entry_name: "{{ openshift_gcp_prefix }}etcd-{{ item.0 }}.{{ public_hosted_zone }}."
-
-- name: Remove GCP Instance Groups
-  gcp_compute_instance_group_manager:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    name: "{{ item[0].name }}"
-    base_instance_name: "{{ item[0].name }}"
-    instance_template: "{{ item[1] }}"
-    state: absent
-  with_nested:
-  - "{{ instance_group_managers['items'] }}"
-  - "{{ instance_templates['items'] }}"
-
-- name: Remove GCP instance templates
-  gcp_compute_instance_template:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ item.name }}"
-    state: absent
-  with_items: "{{ instance_templates['items'] }}"
-
-- name: Remove GCP firewall
-  gcp_compute_firewall:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_gcp_prefix }}{{ item.rule }}"
-    state: absent
-  with_items: "{{ openshift_gcp_firewall_rules }}"
-
-- name: Remove GCP network
-  gcp_compute_network:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_gcp_network_name }}"
-    state: absent
-
-- name: Templatize DNS script
-  template: src=remove.j2.sh dest=/tmp/remove.sh mode=u+rx
-
-- name: Run DNS cleanup script
-  command: /tmp/remove.sh
-  args:
-    chdir: "{{ files_dir }}"

roles/openshift_gcp/tasks/dynamic_inventory.yml

@@ -1,12 +1,5 @@
 ---
 - name: Extract PEM from service account file
-  copy:
-    content: "{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).private_key }}"
-    dest: /tmp/gce.pem
-    mode: 0600
-
+  copy: content="{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).private_key }}" dest=/tmp/gce.pem mode=0600
 - name: Templatize environment script
-  template:
-    src: inventory.j2.sh
-    dest: /tmp/inventory.sh
-    mode: u+rx
+  template: src=inventory.j2.sh dest=/tmp/inventory.sh mode=u+rx

roles/openshift_gcp/tasks/main.yml

@@ -1,255 +1,45 @@
+#
+# This role relies on gcloud invoked via templated bash in order to
+# provide a high performance deployment option. The next logical step
+# is to transition to a deployment manager template which is then instantiated.
+# TODO: use a formal set of role parameters consistent with openshift_aws
+#
 ---
-- name: Create GCP network
-  gcp_compute_network:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_gcp_network_name }}"
-    state: present
-  register: network
-
-- name: Create GCP firewall
-  gcp_compute_firewall:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_gcp_prefix }}{{ item.rule }}"
-    allowed: "{{ item.allowed }}"
-    network: "{{ network.selfLink }}"
-    target_tags: "{{ item.target_tags | default(omit) }}"
-    source_tags: "{{ item.source_tags | default(omit) }}"
-    state: present
-  with_items: "{{ openshift_gcp_firewall_rules }}"
-
-- import_tasks: provision_ssh_keys.yml
-
-- name: Find GCP image
-  gcp_compute_image_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    filters:
-    - "family = {{ openshift_gcp_image }}"
-  register: gcp_node_image
-
-- fail:
-    msg: "No images for family '{{ openshift_gcp_image }}' found"
-  when: gcp_node_image['items'] | length == 0
-
-- name: Provision GCP instance templates
-  gcp_compute_instance_template:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_gcp_prefix }}instance-template-{{ item.name }}"
-    properties:
-      machine_type: "{{ item.machine_type }}"
-      network_interfaces:
-      - network: "{{ network }}"
-        access_configs:
-        - name: "{{ openshift_gcp_prefix }}instance-template-{{ item.name }}-config"
-          type: 'ONE_TO_ONE_NAT'
-      disks:
-      - auto_delete: true
-        boot: true
-        initialize_params:
-          disk_size_gb: "{{ item.boot_disk_size }}"
-          source_image: "{{ gcp_node_image['items'][0].selfLink }}"
-      metadata:
-        "cluster-id": "{{ openshift_gcp_prefix + openshift_gcp_clusterid }}"
-        "node-group": "{{ item.name }}"
-      tags:
-        items:
-        - "ocp"
-        - "{{ openshift_gcp_prefix }}ocp"
-        - "{{ item.tags }}"
-    state: present
-  with_items: "{{ openshift_gcp_node_group_config }}"
-  register: instance_template
-
-- name: Create GCP Instance Groups
-  gcp_compute_instance_group_manager:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    name: "{{ openshift_gcp_prefix }}ig-{{ item.item.suffix }}"
-    base_instance_name: "{{ openshift_gcp_prefix }}ig-{{ item.item.suffix }}"
-    instance_template: "{{ item }}"
-    target_size: "{{ item.item.scale | int}}"
-    named_ports:
-    - name: "{{ openshift_gcp_prefix }}port-kube-api"
-      port: "{{ openshift_gcp_kubernetes_api_port }}"
-    - name: "{{ openshift_gcp_prefix }}port-openshift-api"
-      port: "{{ openshift_master_api_port }}"
-    state: present
-  with_items: "{{ instance_template.results }}"
-  register: instance_groups
-
-- name: Get bootstrap instance group
-  gcp_compute_instance_group_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    filters:
-    - name = "{{ openshift_gcp_prefix }}ig-b"
-  register: bootstrap_instance_group
-
-- name: Get master instance group
-  gcp_compute_instance_group_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    filters:
-    - name = "{{ openshift_gcp_prefix }}ig-m"
-  register: master_instance_group
-
-- set_fact:
-    bootstrap_instance_group: "{{ bootstrap_instance_group['items'][0] }}"
-    master_instance_group: "{{ master_instance_group['items'][0] }}"
-
-- name: Wait for bootstrap instance group to start all instances
-  gcp_compute_instance_group_manager_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    filters: "name = {{ bootstrap_instance_group['name'] }}"
-  register: bootstrap_group_result
-  # Wait for 3 minutes
-  retries: 36
-  delay: 5
-  until:
-  - "bootstrap_group_result['items'][0]['currentActions']['none'] == bootstrap_group_result['items'][0]['targetSize']"
-
-- name: Wait for master instance group to start all instances
-  gcp_compute_instance_group_manager_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    filters: "name = {{ master_instance_group['name'] }}"
-  register: master_group_result
-  # Wait for 3 minutes
-  retries: 36
-  delay: 5
-  until:
-  - "master_group_result['items'][0]['currentActions']['none'] == master_group_result['items'][0]['targetSize']"
-
-- name: Collect a list of instances
-  gcp_compute_instance_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-  register: all_instances
-
-- name: Filter instances to fetch bootstrap
-  set_fact:
-    bootstrap_instances: "{{ item }}"
-  with_items:
-  - "{{ all_instances['items'] }}"
+- name: Templatize DNS script
+  template: src=dns.j2.sh dest=/tmp/openshift_gcp_provision_dns.sh mode=u+rx
+- name: Templatize provision script
+  template: src=provision.j2.sh dest=/tmp/openshift_gcp_provision.sh mode=u+rx
+- name: Templatize de-provision script
+  template: src=remove.j2.sh dest=/tmp/openshift_gcp_provision_remove.sh mode=u+rx
   when:
-  - "'tags' in item"
-  - "'items' in item['tags']"
-  - "cluster_tag in item['tags']['items']"
-  - "'ocp-bootstrap' in item['tags']['items']"
-  vars:
-    cluster_tag: "{{ openshift_gcp_prefix }}ocp"
+  - state | default('present') == 'absent'
 
-- name: Filter instances to fetch masters
-  set_fact:
-    master_instances: "{{ master_instances | default([]) }} + [ {{ item }} ]"
-  with_items:
-  - "{{ all_instances['items'] }}"
+- name: Provision GCP DNS domain
+  command: /tmp/openshift_gcp_provision_dns.sh
+  args:
+    chdir: "{{ files_dir }}"
+  register: dns_provision
   when:
-  - "'tags' in item"
-  - "'items' in item['tags']"
-  - "cluster_tag in item['tags']['items']"
-  - "'ocp-master' in item['tags']['items']"
-  vars:
-    cluster_tag: "{{ openshift_gcp_prefix }}ocp"
-
-- set_fact:
-    etcd_discovery_targets: "{{ etcd_discovery_targets | default([]) }} + ['0 0 2380 {{ entry_name }}']"
-    master_external_ips: "{{ master_external_ips | default([]) }} + ['{{ master_ip }}']"
-  with_indexed_items: "{{ master_instances }}"
-  vars:
-    entry_name: "{{ openshift_gcp_prefix }}etcd-{{ item.0 }}.{{ public_hosted_zone }}."
-    master_ip: "{{ item.1.networkInterfaces[0].accessConfigs[0].natIP }}"
-
-- set_fact:
-    bootstrap_and_masters: "{{ master_external_ips | list }} + ['{{ bootstrap_instances.networkInterfaces[0].accessConfigs[0].natIP }}']"
-
-- name: Get managed zone
-  gcp_dns_managed_zone:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
-    state: present
-  register: managed_zone
-
-- name: Create public API hostname
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_master_cluster_public_hostname }}."
-    managed_zone: "{{ managed_zone }}"
-    type: A
-    ttl: 600
-    target: "{{ bootstrap_and_masters }}"
-    state: present
+  - state | default('present') == 'present'
 
-- name: Create etcd records for masters
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ entry_name }}"
-    managed_zone: "{{ managed_zone }}"
-    type: A
-    ttl: 600
-    target: "{{ master_ip }}"
-    state: present
-  with_indexed_items: "{{ master_instances }}"
-  vars:
-    entry_name: "{{ openshift_gcp_prefix }}etcd-{{ item.0 }}.{{ public_hosted_zone }}."
-    master_ip: "{{ item.1.networkInterfaces[0].networkIP }}"
+- name: Ensure that DNS resolves to the hosted zone
+  assert:
+    that:
+    - "lookup('dig', public_hosted_zone, 'qtype=NS', wantlist=True) | sort | join(',') == dns_provision.stdout"
+    msg: "The DNS domain {{ public_hosted_zone }} defined in 'public_hosted_zone' must have NS records pointing to the Google nameservers: '{{ dns_provision.stdout }}' instead of '{{ lookup('dig', public_hosted_zone, 'qtype=NS') }}'."
+  when:
+  - state | default('present') == 'present'
 
-- name: Templatize DNS script
-  template: src=additional_settings.j2.sh dest=/tmp/additional_settings.sh mode=u+rx
+- import_tasks: provision_ssh_keys.yml
 
-- name: Run addition provision GCP script
-  command: /tmp/additional_settings.sh
+- name: Provision GCP resources
+  command: /tmp/openshift_gcp_provision.sh
   args:
     chdir: "{{ files_dir }}"
+  when:
+  - state | default('present') == 'present'
+
+- name: De-provision GCP resources
+  command: /tmp/openshift_gcp_provision_remove.sh
+  when:
+  - state | default('present') == 'absent'

roles/openshift_gcp/tasks/provision_ssh_keys.yml

@@ -1,9 +1,6 @@
 ---
 - name: Templatize SSH key provision script
-  template:
-    src: provision_ssh.j2.sh
-    dest: /tmp/openshift_gcp_provision_ssh.sh
-    mode: u+rx
+  template: src=provision_ssh.j2.sh dest=/tmp/openshift_gcp_provision_ssh.sh mode=u+rx
 
 - name: Provision GCP SSH key resources
   command: /tmp/openshift_gcp_provision_ssh.sh

roles/openshift_gcp/tasks/publish_image.yml

@@ -0,0 +1,32 @@
+---
+- name: Require openshift_gcp_image
+  fail:
+    msg: "A source image name or family is required for image publishing.  Please ensure `openshift_gcp_image` is defined."
+  when: openshift_gcp_image is undefined
+
+- name: Require openshift_gcp_target_image
+  fail:
+    msg: "A target image name or family is required for image publishing.  Please ensure `openshift_gcp_target_image` is defined."
+  when: openshift_gcp_target_image is undefined
+
+- block:
+  - name: Retrieve images in the {{ openshift_gcp_target_image }} family
+    command: >
+      gcloud --project "{{ openshift_gcp_project }}" compute images list
+        "--filter=family={{ openshift_gcp_target_image }}"
+        --format=json --sort-by ~creationTimestamp
+    register: images
+  - name: Prune oldest images
+    command: >
+      gcloud --project "{{ openshift_gcp_project }}" compute images delete "{{ item['name'] }}"
+    with_items: "{{ (images.stdout | default('[]') | from_json )[( openshift_gcp_keep_images | int ):] }}"
+  when: openshift_gcp_keep_images is defined
+
+- name: Copy the latest image in the family {{ openshift_gcp_image }} to {{ openshift_gcp_target_image }}
+  command: >
+    gcloud --project "{{ openshift_gcp_target_project | default(openshift_gcp_project) }}"
+      beta compute images create
+      "{{ openshift_gcp_target_image_name | default(openshift_gcp_target_image + '-' + lookup('pipe','date +%Y%m%d-%H%M%S')) }}"
+      --family "{{ openshift_gcp_target_image }}"
+      --source-image-family "{{ openshift_gcp_image }}"
+      --source-image-project "{{ openshift_gcp_project }}"

roles/openshift_gcp/tasks/remove_bootstrap.yml

@@ -1,93 +0,0 @@
----
-- name: Get bootstrap instance group
-  gcp_compute_instance_group_manager_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    filters:
-    - name = "{{ openshift_gcp_prefix }}ig-b"
-  register: bootstrap_instance_group
-
-- name: Get bootstrap instance template
-  gcp_compute_instance_template_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    filters:
-    - "name : {{ openshift_gcp_prefix }}instance-template-bootstrap"
-  register: bootstrap_instance_template
-
-- name: Collect a list of instances
-  gcp_compute_instance_facts:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-  register: all_instances
-
-- name: Filter instances to fetch masters
-  set_fact:
-    master_instances: "{{ master_instances | default([]) }} + [ {{ item }} ]"
-  with_items:
-  - "{{ all_instances['items'] }}"
-  when:
-  - "'tags' in item"
-  - "'items' in item['tags']"
-  - "cluster_tag in item['tags']['items']"
-  - "'ocp-master' in item['tags']['items']"
-  vars:
-    cluster_tag: "{{ openshift_gcp_prefix }}ocp"
-
-- set_fact:
-    master_external_ips: "{{ master_external_ips | default([]) }}  + [ '{{ master_ip }}' ]"
-  with_indexed_items: "{{ master_instances }}"
-  vars:
-    master_ip: "{{ item.1.networkInterfaces[0].accessConfigs[0].natIP }}"
-
-- name: Get a managed zone
-  gcp_dns_managed_zone:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
-    state: present
-  register: managed_zone
-
-- name: Update public API hostname
-  gcp_dns_resource_record_set:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/ndev.clouddns.readwrite
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    name: "{{ openshift_master_cluster_public_hostname }}."
-    managed_zone: "{{ managed_zone }}"
-    type: A
-    ttl: 600
-    target: "{{ master_external_ips }}"
-    state: present
-
-- name: Delete bootstrap instance group
-  gcp_compute_instance_group_manager:
-    auth_kind: serviceaccount
-    scopes:
-    - https://www.googleapis.com/auth/compute
-    service_account_file: "{{ openshift_gcp_iam_service_account_keyfile }}"
-    project: "{{ openshift_gcp_project }}"
-    zone: "{{ openshift_gcp_zone }}"
-    name: "{{ bootstrap_instance_group['items'][0]['name'] }}"
-    base_instance_name: "{{ bootstrap_instance_group['items'][0]['baseInstanceName'] }}"
-    instance_template: "{{ bootstrap_instance_template['items'][0] }}"
-    state: absent
-  when:
-  - bootstrap_instance_group['items'] | length > 0
-  - bootstrap_instance_template['items'] | length > 0

roles/openshift_gcp/tasks/setup_scale_group_facts.yml

@@ -1,25 +1,38 @@
 ---
-- name: Add bootstrap instances
+- name: Set var to exclude bootstrapped nodes
+  set_fact:
+    bootstrapped_nodes: "{{ all_nodes | default(false) | ternary([], groups['tag_ocp-bootstrap']) | default([]) }}"
+
+- name: Add node instances to node group
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups:
-    - bootstrap
-    - nodes
-    ignition_file: "{{ openshift_bootstrap_ignition_file }}"
+    groups: nodes, new_nodes
+    openshift_node_group_name: "{{ openshift_gcp_node_group_mapping['compute'] }}"
+  with_items: "{{ groups['tag_ocp-node'] | default([]) | difference(bootstrapped_nodes) }}"
+
+- name: Add bootstrap node instances as nodes
+  add_host:
+    name: "{{ item }}"
+    groups: nodes, new_nodes
   with_items: "{{ groups['tag_ocp-bootstrap'] | default([]) }}"
+  when: all_nodes | default(False)
+
+- name: Add non-bootstrapping master node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+  with_items: "{{ groups['tag_ocp-master'] | default([]) | difference(bootstrapped_nodes) }}"
 
-- name: Add master instances
+- name: Add infra node instances to node group
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups:
-    - masters
-    - nodes
-  with_items: "{{ groups['tag_ocp-master'] | default([]) }}"
+    groups: nodes, new_nodes
+    openshift_node_group_name: "{{ openshift_gcp_node_group_mapping['infra'] }}"
+  with_items: "{{ groups['tag_ocp-infra-node'] | default([]) | difference(bootstrapped_nodes) }}"
 
-- name: Add worker instances
+- name: Add masters to requisite groups
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups:
-    - workers
-    - nodes
-  with_items: "{{ groups['tag_ocp-worker'] | default([]) }}"
+    groups: masters, etcd
+    openshift_node_group_name: "{{ openshift_gcp_node_group_mapping['masters'] }}"
+  with_items: "{{ groups['tag_ocp-master'] }}"

roles/openshift_gcp/templates/additional_settings.j2.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-
-set -euxo pipefail
-
-dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
-# configure DNS
-(
-# Retry DNS changes until they succeed since this may be a shared resource
-while true; do
-    dns="${TMPDIR:-/tmp}/dns.yaml"
-    rm -f $dns
-
-    # DNS records for etcd discovery
-    ETCD_DNS_NAME="_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
-    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "${ETCD_DNS_NAME}" 2>/dev/null | grep -q "${ETCD_DNS_NAME}"; then
-        if [[ ! -f $dns ]]; then
-            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
-        fi
-        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "${ETCD_DNS_NAME}" --type SRV {% for etcd in etcd_discovery_targets %}'{{ etcd }}' {% endfor %}
-
-    else
-        echo "DNS record for '${ETCD_DNS_NAME}' already exists"
-    fi
-
-    # Commit all DNS changes, retrying if preconditions are not met
-    if [[ -f $dns ]]; then
-        if ! out="$( gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns execute -z "${dns_zone}" 2>&1 )"; then
-            rc=$?
-            if [[ "${out}" == *"HTTPError 412: Precondition not met"* ]]; then
-                continue
-            fi
-            exit $rc
-        fi
-    fi
-    break
-done
-) &
-
-for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/dns.j2.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
+
+# Check the DNS managed zone in Google Cloud DNS, create it if it doesn't exist
+if ! gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" dns managed-zones create "${dns_zone}" --dns-name "{{ public_hosted_zone }}" --description "{{ public_hosted_zone }} domain" >/dev/null
+fi
+
+# Always output the expected nameservers as a comma delimited list
+gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" --format='value(nameServers)' | tr ';' ','

roles/openshift_gcp/templates/master_healthcheck.j2

@@ -60,9 +60,9 @@ defaults
 #---------------------------------------------------------------------
 # main frontend which proxys to the backends
 #---------------------------------------------------------------------
-frontend  http-proxy *:{{ openshift_gcp_master_healthcheck_port }}
+frontend  http-proxy *:8080
     acl          url_healthz  path_beg  -i /healthz
     use_backend  ocp          if url_healthz
 
 backend ocp
-    server       ocp localhost:{{ openshift_gcp_kubernetes_api_port }} ssl verify none
+    server       ocp localhost:{{ internal_console_port }} ssl verify none

roles/openshift_gcp/templates/openshift-bootstrap-update.j2

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+oc serviceaccounts create-kubeconfig -n openshift-infra node-bootstrapper > /root/bootstrap.kubeconfig
+gcloud compute project-info --project '{{ openshift_gcp_project }}' add-metadata --metadata-from-file '{{ openshift_gcp_prefix + openshift_gcp_clusterid | default("default") }}-bootstrap-config=/root/bootstrap.kubeconfig'
+rm -f /root/bootstrap.kubeconfig

roles/openshift_gcp/templates/provision.j2.sh

@@ -0,0 +1,304 @@
+#!/bin/bash
+
+set -euo pipefail
+
+metadata=""
+if [[ -n "{{ openshift_gcp_startup_script_file }}" ]]; then
+    if [[ ! -f "{{ openshift_gcp_startup_script_file }}" ]]; then
+        echo "Startup script file missing at {{ openshift_gcp_startup_script_file }} from=$(pwd)"
+        exit 1
+    fi
+    metadata+="--metadata-from-file=startup-script={{ openshift_gcp_startup_script_file }}"
+fi
+if [[ -n "{{ openshift_gcp_user_data_file }}" ]]; then
+    if [[ ! -f "{{ openshift_gcp_user_data_file }}" ]]; then
+        echo "User data file missing at {{ openshift_gcp_user_data_file }}"
+        exit 1
+    fi
+    if [[ -n "${metadata}" ]]; then
+        metadata+=","
+    else
+        metadata="--metadata-from-file="
+    fi
+    metadata+="user-data={{ openshift_gcp_user_data_file }}"
+fi
+
+# Select image or image family
+image="{{ openshift_gcp_image }}"
+if ! gcloud --project "{{ openshift_gcp_project }}" compute images describe "${image}" &>/dev/null; then
+    if ! gcloud --project "{{ openshift_gcp_project }}" compute images describe-from-family "${image}" &>/dev/null; then
+        echo "No compute image or image-family found, create an image named '{{ openshift_gcp_image }}' to continue'"
+        exit 1
+    fi
+    image="family/${image}"
+fi
+
+### PROVISION THE INFRASTRUCTURE ###
+
+dns_zone="{{ dns_managed_zone | default(openshift_gcp_prefix + 'managed-zone') }}"
+
+# Check the DNS managed zone in Google Cloud DNS, create it if it doesn't exist and exit after printing NS servers
+if ! gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${dns_zone}" &>/dev/null; then
+    echo "DNS zone '${dns_zone}' doesn't exist. Must be configured prior to running this script"
+    exit 1
+fi
+
+# Create network
+if ! gcloud --project "{{ openshift_gcp_project }}" compute networks describe "{{ openshift_gcp_network_name }}" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute networks create "{{ openshift_gcp_network_name }}" --mode "auto"
+else
+    echo "Network '{{ openshift_gcp_network_name }}' already exists"
+fi
+
+# Firewall rules in a form:
+# ['name']='parameters for "gcloud compute firewall-rules create"'
+# For all possible parameters see: gcloud compute firewall-rules create --help
+range=""
+if [[ -n "{{ openshift_node_port_range }}" ]]; then
+    range=",tcp:{{ openshift_node_port_range }},udp:{{ openshift_node_port_range }}"
+fi
+declare -A FW_RULES=(
+  ['icmp']='--allow icmp'
+  ['ssh-external']='--allow tcp:22'
+  ['ssh-internal']='--allow tcp:22 --source-tags bastion'
+  ['master-internal']="--allow tcp:2224,tcp:2379,tcp:2380,tcp:4001,udp:4789,udp:5404,udp:5405,tcp:8053,udp:8053,tcp:8444,tcp:10250,tcp:10255,udp:10255,tcp:24224,udp:24224 --source-tags ocp --target-tags ocp-master"
+  ['master-external']="--allow tcp:80,tcp:443,tcp:1936,tcp:8080,tcp:8443${range} --target-tags ocp-master"
+  ['node-internal']="--allow udp:4789,tcp:10250,tcp:10255,udp:10255,tcp:9000-10000 --source-tags ocp --target-tags ocp-node,ocp-infra-node"
+  ['infra-node-internal']="--allow tcp:5000 --source-tags ocp --target-tags ocp-infra-node"
+  ['infra-node-external']="--allow tcp:80,tcp:443,tcp:1936${range} --target-tags ocp-infra-node"
+)
+for rule in "${!FW_RULES[@]}"; do
+    ( if ! gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules describe "{{ openshift_gcp_prefix }}$rule" &>/dev/null; then
+        gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules create "{{ openshift_gcp_prefix }}$rule" --network "{{ openshift_gcp_network_name }}" ${FW_RULES[$rule]}
+    else
+        echo "Firewall rule '{{ openshift_gcp_prefix }}${rule}' already exists"
+    fi ) &
+done
+
+
+# Master IP
+( if ! gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute addresses create "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global
+else
+    echo "IP '{{ openshift_gcp_prefix }}master-ssl-lb-ip' already exists"
+fi ) &
+
+# Internal master IP
+( if ! gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute addresses create "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}"
+else
+    echo "IP '{{ openshift_gcp_prefix }}master-network-lb-ip' already exists"
+fi ) &
+
+# Router IP
+( if ! gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute addresses create "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}"
+else
+    echo "IP '{{ openshift_gcp_prefix }}router-network-lb-ip' already exists"
+fi ) &
+
+
+{% for node_group in openshift_gcp_node_group_config %}
+# configure {{ node_group.name }}
+(
+    if ! gcloud --project "{{ openshift_gcp_project }}" compute instance-templates describe "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" &>/dev/null; then
+        gcloud --project "{{ openshift_gcp_project }}" compute instance-templates create "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" \
+                --machine-type "{{ node_group.machine_type }}" --network "{{ openshift_gcp_network_name }}" \
+                --tags "{{ openshift_gcp_prefix }}ocp,ocp,ocp-bootstrap,{{ node_group.tags }}" \
+                --boot-disk-size "{{ node_group.boot_disk_size }}" --boot-disk-type "pd-ssd" \
+                --scopes "logging-write,monitoring-write,useraccounts-ro,service-control,service-management,storage-ro,compute-rw" \
+                --image "{{ node_group.image | default('${image}') }}" ${metadata}  \
+                --metadata "bootstrap={{ node_group.bootstrap | default(False) | bool | to_json }},cluster-id={{ openshift_gcp_prefix + openshift_gcp_clusterid }},node-group={{ node_group.name }}"
+    else
+        echo "Instance template '{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}' already exists"
+    fi
+
+    # Create instance group
+    if ! gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed describe "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" &>/dev/null; then
+        gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed create "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" \
+                --zone "{{ openshift_gcp_zone }}" --template "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" --size "{{ node_group.scale }}"
+    else
+        echo "Instance group '{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}' already exists"
+    fi
+) &
+{% endfor %}
+
+for i in `jobs -p`; do wait $i; done
+
+
+# Configure the master external LB rules
+(
+# Master health check
+if ! gcloud --project "{{ openshift_gcp_project }}" compute health-checks describe "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute health-checks create https "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" --port "{{ internal_console_port }}" --request-path "/healthz"
+else
+    echo "Health check '{{ openshift_gcp_prefix }}master-ssl-lb-health-check' already exists"
+fi
+
+gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-named-ports "{{ openshift_gcp_prefix }}ig-m" \
+        --zone "{{ openshift_gcp_zone }}" --named-ports "{{ openshift_gcp_prefix }}port-name-master:{{ internal_console_port }}"
+
+# Master backend service
+if ! gcloud --project "{{ openshift_gcp_project }}" compute backend-services describe "{{ openshift_gcp_prefix }}master-ssl-lb-backend" --global &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute backend-services create "{{ openshift_gcp_prefix }}master-ssl-lb-backend" --health-checks "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" --port-name "{{ openshift_gcp_prefix }}port-name-master" --protocol "TCP" --global --timeout="{{ openshift_gcp_master_lb_timeout }}"
+    gcloud --project "{{ openshift_gcp_project }}" compute backend-services add-backend "{{ openshift_gcp_prefix }}master-ssl-lb-backend" --instance-group "{{ openshift_gcp_prefix }}ig-m" --global --instance-group-zone "{{ openshift_gcp_zone }}"
+else
+    echo "Backend service '{{ openshift_gcp_prefix }}master-ssl-lb-backend' already exists"
+fi
+
+# Master tcp proxy target
+if ! gcloud --project "{{ openshift_gcp_project }}" compute target-tcp-proxies describe "{{ openshift_gcp_prefix }}master-ssl-lb-target" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute target-tcp-proxies create "{{ openshift_gcp_prefix }}master-ssl-lb-target" --backend-service "{{ openshift_gcp_prefix }}master-ssl-lb-backend"
+else
+    echo "Proxy target '{{ openshift_gcp_prefix }}master-ssl-lb-target' already exists"
+fi
+
+# Master forwarding rule
+if ! gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules describe "{{ openshift_gcp_prefix }}master-ssl-lb-rule" --global &>/dev/null; then
+    IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global --format='value(address)')
+    gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules create "{{ openshift_gcp_prefix }}master-ssl-lb-rule" --address "$IP" --global --ports "{{ console_port }}" --target-tcp-proxy "{{ openshift_gcp_prefix }}master-ssl-lb-target"
+else
+    echo "Forwarding rule '{{ openshift_gcp_prefix }}master-ssl-lb-rule' already exists"
+fi
+) &
+
+
+# Configure the master internal LB rules
+(
+# Internal master health check
+if ! gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks describe "{{ openshift_gcp_prefix }}master-network-lb-health-check" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks create "{{ openshift_gcp_prefix }}master-network-lb-health-check" --port "8080" --request-path "/healthz"
+else
+    echo "Health check '{{ openshift_gcp_prefix }}master-network-lb-health-check' already exists"
+fi
+
+# Internal master target pool
+if ! gcloud --project "{{ openshift_gcp_project }}" compute target-pools describe "{{ openshift_gcp_prefix }}master-network-lb-pool" --region "{{ openshift_gcp_region }}" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute target-pools create "{{ openshift_gcp_prefix }}master-network-lb-pool" --http-health-check "{{ openshift_gcp_prefix }}master-network-lb-health-check" --region "{{ openshift_gcp_region }}"
+else
+    echo "Target pool '{{ openshift_gcp_prefix }}master-network-lb-pool' already exists"
+fi
+
+# Internal master forwarding rule
+if ! gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules describe "{{ openshift_gcp_prefix }}master-network-lb-rule" --region "{{ openshift_gcp_region }}" &>/dev/null; then
+    IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
+    gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules create "{{ openshift_gcp_prefix }}master-network-lb-rule" --address "$IP" --region "{{ openshift_gcp_region }}" --target-pool "{{ openshift_gcp_prefix }}master-network-lb-pool"
+else
+    echo "Forwarding rule '{{ openshift_gcp_prefix }}master-network-lb-rule' already exists"
+fi
+) &
+
+
+# Configure the infra node rules
+(
+# Router health check
+if ! gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks describe "{{ openshift_gcp_prefix }}router-network-lb-health-check" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute http-health-checks create "{{ openshift_gcp_prefix }}router-network-lb-health-check" --port "1936" --request-path "/healthz"
+else
+    echo "Health check '{{ openshift_gcp_prefix }}router-network-lb-health-check' already exists"
+fi
+
+# Router target pool
+if ! gcloud --project "{{ openshift_gcp_project }}" compute target-pools describe "{{ openshift_gcp_prefix }}router-network-lb-pool" --region "{{ openshift_gcp_region }}" &>/dev/null; then
+    gcloud --project "{{ openshift_gcp_project }}" compute target-pools create "{{ openshift_gcp_prefix }}router-network-lb-pool" --http-health-check "{{ openshift_gcp_prefix }}router-network-lb-health-check" --region "{{ openshift_gcp_region }}"
+else
+    echo "Target pool '{{ openshift_gcp_prefix }}router-network-lb-pool' already exists"
+fi
+
+# Router forwarding rule
+if ! gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules describe "{{ openshift_gcp_prefix }}router-network-lb-rule" --region "{{ openshift_gcp_region }}" &>/dev/null; then
+    IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
+    gcloud --project "{{ openshift_gcp_project }}" compute forwarding-rules create "{{ openshift_gcp_prefix }}router-network-lb-rule" --address "$IP" --region "{{ openshift_gcp_region }}" --target-pool "{{ openshift_gcp_prefix }}router-network-lb-pool"
+else
+    echo "Forwarding rule '{{ openshift_gcp_prefix }}router-network-lb-rule' already exists"
+fi
+) &
+
+for i in `jobs -p`; do wait $i; done
+
+# set the target pools
+(
+if [[ "ig-m" == "{{ openshift_gcp_infra_network_instance_group }}" ]]; then
+    gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}ig-m" --target-pools "{{ openshift_gcp_prefix }}master-network-lb-pool,{{ openshift_gcp_prefix }}router-network-lb-pool" --zone "{{ openshift_gcp_zone }}"
+else
+    gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}ig-m" --target-pools "{{ openshift_gcp_prefix }}master-network-lb-pool" --zone "{{ openshift_gcp_zone }}"
+    gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed set-target-pools "{{ openshift_gcp_prefix }}{{ openshift_gcp_infra_network_instance_group }}" --target-pools "{{ openshift_gcp_prefix }}router-network-lb-pool" --zone "{{ openshift_gcp_zone }}"
+fi
+) &
+
+# configure DNS
+(
+# Retry DNS changes until they succeed since this may be a shared resource
+while true; do
+    dns="${TMPDIR:-/tmp}/dns.yaml"
+    rm -f $dns
+
+    # DNS record for master lb
+    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ openshift_master_cluster_public_hostname }}" 2>/dev/null | grep -q "{{ openshift_master_cluster_public_hostname }}"; then
+        IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-ssl-lb-ip" --global --format='value(address)')
+        if [[ ! -f $dns ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+        fi
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ openshift_master_cluster_public_hostname }}." --type A "$IP"
+    else
+        echo "DNS record for '{{ openshift_master_cluster_public_hostname }}' already exists"
+    fi
+
+    # DNS record for internal master lb
+    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ openshift_master_cluster_hostname }}" 2>/dev/null | grep -q "{{ openshift_master_cluster_hostname }}"; then
+        IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}master-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
+        if [[ ! -f $dns ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+        fi
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ openshift_master_cluster_hostname }}." --type A "$IP"
+    else
+        echo "DNS record for '{{ openshift_master_cluster_hostname }}' already exists"
+    fi
+
+    # DNS record for router lb
+    if ! gcloud --project "{{ openshift_gcp_project }}" dns record-sets list -z "${dns_zone}" --name "{{ wildcard_zone }}" 2>/dev/null | grep -q "{{ wildcard_zone }}"; then
+        IP=$(gcloud --project "{{ openshift_gcp_project }}" compute addresses describe "{{ openshift_gcp_prefix }}router-network-lb-ip" --region "{{ openshift_gcp_region }}" --format='value(address)')
+        if [[ ! -f $dns ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
+        fi
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "{{ wildcard_zone }}." --type A "$IP"
+        gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns add -z "${dns_zone}" --ttl {{ openshift_gcp_master_dns_ttl }} --name "*.{{ wildcard_zone }}." --type CNAME "{{ wildcard_zone }}."
+    else
+        echo "DNS record for '{{ wildcard_zone }}' already exists"
+    fi
+
+    # Commit all DNS changes, retrying if preconditions are not met
+    if [[ -f $dns ]]; then
+        if ! out="$( gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns execute -z "${dns_zone}" 2>&1 )"; then
+            rc=$?
+            if [[ "${out}" == *"HTTPError 412: Precondition not met"* ]]; then
+                continue
+            fi
+            exit $rc
+        fi
+    fi
+    break
+done
+) &
+
+# Create bucket for registry
+(
+if ! gsutil ls -p "{{ openshift_gcp_project }}" "gs://{{ openshift_gcp_registry_bucket_name }}" &>/dev/null; then
+    gsutil mb -p "{{ openshift_gcp_project }}" -l "{{ openshift_gcp_region }}" "gs://{{ openshift_gcp_registry_bucket_name }}"
+else
+    echo "Bucket '{{ openshift_gcp_registry_bucket_name }}' already exists"
+fi
+) &
+
+# wait until all node groups are stable
+{% for node_group in openshift_gcp_node_group_config %}
+{% if node_group.wait_for_stable | default(False) or not (node_group.bootstrap | default(False)) %}
+# wait for stable {{ node_group.name }}
+( gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed wait-until-stable "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --timeout=600 ) &
+{% else %}
+# not waiting for {{ node_group.name }} due to bootstrapping
+{% endif %}
+{% endfor %}
+
+
+for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/remove.j2.sh

@@ -1,6 +1,78 @@
 #!/bin/bash
 
-set -euxo pipefail
+set -euo pipefail
+
+function teardown_cmd() {
+    a=( $@ )
+    local name=$1
+    a=( "${a[@]:1}" )
+    local flag=0
+    local found=
+    for i in ${a[@]}; do
+        if [[ "$i" == "--"* ]]; then
+            found=true
+            break
+        fi
+        flag=$((flag+1))
+    done
+    if [[ -z "${found}" ]]; then
+      flag=$((flag+1))
+    fi
+    if gcloud --project "{{ openshift_gcp_project }}" ${a[@]::$flag} describe "${name}" ${a[@]:$flag} &>/dev/null; then
+        gcloud --project "{{ openshift_gcp_project }}" ${a[@]::$flag} delete -q "${name}" ${a[@]:$flag}
+    fi
+}
+
+function teardown() {
+    for i in `seq 1 20`; do
+        if teardown_cmd $@; then
+            break
+        fi
+        sleep 0.5
+    done
+}
+
+# Preemptively spin down the instances
+{% for node_group in openshift_gcp_node_group_config %}
+# scale down {{ node_group.name }}
+(
+    # performs a delete and scale down as one operation to ensure maximum parallelism
+    if ! instances=$( gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed list-instances "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --format='value[terminator=","](instance)' 2>/dev/null ); then
+        exit 0
+    fi
+    instances="${instances%?}"
+    if [[ -z "${instances}" ]]; then
+        echo "warning: No instances in {{ node_group.name }}" 1>&2
+        exit 0
+    fi
+    if ! gcloud --project "{{ openshift_gcp_project }}" compute instance-groups managed delete-instances "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" --zone "{{ openshift_gcp_zone }}" --instances "${instances}"; then
+        echo "warning: Unable to scale down the node group {{ node_group.name }}" 1>&2
+        exit 0
+    fi
+) &
+{% endfor %}
+
+# Bucket for registry
+(
+if gsutil ls -p "{{ openshift_gcp_project }}" "gs://{{ openshift_gcp_registry_bucket_name }}" &>/dev/null; then
+    gsutil -m rm -r "gs://{{ openshift_gcp_registry_bucket_name }}"
+fi
+) &
+
+# Project metadata prefixed with {{ openshift_gcp_prefix }}
+(
+    for key in $( gcloud --project "{{ openshift_gcp_project }}" compute project-info describe --flatten=commonInstanceMetadata.items[] '--format=value(commonInstanceMetadata.items.key)' ); do
+        if [[ "${key}" == "{{ openshift_gcp_prefix }}"* ]]; then
+            gcloud --project "{{ openshift_gcp_project }}" compute project-info remove-metadata "--keys=${key}"
+        fi
+    done
+) &
+
+# Instances and disks used for image building
+(
+    teardown "{{ openshift_gcp_prefix }}build-image-instance" compute instances --zone "{{ openshift_gcp_zone }}"
+    teardown "{{ openshift_gcp_prefix }}build-image-instance" compute disks --zone "{{ openshift_gcp_zone }}"
+) &
 
 # DNS
 (
@@ -14,16 +86,9 @@ if gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${
         # export all dns records that match into a zone format, and turn each line into a set of args for
         # record-sets transaction.
         gcloud dns record-sets export --project "{{ openshift_gcp_project }}" -z "${dns_zone}" --zone-file-format "${dns}"
-
-        # Write the header
-        ETCD_DNS_NAME="_etcd-server-ssl._tcp.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.{{ public_hosted_zone }}."
-        grep -F -e "${ETCD_DNS_NAME}" "${dns}" | awk '{ print "--name", $1, "--ttl", $2, "--type", $4 }' | head -n1 | xargs echo -n > "${dns}.input"
-        # Append all etcd records
-        grep -F -e "${ETCD_DNS_NAME}" "${dns}" | awk '{ print " \x27"$5" "$6" "$7" "$8"\x27"; }' | tr -d '\n\r' >> "${dns}.input" || true
-        echo >> "${dns}.input"
-
-
-        if [ -s "${dns}.input" ]; then
+        if grep -F -e '{{ openshift_master_cluster_hostname }}' -e '{{ openshift_master_cluster_public_hostname }}' -e '{{ wildcard_zone }}' "${dns}" | \
+                awk '{ print "--name", $1, "--ttl", $2, "--type", $4, $5; }' > "${dns}.input"
+        then
             rm -f "${dns}"
             gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file=$dns start -z "${dns_zone}"
             cat "${dns}.input" | xargs -L1 gcloud --project "{{ openshift_gcp_project }}" dns record-sets transaction --transaction-file="${dns}" remove -z "${dns_zone}"
@@ -43,4 +108,72 @@ if gcloud --project "{{ openshift_gcp_project }}" dns managed-zones describe "${
 fi
 ) &
 
+(
+# Router network rules
+teardown "{{ openshift_gcp_prefix }}router-network-lb-rule" compute forwarding-rules --region "{{ openshift_gcp_region }}"
+teardown "{{ openshift_gcp_prefix }}router-network-lb-pool" compute target-pools --region "{{ openshift_gcp_region }}"
+teardown "{{ openshift_gcp_prefix }}router-network-lb-health-check" compute http-health-checks
+teardown "{{ openshift_gcp_prefix }}router-network-lb-ip" compute addresses --region "{{ openshift_gcp_region }}"
+
+# Internal master network rules
+teardown "{{ openshift_gcp_prefix }}master-network-lb-rule" compute forwarding-rules --region "{{ openshift_gcp_region }}"
+teardown "{{ openshift_gcp_prefix }}master-network-lb-pool" compute target-pools --region "{{ openshift_gcp_region }}"
+teardown "{{ openshift_gcp_prefix }}master-network-lb-health-check" compute http-health-checks
+teardown "{{ openshift_gcp_prefix }}master-network-lb-ip" compute addresses --region "{{ openshift_gcp_region }}"
+) &
+
+(
+# Master SSL network rules
+teardown "{{ openshift_gcp_prefix }}master-ssl-lb-rule" compute forwarding-rules --global
+teardown "{{ openshift_gcp_prefix }}master-ssl-lb-target" compute target-tcp-proxies
+teardown "{{ openshift_gcp_prefix }}master-ssl-lb-ip" compute addresses --global
+teardown "{{ openshift_gcp_prefix }}master-ssl-lb-backend" compute backend-services --global
+teardown "{{ openshift_gcp_prefix }}master-ssl-lb-health-check" compute health-checks
+) &
+
+# Firewall rules
+(
+    if ! firewalls=$( gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules list --filter="network~projects/{{ openshift_gcp_project }}/global/networks/{{ openshift_gcp_network_name }}" --format="value[terminator=' '](name)" 2>/dev/null ); then
+        exit 0
+    fi
+    firewalls="${firewalls%?}"
+    if [[ -z "${firewalls}" ]]; then
+        exit 0
+    fi
+    gcloud --project "{{ openshift_gcp_project }}" compute firewall-rules delete -q ${firewalls}
+) &
+
 for i in `jobs -p`; do wait $i; done
+
+{% for node_group in openshift_gcp_node_group_config %}
+# teardown {{ node_group.name }} - any load balancers referencing these groups must be removed
+(
+    teardown "{{ openshift_gcp_prefix }}ig-{{ node_group.suffix }}" compute instance-groups managed --zone "{{ openshift_gcp_zone }}"
+    teardown "{{ openshift_gcp_prefix }}instance-template-{{ node_group.name }}" compute instance-templates
+) &
+{% endfor %}
+
+for i in `jobs -p`; do wait $i; done
+
+# Images specifically located under this cluster prefix family
+for name in $( gcloud --project "{{ openshift_gcp_project }}" compute images list "--filter=family={{ openshift_gcp_prefix }}images" '--format=value(name)' ); do
+    ( gcloud --project "{{ openshift_gcp_project }}" compute images delete "${name}" ) &
+done
+
+# Disks
+(
+    if ! disks=$( gcloud --project "{{ openshift_gcp_project }}" compute disks list --filter="users~projects/{{ openshift_gcp_project }}/zones/{{ openshift_gcp_zone }}/instances/{{ openshift_gcp_prefix }}.*" --format="value[terminator=' '](name)" 2>/dev/null ); then
+        exit 0
+    fi
+    disks="${disks%?}"
+    if [[ -z "${disks}" ]]; then
+        echo "warning: No disks in use by {{ openshift_gcp_prefix }}" 1>&2
+        exit 0
+    fi
+    gcloud --project "{{ openshift_gcp_project }}" compute disks delete -q "${disks}"
+) &
+
+# Network
+( teardown "{{ openshift_gcp_network_name }}" compute networks ) &
+
+for i in `jobs -p`; do wait $i; done

roles/openshift_gcp/templates/yum_repo.j2

@@ -0,0 +1,20 @@
+{% for repo in provision_custom_repositories %}
+[{{ repo.id | default(repo.name) }}]
+name={{ repo.name | default(repo.id) }}
+baseurl={{ repo.baseurl }}
+{% set enable_repo = repo.enabled | default(1) %}
+enabled={{ 1 if ( enable_repo == 1 or enable_repo == True ) else 0 }}
+{% set enable_gpg_check = repo.gpgcheck | default(1) %}
+gpgcheck={{ 1 if ( enable_gpg_check == 1 or enable_gpg_check == True ) else 0 }}
+{% if 'sslclientcert' in repo %}
+sslclientcert={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_cert" if repo.sslclientcert }}
+{% endif %}
+{% if 'sslclientkey' in repo %}
+sslclientkey={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_key" if repo.sslclientkey }}
+{% endif %}
+{% for key, value in repo.iteritems() %}
+{% if key not in ['id', 'name', 'baseurl', 'enabled', 'gpgcheck', 'sslclientkey', 'sslclientcert'] and value is defined %}
+{{ key }}={{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}

roles/openshift_node40/README.md

@@ -1,61 +0,0 @@
-OpenShift Node
-================================
-
-Node service installation
-
-Requirements
-------------
-
-* Ansible 2.2
-* One or more Master servers
-* A RHEL 7.1 host pre-configured with access to the rhel-7-server-rpms,
-rhel-7-server-extras-rpms, and rhel-7-server-ose-3.0-rpms repos
-
-Role Variables
---------------
-From this role:
-
-| Name                                     | Default value         |                                                          |
-|------------------------------------------|-----------------------|----------------------------------------------------------|
-| openshift_node_start_options             | UNDEF (Optional)      | Options to pass to node start cmdline                    |
-| oreg_url                                 | UNDEF (Optional)      | Default docker registry to use                           |
-| openshift_persistentlocalstorage_enabled | false                 | Enable the persistent local storage                      |
-
-openshift_node_start_options can be used for passing any start node option, e.g.:
-
---enable=kubelet,plugins
-
-Which would have a node running without kube-proxy and dns.
-
-Dependencies
-------------
-
-
-Example Playbook
-----------------
-
-Notes
------
-
-Currently we support re-labeling nodes but we don't re-schedule running pods nor remove existing labels. That means you will have to trigger the re-schedulling manually. To re-schedule your pods, just follow the steps below:
-
-```
-oc adm manage-node --schedulable=false ${NODE}
-oc adm manage-node --drain ${NODE}
-oc adm manage-node --schedulable=true ${NODE}
-````
-
-> If you are using version less than 1.5/3.5 you must replace `--drain` with `--evacuate`.
-
-
-TODO
-
-License
--------
-
-Apache License, Version 2.0
-
-Author Information
-------------------
-
-TODO

roles/openshift_node40/defaults/main.yml

@@ -1,160 +0,0 @@
----
-openshift_node_debug_level: "{{ debug_level | default(2) }}"
-openshift_node_iptables_sync_period: '30s'
-osn_storage_plugin_deps:
-- ceph
-- glusterfs
-- iscsi
-openshift_node_local_quota_per_fsgroup: ""
-openshift_node_proxy_mode: iptables
-openshift_set_node_ip: False
-openshift_config_base: '/etc/origin'
-
-
-# Assume the images are already downloaded on the machine
-system_images_registry: "docker"
-l_osn_image: "{{ (system_images_registry == 'docker') | ternary(osn_image, (osn_image.split('/')|length==2) | ternary(system_images_registry + '/' + osn_image, osn_image)) }}"
-system_osn_image: "{{ (system_images_registry == 'docker') | ternary('docker:' + l_osn_image, l_osn_image) }}"
-
-openshift_node_env_vars: {}
-
-# lo must always be present in this list or dnsmasq will conflict with
-# the node's dns service.
-openshift_node_dnsmasq_except_interfaces:
-- lo
-
-# dnsmasq defaults to neg caching disabled
-openshift_node_dnsmasq_no_negcache: true
-# When openshift_node_dnsmasq_no_negcache is set to false, how many seconds to cache negative lookups.
-openshift_node_dnsmasq_neg_ttl: '1'
-
-r_openshift_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
-r_openshift_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
-
-openshift_node_syscon_auth_mounts_l:
-- type: bind
-  source: "{{ oreg_auth_credentials_path }}"
-  destination: "/root/.docker"
-  options:
-  - ro
-  - bind
-
-# If we need to add new mounts in the future, or the user wants to mount data.
-# This should be in the same format as auth_mounts_l above.
-openshift_node_syscon_add_mounts_l: []
-
-default_r_openshift_node_image_prep_packages:
-- "{{ openshift_service_type }}-node"
-- ansible
-- bash-completion
-- dnsmasq
-- ntp
-- logrotate
-- httpd-tools
-- bind-utils
-- firewalld
-- libselinux-python
-- conntrack-tools
-- openssl
-- iproute
-- python-dbus
-- PyYAML
-- yum-utils
-- glusterfs-fuse
-- device-mapper-multipath
-- nfs-utils
-- cockpit-ws
-- cockpit-system
-- cockpit-bridge
-- cockpit-docker
-- iscsi-initiator-utils
-- ceph-common
-- atomic
-r_openshift_node_image_prep_packages: "{{ default_r_openshift_node_image_prep_packages | union(openshift_node_image_prep_packages | default([])) }}"
-
-r_openshift_node_os_firewall_deny: []
-default_r_openshift_node_os_firewall_allow:
-- service: Kubernetes kubelet
-  port: 10250/tcp
-- service: Kubernetes kube-proxy health check for service load balancers
-  port: 10256/tcp
-- service: http
-  port: 80/tcp
-- service: https
-  port: 443/tcp
-- service: OpenShift OVS sdn
-  port: 4789/udp
-  cond: openshift_use_openshift_sdn | bool
-- service: Calico BGP Port
-  port: 179/tcp
-  cond: "{{ openshift_node_use_calico }}"
-- service: Kubernetes service NodePort TCP
-  port: "{{ openshift_node_port_range | default('') }}/tcp"
-  cond: "{{ openshift_node_port_range is defined }}"
-- service: Kubernetes service NodePort UDP
-  port: "{{ openshift_node_port_range | default('') }}/udp"
-  cond: "{{ openshift_node_port_range is defined }}"
-- service: Prometheus monitoring
-  port: 9000-10000/tcp
-# Allow multiple port ranges to be added to the role
-r_openshift_node_os_firewall_allow: "{{ default_r_openshift_node_os_firewall_allow | union(openshift_node_open_ports | default([])) }}"
-
-# oreg_url is defined by user input
-oreg_auth_credentials_path: "{{ openshift_node_data_dir }}/.docker"
-l_bind_docker_reg_auth: False
-
-openshift_docker_service_name: "docker"
-
-# These defaults assume forcing journald persistence, fsync to disk once
-# a second, rate-limiting to 10,000 logs a second, no forwarding to
-# syslog or wall, using 8GB of disk space maximum, using 10MB journal
-# files, keeping only a days worth of logs per journal file, and
-# retaining journal files no longer than a month.
-journald_vars_to_replace:
-- { var: Storage, val: persistent }
-- { var: Compress, val: yes }
-- { var: SyncIntervalSec, val: 1s }
-- { var: RateLimitInterval, val: 1s }
-- { var: RateLimitBurst, val: 10000 }
-- { var: SystemMaxUse, val: 8G }
-- { var: SystemKeepFree, val: 20% }
-- { var: SystemMaxFileSize, val: 10M }
-- { var: MaxRetentionSec, val: 1month }
-- { var: MaxFileSec, val: 1day }
-- { var: ForwardToSyslog, val: no }
-- { var: ForwardToWall, val: no }
-
-# NOTE
-# r_openshift_node_*_default may be defined external to this role.
-# openshift_use_*, if defined, may affect other roles or play behavior.
-openshift_node_use_openshift_sdn_default: "{{ openshift_use_openshift_sdn | default(True) }}"
-openshift_node_use_openshift_sdn: "{{ openshift_node_use_openshift_sdn_default }}"
-
-openshift_node_sdn_network_plugin_name_default: "{{ os_sdn_network_plugin_name }}"
-openshift_node_sdn_network_plugin_name: "{{ openshift_node_sdn_network_plugin_name_default }}"
-
-openshift_node_use_calico_default: "{{ openshift_use_calico | default(False) }}"
-openshift_node_use_calico: "{{ openshift_node_use_calico_default }}"
-
-openshift_node_use_nuage_default: "{{ openshift_use_nuage | default(False) }}"
-openshift_node_use_nuage: "{{ openshift_node_use_nuage_default }}"
-
-openshift_node_use_contiv_default: "{{ openshift_use_contiv | default(False) }}"
-openshift_node_use_contiv: "{{ openshift_node_use_contiv_default }}"
-
-openshift_node_use_kuryr_default: "{{ openshift_use_kuryr | default(False) }}"
-openshift_node_use_kuryr: "{{ openshift_node_use_kuryr_default }}"
-
-openshift_node_data_dir_default: "{{ openshift_data_dir | default('/var/lib/origin') }}"
-openshift_node_data_dir: "{{ openshift_node_data_dir_default }}"
-
-openshift_node_config_dir_default: "/etc/origin/node"
-openshift_node_config_dir: "{{ openshift_node_config_dir_default }}"
-
-openshift_node_image_config_latest_default: "{{ openshift_image_config_latest | default(False) }}"
-openshift_node_image_config_latest: "{{ openshift_node_image_config_latest_default }}"
-
-
-openshift_node_use_instance_profiles: False
-
-openshift_node_use_persistentlocalvolumes: "{{ openshift_persistentlocalstorage_enabled | default(False) | bool }}"

roles/openshift_node40/files/clean-up-crio-pods.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-for c in $(runc list -q); do
-        output=$(runc state $c | grep io.kubernetes.cri-o.ContainerType)
-        if [[ "$output" =~ "container" ]]; then
-                runc delete -f $c
-        fi
-        for m in $(mount | grep $c | awk '{print $3}'); do
-                umount -R $m
-        done
-done
-for c in $(runc list -q); do
-        output=$(runc state $c | grep io.kubernetes.cri-o.ContainerType)
-        if [[ "$output" =~ "sandbox" ]]; then
-                runc delete -f $c
-        fi
-        for m in $(mount | grep $c | awk '{print $3}'); do
-                umount -R $m
-        done
-done
-mount | grep overlay | awk '{print $3}' | xargs umount | true
-umount -R /var/lib/containers/storage/overlay
-umount -R /var/lib/containers/storage
-rm -rf /var/run/containers/storage/*
-rm -rf /var/lib/containers/storage/*

roles/openshift_node40/files/networkmanager/99-origin-dns.sh

@@ -1,128 +0,0 @@
-#!/bin/bash -x
-# -*- mode: sh; sh-indentation: 2 -*-
-
-# This NetworkManager dispatcher script replicates the functionality of
-# NetworkManager's dns=dnsmasq  however, rather than hardcoding the listening
-# address and /etc/resolv.conf to 127.0.0.1 it pulls the IP address from the
-# interface that owns the default route. This enables us to then configure pods
-# to use this IP address as their only resolver, where as using 127.0.0.1 inside
-# a pod would fail.
-#
-# To use this,
-# - If this host is also a master, reconfigure master dnsConfig to listen on
-#   8053 to avoid conflicts on port 53 and open port 8053 in the firewall
-# - Drop this script in /etc/NetworkManager/dispatcher.d/
-# - systemctl restart NetworkManager
-#
-# Test it:
-# host kubernetes.default.svc.cluster.local
-# host google.com
-#
-# TODO: I think this would be easy to add as a config option in NetworkManager
-# natively, look at hacking that up
-
-cd /etc/sysconfig/network-scripts
-. ./network-functions
-
-[ -f ../network ] && . ../network
-
-if [[ $2 =~ ^(up|dhcp4-change|dhcp6-change)$ ]]; then
-  # If the origin-upstream-dns config file changed we need to restart
-  NEEDS_RESTART=0
-  UPSTREAM_DNS='/etc/dnsmasq.d/origin-upstream-dns.conf'
-  # We'll regenerate the dnsmasq origin config in a temp file first
-  UPSTREAM_DNS_TMP=`mktemp`
-  UPSTREAM_DNS_TMP_SORTED=`mktemp`
-  CURRENT_UPSTREAM_DNS_SORTED=`mktemp`
-  NEW_RESOLV_CONF=`mktemp`
-  NEW_NODE_RESOLV_CONF=`mktemp`
-
-
-  ######################################################################
-  # couldn't find an existing method to determine if the interface owns the
-  # default route
-  def_route=$(/sbin/ip route list match 0.0.0.0/0 | awk '{print $3 }')
-  def_route_int=$(/sbin/ip route get to ${def_route} | awk -F 'dev' '{print $2}' | head -n1 | awk '{print $1}')
-  def_route_ip=$(/sbin/ip route get to ${def_route}  | awk -F 'src' '{print $2}' | head -n1 | awk '{print $1}')
-  if [[ ${DEVICE_IFACE} == ${def_route_int} ]]; then
-    if [ ! -f /etc/dnsmasq.d/origin-dns.conf ]; then
-      cat << EOF > /etc/dnsmasq.d/origin-dns.conf
-no-resolv
-domain-needed
-server=/cluster.local/172.30.0.1
-server=/30.172.in-addr.arpa/172.30.0.1
-enable-dbus
-dns-forward-max=5000
-cache-size=5000
-min-port=1024
-EOF
-      # New config file, must restart
-      NEEDS_RESTART=1
-    fi
-
-    # If network manager doesn't know about the nameservers then the best
-    # we can do is grab them from /etc/resolv.conf but only if we've got no
-    # watermark
-    if ! grep -q '99-origin-dns.sh' /etc/resolv.conf; then
-      if [[ -z "${IP4_NAMESERVERS}" || "${IP4_NAMESERVERS}" == "${def_route_ip}" ]]; then
-            IP4_NAMESERVERS=`grep '^nameserver[[:blank:]]' /etc/resolv.conf | awk '{ print $2 }'`
-      fi
-      ######################################################################
-      # Write out default nameservers for /etc/dnsmasq.d/origin-upstream-dns.conf
-      # and /etc/origin/node/resolv.conf in their respective formats
-      for ns in ${IP4_NAMESERVERS}; do
-        if [[ ! -z $ns ]]; then
-          echo "server=${ns}" >> $UPSTREAM_DNS_TMP
-          echo "nameserver ${ns}" >> $NEW_NODE_RESOLV_CONF
-        fi
-      done
-      # Sort it in case DNS servers arrived in a different order
-      sort $UPSTREAM_DNS_TMP > $UPSTREAM_DNS_TMP_SORTED
-      sort $UPSTREAM_DNS > $CURRENT_UPSTREAM_DNS_SORTED
-      # Compare to the current config file (sorted)
-      NEW_DNS_SUM=`md5sum ${UPSTREAM_DNS_TMP_SORTED} | awk '{print $1}'`
-      CURRENT_DNS_SUM=`md5sum ${CURRENT_UPSTREAM_DNS_SORTED} | awk '{print $1}'`
-      if [ "${NEW_DNS_SUM}" != "${CURRENT_DNS_SUM}" ]; then
-        # DNS has changed, copy the temp file to the proper location (-Z
-        # sets default selinux context) and set the restart flag
-        cp -Z $UPSTREAM_DNS_TMP $UPSTREAM_DNS
-        NEEDS_RESTART=1
-      fi
-      # compare /etc/origin/node/resolv.conf checksum and replace it if different
-      NEW_NODE_RESOLV_CONF_MD5=`md5sum ${NEW_NODE_RESOLV_CONF}`
-      OLD_NODE_RESOLV_CONF_MD5=`md5sum /etc/origin/node/resolv.conf`
-      if [ "${NEW_NODE_RESOLV_CONF_MD5}" != "${OLD_NODE_RESOLV_CONF_MD5}" ]; then
-        cp -Z $NEW_NODE_RESOLV_CONF /etc/origin/node/resolv.conf
-      fi
-    fi
-
-    if ! `systemctl -q is-active dnsmasq.service`; then
-      NEEDS_RESTART=1
-    fi
-
-    ######################################################################
-    if [ "${NEEDS_RESTART}" -eq "1" ]; then
-      systemctl restart dnsmasq
-    fi
-
-    # Only if dnsmasq is running properly make it our only nameserver and place
-    # a watermark on /etc/resolv.conf
-    if `systemctl -q is-active dnsmasq.service`; then
-      if ! grep -q '99-origin-dns.sh' /etc/resolv.conf; then
-          echo "# nameserver updated by /etc/NetworkManager/dispatcher.d/99-origin-dns.sh" >> ${NEW_RESOLV_CONF}
-      fi
-      sed -e '/^nameserver.*$/d' /etc/resolv.conf >> ${NEW_RESOLV_CONF}
-      echo "nameserver "${def_route_ip}"" >> ${NEW_RESOLV_CONF}
-      if ! grep -qw search ${NEW_RESOLV_CONF}; then
-        echo 'search cluster.local' >> ${NEW_RESOLV_CONF}
-      elif ! grep -q 'search cluster.local' ${NEW_RESOLV_CONF}; then
-        # cluster.local should be in first three DNS names so that glibc resolver would work
-        sed -i -e 's/^search[[:blank:]]\(.\+\)\( cluster\.local\)\{0,1\}$/search cluster.local \1/' ${NEW_RESOLV_CONF}
-      fi
-      cp -Z ${NEW_RESOLV_CONF} /etc/resolv.conf
-    fi
-  fi
-
-  # Clean up after yourself
-  rm -f $UPSTREAM_DNS_TMP $UPSTREAM_DNS_TMP_SORTED $CURRENT_UPSTREAM_DNS_SORTED $NEW_RESOLV_CONF
-fi

roles/openshift_node40/files/openshift-node

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-# This launches the Kubelet by converting the node configuration into kube flags.
-
-set -euo pipefail
-
-if ! [[ -f /etc/origin/node/client-ca.crt ]]; then
-  if [[ -f /etc/origin/node/bootstrap.kubeconfig ]]; then
-    oc config --config=/etc/origin/node/bootstrap.kubeconfig view --raw --minify -o go-template='{{ index .clusters 0 "cluster" "certificate-authority-data" }}' | base64 -d - > /etc/origin/node/client-ca.crt
-  fi
-fi
-config=/etc/origin/node/bootstrap-node-config.yaml
-# TODO: remove when dynamic kubelet config is delivered
-if [[ -f /etc/origin/node/node-config.yaml ]]; then
-  config=/etc/origin/node/node-config.yaml
-fi
-flags=$( /usr/bin/openshift-node-config "--config=${config}" )
-eval "exec /usr/bin/hyperkube kubelet --v=${DEBUG_LOGLEVEL:-2} ${flags}"

roles/openshift_node40/handlers/main.yml

@@ -1,20 +0,0 @@
----
-- name: reload systemd units
-  command: systemctl daemon-reload
-  when:
-  - (not skip_node_svc_handlers | default(False) | bool)
-
-- name: restart NetworkManager
-  systemd:
-    name: NetworkManager
-    state: restarted
-    enabled: True
-  when:
-  - (not skip_node_svc_handlers | default(False) | bool)
-
-- name: restart dnsmasq
-  systemd:
-    name: dnsmasq
-    state: restarted
-  when:
-  - (not skip_node_svc_handlers | default(False) | bool)

roles/openshift_node40/meta/main.yml

@@ -1,17 +0,0 @@
----
-galaxy_info:
-  author: Jhon Honce
-  description: OpenShift Node
-  company: Red Hat, Inc.
-  license: Apache License, Version 2.0
-  min_ansible_version: 2.1
-  platforms:
-  - name: EL
-    versions:
-    - 7
-  categories:
-  - cloud
-dependencies:
-- role: lib_openshift
-- role: lib_utils
-- role: openshift_facts

roles/openshift_node40/tasks/bootkube_config.yml

@@ -1,32 +0,0 @@
----
-- name: parse ignition file
-  parse_ignition:
-    ign_file_contents: "{{ openshift_bootkube_ign_contents }}"
-  register: l_parse_ignition_boot_kube_res
-
-- import_tasks: create_files_from_ignition.yml
-  vars:
-    l_parse_ignition_dict: "{{ l_parse_ignition_boot_kube_res }}"
-
-#### Disable SWAP #####
-# https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory
-# swapoff is a custom module in lib_utils that comments out swap entries in
-# /etc/fstab and runs swapoff -a, if necessary.
-- name: Disable swap
-  swapoff: {}
-  when: openshift_disable_swap | default(true) | bool
-
-# The atomic-openshift-node service will set this parameter on
-# startup, but if the network service is restarted this setting is
-# lost. Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1372388
-- sysctl:
-    name: net.ipv4.ip_forward
-    value: 1
-    sysctl_file: "/etc/sysctl.d/99-openshift.conf"
-    reload: yes
-
-- name: Setting sebool container_manage_cgroup
-  seboolean:
-    name: container_manage_cgroup
-    state: yes
-    persistent: yes

roles/openshift_node40/tasks/config.yml

@@ -1,49 +0,0 @@
----
-- name: get worker ignition file
-  command: >
-    curl -k {{ openshift_bootstrap_endpoint }}
-  register: l_worker_bootstrap
-  when: openshift_bootstrap_endpoint is defined
-
-- set_fact:
-    ign_contents: "{{ l_worker_bootstrap.stdout }}"
-  when: openshift_bootstrap_endpoint is defined
-
-- set_fact:
-    ign_contents: "{{ lookup('file', ignition_file) }}"
-  when: ignition_file is defined
-
-- debug:
-    var: ign_contents
-
-- name: parse ignition file
-  parse_ignition:
-    ign_file_contents: "{{ ign_contents }}"
-  register: l_parse_ignition_res
-
-- import_tasks: create_files_from_ignition.yml
-  vars:
-    l_parse_ignition_dict: "{{ l_parse_ignition_res }}"
-
-#### Disable SWAP #####
-# https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory
-# swapoff is a custom module in lib_utils that comments out swap entries in
-# /etc/fstab and runs swapoff -a, if necessary.
-- name: Disable swap
-  swapoff: {}
-  when: openshift_disable_swap | default(true) | bool
-
-# The atomic-openshift-node service will set this parameter on
-# startup, but if the network service is restarted this setting is
-# lost. Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1372388
-- sysctl:
-    name: net.ipv4.ip_forward
-    value: 1
-    sysctl_file: "/etc/sysctl.d/99-openshift.conf"
-    reload: yes
-
-- name: Setting sebool container_manage_cgroup
-  seboolean:
-    name: container_manage_cgroup
-    state: yes
-    persistent: yes

roles/openshift_node40/tasks/create_files_from_ignition.yml

@@ -1,16 +0,0 @@
----
-- name: Create all the directories we will need
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items: "{{ l_parse_ignition_dict.dir_list }}"
-
-- name: create files from ignition contents
-  copy:
-    content: "{{ item.value.contents }}"
-    dest: "{{ item.key }}"
-    mode: "{{ l_file_mode }}"
-  with_dict: "{{ l_parse_ignition_dict.files_dict }}"
-  vars:
-    l_mode_prepend: "{{ '0' if (item.value.mode | length < 4) else '' }}"
-    l_file_mode: "{{ l_mode_prepend ~ item.value.mode }}"

roles/openshift_node40/tasks/install.yml

@@ -1,15 +0,0 @@
----
-
-- name: Install openshift packages
-  package:
-    name: "{{ l_node_packages | join(',') }}"
-    update_cache: true
-  register: install_openshift
-  until: install_openshift.rc == 0
-  retries: 3
-  delay: 1
-  vars:
-    l_node_packages:
-    - "origin-node{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
-    - "origin-clients{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"
-    - "origin-hyperkube{{ (openshift_pkg_version | default('')) | lib_utils_oo_image_tag_to_rpm_version(include_dash=True) }}"

roles/openshift_node40/tasks/systemd.yml

@@ -1,15 +0,0 @@
----
-
-- name: daemon reload
-  systemd:
-    daemon_reload: yes
-
-# dictionary of kv pairs, servicename: enabled, eg:
-# {'kubernetes': "true"}
-- name: Start and enable services
-  systemd:
-    name: "{{ item.key }}"
-    state: "{{ 'restarted' if (item.value | bool) else 'stopped' }}"
-    enabled: "{{ item.value | bool }}"
-  with_dict: "{{ l_parse_ignition_res.systemd_dict }}"
-  when: item.key not in excluded_services | default([])

roles/openshift_node40/templates/bootstrap.yml.j2

@@ -1,69 +0,0 @@
-{% raw -%}
-#!/usr/bin/ansible-playbook
----
-- hosts: localhost
-  gather_facts: yes
-  vars:
-    origin_dns:
-      file: /etc/dnsmasq.d/origin-dns.conf
-      lines:
-      - regex: ^listen-address
-        state: present
-        line: "listen-address={{ ansible_default_ipv4.address }}"
-
-  tasks:
-  - include_vars: openshift_settings.yaml
-
-  - name: set the data for origin_dns
-    lineinfile:
-      create: yes
-      state: "{{ item.state | default('present') }}"
-      insertafter: "{{ item.after | default(omit) }}"
-      path: "{{ origin_dns.file }}"
-      regexp: "{{ item.regex }}"
-      line: "{{ item.line | default(omit)}}"
-    with_items: "{{ origin_dns.lines }}"
-
-  - when:
-    - openshift_node_config_name is defined
-    - openshift_node_config_name != ''
-    block:
-    - name: determine the openshift_service_type
-      stat:
-        path: /etc/sysconfig/atomic-openshift-node
-        get_checksum: false
-        get_attributes: false
-        get_mime: false
-      register: service_type_results
-
-    - name: set openshift_service_type fact based on stat results
-      set_fact:
-        openshift_service_type: "{{ service_type_results.stat.exists | ternary('atomic-openshift', 'origin') }}"
-
-    - name: update the sysconfig to have necessary variables
-      lineinfile:
-        dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
-        line: "{{ item.line }}"
-        regexp: "{{ item.regexp }}"
-      with_items:
-      - line: "BOOTSTRAP_CONFIG_NAME={{ openshift_node_config_name }}"
-        regexp: "^BOOTSTRAP_CONFIG_NAME=.*"
-{% endraw %}
-
-{% if openshift_cloudprovider_kind | default('') == 'aws' %}
-  # need to update aws.conf file if the instance has come up in a new region
-  - name: set up aws.conf
-    block:
-    - name: get current AZ
-      uri:
-        url: http://169.254.169.254/latest/meta-data/placement/availability-zone
-        return_content: yes
-      register: aws_out
-
-    - name: set AZ in aws.conf
-      ini_file:
-        path: /etc/origin/cloudprovider/aws.conf
-        section: Global
-        option: Zone
-        value: "{% raw %}{{ aws_out.content }}{% endraw %}"
-{% endif %}

roles/openshift_node40/templates/multipath.conf.j2

@@ -1,20 +0,0 @@
-# LIO iSCSI
-# TODO: Add env variables for tweaking
-devices {
-        device {
-                vendor "LIO-ORG"
-                user_friendly_names "yes" 
-                path_grouping_policy "failover"
-                path_selector "round-robin 0"
-                failback immediate
-                path_checker "tur"
-                prio "alua"
-                no_path_retry 120
-                rr_weight "uniform"
-        }
-}
-defaults {
-	user_friendly_names yes
-	find_multipaths yes
-}
-

roles/openshift_node40/templates/node.service.j2

@@ -1,26 +0,0 @@
-[Unit]
-Description=OpenShift Node
-After={{ openshift_docker_service_name }}.service
-After=chronyd.service
-After=ntpd.service
-Wants={{ openshift_docker_service_name }}.service
-Documentation=https://github.com/openshift/origin
-Wants=dnsmasq.service
-After=dnsmasq.service
-{% if openshift_use_crio | bool %}Wants=cri-o.service{% endif %}
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/{{ openshift_service_type }}-node
-ExecStart=/usr/local/bin/openshift-node
-LimitNOFILE=65536
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier={{ openshift_service_type }}-node
-Restart=always
-RestartSec=5s
-TimeoutStartSec=300
-OOMScoreAdjust=-999
-
-[Install]
-WantedBy=multi-user.target

roles/openshift_node40/templates/origin-dns.conf.j2

@@ -1,17 +0,0 @@
-no-resolv
-domain-needed
-{% if openshift_node_dnsmasq_no_negcache %}
-no-negcache
-{% else %}
-neg-ttl={{ openshift_node_dnsmasq_neg_ttl }}
-{% endif %}
-max-cache-ttl=1
-enable-dbus
-dns-forward-max=10000
-cache-size=10000
-bind-dynamic
-min-port=1024
-{% for interface in openshift_node_dnsmasq_except_interfaces %}
-except-interface={{ interface }}
-{% endfor %}
-# End of config

roles/openshift_version/tasks/first_master.yml

@@ -1,5 +1,15 @@
 ---
-# Determine the openshift_version
+# Determine the openshift_version to configure if none has been specified or set previously.
+
+# Protect the installed version by default unless explicitly told not to, or given an
+# openshift_version already.
+- name: Use openshift_current_version fact as version to configure if already installed
+  set_fact:
+    openshift_version: "{{ openshift_current_version }}"
+  when:
+  - openshift_current_version is defined
+  - openshift_version is not defined or openshift_version == ""
+  - openshift_protect_installed_version | bool
 
 - name: Set openshift_version to openshift_release if undefined
   set_fact:

test/ci/README.md

@@ -0,0 +1,7 @@
+* Copy `test/ci/vars.yml.sample` to `test/ci/vars.yml`
+* Adjust it your liking - this would be the host configuration
+* Adjust `inventory/group_vars/OSEv3/vars.yml` - this would be Origin-specific config
+* Provision instances via `ansible-playbook -vv -i test/ci/inventory/ test/ci/launch.yml`
+  This would place inventory file in `test/ci/inventory/hosts` and run prerequisites and deploy.
+
+* Once the setup is complete run `ansible-playbook -vv -i test/ci/inventory/ test/ci/deprovision.yml`

test/ci/deprovision.yml

@@ -0,0 +1,45 @@
+---
+- hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+    - include_vars: "{{ item }}"
+      with_first_found:
+        - vars.yml
+        - vars.yaml
+
+    - name: Gather ec2 facts
+      ec2_instance_facts:
+        region: "{{ aws_region }}"
+        filters:
+          tag-key: "kubernetes.io/cluster/{{ aws_cluster_id }}"
+      register: ec2
+
+    - name: Terminate instances
+      ec2:
+        instance_ids: "{{ item.instance_id }}"
+        region: "{{ aws_region }}"
+        state: absent
+        wait: no
+      with_items: "{{ ec2.instances }}"
+      when: not aws_use_auto_terminator | default(true)
+
+    - when: aws_use_auto_terminator | default(true)
+      block:
+        - name: Stop VMs
+          ec2:
+            instance_ids: "{{ item.instance_id }}"
+            region: "{{ aws_region }}"
+            state: stopped
+            wait: no
+          with_items: "{{ ec2.instances }}"
+          ignore_errors: true
+
+        - name: Rename VMs
+          ec2_tag:
+            resource: "{{ item.instance_id }}"
+            region: "{{ aws_region }}"
+            tags:
+              Name: "{{ item.tags.Name }}-terminate"
+          when: "'-terminate' not in item.tags.Name"
+          with_items: "{{ ec2.instances }}"

test/ci/inventory/group_vars/OSEv3/vars.yml

@@ -0,0 +1,113 @@
+---
+ansible_become: true
+ansible_become_sudo: true
+
+openshift_deployment_type: origin
+openshift_repos_enable_testing: false
+
+#Minimal set of services
+openshift_web_console_install: true
+openshift_console_install: true
+openshift_metrics_install_metrics: false
+openshift_metrics_install_logging: false
+openshift_logging_install_logging: false
+openshift_management_install_management: false
+template_service_broker_install: false
+ansible_service_broker_install: false
+openshift_enable_service_catalog: false
+osm_use_cockpit: false
+openshift_monitoring_deploy: false
+openshift_metering_install: false
+openshift_metrics_server_install: false
+openshift_monitor_availability_install: false
+openshift_enable_olm: false
+openshift_descheduler_install: false
+openshift_node_problem_detector_install: false
+openshift_autoheal_deploy: false
+openshift_cluster_autoscaler_install: false
+
+# debugging
+debug_level: 4
+etcd_debug: true
+etcd_log_package_levels: 'auth=INFO,etcdmain=DEBUG,etcdserver=DEBUG'
+openshift_docker_options: "--log-driver=journald"
+
+#Disable journald persistence
+journald_vars_to_replace:
+  - { var: Storage, val: volatile }
+  - { var: Compress, val: no }
+  - { var: SyncIntervalSec, val: 1s }
+  - { var: RateLimitInterval, val: 1s }
+  - { var: RateLimitBurst, val: 10000 }
+  - { var: SystemMaxUse, val: 8G }
+  - { var: SystemKeepFree, val: 20% }
+  - { var: SystemMaxFileSize, val: 10M }
+  - { var: MaxRetentionSec, val: 1month }
+  - { var: MaxFileSec, val: 1day }
+  - { var: ForwardToSyslog, val: no }
+  - { var: ForwardToWall, val: no }
+
+#Other settings
+openshift_enable_origin_repo: false
+osm_default_node_selector: "node-role.kubernetes.io/compute=true"
+openshift_hosted_infra_selector: "node-role.kubernetes.io/infra=true"
+openshift_logging_es_nodeselector:
+  node-role.kubernetes.io/infra: "true"
+openshift_logging_es_ops_nodeselector:
+  node-role.kubernetes.io/infra: "true"
+osm_controller_args:
+  enable-hostpath-provisioner:
+    - "true"
+openshift_hosted_router_create_certificate: true
+openshift_master_audit_config:
+  enabled: true
+openshift_master_identity_providers:
+  - name: "allow_all"
+    login: "true"
+    challenge: "true"
+    kind: "AllowAllPasswordIdentityProvider"
+openshift_template_service_broker_namespaces:
+  - "openshift"
+enable_excluders: "true"
+osm_cluster_network_cidr: "10.128.0.0/14"
+openshift_portal_net: "172.30.0.0/16"
+osm_host_subnet_length: 9
+openshift_check_min_host_disk_gb: 1.5
+openshift_check_min_host_memory_gb: 1.9
+openshift_disable_check: package_update,package_availability,memory_availability,disk_availability
+
+openshift_logging_use_mux: false
+openshift_logging_use_ops: true
+openshift_logging_es_log_appenders:
+  - "console"
+openshift_logging_fluentd_journal_read_from_head: false
+openshift_logging_fluentd_audit_container_engine: true
+
+openshift_logging_curator_cpu_request: "100m"
+openshift_logging_curator_memory_limit: "32Mi"
+openshift_logging_curator_ops_cpu_request: "100m"
+openshift_logging_curator_ops_memory_limit: "32Mi"
+openshift_logging_elasticsearch_proxy_cpu_request: "100m"
+openshift_logging_elasticsearch_proxy_memory_limit: "32Mi"
+openshift_logging_es_cpu_request: "400m"
+openshift_logging_es_memory_limit: "4Gi"
+openshift_logging_es_ops_cpu_request: "400m"
+openshift_logging_es_ops_memory_limit: "4Gi"
+openshift_logging_eventrouter_cpu_request: "100m"
+openshift_logging_eventrouter_memory_limit: "64Mi"
+openshift_logging_fluentd_cpu_request: "100m"
+openshift_logging_fluentd_memory_limit: "256Mi"
+openshift_logging_kibana_cpu_request: "100m"
+openshift_logging_kibana_memory_limit: "128Mi"
+openshift_logging_kibana_ops_cpu_request: "100m"
+openshift_logging_kibana_ops_memory_limit: "128Mi"
+openshift_logging_kibana_ops_proxy_cpu_request: "100m"
+openshift_logging_kibana_ops_proxy_memory_limit: "64Mi"
+openshift_logging_kibana_proxy_cpu_request: "100m"
+openshift_logging_kibana_proxy_memory_limit: "64Mi"
+openshift_logging_mux_cpu_request: "400m"
+openshift_logging_mux_memory_limit: "256Mi"
+
+openshift_master_cluster_method: native
+
+openshift_node_port_range: '30000-32000'

test/ci/launch.yml

@@ -0,0 +1,112 @@
+---
+- hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+    - include_vars: "{{ item }}"
+      with_first_found:
+        - vars.yml
+        - vars.yaml
+
+    - name: list available AMIs
+      ec2_ami_facts:
+        region: "{{ aws_region }}"
+        filters: "{{ aws_ami_tags }}"
+      register: ami_facts
+      when: aws_image is not defined
+
+    - name: determine which AMI to use
+      set_fact:
+        aws_image: "{{ ami_facts.images[-1].image_id }}"
+      when: aws_image is not defined
+
+    - name: determine which AMI to use
+      set_fact:
+        aws_image: "{{ ami_facts.images[-1].image_id }}"
+      when: aws_image is not defined
+
+    - name: Create EC2 instance
+      ec2:
+        region: "{{ aws_region }}"
+        key_name: "{{ aws_key }}"
+        instance_type: "{{ item.aws_flavor }}"
+        image: "{{ item.aws_image | default(aws_image) }}"
+        wait: yes
+        group: "{{ item.aws_security_group }}"
+        count: 1
+        vpc_subnet_id: "{{ aws_subnet }}"
+        assign_public_ip: yes
+        instance_tags: "{{ aws_instance_tags }}"
+        volumes: "{{ item.aws_volumes | default(omit) }}"
+      register: ec2
+      with_items: "{{ aws_instances }}"
+      vars:
+        aws_instance_tags: |
+          {
+            "kubernetes.io/cluster/{{ aws_cluster_id }}": "true",
+            "Name": "{{ item.name }}",
+            "ansible-groups": "{{ item.ansible_groups | join(',') }}",
+            "ansible-node-group": "{{ item.node_group }}",
+            "expirationDate": "{{ item.aws_expiration_date | default(aws_expiration_date) }}"
+          }
+
+    - name: Add machine to inventory
+      add_host:
+        name: "{{ item.instances.0.tags['Name'] }}"
+        ansible_host: "{{ item.instances.0.dns_name }}"
+        ansible_user: "{{ item.instances.0.aws_user | default(aws_user)}}"
+        groups: "{{ item.instances.0.tags['ansible-groups'].split(',') }}"
+        aws_region: "{{ aws_region }}"
+        aws_ip: "{{ item.instances.0.public_ip }}"
+        aws_id: "{{ item.instances.0.id }}"
+        openshift_node_group_name: "{{ item.instances.0.tags['ansible-node-group'] }}"
+      with_items: "{{ ec2.results }}"
+
+    - name: write the inventory
+      template:
+        src: ./template-inventory.j2
+        dest: "inventory/hosts"
+
+    - name: Refresh inventory to ensure new instances exist in inventory
+      meta: refresh_inventory
+
+- hosts: all
+  gather_facts: no
+  become: true
+  tasks:
+    - wait_for_connection: {}
+    - name: Make sure hostname is set to public ansible host
+      hostname:
+        name: "{{ ansible_host }}"
+    - name: Detecting Operating System
+      shell: ls /run/ostree-booted
+      ignore_errors: yes
+      failed_when: false
+      register: ostree_output
+    - name: Update all packages
+      package:
+        name: '*'
+        state: latest
+      when: ostree_output.rc != 0
+      register: yum_update
+    - name: Update Atomic system
+      command: atomic host upgrade
+      when: ostree_output.rc == 0
+      register: ostree_update
+    - name: Reboot machines
+      shell: sleep 5 && systemctl reboot
+      async: 1
+      poll: 0
+      ignore_errors: true
+      when: yum_update | changed or ostree_update | changed
+    - name: Wait for connection
+      wait_for_connection:
+        connect_timeout: 20
+        sleep: 5
+        delay: 5
+        timeout: 300
+    - setup: {}
+
+- import_playbook: ../../playbooks/openshift-node/network_manager.yml
+- import_playbook: ../../playbooks/prerequisites.yml
+- import_playbook: ../../playbooks/deploy_cluster.yml

test/ci/template-inventory.j2

@@ -0,0 +1,26 @@
+[OSEv3:vars]
+ansible_python_interpreter="{{ python }}"
+ansible_user="{{ aws_user }}"
+aws_region="{{ aws_region }}"
+openshift_master_default_subdomain="{{ hostvars[groups[('lb' in groups) | ternary('lb', 'masters')][0]]["aws_ip"] }}.xip.io"
+
+[OSEv3:children]
+{% for group in groups %}
+{% if group not in ["all", "ungrouped", "OSEv3"] %}
+{{group}}
+{% endif %}
+{% endfor %}
+
+{% for group in groups %}
+{% if group not in ["all", "ungrouped", "OSEv3"] %}
+[{{group}}]
+{% for entry in groups[group] %}
+{% set addon_opts = "" %}
+{% if group == "nodes" %}
+{% set addon_opts = addon_opts + " openshift_node_group_name='" + hostvars[entry]['openshift_node_group_name'] + "'" %}
+{% endif %}
+{{ entry }} ansible_host='{{ hostvars[entry]['ansible_host'] }}' aws_id='{{ hostvars[entry]['aws_id'] }}' {{ addon_opts }}
+{% endfor %}
+{% endif %}
+
+{% endfor %}

test/ci/vars.yml.sample

@@ -0,0 +1,46 @@
+---
+vm_prefix: "ci_test"
+#aws_use_auto_terminator is set to True by default, as rh-dev account doesn't have permission
+# to terminate instances. These should be stopped and renamed to include 'terminate' instead
+#aws_use_auto_terminator: false
+
+type: aws
+aws_user: "ec2-user"
+python: "/usr/bin/python"
+
+aws_key: "libra"
+aws_region: "us-east-1"
+aws_cluster_id: "ci"
+# us-east-1d
+aws_subnet: "subnet-cf57c596"
+
+aws_expiration_date: "{{ lookup('pipe','date -d \"4 hours\" --iso=minutes --utc') }}"
+
+aws_ami_tags:
+  "tag:operating_system": "rhel"
+  "tag:image_stage": "base"
+  "tag:ready": "yes"
+
+aws_instances:
+- name: "{{ vm_prefix }}-master"
+  ansible_groups:
+    - masters
+    - etcd
+    - nodes
+  aws_flavor: t2.large
+  aws_security_group: public
+  node_group: "node-config-all-in-one"
+  # Use custom AMI tags
+  # aws_ami_tags:
+  #   operating_system: "rhel"
+  #   image_stage: "base"
+  #   ready: "yes"
+  # Use custom AMI
+  #aws_image: "ami-70e8fd66"
+  # Attach custom volumes
+  #aws_volumes:
+  # - device_name: /dev/sdb
+  #   volume_size: 50
+  #   delete_on_termination: yes
+  #Set expiration date for instances on CI namespace
+  #aws_expiration_date: "{{ lookup('pipe','date -d \"4 hours\" --iso=minutes --utc') }}"

test/gcp/install.yml

@@ -1,29 +0,0 @@
-# This playbook installs onto a provisioned cluster
-#TODO: split into parts: nodes.yml, bootstrap.yml, masters.yml, workers.yml, bootkube/post_setup.yml
----
-- hosts: localhost
-  connection: local
-  tasks:
-  - name: place all scale groups into Ansible groups
-    include_role:
-      name: openshift_gcp
-      tasks_from: setup_scale_group_facts.yml
-
-- hosts: nodes
-  tasks:
-  - name: Disable google hostname updater
-    file:
-      path: /etc/dhcp/dhclient.d/google_hostname.sh
-      mode: 0644
-
-- name: run the deploy_cluster_40
-  import_playbook: ../../playbooks/deploy_cluster_40.yml
-
-- name: destroy bootstrap node
-  hosts: localhost
-  connection: local
-  tasks:
-  - name: Scale down bootstrap node and update public API DNS record
-    include_role:
-      name: openshift_gcp
-      tasks_from: remove_bootstrap.yml

test/gcp/roles

@@ -1 +0,0 @@
-../../roles