Merge pull request #7723 from smarterclayton/upgrade

Upgrade to 3.10 with static pods

inventory/dynamic/gcp/group_vars/all/00_defaults.yml

@@ -34,10 +34,3 @@ openshift_node_sdn_mtu: 1410
 osm_cluster_network_cidr: 172.16.0.0/16
 osm_host_subnet_length: 9
 openshift_portal_net: 172.30.0.0/16
-
-# Default cluster configuration
-openshift_master_cluster_method: native
-openshift_schedulable: true
-# TODO: change to upstream conventions
-openshift_hosted_infra_selector: "role=infra"
-osm_default_node_selector: "role=app"

playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml

@@ -57,6 +57,8 @@
   - import_role:
       name: openshift_node
       tasks_from: upgrade.yml
+    vars:
+      r_node_dynamic_config_name: "{{ openshift_node_group_name | default('node-config-compute') }}"
 
   # Run the upgrade hook prior to make the node schedulable again.
   - debug: msg="Running node upgrade hook {{ openshift_node_upgrade_hook }}"

playbooks/common/openshift-cluster/upgrades/v3_10/label_nodes.yml

@@ -0,0 +1,25 @@
+---
+
+- name: Update all labels
+  hosts: oo_masters_to_config
+  roles:
+    - openshift_facts
+    - lib_openshift
+  tasks:
+    - import_role:
+        name: openshift_manage_node
+        tasks_from: config.yml
+      vars:
+        openshift_master_host: '{{ groups.oo_first_master.0 }}'
+
+- name: Update node labels to differentiate from (now-schedulable) masters
+  hosts: oo_first_master
+  roles:
+    - openshift_facts
+    - lib_openshift
+  tasks:
+    - import_role:
+        name: openshift_manage_node
+        tasks_from: set_default_node_role.yml
+      vars:
+        openshift_master_host: '{{ groups.oo_first_master.0 }}'

playbooks/common/openshift-cluster/upgrades/v3_10/upgrade_control_plane.yml

@@ -15,6 +15,7 @@
   vars:
     l_upgrade_no_switch_firewall_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
     l_init_fact_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
+    l_base_packages_hosts: "oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config"
 
 - name: Configure the upgrade target for the common upgrade tasks 3.10
   hosts: oo_masters_to_config:oo_etcd_to_config:oo_lb_to_config
@@ -44,6 +45,52 @@
   - set_fact:
       pre_upgrade_complete: True
 
+- import_playbook: label_nodes.yml
+
+# To upgrade, we need masters to be capable of signing certificates
+- hosts: oo_masters
+  serial: 1
+  tasks:
+  - name: Enable core bootstrapping components
+    include_tasks: ../../../../openshift-master/private/tasks/enable_bootstrap.yml
+  - name: Place shim commands on the masters before we begin the upgrade
+    import_role:
+      name: openshift_control_plane
+      tasks_from: static_shim
+
+# TODO: need to verify settings about the bootstrap configs
+# 1. Does network policy match the master config
+
+- name: Configure components that must be available prior to upgrade
+  hosts: oo_first_master
+  pre_tasks:
+  - name: Enable core bootstrapping components
+    include_tasks: ../../../../openshift-master/private/tasks/enable_bootstrap_config.yml
+  - name: Ensure the master bootstrap config has bootstrapping config
+    import_role:
+      name: openshift_node_group
+      tasks_from: upgrade
+  - name: Enable node configuration reconciliation
+    import_role:
+      name: openshift_node_group
+      tasks_from: sync
+  roles:
+  - role: openshift_sdn
+    when: openshift_use_openshift_sdn | default(True) | bool
+
+- name: Update master nodes
+  hosts: oo_masters
+  serial: 1
+  tasks:
+  - import_role:
+      name: openshift_node
+      tasks_from: upgrade_pre
+  - import_role:
+      name: openshift_node
+      tasks_from: upgrade
+    vars:
+      r_node_dynamic_config_name: "{{ openshift_node_group_name | default('node-config-master') }}"
+
 - import_playbook: ../upgrade_control_plane.yml
   vars:
     openshift_release: '3.10'

playbooks/gcp/openshift-cluster/upgrade.yml

@@ -0,0 +1,14 @@
+# This playbook installs onto a provisioned cluster
+---
+- hosts: localhost
+  connection: local
+  tasks:
+  - name: place all scale groups into Ansible groups
+    include_role:
+      name: openshift_gcp
+      tasks_from: setup_scale_group_facts.yml
+    vars:
+      all_nodes: true
+
+- name: run the upgrade
+  import_playbook: ../../common/openshift-cluster/upgrades/v3_10/upgrade.yml

playbooks/init/basic_facts.yml

@@ -35,7 +35,7 @@
     register: bootstrap_node_config_path_check
   - name: initialize_facts set fact openshift_is_bootstrapped
     set_fact:
-      openshift_is_bootstrapped: "{{ bootstrap_node_config_path_check.stat.exists }}"
+      openshift_is_bootstrapped: "{{ openshift_is_bootstrapped|default(False) or bootstrap_node_config_path_check.stat.exists }}"
 
   - name: initialize_facts set fact openshift_is_atomic and openshift_is_containerized
     set_fact:

playbooks/init/cluster_facts.yml

@@ -45,6 +45,7 @@
       role: node
       local_facts:
         sdn_mtu: "{{ openshift_node_sdn_mtu | default(None) }}"
+        bootstrapped: "{{ openshift_is_bootstrapped }}"
 
 - name: Initialize etcd host variables
   hosts: oo_masters_to_config

playbooks/init/evaluate_groups.yml

@@ -143,15 +143,23 @@
     with_items: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}"
     changed_when: no
 
-  # Skip adding the master to oo_nodes_to_config when g_new_node_hosts is
-  - name: Add master to oo_nodes_to_config
+  - name: Evaluate oo_nodes_to_bootstrap
     add_host:
       name: "{{ item }}"
-      groups: oo_nodes_to_config
+      groups: oo_nodes_to_bootstrap
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_become: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}"
+    changed_when: no
+    when: hostvars[item].openshift_node_bootstrap | default(True) | bool
+
+  - name: Add masters to oo_nodes_to_bootstrap
+    add_host:
+      name: "{{ item }}"
+      groups: oo_nodes_to_bootstrap
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_become: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ g_master_hosts | default([]) }}"
-    when: g_nodeonmaster | default(false) | bool and not g_new_node_hosts | default(false) | bool
+    with_items: "{{ groups.oo_masters | default([]) }}"
     changed_when: no
 
   - name: Evaluate oo_lb_to_config

playbooks/openshift-etcd/private/upgrade_backup.yml

@@ -8,6 +8,7 @@
     vars:
       r_etcd_common_backup_tag: "{{ etcd_backup_tag }}"
       r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+      r_etcd_common_skip_command_shim: true
 
 - name: Gate on etcd backup
   hosts: localhost

playbooks/openshift-master/private/additional_config.yml

@@ -21,6 +21,7 @@
   # TODO: this is currently required in order to schedule pods onto the masters, but
   #   should be moved into components once nodes are using dynamic config
   - role: openshift_sdn
+    when: openshift_use_openshift_sdn | default(True) | bool
   - role: openshift_project_request_template
     when: openshift_project_request_template_manage
   - role: openshift_examples

playbooks/openshift-master/private/config.yml

@@ -71,6 +71,10 @@
         console_url: "{{ openshift_master_console_url | default(None) }}"
         console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
         public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
+  - openshift_facts:
+      role: node
+      local_facts:
+        bootstrapped: true
 
 - name: Inspect state of first master config settings
   hosts: oo_first_master
@@ -79,7 +83,7 @@
   tasks:
   - import_role:
       name: openshift_control_plane
-      tasks_from: check_existing_config.yml
+      tasks_from: check_existing_config
 
 - name: Configure masters
   hosts: oo_masters_to_config
@@ -96,7 +100,8 @@
       name: openshift_node_group
       tasks_from: bootstrap_config
     vars:
-      r_node_bootstrap_config_name: "{{ openshift_node_group_master | default('node-config-master') }}"
+      r_node_dynamic_config_name: "{{ openshift_node_group_name | default('node-config-master') }}"
+      r_node_dynamic_config_force: True
 
   roles:
   - role: openshift_master_facts
@@ -119,6 +124,7 @@
     when: openshift_use_calico | default(false) | bool
   tasks:
   - name: Set up automatic node config reconcilation
+    run_once: True
     import_role:
       name: openshift_node_group
       tasks_from: sync
@@ -135,6 +141,22 @@
     group_by: key=oo_masters_deployment_type_{{ openshift_deployment_type }}
     changed_when: False
 
+- name: Deploy the central bootstrap configuration
+  hosts: oo_first_master
+  gather_facts: no
+  tasks:
+  - name: setup bootstrap settings
+    import_tasks: tasks/enable_bootstrap_config.yml
+
+- name: Ensure inventory labels are assigned to masters
+  hosts: oo_masters_to_config
+  gather_facts: no
+  roles:
+  - role: openshift_manage_node
+    openshift_master_host: "{{ groups.oo_first_master.0 }}"
+    openshift_manage_node_is_master: "{{ ('oo_masters_to_config' in group_names) | bool }}"
+    openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
+
 - name: Re-enable excluder if it was previously enabled
   hosts: oo_masters_to_config
   gather_facts: no

playbooks/openshift-master/private/enable_bootstrap.yml

@@ -5,3 +5,10 @@
   tasks:
   - name: include bootstrapping tasks
     import_tasks: tasks/enable_bootstrap.yml
+
+- name: Enable bootstrapping configuration
+  hosts: oo_first_master
+  gather_facts: no
+  tasks:
+  - name: include bootstrapping tasks
+    import_tasks: tasks/enable_bootstrap_config.yml

playbooks/openshift-master/private/tasks/enable_bootstrap.yml

@@ -8,15 +8,3 @@
   import_role:
     name: openshift_control_plane
     tasks_from: bootstrap.yml
-
-- name: Setup the node group config maps
-  import_role:
-    name: openshift_node_group
-  run_once: True
-
-- name: Setup the node bootstrap auto approver
-  import_role:
-    name: openshift_bootstrap_autoapprover
-  run_once: True
-  when:
-  - openshift_master_bootstrap_auto_approve | default(False) | bool

playbooks/openshift-master/private/tasks/enable_bootstrap_config.yml

@@ -0,0 +1,10 @@
+---
+- name: Setup the node group config maps
+  import_role:
+    name: openshift_node_group
+
+- name: Setup the node bootstrap auto approver
+  import_role:
+    name: openshift_bootstrap_autoapprover
+  when:
+  - openshift_master_bootstrap_auto_approve | default(False) | bool

playbooks/openshift-master/private/upgrade.yml

@@ -69,7 +69,7 @@
 
   - import_role:
       name: openshift_control_plane
-      tasks_from: upgrade.yml
+      tasks_from: upgrade
 
   - name: update vsphere provider master config
     import_role:
@@ -80,9 +80,6 @@
     - openshift_cloudprovider_kind == 'vsphere'
     - openshift_version | version_compare('3.9', '>=')
 
-  - name: Setup and enable bootstrapping options
-    import_tasks: tasks/enable_bootstrap.yml
-
   # Run the upgrade hook prior to restarting services/system if defined:
   - debug: msg="Running master upgrade hook {{ openshift_master_upgrade_hook }}"
     when: openshift_master_upgrade_hook is defined
@@ -90,11 +87,10 @@
   - include_tasks: "{{ openshift_master_upgrade_hook }}"
     when: openshift_master_upgrade_hook is defined
 
-  - name: Disable master controller
-    service:
-      name: "{{ openshift_service_type }}-master-controllers"
-      enabled: false
-    when: openshift_rolling_restart_mode | default('services') == 'system'
+  - name: Lay down the static configuration
+    import_role:
+      name: openshift_control_plane
+      tasks_from: static.yml
 
   - import_tasks: tasks/restart_hosts.yml
     when: openshift_rolling_restart_mode | default('services') == 'system'
@@ -290,44 +286,11 @@
     import_role:
       name: lib_openshift
 
-  # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node
-  # or docker actually needs an upgrade before proceeding. Perhaps best to save this until
-  # we merge upgrade functionality into the base roles and a normal config.yml playbook run.
-  - name: Mark node unschedulable
-    oc_adm_manage_node:
-      node: "{{ openshift.node.nodename | lower }}"
-      schedulable: False
-    delegate_to: "{{ groups.oo_first_master.0 }}"
-    retries: 10
-    delay: 5
-    register: node_unschedulable
-    until: node_unschedulable is succeeded
-
-  - name: Drain Node for Kubelet upgrade
-    command: >
-      {{ hostvars[groups.oo_first_master.0]['first_master_client_binary'] }} adm drain {{ openshift.node.nodename | lower }}
-      --config={{ openshift.common.config_base }}/master/admin.kubeconfig
-      --force --delete-local-data --ignore-daemonsets
-      --timeout={{ openshift_upgrade_nodes_drain_timeout | default(0) }}s
-    delegate_to: "{{ groups.oo_first_master.0 }}"
-    register: l_upgrade_control_plane_drain_result
-    until: not (l_upgrade_control_plane_drain_result is failed)
-    retries: "{{ 1 if ( openshift_upgrade_nodes_drain_timeout | default(0) | int ) == 0 else 0 }}"
-    delay: 5
-    failed_when:
-    - l_upgrade_control_plane_drain_result is failed
-    - openshift_upgrade_nodes_drain_timeout | default(0) | int == 0
-
   roles:
   - openshift_facts
+
   post_tasks:
   - import_role:
-      name: openshift_node
-      tasks_from: upgrade_pre.yml
-  - import_role:
-      name: openshift_node
-      tasks_from: upgrade.yml
-  - import_role:
       name: openshift_manage_node
       tasks_from: config.yml
     vars:

playbooks/openshift-node/private/additional_config.yml

@@ -1,6 +1,6 @@
 ---
 - name: create additional node network plugin groups
-  hosts: "{{ openshift_node_scale_up_group | default('oo_nodes_to_config') }}:!oo_exclude_nodes_to_config"
+  hosts: "{{ openshift_node_scale_up_group | default('oo_nodes_to_config') }}:!oo_nodes_to_bootstrap"
   tasks:
   # Creating these node groups will prevent a ton of skipped tasks.
   # Create group for flannel nodes

playbooks/openshift-node/private/bootstrap.yml

@@ -11,31 +11,35 @@
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
-- import_playbook: setup_bootstrap.yml
+- name: Only target nodes that have not yet been bootstrapped
+  hosts: localhost
+  tasks:
+  - add_host:
+      name: "{{ item }}"
+      groups: oo_exclude_bootstrapped_nodes
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_become: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ groups.oo_nodes_to_bootstrap | default([]) }}"
+    changed_when: no
+    when: hostvars[item].openshift_is_bootstrapped | default(False) | bool
+
+- import_playbook: disable_excluders.yml
+  vars:
+    l_node_group: oo_nodes_to_bootstrap:!oo_exclude_bootstrapped_nodes
 
 - import_playbook: configure_nodes.yml
+  vars:
+    l_node_group: oo_nodes_to_bootstrap:!oo_exclude_bootstrapped_nodes
 
-- name: node bootstrap config
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
-  tasks:
-  - import_role:
-      name: openshift_node
-      tasks_from: bootstrap
-  - import_role:
-      name: openshift_node_group
-      tasks_from: bootstrap
-  - name: Have the nodes automatically pull their configuration
-    import_role:
-      name: openshift_node_group
-      tasks_from: bootstrap_config
-    vars:
-      r_node_bootstrap_config_name: "{{ openshift_node_group_name | default('node-config-compute') }}"
-  - set_fact:
-      openshift_is_bootstrapped: True
+- import_playbook: configure_bootstrap.yml
 
 - import_playbook: enable_excluders.yml
+  vars:
+    l_node_group: oo_nodes_to_bootstrap:!oo_exclude_bootstrapped_nodes
 
 - import_playbook: clean_image.yml
+  vars:
+    l_node_group: oo_nodes_to_bootstrap:!oo_exclude_bootstrapped_nodes
 
 - name: Node Preparation Checkpoint End
   hosts: all

playbooks/openshift-node/private/certificates-backup.yml

@@ -14,7 +14,7 @@
       warn: no
 
 - name: Redeploy node certificates
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
+  hosts: oo_nodes_to_config:!oo_nodes_to_bootstrap
   pre_tasks:
   - name: Remove CA certificate
     file:

playbooks/openshift-node/private/certificates.yml

@@ -1,8 +1,7 @@
 ---
 - name: Create OpenShift certificates for node hosts
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
+  hosts: oo_nodes_to_config:!oo_nodes_to_bootstrap
   gather_facts: no
   roles:
   - role: openshift_node_certificates
     openshift_ca_host: "{{ groups.oo_first_master.0 }}"
-    when: not openshift_node_bootstrap | default(false) | bool

playbooks/openshift-node/private/clean_image.yml

@@ -1,6 +1,6 @@
 ---
 - name: Configure nodes
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
+  hosts: "{{ l_node_group }}"
   tasks:
   - name: Remove any ansible facts created during AMI creation
     file:

playbooks/openshift-node/private/config.yml

@@ -11,17 +11,23 @@
           status: "In Progress"
           start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
-- import_playbook: setup.yml
+- import_playbook: disable_excluders.yml
+  vars:
+    l_node_group: oo_nodes_to_config:!oo_nodes_to_bootstrap
 
 - import_playbook: certificates.yml
 
 - import_playbook: configure_nodes.yml
+  vars:
+    l_node_group: oo_nodes_to_config:!oo_nodes_to_bootstrap
 
 - import_playbook: additional_config.yml
 
 - import_playbook: manage_node.yml
 
 - import_playbook: enable_excluders.yml
+  vars:
+    l_node_group: oo_nodes_to_config:!oo_nodes_to_bootstrap
 
 - name: Node Install Checkpoint End
   hosts: all

playbooks/openshift-node/private/configure_bootstrap.yml

@@ -0,0 +1,18 @@
+---
+- name: node bootstrap config
+  hosts: oo_nodes_to_bootstrap:!oo_exclude_bootstrapped_nodes
+  tasks:
+  - import_role:
+      name: openshift_node
+      tasks_from: bootstrap
+  - import_role:
+      name: openshift_node_group
+      tasks_from: bootstrap
+  - name: Have the nodes automatically pull their configuration
+    import_role:
+      name: openshift_node_group
+      tasks_from: bootstrap_config
+    vars:
+      r_node_dynamic_config_name: "{{ openshift_node_group_name | default('node-config-compute') }}"
+  - set_fact:
+      openshift_is_bootstrapped: True

playbooks/openshift-node/private/configure_nodes.yml

@@ -1,6 +1,6 @@
 ---
 - name: Configure nodes
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
+  hosts: "{{ l_node_group }}"
   vars:
     openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
     openshift_node_first_master_ip: "{{ hostvars[groups.oo_first_master.0].openshift.common.ip }}"

playbooks/openshift-node/private/disable_excluders.yml

@@ -0,0 +1,7 @@
+---
+- name: Disable excluders
+  hosts: "{{ l_node_group }}"
+  gather_facts: no
+  roles:
+  - role: openshift_excluder
+    r_openshift_excluder_action: disable

playbooks/openshift-node/private/enable_excluders.yml

@@ -1,6 +1,6 @@
 ---
 - name: Re-enable excluder if it was previously enabled
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
+  hosts: "{{ l_node_group }}"
   gather_facts: no
   roles:
   - role: openshift_excluder

playbooks/openshift-node/private/image_prep.yml

@@ -9,10 +9,14 @@
     l_openshift_version_check_hosts: "all:!all"
 
 - name: run node config setup
-  import_playbook: setup.yml
+  import_playbook: disable_excluders.yml
+  vars:
+    l_node_group: oo_nodes_to_config
 
 - name: run node config
   import_playbook: configure_nodes.yml
+  vars:
+    l_node_group: oo_nodes_to_config
 
 - name: node bootstrap config
   hosts: oo_nodes_to_config
@@ -24,9 +28,12 @@
         name: openshift_node_group
         tasks_from: bootstrap.yml
 
-
 - name: Re-enable excluders
   import_playbook: enable_excluders.yml
+  vars:
+    l_node_group: oo_nodes_to_config
 
 - name: Remove any undesired artifacts from build
   import_playbook: clean_image.yml
+  vars:
+    l_node_group: oo_nodes_to_config

playbooks/openshift-node/private/join.yml

@@ -1,22 +1,4 @@
 ---
-- name: Evaluate bootstrapped nodes
-  hosts: localhost
-  gather_facts: no
-  connection: local
-  tasks:
-  - name: Add all nodes that are bootstrapped
-    add_host:
-      name: "{{ item }}"
-      groups: oo_nodes_to_bootstrap
-      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
-      ansible_become: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ groups.oo_nodes_to_config | default([]) }}"
-    when:
-    - hostvars[item].openshift is defined
-    - hostvars[item].openshift.common is defined
-    - (hostvars[item].openshift_is_bootstrapped | bool) or (hostvars[item].openshift_node_bootstrap | default(True) | bool)
-    changed_when: False
-
 - name: Distribute bootstrap and start nodes
   hosts: oo_nodes_to_bootstrap
   gather_facts: no

playbooks/openshift-node/private/manage_node.yml

@@ -1,6 +1,6 @@
 ---
 - name: Additional node config
-  hosts: "{{ openshift_node_scale_up_group | default('oo_nodes_to_config') }}:!oo_exclude_nodes_to_config"
+  hosts: "{{ openshift_node_scale_up_group | default('oo_nodes_to_config') }}:!oo_nodes_to_bootstrap"
   vars:
     openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
   roles:

playbooks/openshift-node/private/setup.yml

@@ -1,24 +0,0 @@
----
-- name: Evaluate node groups
-  hosts: localhost
-  connection: local
-  tasks:
-  - name: Evaluate oo_exclude_nodes_to_config as all nodes that have already been bootstrapped
-    add_host:
-      name: "{{ item }}"
-      groups: oo_exclude_nodes_to_config
-      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
-      ansible_become: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ groups.oo_nodes_to_config | default([]) }}"
-    when:
-    - hostvars[item].openshift is defined
-    - hostvars[item].openshift.common is defined
-    - hostvars[item].openshift_is_bootstrapped | bool
-    changed_when: False
-
-- name: Disable excluders
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
-  gather_facts: no
-  roles:
-  - role: openshift_excluder
-    r_openshift_excluder_action: disable

playbooks/openshift-node/private/setup_bootstrap.yml

@@ -1,23 +0,0 @@
----
-# We exclude all nodes that have already been bootstrapped or have requested not to be bootstrapped
-- name: Evaluate node groups
-  hosts: localhost
-  connection: local
-  tasks:
-  - name: Evaluate oo_exclude_nodes_to_config as all nodes that shouldn't be configured for bootstrapping
-    add_host:
-      name: "{{ item }}"
-      groups: oo_exclude_nodes_to_config
-      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
-      ansible_become: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ groups.oo_nodes_to_config | default([]) }}"
-    when:
-    - (not (hostvars[item].openshift_node_bootstrap | default(True) | bool)) or (hostvars[item].openshift_is_bootstrapped | bool)
-    changed_when: False
-
-- name: Disable excluders
-  hosts: oo_nodes_to_config:!oo_exclude_nodes_to_config
-  gather_facts: no
-  roles:
-  - role: openshift_excluder
-    r_openshift_excluder_action: disable

roles/etcd/defaults/main.yaml

@@ -5,18 +5,19 @@ r_etcd_common_backup_sufix_name: ''
 l_is_etcd_system_container: "{{ (openshift_use_etcd_system_container | default(openshift_use_system_containers | default(false)) | bool) }}"
 
 # runc, docker, host
-r_etcd_common_etcd_runtime: "{{ 'runc' if l_is_etcd_system_container else 'docker' if openshift_is_containerized else 'host' }}"
+r_etcd_common_etcd_runtime: "{{ 'runc' if l_is_etcd_system_container else ('docker' if openshift_is_containerized else 'host') }}"
 
-osm_etcd_image: 'registry.access.redhat.com/rhel7/etcd'
+r_etcd_default_version: "3.2.15"
+osm_etcd_image: "registry.access.redhat.com/rhel7/etcd:{{ r_etcd_upgrade_version | default(r_etcd_default_version) }}"
 etcd_image_dict:
-  origin: "registry.fedoraproject.org/latest/etcd"
+  origin: "quay.io/coreos/etcd:v{{ r_etcd_upgrade_version | default(r_etcd_default_version) }}"
   openshift-enterprise: "{{ osm_etcd_image }}"
 etcd_image: "{{ etcd_image_dict[openshift_deployment_type | default('origin')] }}"
 
 # etcd run on a host => use etcdctl command directly
 # etcd run as a docker container => use docker exec
 # etcd run as a runc container => use runc exec
-r_etcd_common_etcdctl_command: "{{ 'etcdctl' if r_etcd_common_etcd_runtime == 'host' | bool else 'docker exec etcd_container etcdctl' if r_etcd_common_etcd_runtime == 'docker' else 'runc exec etcd etcdctl' }}"
+r_etcd_common_etcdctl_command: "{{ 'etcdctl' if (r_etcd_common_etcd_runtime == 'host') else ('docker exec etcd_container etcdctl' if (r_etcd_common_etcd_runtime == 'docker') else 'runc exec etcd etcdctl') }}"
 
 # etcd server vars
 etcd_conf_dir: '/etc/etcd'

roles/etcd/tasks/backup/backup.yml

@@ -49,10 +49,17 @@
   - l_etcd_selinux_labels.rc == 0
   - "'svirt_sandbox_file_t' not in l_etcd_selinux_labels.stdout"
 
-- name: Generate etcd backup
+- name: Generate etcd backup (legacy)
   command: >
     {{ r_etcd_common_etcdctl_command }} backup --data-dir={{ l_etcd_incontainer_data_dir }}
     --backup-dir={{ l_etcd_incontainer_backup_dir }}
+  when: r_etcd_common_skip_command_shim | default(False) | bool
+
+- name: Generate etcd backup (static pod)
+  command: >
+    /usr/local/bin/master-exec etcd etcd etcdctl backup --data-dir={{ l_etcd_incontainer_data_dir }}
+    --backup-dir={{ l_etcd_incontainer_backup_dir }}
+  when: not (r_etcd_common_skip_command_shim | default(False) | bool)
 
 # According to the docs change you can simply copy snap/db
 # https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6

roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml

@@ -180,8 +180,6 @@
   file:
     path: "{{ item }}"
     mode: 0600
-    owner: "{{ 'etcd' if not openshift_is_bootstrapped else omit }}"
-    group: "{{ 'etcd' if not openshift_is_bootstrapped else omit }}"
   when: etcd_url_scheme == 'https'
   with_items:
   - "{{ etcd_ca_file }}"
@@ -192,8 +190,6 @@
   file:
     path: "{{ item }}"
     mode: 0600
-    owner: "{{ 'etcd' if not openshift_is_bootstrapped else omit }}"
-    group: "{{ 'etcd' if not openshift_is_bootstrapped else omit }}"
   when: etcd_peer_url_scheme == 'https'
   with_items:
   - "{{ etcd_peer_ca_file }}"
@@ -204,6 +200,4 @@
   file:
     path: "{{ etcd_conf_dir }}"
     state: directory
-    owner: "{{ 'etcd' if not openshift_is_bootstrapped else omit }}"
-    group: "{{ 'etcd' if not openshift_is_bootstrapped else omit }}"
     mode: 0700

roles/etcd/tasks/upgrade/upgrade_static.yml

@@ -0,0 +1,26 @@
+---
+# PREREQ Node service is ready to run static pods
+
+# INPUT r_etcd_upgrade_version
+- name: Verify cluster is healthy pre-upgrade
+  command: "{{ etcdctlv2 }} cluster-health"
+
+- name: Stop etcd
+  systemd:
+    name: "{{ etcd_service }}"
+    state: stopped
+
+- name: Remove old service file
+  file:
+    path: "{{ etcd_service }}"
+    state: absent
+
+- name: Configure static definition
+  import_tasks: ../static.yml
+
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 3
+  delay: 10

roles/etcd/tasks/upgrade_image.yml

@@ -1,2 +1,6 @@
 ---
+- include_tasks: upgrade/upgrade_static.yml
+  when: openshift_master_bootstrap_enabled | default(False) | bool
+
 - include_tasks: upgrade/upgrade_image.yml
+  when: not openshift_master_bootstrap_enabled | default(False) | bool

roles/etcd/tasks/upgrade_rpm.yml

@@ -1,2 +1,6 @@
 ---
+- include_tasks: upgrade/upgrade_static.yml
+  when: openshift_master_bootstrap_enabled | default(False) | bool
+
 - include_tasks: upgrade/upgrade_rpm.yml
+  when: not openshift_master_bootstrap_enabled | default(False) | bool

roles/lib_utils/filter_plugins/openshift_master.py

@@ -488,6 +488,11 @@ class FilterModule(object):
         certs = ['admin.crt',
                  'admin.key',
                  'admin.kubeconfig',
+                 'aggregator-front-proxy.crt',
+                 'aggregator-front-proxy.key',
+                 'aggregator-front-proxy.kubeconfig',
+                 'front-proxy-ca.crt',
+                 'front-proxy-ca.key',
                  'master.kubelet-client.crt',
                  'master.kubelet-client.key',
                  'master.proxy-client.crt',

roles/openshift_bootstrap_autoapprover/tasks/main.yml

@@ -12,7 +12,6 @@
     - "files/*.yaml"
 
 - name: Update the image tag
-  run_once: true
   yedit:
     src: "{{ mktemp.stdout }}/openshift-bootstrap-images.yaml"
     key: 'tag.from.name'
@@ -25,13 +24,11 @@
     --config={{ openshift.common.config_base }}/master/admin.kubeconfig
 
 - name: Apply the config
-  run_once: true
   command: >
     {{ openshift_client_binary }} apply -f "{{ mktemp.stdout }}"
     --config={{ openshift.common.config_base }}/master/admin.kubeconfig
 
 - name: Remove temp directory
-  run_once: true
   file:
     state: absent
     name: "{{ mktemp.stdout }}"

roles/openshift_control_plane/files/scripts/docker/master-exec

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+# Exec a file in the named component by component name and container name.
+# Remaining arguments are passed to the command. If no static pods have been
+# created yet this will execute on the host.
+if [[ -z "${1-}" || -z "${2-}" ]]; then
+  echo "A component name like 'api', 'etcd', or 'controllers' must be specified along with the container name within that component." 1>&2
+  exit 1
+fi
+
+# We haven't started using static pods yet, assume this command is to be direct executed
+if [[ ! -d /etc/origin/node/pods || -z "$( ls -A /etc/origin/node/pods )" ]]; then
+  exec "${@:3}"
+fi
+
+# TODO: move to cri-ctl
+# TODO: short term hack for cri-o
+
+uid=$(docker ps -l -a --filter "label=openshift.io/component=${1}" --filter "label=io.kubernetes.container.name=POD" --format '{{ .Label "io.kubernetes.pod.uid" }}')
+if [[ -z "${uid}" ]]; then
+  echo "Component ${1} is stopped or not running" 1>&2
+  exit 0
+fi
+container=$(docker ps -l -a -q --filter "label=io.kubernetes.pod.uid=${uid}" --filter "label=io.kubernetes.container.name=${2}")
+exec docker exec "${container}" "${@:3}"

roles/openshift_control_plane/tasks/generate_session_secrets.yml

@@ -13,6 +13,13 @@
   register: l_osm_session_secrets_slurp
   when: l_osm_session_secrets_stat.stat.exists
 
+- name: slurp session secrets if defined
+  slurp:
+    src: "{{ openshift_master_session_secrets_file }}"
+  register: osm_session_secrets
+  no_log: true
+  when: l_osm_session_secrets_stat.stat.exists
+
 # lib_utils_oo_collect is a custom filter in
 # roles/lib_utils/filter_plugins/oo_filters.py
 - name: Gather existing session secrets from first master

roles/openshift_control_plane/tasks/main.yml

@@ -153,6 +153,18 @@
 
 - import_tasks: static.yml
 
+- name: Establish the default bootstrap kubeconfig for masters
+  copy:
+    remote_src: true
+    src: "/etc/origin/master/admin.kubeconfig"
+    dest: "{{ item }}"
+    mode: 0600
+  with_items:
+  # bootstrap as an admin
+  - /etc/origin/node/bootstrap.kubeconfig
+  # copy to this location to bypass initial bootstrap request
+  - /etc/origin/node/node.kubeconfig
+
 - name: Start and enable self-hosting node
   systemd:
     name: "{{ openshift_service_type }}-node"

roles/openshift_control_plane/tasks/static.yml

@@ -49,15 +49,3 @@
     name: "{{ mktemp.stdout }}"
     state: absent
   changed_when: False
-
-- name: Establish the default bootstrap kubeconfig for masters
-  copy:
-    remote_src: true
-    src: "/etc/origin/master/admin.kubeconfig"
-    dest: "{{ item }}"
-    mode: 0600
-  with_items:
-  # bootstrap as an admin
-  - /etc/origin/node/bootstrap.kubeconfig
-  # copy to this location to bypass initial bootstrap request
-  - /etc/origin/node/node.kubeconfig

roles/openshift_control_plane/tasks/upgrade.yml

@@ -1,6 +1,14 @@
 ---
-- import_tasks: upgrade/rpm_upgrade.yml
-  when: not openshift_is_containerized | bool
+- name: Stop control plane services
+  service:
+    name: "{{ item }}"
+    state: stopped
+  with_items:
+  - "{{ openshift_service_type }}-master-api"
+  - "{{ openshift_service_type }}-master-controllers"
+  failed_when: false
+
+- import_tasks: static_shim.yml
 
 - import_tasks: upgrade/upgrade_scheduler.yml
 
@@ -8,8 +16,6 @@
 - include_tasks: "upgrade/{{ master_config_hook }}"
   when: master_config_hook is defined
 
-- import_tasks: systemd_units.yml
-
 - import_tasks: set_loopback_context.yml
 
 - name: Check for ca-bundle.crt
@@ -37,6 +43,37 @@
     state: link
   when: ca_crt_stat.stat.isreg and not ca_bundle_stat.stat.exists
 
+- name: Find existing master sysconfig
+  find:
+    paths:
+    - /etc/sysconfig
+    patterns:
+    - openshift-master-api
+    - openshift-master-controllers
+    - origin-master-api
+    - origin-master-controllers
+  register: sysconfigs
+- when: sysconfigs|succeeded and sysconfigs.matched > 0
+  name: Migrate OPENSHIFT_DEFAULT_REGISTRY from master sysconfig to master-config.yaml
+  block:
+  - name: Get master sysconfig contents
+    slurp:
+      src: "{{ sysconfigs.files[0].path }}"
+    register: sysconfig
+  # TODO: surely there is a better way
+  - name: Update imagePolicyConfig.internalRegistryHostname
+    yedit:
+      src: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      key: "imagePolicyConfig.internalRegistryHostname"
+      value: "{{ item | regex_replace('^OPENSHIFT_DEFAULT_REGISTRY=\\s*([^#\\s]+).*','\\1') }}"
+    with_items: "{{ (sysconfig.content | b64decode).split('\n') | select('match','^OPENSHIFT_DEFAULT_REGISTRY=\\s*.+') | list }}"
+
+- name: Create the master service env file if it does not exist
+  template:
+    src: "master.env.j2"
+    dest: "{{ openshift.common.config_base }}/master/master.env"
+    force: no
+
 - name: Update oreg value
   yedit:
     src: "{{ openshift.common.config_base }}/master/master-config.yaml"
@@ -50,3 +87,38 @@
     key: 'projectConfig.defaultNodeSelector'
     value: '{{ hostvars[groups.oo_first_master.0].l_osm_default_node_selector }}'
   when: openshift_upgrade_target | version_compare('3.9', '>=')
+
+- name: Remove use of pod presets from master config
+  yedit:
+    src: "{{ openshift.common.config_base }}/master/master-config.yaml"
+    key: 'admissionConfig.pluginConfig.PodPreset'
+    state: absent
+- name: Find current value for runtime-config
+  yedit:
+    src: "/tmp/master-config.yaml"
+    key: "kubernetesMasterConfig.apiServerArguments.runtime-config"
+    state: list
+  register: runtime_config
+- name: Set the runtime-config to exclude pod presets
+  yedit:
+    src: "{{ openshift.common.config_base }}/master/master-config.yaml"
+    key: "kubernetesMasterConfig.apiServerArguments.runtime-config"
+    value: "{{ runtime_config.result | join(',') | regex_replace('(?:,)*apis/settings\\.k8s\\.io/v1alpha1=true','') }}"
+  when: runtime_config.result
+
+- name: Remove old service information
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - /etc/systemd/system/atomic-openshift-master-api.service
+  - /etc/systemd/system/atomic-openshift-master-controllers.service
+  - /etc/systemd/system/origin-master-api.service
+  - /etc/systemd/system/origin-master-controllers.service
+  - /usr/lib/systemd/system/atomic-openshift-master-api.service
+  - /usr/lib/systemd/system/atomic-openshift-master-controllers.service
+  - /usr/lib/systemd/system/origin-master-api.service
+  - /usr/lib/systemd/system/origin-master-controllers.service
+
+- name: reload systemd units
+  command: systemctl daemon-reload

roles/openshift_gcp/tasks/setup_scale_group_facts.yml

@@ -3,6 +3,8 @@
   add_host:
     name: "{{ hostvars[item].gce_name }}"
     groups: masters, etcd
+    openshift_node_labels:
+      node-role.kubernetes.io/master: "true"
   with_items: "{{ groups['tag_ocp-master'] }}"
 
 - name: Add a master to the primary masters group
@@ -15,24 +17,21 @@
   add_host:
     name: "{{ hostvars[item].gce_name }}"
     groups: nodes
-    openshift_node_labels:
-      role: infra
   with_items: "{{ groups['tag_ocp-master'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
 
 - name: Add infra node instances to node group
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups: nodes
+    groups: nodes, new_nodes
     openshift_node_labels:
-      role: infra
+      node-role.kubernetes.io/infra: "true"
   with_items: "{{ groups['tag_ocp-infra-node'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
 
 - name: Add node instances to node group
   add_host:
     name: "{{ hostvars[item].gce_name }}"
-    groups: nodes
-    openshift_node_labels:
-      role: app
+    groups: nodes, new_nodes
+    openshift_node_bootstrap: False
   with_items: "{{ groups['tag_ocp-node'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
 
 - name: Add bootstrap node instances
@@ -40,5 +39,14 @@
     name: "{{ hostvars[item].gce_name }}"
     groups: bootstrap_nodes
     openshift_node_bootstrap: True
+    openshift_is_bootstrapped: True
   with_items: "{{ groups['tag_ocp-node'] | default([]) | intersect(groups['tag_ocp-bootstrap'] | default([])) }}"
-  when: not (openshift_node_bootstrap | default(True))
+
+- name: Add bootstrap node instances as nodes
+  add_host:
+    name: "{{ item }}"
+    groups: nodes, new_nodes
+    openshift_node_bootstrap: True
+    openshift_is_bootstrapped: True
+  with_items: "{{ groups['tag_ocp-bootstrap'] | default([]) }}"
+  when: all_nodes | default(False)

roles/openshift_gcp/templates/openshift-bootstrap-update.j2

@@ -2,6 +2,6 @@
 
 set -euo pipefail
 
-"{{ openshift_client_binary }}" serviceaccounts create-kubeconfig -n openshift-infra node-bootstrapper > /root/bootstrap.kubeconfig
+oc serviceaccounts create-kubeconfig -n openshift-infra node-bootstrapper > /root/bootstrap.kubeconfig
 gcloud compute project-info --project '{{ openshift_gcp_project }}' add-metadata --metadata-from-file '{{ openshift_gcp_prefix + openshift_gcp_clusterid | default("default") }}-bootstrap-config=/root/bootstrap.kubeconfig'
 rm -f /root/bootstrap.kubeconfig

roles/openshift_manage_node/tasks/set_default_node_role.yml

@@ -3,36 +3,36 @@
 - delegate_to: '{{ openshift_master_host }}'
   run_once: true
   block:
-    - name: Retrieve non-infra, non-master nodes that are not yet labeled compute
+    - name: Retrieve nodes that are marked with the infra selector or the legacy infra selector
       oc_obj:
         state: list
         kind: Node
-        selector: '{{ (openshift_hosted_infra_selector | default("node-role.kubernetes.io/infra=true")) | regex_replace("=", "!=") }},node-role.kubernetes.io/master!=true,node-role.kubernetes.io/compute!=true'
-      register: non_master_non_infra_nodes_result
+        selector: '{{ openshift_hosted_infra_selector | default("region=infra") }}'
+      register: infra_nodes_by_selector
 
-    - name: label non-master non-infra nodes compute
+    - name: Label infra or legacy infra nodes with the new role label
       oc_label:
         name: '{{ item }}'
         kind: node
         state: add
         labels:
-          - key: node-role.kubernetes.io/compute
+          - key: node-role.kubernetes.io/infra
             value: 'true'
-      with_items: "{{ non_master_non_infra_nodes_result.results.results.0['items'] | map(attribute='metadata') | map(attribute='name') | list }}"
+      with_items: "{{ infra_nodes_by_selector.results.results.0['items'] | map(attribute='metadata') | map(attribute='name') | list }}"
 
-    - name: Retrieve nodes that are marked with the infra selector or the legacy infra selector
+    - name: Retrieve non-infra, non-master nodes that are not yet labeled compute
       oc_obj:
         state: list
         kind: Node
-        selector: '{{ openshift_hosted_infra_selector | default("region=infra") }}'
-      register: infra_nodes_by_selector
+        selector: '{{ (openshift_hosted_infra_selector | default("node-role.kubernetes.io/infra=true")) | regex_replace("=", "!=") }},node-role.kubernetes.io/infra!=true,node-role.kubernetes.io/master!=true,node-role.kubernetes.io/compute!=true'
+      register: non_master_non_infra_nodes_result
 
-    - name: Label infra or legacy infra nodes with the new role label
+    - name: label non-master non-infra nodes compute
       oc_label:
         name: '{{ item }}'
         kind: node
         state: add
         labels:
-          - key: node-role.kubernetes.io/infra
+          - key: node-role.kubernetes.io/compute
             value: 'true'
-      with_items: "{{ infra_nodes_by_selector.results.results.0['items'] | map(attribute='metadata') | map(attribute='name') | list }}"
+      with_items: "{{ non_master_non_infra_nodes_result.results.results.0['items'] | map(attribute='metadata') | map(attribute='name') | list }}"

roles/openshift_node/tasks/config.yml

@@ -4,8 +4,7 @@
 
 - name: Install Node system container
   import_tasks: node_system_container.yml
-  when:
-    - l_is_node_system_container | bool
+  when: l_is_node_system_container | bool
 
 - file:
     dest: "{{ l2_openshift_node_kubelet_args['config'] }}"

roles/openshift_node/tasks/node_system_container.yml

@@ -9,8 +9,11 @@
 - name: Ensure old system path is set
   file:
     state: directory
-    path: "/etc/origin/openvswitch"
+    path: "{{ item }}"
     mode: '0750'
+  with_items:
+  - "/etc/origin/openvswitch"
+  - "/var/lib/kubelet"
 
 - name: Pre-pull node system container image
   command: >

roles/openshift_node/tasks/upgrade.yml

@@ -27,7 +27,6 @@
     openshift_version: "{{ openshift_pkg_version | default('') }}"
   when: not openshift_is_containerized | bool
 
-
 - include_tasks: "{{ node_config_hook }}"
   when: node_config_hook is defined
 
@@ -39,6 +38,15 @@
 # Restart all services
 - import_tasks: upgrade/restart.yml
 
+- name: Approve node certificates when bootstrapping
+  oc_adm_csr:
+    nodes: "{{ openshift.common.hostname | lower }}"
+    timeout: 180
+    fail_on_timeout: true
+  delegate_to: "{{ groups.oo_first_master.0 }}"
+  ignore_errors: true
+  when: openshift_node_bootstrap | default(True) | bool
+
 - name: Wait for node to be ready
   oc_obj:
     state: list
@@ -47,8 +55,8 @@
   register: node_output
   delegate_to: "{{ groups.oo_first_master.0 }}"
   until: node_output.results.returncode == 0 and node_output.results.results[0].status.conditions | selectattr('type', 'match', '^Ready$') | map(attribute='status') | join | bool == True
-  # Give the node two minutes to come back online.
-  retries: 24
+  # Give the node three minutes to come back online.
+  retries: 36
   delay: 5
 
 - import_tasks: journald.yml

roles/openshift_node/tasks/upgrade/bootstrap_changes.yml

@@ -0,0 +1,112 @@
+---
+- name: Update node-config to prepare for bootstrapping
+  yedit:
+    src: "{{ openshift.common.config_base }}/node/node-config.yaml"
+    edits:
+    - key: servingInfo.certFile
+      value: ""
+    - key: servingInfo.keyFile
+      value: ""
+    - key: kubeletArguments.bootstrap-kubeconfig
+      value:
+      - "{{ openshift.common.config_base }}/node/bootstrap.kubeconfig"
+    - key: kubeletArguments.rotate-certificates
+      value:
+      - "true"
+    - key: kubeletArguments.cert-dir
+      value:
+      - "{{ openshift.common.config_base }}/node/certificates"
+    - key: kubeletArguments.feature-gates
+      value:
+      - RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true
+
+- name: Check for existing node-config.yaml
+  stat:
+    path: "{{ openshift.common.config_base }}/node/node-config.yaml"
+  register: existing_node_config
+
+- name: Copy existing configuration to bootstrap configuration
+  copy:
+    remote_src: true
+    src: "{{ openshift.common.config_base }}/node/node-config.yaml"
+    dest: "{{ openshift.common.config_base }}/node/bootstrap-node-config.yaml"
+    force: no
+    owner: root
+    group: root
+    mode: 0600
+  when: existing_node_config.stat.exists
+
+- name: Find existing credentials
+  find:
+    paths:
+    - "{{ openshift.common.config_base }}/node"
+    patterns:
+    - system*.kubeconfig
+  register: system_kubeconfigs
+
+- name: Copy existing credentials to bootstrap credentials
+  copy:
+    remote_src: true
+    src: "{{ item }}"
+    dest: "{{ openshift.common.config_base }}/node/bootstrap.kubeconfig"
+    force: no
+    owner: root
+    group: root
+    mode: 0600
+  with_items: "{{ system_kubeconfigs.files | default([]) | map(attribute='path') | list + [openshift.common.config_base+'/node/node.kubeconfig'] }}"
+
+- name: Remove non-bootstrap configuration
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+  - "{{ openshift.common.config_base }}/node/node.kubeconfig"
+  - "{{ openshift.common.config_base }}/node/node-config.yaml"
+
+- name: Use the admin.kubeconfig for the kubelet bootstrap identity
+  copy:
+    remote_src: true
+    src: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+    dest: "{{ openshift.common.config_base }}/node/bootstrap.kubeconfig"
+    force: yes
+    owner: root
+    group: root
+    mode: 0600
+  when: inventory_hostname in groups.oo_masters_to_config
+
+- name: Update symlink master CA for docker-registry (name changed)
+  file:
+    src: "{{ item }}"
+    dest: "/etc/docker/certs.d/docker-registry.default.svc:5000/{{ item | basename }}"
+    state: link
+    force: yes
+  with_items:
+  - "{{ openshift.common.config_base }}/node/client-ca.crt"
+
+- name: Remove previous bootstrap certificates
+  file:
+    path: "{{ openshift.common.config_base }}/node/certificates"
+    state: absent
+
+- name: Determine if node already has a dynamic config group
+  command: grep -E '^BOOTSTRAP_CONFIG_NAME=.+' "/etc/sysconfig/{{ openshift_service_type }}-node"
+  ignore_errors: true
+  register: existing
+
+- name: Update the sysconfig to group "{{ r_node_dynamic_config_name }}"
+  lineinfile:
+    dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
+    line: "BOOTSTRAP_CONFIG_NAME={{ r_node_dynamic_config_name }}"
+    regexp: "^BOOTSTRAP_CONFIG_NAME=.*"
+  when: r_node_dynamic_config_force|default(False) or existing is failed
+
+- name: Set up node-config.yml if dynamic configuration is off
+  copy:
+    remote_src: true
+    src: "{{ openshift.common.config_base }}/node/bootstrap-node-config.yaml"
+    dest: "{{ openshift.common.config_base }}/node/node-config.yaml"
+    force: no
+    owner: root
+    group: root
+    mode: 0600
+  when: r_node_dynamic_config_name|length == 0

roles/openshift_node/tasks/upgrade/config_changes.yml

@@ -1,8 +1,4 @@
 ---
-- name: Update systemd units
-  import_tasks: ../systemd_units.yml
-  when: openshift_is_containerized | bool
-
 - name: Update oreg value
   yedit:
     src: "{{ openshift.common.config_base }}/node/node-config.yaml"
@@ -10,47 +6,45 @@
     value: "{{ oreg_url | default(oreg_url_node) }}"
   when: oreg_url is defined or oreg_url_node is defined
 
-- name: Remove obsolete docker-sdn-ovs.conf
+- name: Ensure the node static pod directory exists
   file:
-    path: "/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf"
-    state: absent
+    path: "{{ openshift.common.config_base }}/node/pods"
+    state: directory
+    mode: 0755
 
-# https://bugzilla.redhat.com/show_bug.cgi?id=1513054
-- name: Clean up dockershim data
-  file:
-    path: "/var/lib/dockershim/sandbox/"
-    state: absent
+- name: Update node-config for static pods
+  yedit:
+    src: "{{ openshift.common.config_base }}/node/node-config.yaml"
+    edits:
+    - key: kubeletArguments.pod-manifest-path
+      value:
+      - "{{ openshift.common.config_base }}/node/pods"
 
 # https://bugzilla.redhat.com/show_bug.cgi?id=1518912
-- name: Clean up IPAM data
-  file:
-    path: "/var/lib/cni/networks/openshift-sdn/"
-    state: absent
+# - name: Clean up IPAM data
+#   file:
+#     path: "/var/lib/cni/networks/openshift-sdn/"
+#     state: absent
 
-# https://docs.openshift.com/container-platform/3.4/admin_guide/overcommit.html#disabling-swap-memory
-# swapoff is a custom module in lib_utils that comments out swap entries in
-# /etc/fstab and runs swapoff -a, if necessary.
-- name: Disable swap
-  swapoff: {}
-  when: openshift_disable_swap | default(true) | bool
+- name: Uninstall openvswitch
+  package:
+    name: openvswitch
+    state: absent
+  when: not openshift_is_atomic | bool
 
-- name: Apply 3.6 dns config changes
-  yedit:
-    src: /etc/origin/node/node-config.yaml
-    key: "{{ item.key }}"
-    value: "{{ item.value }}"
+- name: Remove old service information
+  file:
+    path: "{{ item }}"
+    state: absent
   with_items:
-  - key: "dnsBindAddress"
-    value: "127.0.0.1:53"
-  - key: "dnsRecursiveResolvConf"
-    value: "/etc/origin/node/resolv.conf"
+  - /etc/systemd/system/docker.service.d/docker-sdn-ovs.conf
+  - /etc/systemd/system/atomic-openshift-node-dep.service
+  - /etc/systemd/system/origin-node-dep.service
+  - /etc/systemd/system/openvswitch.service
 
-- name: Install Node service file
-  template:
-    dest: "/etc/systemd/system/{{ openshift_service_type }}-node.service"
-    src: "node.service.j2"
-  register: l_node_unit
-  when: not openshift_is_containerized | bool
+- name: Move existing credentials and configuration into bootstrap configuration
+  import_tasks: bootstrap_changes.yml
+  when: openshift_node_bootstrap | default(True) | bool
 
 - name: Reset selinux context
   command: restorecon -RF {{ openshift_node_data_dir }}/openshift.local.volumes
@@ -58,6 +52,9 @@
   - ansible_selinux is defined
   - ansible_selinux.status == 'enabled'
 
+- name: Update systemd units
+  import_tasks: ../systemd_units.yml
+
 # NOTE: This is needed to make sure we are using the correct set
 #       of systemd unit files. The RPMs lay down defaults but
 #       the install/upgrade may override them in /etc/systemd/system/.

roles/openshift_node/tasks/upgrade/stop_services.yml

@@ -4,10 +4,7 @@
     name: "{{ item }}"
     state: stopped
   with_items:
-  - etcd_container
   - openvswitch
-  - "{{ openshift_service_type }}-master-api"
-  - "{{ openshift_service_type }}-master-controllers"
   - "{{ openshift_service_type }}-node"
   failed_when: false
 

roles/openshift_node_group/defaults/main.yml

@@ -13,6 +13,8 @@ openshift_node_groups:
   - 'node-role.kubernetes.io/compute=true'
   edits: []
 
+l_openshift_node_group_master: "{{ openshift_node_group_master | default('node-config-master') }}"
+
 openshift_node_group_edits: []
 openshift_node_group_namespace: openshift-node
 openshift_node_group_labels: []

roles/openshift_node_group/files/sync-policy.yaml

@@ -6,3 +6,15 @@ items:
   metadata:
     name: sync
     namespace: openshift-node
+- apiVersion: authorization.openshift.io/v1
+  kind: RoleBinding
+  metadata:
+    name: sync-node-config-reader-binding
+    namespace: openshift-node
+  roleRef:
+    name: system:node-config-reader
+    namespace: openshift-node
+  subjects:
+  - kind: ServiceAccount
+    name: sync
+    namespace: openshift-node

roles/openshift_node_group/files/sync.yaml

@@ -16,6 +16,8 @@ spec:
       app: sync
   updateStrategy:
     type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 50%
   template:
     metadata:
       labels:
@@ -50,19 +52,10 @@ spec:
           #!/bin/bash
           set -euo pipefail
 
-          # loop until BOOTSTRAP_CONFIG_NAME is set
-          set -o allexport
-          while true; do
-            if [[ -f /etc/sysconfig/origin-node ]]; then
-              source /etc/sysconfig/origin-node
-              if [[ -z "${BOOTSTRAP_CONFIG_NAME-}" ]]; then
-                echo "info: Waiting for BOOTSTRAP_CONFIG_NAME to be set" 2>&1
-                sleep 15
-                continue
-              fi
-              break
-            fi
-          done
+          # set by the node image
+          unset KUBECONFIG
+
+          trap 'kill $(jobs -p); exit 0' TERM
 
           # track the current state of the config
           if [[ -f /etc/origin/node/node-config.yaml ]]; then
@@ -71,12 +64,50 @@ spec:
             touch /tmp/.old
           fi
 
+          # loop until BOOTSTRAP_CONFIG_NAME is set
+          while true; do
+            file=/etc/sysconfig/origin-node
+            if [[ -f /etc/sysconfig/atomic-openshift-node ]]; then
+              file=/etc/sysconfig/atomic-openshift-node
+            elif [[ -f /etc/sysconfig/origin-node ]]; then
+              file=/etc/sysconfig/origin-node
+            else
+              echo "info: Waiting for the node sysconfig file to be created" 2>&1
+              sleep 15 & wait
+              continue
+            fi
+            name="$(sed -nE 's|^BOOTSTRAP_CONFIG_NAME=([^#].+)|\1|p' "${file}" | head -1)"
+            if [[ -z "${name}" ]]; then
+              echo "info: Waiting for BOOTSTRAP_CONFIG_NAME to be set" 2>&1
+              sleep 15 & wait
+              continue
+            fi
+            # in the background check to see if the value changes and exit if so
+            pid=$BASHPID
+            (
+              while true; do
+                if ! updated="$(sed -nE 's|^BOOTSTRAP_CONFIG_NAME=([^#].+)|\1|p' "${file}" | head -1)"; then
+                  echo "error: Unable to check for bootstrap config, exiting" 2>&1
+                  kill $pid
+                  exit 1
+                fi
+                if [[ "${updated}" != "${name}" ]]; then
+                  echo "info: Bootstrap configuration profile name changed, exiting" 2>&1
+                  kill $pid
+                  exit 0
+                fi
+                sleep 15
+              done
+            ) &
+            break
+          done
+
           # periodically refresh both node-config.yaml and relabel the node
           while true; do
-            name=${BOOTSTRAP_CONFIG_NAME}
-            if ! oc extract --config=/etc/origin/node/node.kubeconfig "cm/${BOOTSTRAP_CONFIG_NAME}" -n openshift-node --to=/etc/origin/node --confirm; then
+            if ! oc extract "configmaps/${name}" -n openshift-node --to=/etc/origin/node --confirm --request-timeout=10s --config /etc/origin/node/node.kubeconfig "--token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token )"  > /dev/null; then
               echo "error: Unable to retrieve latest config for node" 2>&1
-              sleep 15
+              sleep 15 &
+              wait $!
               continue
             fi
             # detect whether the node-config.yaml has changed, and if so trigger a restart of the kubelet.
@@ -92,17 +123,22 @@ spec:
                   echo "info: Applying node labels $labels" 2>&1
                   if ! oc label --config=/etc/origin/node/node.kubeconfig "node/${NODE_NAME}" ${labels} --overwrite; then
                     echo "error: Unable to apply labels, will retry in 10" 2>&1
-                    sleep 10
+                    sleep 10 &
+                    wait $!
                     continue
                   fi
                 fi
+              else
+                echo "error: The downloaded node configuration is invalid, exiting" 2>&1
+                exit 1
               fi
               if ! pgrep -U 0 -f 'hyperkube kubelet ' | xargs kill; then
                 echo "error: Unable to restart Kubelet" 2>&1
               fi
             fi
             cp -f /tmp/.new /tmp/.old
-            sleep 180
+            sleep 180 &
+            wait $!
           done
 
         env:
@@ -117,7 +153,7 @@ spec:
         # Directory which contains the host configuration. We read from this directory
         - mountPath: /etc/origin/node/
           name: host-config
-        - mountPath: /etc/sysconfig/origin-node
+        - mountPath: /etc/sysconfig
           name: host-sysconfig-node
           readOnly: true
 
@@ -129,7 +165,4 @@ spec:
           path: /etc/origin/node
       - name: host-sysconfig-node
         hostPath:
-          path: /etc/sysconfig/origin-node
-      - name: host-modules
-        hostPath:
-          path: /lib/modules
+          path: /etc/sysconfig

roles/openshift_node_group/tasks/bootstrap_config.yml

@@ -9,12 +9,13 @@
   with_items:
   - /etc/origin/node/pods
   - /etc/origin/node/certificates
-- name: Update the sysconfig to group "{{ r_node_bootstrap_config_name }}"
+- name: Determine if node already has a dynamic config group
+  command: grep -E '^BOOTSTRAP_CONFIG_NAME=.+' "/etc/sysconfig/{{ openshift_service_type }}-node"
+  ignore_errors: true
+  register: existing
+- name: Update the sysconfig to group "{{ r_node_dynamic_config_name }}"
   lineinfile:
     dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
-    line: "{{ item.line | default(omit) }}"
-    regexp: "{{ item.regexp }}"
-    state: "{{ item.state | default('present') }}"
-  with_items:
-  - line: "BOOTSTRAP_CONFIG_NAME={{ r_node_bootstrap_config_name }}"
+    line: "BOOTSTRAP_CONFIG_NAME={{ r_node_dynamic_config_name }}"
     regexp: "^BOOTSTRAP_CONFIG_NAME=.*"
+  when: r_node_dynamic_config_force|default(False) or existing is failed

roles/openshift_node_group/tasks/upgrade.yml

@@ -0,0 +1,29 @@
+---
+- name: Ensure all node groups have bootstrap settings
+  include_tasks: create_config.yml
+  vars:
+    openshift_node_group_name: "{{ node_group.name }}"
+    openshift_node_group_edits:
+    - key: servingInfo.certFile
+      value: ""
+    - key: servingInfo.keyFile
+      value: ""
+    - key: kubeletArguments.pod-manifest-path
+      value:
+      - /etc/origin/node/pods
+    - key: kubeletArguments.bootstrap-kubeconfig
+      value:
+      - /etc/origin/node/bootstrap.kubeconfig
+    - key: kubeletArguments.feature-gates
+      value:
+      - RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true
+    - key: kubeletArguments.rotate-certificates
+      value:
+      - "true"
+    - key: kubeletArguments.cert-dir
+      value:
+      - /etc/origin/node/certificates
+    openshift_node_group_labels: "{{ node_group.labels | default([]) }}"
+  with_items: "{{ openshift_node_groups }}"
+  loop_control:
+    loop_var: node_group

roles/openshift_node_group/templates/node-config.yaml.j2

@@ -59,7 +59,6 @@ masterKubeConfig: node.kubeconfig
 networkConfig:
   mtu: {{ openshift_node_group_network_mtu }}
   networkPluginName: {{ openshift_node_group_network_plugin }}
-networkPluginName: {{ openshift_node_group_network_plugin }}
 servingInfo:
   bindAddress: 0.0.0.0:10250
   bindNetwork: tcp4

roles/openshift_sdn/files/sdn-ovs.yaml

@@ -28,6 +28,7 @@ spec:
       # as all pods.
       serviceAccountName: sdn
       hostNetwork: true
+      hostPID: true
       containers:
       - name: openvswitch
         image: " "
@@ -37,6 +38,25 @@ spec:
         - |
           #!/bin/bash
           set -euo pipefail
+
+          # if another process is listening on the cni-server socket, wait until it exits
+          trap 'kill $(jobs -p); exit 0' TERM
+          retries=0
+          while true; do
+            if /usr/share/openvswitch/scripts/ovs-ctl status &>/dev/null; then
+              echo "warning: Another process is currently managing OVS, waiting 15s ..." 2>&1
+              sleep 15 & wait
+              (( retries += 1 ))
+            else
+              break
+            fi
+            if [[ "${retries}" -gt 40 ]]; then
+              echo "error: Another process is currently managing OVS, exiting" 2>&1
+              exit 1
+            fi
+          done
+
+          # launch OVS
           function quit {
               /usr/share/openvswitch/scripts/ovs-ctl stop
               exit 0

roles/openshift_sdn/files/sdn.yaml

@@ -42,6 +42,39 @@ spec:
         - |
           #!/bin/bash
           set -euo pipefail
+
+          # if another process is listening on the cni-server socket, wait until it exits
+          trap 'kill $(jobs -p); exit 0' TERM
+          retries=0
+          while true; do
+            if echo 'test' | socat - UNIX-CONNECT:/var/run/openshift-sdn/cni-server.sock >/dev/null; then
+              echo "warning: Another process is currently listening on the CNI socket, waiting 15s ..." 2>&1
+              sleep 15 & wait
+              (( retries += 1 ))
+            else
+              break
+            fi
+            if [[ "${retries}" -gt 40 ]]; then
+              echo "error: Another process is currently listening on the CNI socket, exiting" 2>&1
+              exit 1
+            fi
+          done
+          # if the node config doesn't exist yet, wait until it does
+          retries=0
+          while true; do
+            if [[ ! -f /etc/origin/node/node-config.yaml ]]; then
+              echo "warning: Cannot find existing node-config.yaml, waiting 15s ..." 2>&1
+              sleep 15 & wait
+              (( retries += 1 ))
+            else
+              break
+            fi
+            if [[ "${retries}" -gt 40 ]]; then
+              echo "error: No existing node-config.yaml, exiting" 2>&1
+              exit 1
+            fi
+          done
+
           # Take over network functions on the node
           rm -Rf /etc/cni/net.d/*
           rm -Rf /host/opt/cni/bin/*
@@ -83,6 +116,10 @@ spec:
         - mountPath: /etc/sysconfig/origin-node
           name: host-sysconfig-node
           readOnly: true
+        # Mount the entire run directory for socket access for Docker or CRI-o
+        # TODO: remove
+        - mountPath: /var/run
+          name: host-var-run
         # Run directories where we need to be able to access sockets
         - mountPath: /var/run/dbus/
           name: host-var-run-dbus
@@ -114,12 +151,14 @@ spec:
         ports:
         - name: healthz
           containerPort: 10256
-        livenessProbe:
-          initialDelaySeconds: 10
-          httpGet:
-            path: /healthz
-            port: 10256
-            scheme: HTTP
+        # TODO: Temporarily disabled until we determine how to wait for clean default
+        # config
+        # livenessProbe:
+        #   initialDelaySeconds: 10
+        #   httpGet:
+        #     path: /healthz
+        #     port: 10256
+        #     scheme: HTTP
         lifecycle:
 
       volumes:
@@ -135,15 +174,19 @@ spec:
         hostPath:
           path: /lib/modules
 
+      # TODO: access to the docker socket should be replaced by CRI socket
+      - name: host-var-run
+        hostPath:
+          path: /var/run
+      - name: host-var-run-dbus
+        hostPath:
+          path: /var/run/dbus
       - name: host-var-run-ovs
         hostPath:
           path: /var/run/openvswitch
       - name: host-var-run-kubernetes
         hostPath:
           path: /var/run/kubernetes
-      - name: host-var-run-dbus
-        hostPath:
-          path: /var/run/dbus
       - name: host-var-run-openshift-sdn
         hostPath:
           path: /var/run/openshift-sdn