Merge pull request #11203 from vrutkovs/devel-40-sshbastion

crio 1.12 and ssh-bastion in a pod

inventory/dynamic/aws/ansible.cfg

@@ -39,6 +39,7 @@ unparsed_is_failed=true
 
 # Additional ssh options for OpenShift Ansible
 [ssh_connection]
+retries = 10
 pipelining = True
 ssh_args = -o ControlMaster=auto -o ControlPersist=600s
 timeout = 10

inventory/dynamic/gcp/group_vars/all/00_defaults.yml

@@ -20,7 +20,7 @@ openshift_master_cluster_hostname: "internal-openshift-master.{{ public_hosted_z
 openshift_master_cluster_public_hostname: "openshift-master.{{ public_hosted_zone }}"
 openshift_master_default_subdomain: "{{ wildcard_zone }}"
 
-mcd_port: 49500
+mcd_port: 22623
 mcd_endpoint: "https://{{ openshift_master_cluster_public_hostname }}:{{ mcd_port }}"
 
 # Cloud specific settings

playbooks/init/basic_facts.yml

@@ -30,6 +30,46 @@
     - openshift_deployment_type is undefined
     - deployment_type is defined
 
+- name: Read API URL from infra config
+  hosts: "{{ l_init_fact_hosts | default('nodes') }}"
+  tasks:
+  - oc_obj:
+      name: cluster
+      kind: infrastructures.config.openshift.io
+      kubeconfig: "{{ kubeconfig_path }}"
+      state: present
+    register: clustercfg
+    when: kubeconfig_path is defined
+    until:
+    - clustercfg.results is defined
+    - clustercfg.results.returncode is defined
+    - clustercfg.results.returncode == 0
+    - clustercfg.results.results is defined
+    - clustercfg.results.results | length > 0
+    retries: 36
+    delay: 5
+    delegate_to: localhost
+  - set_fact:
+      openshift_api_prefix: "{{ clustercfg.results.results[0].status.apiServerURL.split(':')[0:-1] | join(':') }}"
+
+- name: Set worker openshift_bootstrap_endpoint if not already defined
+  hosts: "{{ l_init_fact_hosts | default('nodes') }}:!masters:!bootstrap"
+  tasks:
+  - set_fact:
+      openshift_bootstrap_endpoint: "{{ openshift_api_prefix }}:22623/config/worker"
+    when:
+    - kubeconfig_path is defined
+    - openshift_bootstrap_endpoint is not defined
+
+- name: Set master openshift_bootstrap_endpoint if not already defined
+  hosts: "{{ l_init_fact_hosts | default('nodes') }}:&masters"
+  tasks:
+  - set_fact:
+      openshift_bootstrap_endpoint: "{{ openshift_api_prefix }}:22623/config/master"
+    when:
+    - kubeconfig_path is defined
+    - openshift_bootstrap_endpoint is not defined
+
 - name: Read in openshift-install
   hosts: "{{ l_init_fact_hosts | default('nodes') }}"
   tasks:
@@ -47,7 +87,7 @@
   hosts: "{{ l_init_fact_hosts | default('nodes') }}:!masters:!bootstrap"
   tasks:
   - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ openshift_install_config['metadata']['name'] }}-api.{{ openshift_install_config['baseDomain'] }}:49500/config/worker"
+      openshift_bootstrap_endpoint: "https://api.{{ openshift_install_config['metadata']['name'] }}.{{ openshift_install_config['baseDomain'] }}:22623/config/worker"
     when:
     - openshift_install_config_path is defined
     - openshift_bootstrap_endpoint is not defined
@@ -56,7 +96,7 @@
   hosts: "{{ l_init_fact_hosts | default('nodes') }}:&masters"
   tasks:
   - set_fact:
-      openshift_bootstrap_endpoint: "https://{{ openshift_install_config['metadata']['name'] }}-api.{{ openshift_install_config['baseDomain'] }}:49500/config/master"
+      openshift_bootstrap_endpoint: "https://api.{{ openshift_install_config['metadata']['name'] }}.{{ openshift_install_config['baseDomain'] }}:22623/config/master"
     when:
     - openshift_install_config_path is defined
     - openshift_bootstrap_endpoint is not defined

roles/container_runtime/tasks/package_crio.yml

@@ -46,11 +46,19 @@
     - /etc/cni/net.d/200-loopback.conf
     - /etc/cni/net.d/100-crio-bridge.conf
 
-- name: Create the CRI-O configuration
-  template:
+- name: Set pause_image to pod image
+  ini_file:
+    dest: /etc/crio/crio.conf
+    section: "crio.image"
+    option: pause_image
+    value: '"{{ openshift_crio_pause_image }}"'
+
+- name: Set pause_command to pod cmd
+  ini_file:
     dest: /etc/crio/crio.conf
-    src: crio.conf.j2
-    backup: yes
+    section: "crio.image"
+    option: pause_command
+    value: ' "/usr/bin/pod"'
 
 - name: Update crictl.yaml runtime-endpoint
   yedit:
@@ -89,6 +97,18 @@
     state: restarted
     daemon_reload: yes
   register: start_result
+  ignore_errors: true
+
+- when: start_result is failed
+  block:
+    - name: Get crio logs
+      command: journalctl --no-pager -u crio
+      register: crio_logs
+      ignore_errors: true
+    - debug:
+        msg: "{{ crio_logs.stdout_lines }}"
+    - fail:
+        msg: crio start failed.
 
 # If we are using crio only, docker.service might not be available for
 # 'docker login'

roles/container_runtime/templates/crio.conf.j2

@@ -1,166 +0,0 @@
-# {{ ansible_managed }}
-
-# The "crio" table contains all of the server options.
-[crio]
-
-# root is a path to the "root directory". CRIO stores all of its data,
-# including container images, in this directory.
-root = "/var/lib/containers/storage"
-
-# run is a path to the "run directory". CRIO stores all of its state
-# in this directory.
-runroot = "/var/run/containers/storage"
-
-# storage_driver select which storage driver is used to manage storage
-# of images and containers.
-storage_driver = "overlay"
-
-# storage_option is used to pass an option to the storage driver.
-storage_option = [
-{% if ansible_distribution in ['RedHat', 'CentOS'] %}
-	"overlay.override_kernel_check=1"
-{% endif %}
-]
-
-# The "crio.api" table contains settings for the kubelet/gRPC
-# interface (which is also used by crioctl).
-[crio.api]
-
-# listen is the path to the AF_LOCAL socket on which crio will listen.
-listen = "/var/run/crio/crio.sock"
-
-# stream_address is the IP address on which the stream server will listen
-stream_address = ""
-
-# stream_port is the port on which the stream server will listen
-stream_port = "10010"
-
-# file_locking is whether file-based locking will be used instead of
-# in-memory locking
-file_locking = false
-
-# The "crio.runtime" table contains settings pertaining to the OCI
-# runtime used and options for how to set up and manage the OCI runtime.
-[crio.runtime]
-
-# runtime is the OCI compatible runtime used for trusted container workloads.
-# This is a mandatory setting as this runtime will be the default one
-# and will also be used for untrusted container workloads if
-# runtime_untrusted_workload is not set.
-runtime = "/usr/bin/runc"
-
-# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
-# container workloads. This is an optional setting, except if
-# default_container_trust is set to "untrusted".
-runtime_untrusted_workload = ""
-
-# default_workload_trust is the default level of trust crio puts in container
-# workloads. It can either be "trusted" or "untrusted", and the default
-# is "trusted".
-# Containers can be run through different container runtimes, depending on
-# the trust hints we receive from kubelet:
-# - If kubelet tags a container workload as untrusted, crio will try first to
-# run it through the untrusted container workload runtime. If it is not set,
-# crio will use the trusted runtime.
-# - If kubelet does not provide any information about the container workload trust
-# level, the selected runtime will depend on the default_container_trust setting.
-# If it is set to "untrusted", then all containers except for the host privileged
-# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
-# containers are by definition trusted and will always use the trusted container
-# runtime. If default_container_trust is set to "trusted", crio will use the trusted
-# container runtime for all containers.
-default_workload_trust = "trusted"
-
-# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE
-no_pivot = false
-
-# conmon is the path to conmon binary, used for managing the runtime.
-conmon = "/usr/libexec/crio/conmon"
-
-# conmon_env is the environment variable list for conmon process,
-# used for passing necessary environment variable to conmon or runtime.
-conmon_env = [
-	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-]
-
-# selinux indicates whether or not SELinux will be used for pod
-# separation on the host. If you enable this flag, SELinux must be running
-# on the host.
-selinux = true
-
-# seccomp_profile is the seccomp json profile path which is used as the
-# default for the runtime.
-seccomp_profile = "/etc/crio/seccomp.json"
-
-# apparmor_profile is the apparmor profile name which is used as the
-# default for the runtime.
-apparmor_profile = "crio-default"
-
-# cgroup_manager is the cgroup management implementation to be used
-# for the runtime.
-cgroup_manager = "systemd"
-
-# hooks_dir_path is the oci hooks directory for automatically executed hooks
-hooks_dir_path = "/usr/share/containers/oci/hooks.d"
-
-# default_mounts is the mounts list to be mounted for the container when created
-default_mounts = [
-	"/usr/share/rhel/secrets:/run/secrets",
-]
-
-# pids_limit is the number of processes allowed in a container
-pids_limit = 1024
-
-# log_size_max is the max limit for the container log size in bytes.
-# Negative values indicate that no limit is imposed.
-log_size_max = 52428800
-
-# The "crio.image" table contains settings pertaining to the
-# management of OCI images.
-[crio.image]
-
-# default_transport is the prefix we try prepending to an image name if the
-# image name as we receive it can't be parsed as a valid source reference
-default_transport = "docker://"
-
-# pause_image is the image which we use to instantiate infra containers.
-pause_image = "{{ openshift_crio_pause_image }}"
-
-# pause_command is the command to run in a pause_image to have a container just
-# sit there.  If the image contains the necessary information, this value need
-# not be specified.
-pause_command = "/usr/bin/pod"
-
-# signature_policy is the name of the file which decides what sort of policy we
-# use when deciding whether or not to trust an image that we've pulled.
-# Outside of testing situations, it is strongly advised that this be left
-# unspecified so that the default system-wide policy will be used.
-signature_policy = ""
-
-# image_volumes controls how image volumes are handled.
-# The valid values are mkdir and ignore.
-image_volumes = "mkdir"
-
-# CRI-O reads its configured registries defaults from the containers/image configuration
-# file, /etc/containers/registries.conf. Modify registries.conf if you want to
-# change default registries for all tools that use containers/image.  If you
-# want to modify just crio, you can change the registies configuration in this
-# file.
-
-# insecure_registries is used to skip TLS verification when pulling images.
-# insecure_registries = []
-
-# registries is used to specify a comma separated list of registries to be used
-# when pulling an unqualified image (e.g. fedora:rawhide).
-registries = ['docker.io']
-
-# The "crio.network" table contains settings pertaining to the
-# management of CNI plugins.
-[crio.network]
-
-# network_dir is where CNI network configuration
-# files are stored.
-network_dir = "/etc/cni/net.d/"
-
-# plugin_dir is where CNI plugin binaries are stored.
-plugin_dir = "/opt/cni/bin/"

roles/lib_openshift/library/oc_adm_ca_server_cert.py

@@ -1358,6 +1358,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1487,7 +1490,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_adm_ca_server_cert.py -*- -*- -*-

roles/lib_openshift/library/oc_adm_csr.py

@@ -1338,6 +1338,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1467,7 +1470,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_adm_csr.py -*- -*- -*-

roles/lib_openshift/library/oc_adm_manage_node.py

@@ -1344,6 +1344,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1473,7 +1476,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_adm_manage_node.py -*- -*- -*-

roles/lib_openshift/library/oc_adm_policy_group.py

@@ -1336,6 +1336,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1465,7 +1468,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/rolebinding.py -*- -*- -*-

roles/lib_openshift/library/oc_adm_policy_user.py

@@ -1350,6 +1350,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1479,7 +1482,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/rolebinding.py -*- -*- -*-

roles/lib_openshift/library/oc_adm_registry.py

@@ -1449,6 +1449,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1578,7 +1581,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/deploymentconfig.py -*- -*- -*-

roles/lib_openshift/library/oc_adm_router.py

@@ -1462,6 +1462,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1591,7 +1594,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/service.py -*- -*- -*-

roles/lib_openshift/library/oc_clusterrole.py

@@ -1322,6 +1322,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1451,7 +1454,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/rule.py -*- -*- -*-

roles/lib_openshift/library/oc_configmap.py

@@ -1328,6 +1328,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1457,7 +1460,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_configmap.py -*- -*- -*-

roles/lib_openshift/library/oc_edit.py

@@ -1378,6 +1378,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1507,7 +1510,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_edit.py -*- -*- -*-

roles/lib_openshift/library/oc_env.py

@@ -1339,6 +1339,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1468,7 +1471,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/deploymentconfig.py -*- -*- -*-

roles/lib_openshift/library/oc_group.py

@@ -1312,6 +1312,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1441,7 +1444,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/group.py -*- -*- -*-

roles/lib_openshift/library/oc_image.py

@@ -1331,6 +1331,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1460,7 +1463,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_image.py -*- -*- -*-

roles/lib_openshift/library/oc_label.py

@@ -1348,6 +1348,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1477,7 +1480,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_label.py -*- -*- -*-

roles/lib_openshift/library/oc_obj.py

@@ -1358,6 +1358,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1487,7 +1490,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_obj.py -*- -*- -*-

roles/lib_openshift/library/oc_objectvalidator.py

@@ -1283,6 +1283,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1412,7 +1415,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_objectvalidator.py -*- -*- -*-

roles/lib_openshift/library/oc_process.py

@@ -1340,6 +1340,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1469,7 +1472,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_process.py -*- -*- -*-

roles/lib_openshift/library/oc_project.py

@@ -1340,6 +1340,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1469,7 +1472,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/project.py -*- -*- -*-

roles/lib_openshift/library/oc_pvc.py

@@ -1344,6 +1344,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1473,7 +1476,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/pvc.py -*- -*- -*-

roles/lib_openshift/library/oc_route.py

@@ -1391,6 +1391,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1520,7 +1523,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/route.py -*- -*- -*-

roles/lib_openshift/library/oc_scale.py

@@ -1326,6 +1326,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1455,7 +1458,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/deploymentconfig.py -*- -*- -*-

roles/lib_openshift/library/oc_secret.py

@@ -1387,6 +1387,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1516,7 +1519,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/secret.py -*- -*- -*-

roles/lib_openshift/library/oc_service.py

@@ -1394,6 +1394,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1523,7 +1526,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/service.py -*- -*- -*-

roles/lib_openshift/library/oc_serviceaccount.py

@@ -1327,6 +1327,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1456,7 +1459,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/serviceaccount.py -*- -*- -*-

roles/lib_openshift/library/oc_serviceaccount_secret.py

@@ -1327,6 +1327,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1456,7 +1459,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/serviceaccount.py -*- -*- -*-

roles/lib_openshift/library/oc_storageclass.py

@@ -1357,6 +1357,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1486,7 +1489,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/storageclass.py -*- -*- -*-

roles/lib_openshift/library/oc_user.py

@@ -1384,6 +1384,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1513,7 +1516,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/user.py -*- -*- -*-

roles/lib_openshift/library/oc_version.py

@@ -1297,6 +1297,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1426,7 +1429,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: class/oc_version.py -*- -*- -*-

roles/lib_openshift/library/oc_volume.py

@@ -1373,6 +1373,9 @@ class Utils(object):  # pragma: no cover
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -1502,7 +1505,6 @@ class OpenShiftCLIConfig(object):
 
         return rval
 
-
 # -*- -*- -*- End included fragment: lib/base.py -*- -*- -*-
 
 # -*- -*- -*- Begin included fragment: lib/deploymentconfig.py -*- -*- -*-

roles/lib_openshift/src/lib/base.py

@@ -495,6 +495,9 @@ class Utils(object):
     def check_def_equal(user_def, result_def, skip_keys=None, debug=False):
         ''' Given a user defined definition, compare it with the results given back by our query.  '''
 
+        if user_def is None:
+            user_def = {}
+
         # Currently these values are autogenerated and we do not need to check them
         skip = ['metadata', 'status']
         if skip_keys:
@@ -623,4 +626,3 @@ class OpenShiftCLIConfig(object):
                 rval.append('--{}={}'.format(key.replace('_', '-'), val))
 
         return rval
-

test/aws/create_machineset.yml

@@ -57,10 +57,13 @@
   - "'items' in new_machine.results.results[0]"
   - "new_machine.results.results[0]['items'] | length > 0"
   - "'status' in new_machine.results.results[0]['items'][0]"
+  - "'providerStatus' in new_machine.results.results[0]['items'][0]['status']"
+  - "'instanceState' in new_machine.results.results[0]['items'][0]['status']['providerStatus']"
+  - "new_machine.results.results[0]['items'][0]['status']['providerStatus']['instanceState'] == 'running'"
 
 - name: add machine to the inventory
   add_host:
     name: "{{ new_machine.results.results[0]['items'][0].status.addresses | selectattr('type', 'match', '^InternalIP$') | map(attribute='address') | first }}"
     node_name: "{{ new_machine.results.results[0]['items'][0].status.addresses | selectattr('type', 'match', '^InternalDNS$') | map(attribute='address') | first }}"
     groups: new_workers
-    ansible_ssh_common_args: '-o ProxyCommand="ssh -o IdentityFile="/opt/app-root/src/.ssh/id_rsa" -o StrictHostKeyChecking=no -W %h:%p -q core@{{ master_external_dns }}"'
+    ansible_ssh_common_args: "-o ProxyCommand=\"ssh -o IdentityFile='{{ openshift_aws_scaleup_key_path | default('/opt/app-root/src/.ssh/id_rsa') }}' -o StrictHostKeyChecking=no -W %h:%p -q core@{{ ssh_bastion }}\""

test/aws/files/01_service.yml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    run: ssh-bastion
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+spec:
+  externalTrafficPolicy: Local
+  ports:
+  - name: ssh
+    port: 22
+    protocol: TCP
+    targetPort: ssh
+  selector:
+    run: ssh-bastion
+  type: LoadBalancer

test/aws/files/02_serviceaccount.yml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion

test/aws/files/03_role.yml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+rules:
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use
+  resourceNames:
+  - anyuid

test/aws/files/04_rolebinding.yml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  annotations:
+    openshift.io/description: Allows ssh-pod to run as root
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ssh-bastion
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:openshift-ssh-bastion:ssh-bastion

test/aws/files/05_clusterrole.yml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ssh-bastion
+rules:
+- apiGroups:
+  - "machineconfiguration.openshift.io"
+  resources:
+  - "machineconfigs"
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - "nodes"
+  verbs:
+  - list
+  - get

test/aws/files/06_clusterrolebinding.yml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    openshift.io/description: Allows ssh-pod to read nodes and machineconfigs
+  name: ssh-bastion
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ssh-bastion
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:openshift-ssh-bastion:ssh-bastion

test/aws/files/07_deployment.yml

@@ -0,0 +1,41 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    run: ssh-bastion
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: ssh-bastion
+  template:
+    metadata:
+      labels:
+        run: ssh-bastion
+    spec:
+      serviceAccountName: "ssh-bastion"
+      containers:
+      - image: quay.io/eparis/ssh:latest
+        imagePullPolicy: Always
+        name: ssh-bastion
+        ports:
+        - containerPort: 22
+          name: ssh
+          protocol: TCP
+        volumeMounts:
+        - name: ssh-host-keys
+          mountPath: "/etc/ssh/"
+          readOnly: true
+      volumes:
+      - name: ssh-host-keys
+        secret:
+          secretName: ssh-host-keys
+          items:
+          - key: ssh_host_rsa_key
+            path: ssh_host_rsa_key
+            mode: 256
+          - key: sshd_config
+            path: sshd_config
+      restartPolicy: Always

test/aws/files/sshd_config

@@ -0,0 +1,18 @@
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+SyslogFacility AUTHPRIV
+PermitRootLogin no
+AuthorizedKeysFile	/home/core/.ssh/authorized_keys
+PasswordAuthentication no
+ChallengeResponseAuthentication no
+GSSAPIAuthentication yes
+GSSAPICleanupCredentials no
+UsePAM yes
+X11Forwarding yes
+PrintMotd no
+AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
+AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
+AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
+AcceptEnv XMODIFIERS
+Subsystem	sftp	/usr/libexec/openssh/sftp-server

test/aws/get_machinesets.yml

@@ -19,11 +19,6 @@
 - set_fact:
     pre_scaleup_workers_name: "{{ pre_scaleup_workers.results.results[0]['items'] |map(attribute='metadata.name') | list }}"
 
-- name: create temp directory
-  command: mktemp -d /tmp/openshift-ansible-XXXXXXX
-  register: mktemp
-  changed_when: False
-
 - name: get existing worker machinesets
   oc_obj:
     state: list
@@ -43,23 +38,3 @@
 
 - set_fact:
     pre_scaleup_machineset_names: "{{ machineset.results.results[0]['items'] |map(attribute='metadata.name') | list }}"
-
-- name: List existing masters
-  oc_obj:
-    kubeconfig: "{{ kubeconfig_path }}"
-    state: list
-    kind: node
-    selector: "node-role.kubernetes.io/master"
-  delegate_to: localhost
-  register: masters
-  until:
-  - masters.results is defined
-  - masters.results.returncode is defined
-  - masters.results.results is defined
-  - masters.results.returncode == 0
-  retries: 36
-  delay: 5
-
-- name: save first master external DNS name
-  set_fact:
-    master_external_dns: "{{ masters.results.results[0]['items'][0].status.addresses | selectattr('type', 'match', '^ExternalDNS$') | map(attribute='address') | first }}"

test/aws/scaleup.yml

@@ -9,12 +9,17 @@
   hosts: localhost
   connection: local
   tasks:
+  - name: create temp directory
+    command: mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: mktemp
+    changed_when: False
   - name: add localhost as master
     add_host:
       name: localhost
       ansible_connection: local
       groups: masters
 
+  - import_tasks: ssh_bastion.yml
   - import_tasks: get_machinesets.yml
 
   - include_tasks: create_machineset.yml
@@ -29,6 +34,14 @@
   tasks:
   - wait_for_connection: {}
   - setup: {}
+  - name: Copy ops-mirror.pem
+    copy:
+      src: ../../inventory/dynamic/injected/ops-mirror.pem
+      dest: /var/lib/yum/ops-mirror.pem
+      owner: root
+      group: root
+      mode: 0644
+    ignore_errors: true
 
 - import_playbook: ../../playbooks/openshift-node/scaleup.yml
 
@@ -50,14 +63,6 @@
       owner: core
       group: wheel
       mode: 0700
-  - name: Copy a list of authorized ssh keys
-    copy:
-      src: /home/ec2-user/.ssh/authorized_keys
-      dest: /home/core/.ssh/authorized_keys
-      remote_src: true
-      owner: core
-      group: wheel
-      mode: 600
   - name: Install nfs-utils for storage tests
     package:
       name: nfs-utils
@@ -92,10 +97,20 @@
       register: crictl_logs_output
       with_items: "{{ crictl_ps_output.stdout_lines }}"
       ignore_errors: true
+    - name: Get crio logs
+      command: journalctl --no-pager -u cri-o
+      register: crio_logs
+      ignore_errors: true
+    - name: Get kubelet logs
+      command: journalctl --no-pager -u kubelet
+      register: kubelet_logs
+      ignore_errors: tru
     - debug:
         var: crictl_logs_output
     - debug:
-        msg: "{{ bootkube_logs.stdout_lines }}"
+        msg: "{{ kubelet_logs.stdout_lines }}"
+    - debug:
+        msg: "{{ crio_logs.stdout_lines }}"
     - fail:
         msg: Node failed to become Ready
 

test/aws/ssh_bastion.yml

@@ -0,0 +1,50 @@
+---
+- name: Create ssh bastion project
+  oc_project:
+    name: openshift-ssh-bastion
+    description: openshift-ssh-bastion
+    node_selector: ""
+    kubeconfig: "{{ kubeconfig_path }}"
+
+- name: Create ssh keys secret
+  oc_secret:
+    state: present
+    namespace: openshift-ssh-bastion
+    name: ssh-host-keys
+    kubeconfig: "{{ kubeconfig_path }}"
+    files:
+    - name: ssh_host_rsa_key
+      path: ../../inventory/dynamic/injected/ssh-privatekey
+    - name: sshd_config
+      path: files/sshd_config
+  no_log: true
+
+- name: Create ssh bastion
+  command: "oc --kubeconfig={{ kubeconfig_path }} apply -f files/"
+
+- name: Wait for deployment to rollout
+  command: "oc --kubeconfig={{ kubeconfig_path }} -n openshift-ssh-bastion rollout status deployment ssh-bastion -w"
+
+- name: Get ssh bastion address
+  oc_obj:
+    name: ssh-bastion
+    kind: service
+    namespace: openshift-ssh-bastion
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: present
+  register: svc
+  until:
+  - svc.results is defined
+  - svc.results.returncode is defined
+  - svc.results.returncode == 0
+  - svc.results.results is defined
+  - svc.results.results | length > 0
+  - svc.results.results[0].status is defined
+  - svc.results.results[0].status.loadBalancer is defined
+  - svc.results.results[0].status.loadBalancer.ingress | length > 0
+  - svc.results.results[0].status.loadBalancer.ingress[0].hostname is defined
+  retries: 36
+  delay: 5
+
+- set_fact:
+    ssh_bastion: "{{ svc.results.results[0].status.loadBalancer.ingress[0].hostname }}"

test/gcp/launch.yml

@@ -28,4 +28,4 @@
 - name: run worker scaleup
   import_playbook: ../../playbooks/openshift-node/scaleup.yml
   vars:
-    openshift_api_host: "{{ openshift_install_config['metadata']['name'] }}-api.{{ openshift_install_config['baseDomain'] }}"
+    openshift_api_host: "api.{{ openshift_install_config['metadata']['name'] }}.{{ openshift_install_config['baseDomain'] }}"