Setup an ssh bastion in the pod and use it instead of masters

test/aws/create_machineset.yml

@@ -66,4 +66,4 @@
     name: "{{ new_machine.results.results[0]['items'][0].status.addresses | selectattr('type', 'match', '^InternalIP$') | map(attribute='address') | first }}"
     node_name: "{{ new_machine.results.results[0]['items'][0].status.addresses | selectattr('type', 'match', '^InternalDNS$') | map(attribute='address') | first }}"
     groups: new_workers
-    ansible_ssh_common_args: '-o ProxyCommand="ssh -o IdentityFile="/opt/app-root/src/.ssh/id_rsa" -o StrictHostKeyChecking=no -W %h:%p -q core@{{ master_external_dns }}"'
+    ansible_ssh_common_args: '-o ProxyCommand="ssh -o IdentityFile="/opt/app-root/src/.ssh/id_rsa" -o StrictHostKeyChecking=no -W %h:%p -q core@{{ ssh_bastion }}"'

test/aws/files/01_service.yml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    run: ssh-bastion
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+spec:
+  externalTrafficPolicy: Local
+  ports:
+  - name: ssh
+    port: 22
+    protocol: TCP
+    targetPort: ssh
+  selector:
+    run: ssh-bastion
+  type: LoadBalancer

test/aws/files/02_serviceaccount.yml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion

test/aws/files/03_role.yml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+rules:
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use
+  resourceNames:
+  - anyuid

test/aws/files/04_rolebinding.yml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  annotations:
+    openshift.io/description: Allows ssh-pod to run as root
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ssh-bastion
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:openshift-ssh-bastion:ssh-bastion

test/aws/files/05_clusterrole.yml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ssh-bastion
+rules:
+- apiGroups:
+  - "machineconfiguration.openshift.io"
+  resources:
+  - "machineconfigs"
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - "nodes"
+  verbs:
+  - list
+  - get

test/aws/files/06_clusterrolebinding.yml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    openshift.io/description: Allows ssh-pod to read nodes and machineconfigs
+  name: ssh-bastion
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ssh-bastion
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:openshift-ssh-bastion:ssh-bastion

test/aws/files/07_deployment.yml

@@ -0,0 +1,41 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    run: ssh-bastion
+  name: ssh-bastion
+  namespace: openshift-ssh-bastion
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: ssh-bastion
+  template:
+    metadata:
+      labels:
+        run: ssh-bastion
+    spec:
+      serviceAccountName: "ssh-bastion"
+      containers:
+      - image: quay.io/eparis/ssh:latest
+        imagePullPolicy: Always
+        name: ssh-bastion
+        ports:
+        - containerPort: 22
+          name: ssh
+          protocol: TCP
+        volumeMounts:
+        - name: ssh-host-keys
+          mountPath: "/etc/ssh/"
+          readOnly: true
+      volumes:
+      - name: ssh-host-keys
+        secret:
+          secretName: ssh-host-keys
+          items:
+          - key: ssh_host_rsa_key
+            path: ssh_host_rsa_key
+            mode: 256
+          - key: sshd_config
+            path: sshd_config
+      restartPolicy: Always

test/aws/files/sshd_config

@@ -0,0 +1,18 @@
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+SyslogFacility AUTHPRIV
+PermitRootLogin no
+AuthorizedKeysFile	/home/core/.ssh/authorized_keys
+PasswordAuthentication no
+ChallengeResponseAuthentication no
+GSSAPIAuthentication yes
+GSSAPICleanupCredentials no
+UsePAM yes
+X11Forwarding yes
+PrintMotd no
+AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
+AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
+AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
+AcceptEnv XMODIFIERS
+Subsystem	sftp	/usr/libexec/openssh/sftp-server

test/aws/get_machinesets.yml

@@ -19,11 +19,6 @@
 - set_fact:
     pre_scaleup_workers_name: "{{ pre_scaleup_workers.results.results[0]['items'] |map(attribute='metadata.name') | list }}"
 
-- name: create temp directory
-  command: mktemp -d /tmp/openshift-ansible-XXXXXXX
-  register: mktemp
-  changed_when: False
-
 - name: get existing worker machinesets
   oc_obj:
     state: list
@@ -43,23 +38,3 @@
 
 - set_fact:
     pre_scaleup_machineset_names: "{{ machineset.results.results[0]['items'] |map(attribute='metadata.name') | list }}"
-
-- name: List existing masters
-  oc_obj:
-    kubeconfig: "{{ kubeconfig_path }}"
-    state: list
-    kind: node
-    selector: "node-role.kubernetes.io/master"
-  delegate_to: localhost
-  register: masters
-  until:
-  - masters.results is defined
-  - masters.results.returncode is defined
-  - masters.results.results is defined
-  - masters.results.returncode == 0
-  retries: 36
-  delay: 5
-
-- name: save first master external DNS name
-  set_fact:
-    master_external_dns: "{{ masters.results.results[0]['items'][0].status.addresses | selectattr('type', 'match', '^ExternalDNS$') | map(attribute='address') | first }}"

test/aws/scaleup.yml

@@ -9,12 +9,17 @@
   hosts: localhost
   connection: local
   tasks:
+  - name: create temp directory
+    command: mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: mktemp
+    changed_when: False
   - name: add localhost as master
     add_host:
       name: localhost
       ansible_connection: local
       groups: masters
 
+  - import_tasks: ssh_bastion.yml
   - import_tasks: get_machinesets.yml
 
   - include_tasks: create_machineset.yml

test/aws/ssh_bastion.yml

@@ -0,0 +1,50 @@
+---
+- name: Create ssh bastion project
+  oc_project:
+    name: openshift-ssh-bastion
+    description: openshift-ssh-bastion
+    node_selector: ""
+    kubeconfig: "{{ kubeconfig_path }}"
+
+- name: Create ssh keys secret
+  oc_secret:
+    state: present
+    namespace: openshift-ssh-bastion
+    name: ssh-host-keys
+    kubeconfig: "{{ kubeconfig_path }}"
+    files:
+    - name: ssh_host_rsa_key
+      path: ../../inventory/dynamic/injected/ssh-privatekey
+    - name: sshd_config
+      path: files/sshd_config
+  no_log: true
+
+- name: Create ssh bastion
+  command: "oc --kubeconfig={{ kubeconfig_path }} apply -f files/"
+
+- name: Wait for deployment to rollout
+  command: "oc --kubeconfig={{ kubeconfig_path }} -n openshift-ssh-bastion rollout status deployment ssh-bastion -w"
+
+- name: Get ssh bastion address
+  oc_obj:
+    name: ssh-bastion
+    kind: service
+    namespace: openshift-ssh-bastion
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: present
+  register: svc
+  until:
+  - svc.results is defined
+  - svc.results.returncode is defined
+  - svc.results.returncode == 0
+  - svc.results.results is defined
+  - svc.results.results | length > 0
+  - svc.results.results[0].status is defined
+  - svc.results.results[0].status.loadBalancer is defined
+  - svc.results.results[0].status.loadBalancer.ingress | length > 0
+  - svc.results.results[0].status.loadBalancer.ingress[0].hostname is defined
+  retries: 36
+  delay: 5
+
+- set_fact:
+    ssh_bastion: "{{ svc.results.results[0].status.loadBalancer.ingress[0].hostname }}"