Options for bastion, SSH config, static inventory autogeneration

* At the provisioning stage, allow users to auto-generate SSH config,
  when using a static inventory.
* Run playbooks to provsion and post-provision as a separate, when
  using a bastion. This re-applies the SSH config, which ansible can't
  do on the fly.
* Support a pre-installed bastion node, colocated with the 1st infra
  node.
* With a bastion enabled, reduce floating IP footprint to infra and
  dns nodes only, effectively isolating a cluster in a private
  network.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

playbooks/provisioning/openstack/README.md

@@ -40,7 +40,7 @@ Alternatively you can install directly from github:
       -p openshift-ansible-contrib/roles
 
 Notes:
-* This assumes we're in the directory that contains the clonned 
+* This assumes we're in the directory that contains the clonned
 openshift-ansible-contrib repo in its root path.
 * When trying to install a different version, the previous one must be removed first
 (`infra-ansible` directory from [roles](https://github.com/openshift/openshift-ansible-contrib/tree/master/roles)).
@@ -177,16 +177,30 @@ variables for the `inventory/group_vars/OSEv3.yml`, `all.yml`:
     origin_release: 1.5.1
     openshift_deployment_type: "{{ deployment_type }}"
 
-### Configure static inventory
+### Configure static inventory and access via a bastion node
 
 Example inventory variables:
 
+    openstack_use_bastion: true
+    bastion_ingress_cidr: "{{openstack_subnet_prefix}}.0/24"
     openstack_private_ssh_key: ~/.ssh/openshift
     openstack_inventory: static
     openstack_inventory_path: ../../../../inventory
+    openstack_ssh_config_path: /tmp/ssh.config.openshift.ansible.openshift.example.com
 
+The `openstack_subnet_prefix` is the openstack private network for your cluster.
+And the `bastion_ingress_cidr` defines accepted range for SSH connections to nodes
+additionally to the `ssh_ingress_cidr`` (see the security notes above).
 
-In this guide, the latter points to the current directory, where you run ansible commands
+The SSH config will be stored on the ansible control node by the
+gitven path. Ansible uses it automatically. To access the cluster nodes with
+that ssh config, use the `-F` prefix, f.e.:
+
+    ssh -F /tmp/ssh.config.openshift.ansible.openshift.example.com master-0.openshift.example.com echo OK
+
+Note, relative paths will not work for the `openstack_ssh_config_path`, but it
+works for the `openstack_private_ssh_key` and `openstack_inventory_path`. In this
+guide, the latter points to the current directory, where you run ansible commands
 from.
 
 To verify nodes connectivity, use the command:
@@ -194,7 +208,7 @@ To verify nodes connectivity, use the command:
     ansible -v -i inventory/hosts -m ping all
 
 If something is broken, double-check the inventory variables, paths and the
-generated `<openstack_inventory_path>/hosts` file.
+generated `<openstack_inventory_path>/hosts` and `openstack_ssh_config_path` files.
 
 The `inventory: dynamic` can be used instead to access cluster nodes directly via
 floating IPs. In this mode you can not use a bastion node and should specify
@@ -213,6 +227,15 @@ this is how you stat the provisioning process from your ansible control node:
 Note, here you start with an empty inventory. The static inventory will be populated
 with data so you can omit providing additional arguments for future ansible commands.
 
+If bastion enabled, the generates SSH config must be applied for ansible.
+Otherwise, it is auto included by the previous step. In order to execute it
+as a separate playbook, use the following command:
+
+    ansible-playbook openshift-ansible-contrib/playbooks/provisioning/openstack/post-provision-openstack.yml
+
+The first infra node then becomes a bastion node as well and proxies access
+for future ansible commands. The post-provision step also configures Satellite,
+if requested, and DNS server, and ensures other OpenShift requirements to be met.
 
 ### Install OpenShift
 

playbooks/provisioning/openstack/openstack_dns_records.yml

@@ -36,11 +36,13 @@
   set_fact:
     public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': hostvars[item]['ansible_hostname'], 'ip': hostvars[item]['public_v4'] } ] }}"
   with_items: "{{ groups['cluster_hosts'] }}"
+  when: hostvars[item]['public_v4'] is defined
 
 - name: "Add wildcard records to the public A records"
   set_fact:
     public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': '*.' + openshift_app_domain, 'ip': hostvars[item]['public_v4'] } ] }}"
   with_items: "{{ groups['infra_hosts'] }}"
+  when: hostvars[item]['public_v4'] is defined
 
 - name: "Set the public DNS server details to use the external value (if provided)"
   set_fact:

playbooks/provisioning/openstack/post-provision-openstack.yml

@@ -4,7 +4,11 @@
   become: False
   gather_facts: False
   tasks:
-    - wait_for_connection:
+    - when: not openstack_use_bastion|default(False)|bool
+      wait_for_connection:
+    - when: openstack_use_bastion|default(False)|bool
+      delegate_to: bastion
+      wait_for_connection:
 
 - hosts: cluster_hosts
   gather_facts: True

playbooks/provisioning/openstack/provision-openstack.yml

@@ -12,13 +12,20 @@
       when: openstack_inventory|default('static') == 'static'
       inventory_path: "{{ openstack_inventory_path|default(inventory_dir) }}"
       private_ssh_key: "{{ openstack_private_ssh_key|default('~/.ssh/id_rsa') }}"
+      ssh_config_path: "{{ openstack_ssh_config_path|default('/tmp/ssh.config.openshift.ansible' + '.' + stack_name) }}"
+      ssh_user: "{{ ansible_user }}"
 
-- name: Refresh Server inventory
+- name: Refresh Server inventory or exit to apply SSH config
   hosts: localhost
   connection: local
   become: False
   gather_facts: False
   tasks:
-    - meta: refresh_inventory
+    - name: Exit to apply SSH config for a bastion
+      meta: end_play
+      when: openstack_use_bastion|default(False)|bool
+    - name: Refresh Server inventory
+      meta: refresh_inventory
 
 - include: post-provision-openstack.yml
+  when: not openstack_use_bastion|default(False)|bool

playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml

@@ -69,5 +69,12 @@ ansible_user: openshift
 # # The path to checkpoint the static inventory from the in-memory one
 #openstack_inventory_path: ../../../../inventory
 
+# # Use bastion node to access cluster nodes (Defaults to False).
+# # Requires a static inventory.
+#openstack_use_bastion: False
+#bastion_ingress_cidr: "{{openstack_subnet_prefix}}.0/24"
+#
 # # The Nova key-pair's private SSH key to access inventory nodes
 #openstack_private_ssh_key: ~/.ssh/openshift
+# # The path for the SSH config to access all nodes
+#openstack_ssh_config_path: /tmp/ssh.config.openshift.ansible.{{ env_id }}.{{ public_dns_domain }}

playbooks/provisioning/openstack/stack_params.yaml

@@ -21,3 +21,4 @@ master_volume_size: "{{ docker_volume_size }}"
 app_volume_size: "{{ docker_volume_size }}"
 infra_volume_size: "{{ docker_volume_size }}"
 nodes_to_remove: "{{ openstack_nodes_to_remove | default([]) |  to_yaml }}"
+use_bastion: "{{ openstack_use_bastion|default(False) }}"

roles/openstack-stack/defaults/main.yml

@@ -4,6 +4,7 @@ ssh_ingress_cidr: 0.0.0.0/0
 node_ingress_cidr: 0.0.0.0/0
 master_ingress_cidr: 0.0.0.0/0
 lb_ingress_cidr: 0.0.0.0/0
+bastion_ingress_cidr: 0.0.0.0/0
 num_etcd: 0
 num_masters: 1
 num_nodes: 1
@@ -11,3 +12,4 @@ num_dns: 1
 num_infra: 1
 nodes_to_remove: []
 etcd_volume_size: 2
+use_bastion: False

roles/openstack-stack/tasks/main.yml

@@ -8,7 +8,6 @@
 - name: set template paths
   set_fact:
     stack_template_path: "{{ stack_template_pre.path }}/stack.yaml"
-    server_template_path: "{{ stack_template_pre.path }}/server.yaml"
     user_data_template_path: "{{ stack_template_pre.path }}/user-data"
 
 - name: generate HOT stack template from jinja2 template
@@ -19,7 +18,13 @@
 - name: generate HOT server template from jinja2 template
   template:
     src: heat_stack_server.yaml.j2
-    dest: "{{ server_template_path }}"
+    dest: "{{ stack_template_pre.path }}/server.yaml"
+
+- name: generate HOT server w/o floating IPs template from jinja2 template
+  template:
+    src: heat_stack_server_nofloating.yaml.j2
+    dest: "{{ stack_template_pre.path }}/server_nofloating.yaml"
+  when: use_bastion|bool
 
 - name: generate user_data from jinja2 template
   template:

roles/openstack-stack/templates/heat_stack.yaml.j2

@@ -156,6 +156,13 @@ resources:
           port_range_min: 22
           port_range_max: 22
           remote_ip_prefix: {{ ssh_ingress_cidr }}
+{% if use_bastion|bool %}
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 22
+          port_range_max: 22
+          remote_ip_prefix: {{ bastion_ingress_cidr }}
+{% endif %}
         - direction: ingress
           protocol: icmp
           remote_ip_prefix: {{ ssh_ingress_cidr }}
@@ -458,7 +465,11 @@ resources:
     properties:
       count: {{ num_etcd }}
       resource_def:
+{% if use_bastion|bool %}
+        type: server_nofloating.yaml
+{% else %}
         type: server.yaml
+{% endif %}
         properties:
           name:
             str_replace:
@@ -483,7 +494,9 @@ resources:
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}etcd-secgrp{% endif %} }
             - { get_resource: common-secgrp }
+{% if not use_bastion|bool %}
           floating_network: {{ external_network }}
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -540,7 +553,11 @@ resources:
     properties:
       count: {{ num_masters }}
       resource_def:
+{% if use_bastion|bool %}
+        type: server_nofloating.yaml
+{% else %}
         type: server.yaml
+{% endif %}
         properties:
           name:
             str_replace:
@@ -573,7 +590,9 @@ resources:
 {% endif %}
 {% endif %}
             - { get_resource: common-secgrp }
+{% if not use_bastion|bool %}
           floating_network: {{ external_network }}
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -590,7 +609,11 @@ resources:
       removal_policies:
       - resource_list: {{ nodes_to_remove }}
       resource_def:
+{% if use_bastion|bool %}
+        type: server_nofloating.yaml
+{% else %}
         type: server.yaml
+{% endif %}
         properties:
           name:
             str_replace:
@@ -621,7 +644,9 @@ resources:
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}node-secgrp{% endif %} }
             - { get_resource: common-secgrp }
+{% if not use_bastion|bool %}
           floating_network: {{ external_network }}
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net

roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2

@@ -0,0 +1,149 @@
+heat_template_version: 2016-10-14
+
+description: OpenShift cluster server w/o floating IP
+
+parameters:
+
+  name:
+    type: string
+    label: Name
+    description: Name
+
+  group:
+    type: string
+    label: Host Group
+    description: The Primary Ansible Host Group
+    default: host
+
+  cluster_env:
+    type: string
+    label: Cluster environment
+    description: Environment of the cluster
+
+  cluster_id:
+    type: string
+    label: Cluster ID
+    description: Identifier of the cluster
+
+  type:
+    type: string
+    label: Type
+    description: Type master or node
+
+  subtype:
+    type: string
+    label: Sub-type
+    description: Sub-type compute or infra for nodes, default otherwise
+    default: default
+
+  key_name:
+    type: string
+    label: Key name
+    description: Key name of keypair
+
+  image:
+    type: string
+    label: Image
+    description: Name of the image
+
+  flavor:
+    type: string
+    label: Flavor
+    description: Name of the flavor
+
+  net:
+    type: string
+    label: Net ID
+    description: Net resource
+
+  net_name:
+    type: string
+    label: Net name
+    description: Net name
+
+  subnet:
+    type: string
+    label: Subnet ID
+    description: Subnet resource
+
+  secgrp:
+    type: comma_delimited_list
+    label: Security groups
+    description: Security group resources
+
+  availability_zone:
+    type: string
+    description: The Availability Zone to launch the instance.
+    default: nova
+
+  volume_size:
+    type: number
+    description: Size of the volume to be created.
+    default: 1
+    constraints:
+      - range: { min: 1, max: 1024 }
+        description: must be between 1 and 1024 Gb.
+
+  node_labels:
+    type: json
+    description: OpenShift Node Labels
+    default: {"region": "default" }
+
+outputs:
+
+  name:
+    description: Name of the server
+    value: { get_attr: [ server_nofloating, name ] }
+
+  private_ip:
+    description: Private IP of the server
+    value:
+      get_attr:
+        - server_nofloating
+        - addresses
+        - { get_param: net_name }
+        - 0
+        - addr
+
+resources:
+
+  server_nofloating:
+    type: OS::Nova::Server
+    properties:
+      name:      { get_param: name }
+      key_name:  { get_param: key_name }
+      image:     { get_param: image }
+      flavor:    { get_param: flavor }
+      networks:
+        - port:  { get_resource: port }
+      user_data:
+        get_file: user-data
+      user_data_format: RAW
+      metadata:
+        group: { get_param: group }
+        environment: { get_param: cluster_env }
+        clusterid: { get_param: cluster_id }
+        host-type: { get_param: type }
+        sub-host-type:    { get_param: subtype }
+        node_labels: { get_param: node_labels }
+
+  port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_param: net }
+      fixed_ips:
+        - subnet: { get_param: subnet }
+      security_groups: { get_param: secgrp }
+
+  cinder_volume:
+    type: OS::Cinder::Volume
+    properties:
+      size: { get_param: volume_size }
+      availability_zone: { get_param: availability_zone }
+
+  volume_attachment:
+    type: OS::Cinder::VolumeAttachment
+    properties:
+      volume_id: { get_resource: cinder_volume }
+      instance_uuid: { get_resource: server_nofloating }
+      mountpoint: /dev/sdb

roles/static_inventory/tasks/openstack.yml

@@ -23,11 +23,9 @@
         q2: "[] | [?metadata.clusterid=='{{stack_name}}'] | [?public_v4!='']"
       when:
         - refresh_inventory|bool
-        - use_bastion|bool
 
     - name: Add cluster nodes w/o floating IPs to inventory
-      with_items: "{{ registered_nodes }}"
-      when: not item in registered_nodes_floating
+      with_items: "{{ registered_nodes|difference(registered_nodes_floating) }}"
       add_host:
         name: '{{ item.name }}'
         groups: '{{ item.metadata.group }}'
@@ -40,11 +38,10 @@
 
     - name: Add cluster nodes with floating IPs to inventory
       with_items: "{{ registered_nodes_floating }}"
-      when: item in registered_nodes_floating
       add_host:
         name: '{{ item.name }}'
         groups: '{{ item.metadata.group }}'
-        ansible_host: "{% if use_bastion|bool %}{{ item.name }}{% else %}{{ item.private_v4 }}{% endif %}"
+        ansible_host: "{% if use_bastion|bool %}{{ item.name }}{% else %}{{ item.public_v4 }}{% endif %}"
         ansible_fqdn: '{{ item.name }}'
         ansible_user: '{{ ssh_user }}'
         ansible_private_key_file: '{{ private_ssh_key }}'

roles/static_inventory/templates/inventory.j2

@@ -14,9 +14,8 @@
 %} ansible_user={{ hostvars[host]['ansible_user'] }}{% endif %}
 {% if 'ansible_private_key_file' in hostvars[host]
 %} ansible_private_key_file={{ hostvars[host]['ansible_private_key_file'] }}{% endif %}
-{% if 'ansible_ssh_extra_args' in hostvars[host]
-%} ansible_ssh_extra_args={{ hostvars[host]['ansible_ssh_extra_args']|quote }}{% endif %}
- openshift_hostname={{ host }}
+{% if use_bastion|bool and 'ansible_ssh_extra_args' in hostvars[host]
+%} ansible_ssh_extra_args={{ hostvars[host]['ansible_ssh_extra_args']|quote }}{% endif %} openshift_hostname={{ host }}
 
 {% endif %}
 {% endfor %}