Kuryr var generation in OSt dynamic inventory

This patch adds the necessary network resources for Kuryr in the heat
stack, including putting a l4 load balancer in front of the API.

It then automatically uses the resources to generate the OSEv3 variables
needed for the already existing kuryr role.

Change-Id: I6fc4c6bc9835217334db1289987daf358dcf287b
Signed-off-by: Antoni Segura Puimedon <antonisp@celebdor.com>

playbooks/openstack/README.md

@@ -144,7 +144,18 @@ $ vi inventory/group_vars/all.yml
 4. Set the `openshift_openstack_default_flavor` to the flavor you want your
    OpenShift VMs to use.
    - See `openstack flavor list` for the list of available flavors.
+5. If you opt to use Kuryr for the networking, make sure that you review all
+   the kuryr options in the file. At the very least, if you use Kuryr, you
+   should uncomment:
 
+```bash
+#openshift_use_kuryr: True
+#use_trunk_ports: True
+#openshift_use_openshift_sdn: False
+#os_sdn_network_plugin_name: cni
+#openshift_node_proxy_mode: userspace
+#openshift_hosted_manage_registry: false
+```
 
 
 #### OpenShift configuration

playbooks/openstack/inventory.py

@@ -11,6 +11,7 @@ from __future__ import print_function
 
 from collections import Mapping
 import json
+import os
 
 import shade
 
@@ -140,8 +141,56 @@ def build_inventory():
                 hostvars['docker_storage_mountpoints'] = ' '.join(docker_storage_mountpoints[server.id])
 
         inventory['_meta']['hostvars'][server.name] = hostvars
+
+    kuryr_vars = _get_kuryr_vars(cloud)
+    if kuryr_vars:
+        inventory['OSEv3']['vars'] = kuryr_vars
+
     return inventory
 
 
+def _get_kuryr_vars(cloud_client):
+    """Returns a dictionary of Kuryr variables resulting of heat stacking"""
+    # TODO: Filter the cluster stack with tags once it is supported in shade
+    cluster_name = os.getenv('OPENSHIFT_CLUSTER', 'openshift-cluster')
+
+    stack = cloud_client.get_stack(cluster_name)
+    if stack is None or stack['stack_status'] not in (
+            'CREATE_COMPLETE', 'UPDATE_COMPLETE'):
+        return None
+
+    data = {}
+    for output in stack['outputs']:
+        data[output['output_key']] = output['output_value']
+
+    settings = {}
+    settings['kuryr_openstack_pod_subnet_id'] = data['pod_subnet']
+    settings['kuryr_openstack_worker_nodes_subnet_id'] = data['vm_subnet']
+    settings['kuryr_openstack_service_subnet_id'] = data['service_subnet']
+    settings['kuryr_openstack_pod_sg_id'] = data['pod_access_sg_id']
+    settings['kuryr_openstack_pod_project_id'] = (
+        cloud_client.current_project_id)
+
+    settings['kuryr_openstack_auth_url'] = cloud_client.auth['auth_url']
+    settings['kuryr_openstack_username'] = cloud_client.auth['username']
+    settings['kuryr_openstack_password'] = cloud_client.auth['password']
+    if 'user_domain_id' in cloud_client.auth:
+        settings['kuryr_openstack_user_domain_name'] = (
+            cloud_client.auth['user_domain_id'])
+    else:
+        settings['kuryr_openstack_user_domain_name'] = (
+            cloud_client.auth['user_domain_name'])
+    # FIXME(apuimedo): consolidate kuryr controller credentials into the same
+    #                  vars the openstack playbook uses.
+    settings['kuryr_openstack_project_id'] = cloud_client.current_project_id
+    if 'project_domain_id' in cloud_client.auth:
+        settings['kuryr_openstack_project_domain_name'] = (
+            cloud_client.auth['project_domain_id'])
+    else:
+        settings['kuryr_openstack_project_domain_name'] = (
+            cloud_client.auth['project_domain_name'])
+    return settings
+
+
 if __name__ == '__main__':
     print(json.dumps(build_inventory(), indent=4, sort_keys=True))

playbooks/openstack/openshift-cluster/provision.yml

@@ -2,6 +2,9 @@
 - name: Create the OpenStack resources for cluster installation
   hosts: localhost
   tasks:
+  - name: retrieve cluster name from the environment if present
+    set_fact:
+      openshift_openstack_stack_name: "{{ lookup('env', 'OPENSHIFT_CLUSTER') | ternary (lookup('env', 'OPENSHIFT_CLUSTER'), omit) }}"
   - name: provision cluster
     import_role:
       name: openshift_openstack

playbooks/openstack/sample-inventory/group_vars/all.yml

@@ -20,6 +20,33 @@ openshift_openstack_external_network_name: "public"
 # # NOTE: this is only supported with Flannel SDN yet
 #openstack_private_data_network_name: "openshift-ansible-{{ openshift_openstack_stack_name }}-data-net"
 
+## Kuryr networking
+# TODO: Allow the user to specify pre-existing subnets for pod and services
+#openshift_openstack_kuryr_service_subnet_cidr: "172.30.0.0/16"
+#
+## You can alter the port pooling defaults here
+#kuryr_openstack_enable_pools: True
+#kuryr_openstack_pool_max: 0
+#kuryr_openstack_pool_min: 1
+#kuryr_openstack_pool_batch: 5
+#kuryr_openstack_pool_update_frequency: 20
+#
+## You should set the following if you want to use Kuryr/Neutron as your SDN
+#openshift_use_kuryr: True
+#openshift_use_openshift_sdn: False
+#use_trunk_ports: True
+#os_sdn_network_plugin_name: cni
+#openshift_node_proxy_mode: userspace
+# # Kuryr needs to have the pod based registry (if desired in the cluster)
+# deployed after kuryr is up and running. This can be done with oadm
+# #Disable management of the OpenShift Registry
+#openshift_hosted_manage_registry: false
+
+# If you VM images will name the ethernet device different than 'eth0',
+# override this
+#kuryr_cni_link_interface: eth0
+
+
 ## If you want to use a provider network, set its name here.
 ## NOTE: the `openshift_openstack_external_network_name` and
 ## `openshift_openstack_private_network_name` options will be ignored when using a

roles/openshift_openstack/defaults/main.yml

@@ -53,10 +53,14 @@ openshift_openstack_app_subdomain: "apps"
 
 # heat vars
 openshift_openstack_clusterid: openshift
-openshift_openstack_stack_name: "{{ openshift_openstack_clusterid }}.{{ openshift_openstack_public_dns_domain }}"
+openshift_openstack_stack_name: "openshift-cluster"
 openshift_openstack_subnet_cidr: "192.168.99.0/24"
 openshift_openstack_pool_start: "192.168.99.3"
 openshift_openstack_pool_end: "192.168.99.254"
+openshift_openstack_kuryr_service_subnet_cidr: "172.30.0.0/16"
+openshift_openstack_kuryr_service_pool_start: "172.30.128.1"
+openshift_openstack_kuryr_service_pool_end: "172.30.255.253"
+openshift_openstack_kuryr_pod_subnet_cidr: "10.11.0.0/16"
 openshift_openstack_master_hostname: master
 openshift_openstack_infra_hostname: infra-node
 openshift_openstack_cns_hostname: cns

roles/openshift_openstack/templates/heat_stack.yaml.j2

@@ -54,12 +54,118 @@ outputs:
     description: Floating IPs of the nodes
     value: { get_attr: [ infra_nodes, floating_ip ] }
 
+{% if openshift_use_kuryr|default(false)|bool %}
+  vm_subnet:
+    description: ID of the subnet the Pods will be on
+    value: { get_resource: subnet }
+
+  pod_subnet:
+    description: ID of the subnet the Pods will be on
+    value: { get_resource: pod_subnet }
+
+  service_subnet:
+    description: ID of the subnet the services will be on
+    value: { get_resource: service_subnet }
+
+  pod_access_sg_id:
+    description: Id of the security group for services to be able to reach pods
+    value: { get_resource: pod_access_sg }
+{% endif %}
+
 conditions:
   no_floating: {% if openshift_openstack_provider_network_name %}true{% else %}false{% endif %}
 
 resources:
 
 {% if not openshift_openstack_provider_network_name %}
+{% if openshift_use_kuryr|default(false)|bool %}
+  api_lb:
+    type: OS::Neutron::LBaaS::LoadBalancer
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-api-lb
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+      vip_address: {{ openshift_openstack_kuryr_service_subnet_cidr | ipaddr('1') | ipaddr('address') }}
+      vip_subnet: { get_resource: service_subnet }
+
+  api_lb_listener:
+    type: OS::Neutron::LBaaS::Listener
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-api-lb-listener
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+      loadbalancer: { get_resource: api_lb }
+      protocol: HTTPS
+      protocol_port: 443
+
+  api_lb_pool:
+    type: OS::Neutron::LBaaS::Pool
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-api-lb-pool
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+      protocol: HTTPS
+      lb_algorithm: ROUND_ROBIN
+      listener: { get_resource: api_lb_listener }
+
+  pod_net:
+    type: OS::Neutron::Net
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-pod-net
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+
+  pod_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: pod_net }
+      cidr: {{ openshift_openstack_kuryr_pod_subnet_cidr }}
+      enable_dhcp: False
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-pod-subnet
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+      dns_nameservers:
+{% for nameserver in openshift_openstack_dns_nameservers %}
+        - {{ nameserver }}
+{% endfor %}
+
+  service_net:
+    type: OS::Neutron::Net
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-service-net
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+
+  service_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: service_net }
+      cidr: {{ openshift_openstack_kuryr_service_subnet_cidr }}
+      gateway_ip: {{ openshift_openstack_kuryr_service_subnet_cidr | ipaddr('-2') | ipaddr('address') }}
+      enable_dhcp: False
+      allocation_pools:
+        - start: {{ openshift_openstack_kuryr_service_pool_start }}
+          end: {{ openshift_openstack_kuryr_service_pool_end }}
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-service-subnet
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+
+{% endif %}
+
   net:
     type: OS::Neutron::Net
     properties:
@@ -120,6 +226,33 @@ resources:
       router_id: { get_resource: router }
       subnet_id: { get_resource: subnet }
 
+{% if openshift_use_kuryr|default(false)|bool %}
+  pod_subnet_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: pod_subnet }
+
+  service_router_port:
+      type: OS::Neutron::Port
+      properties:
+        network: { get_resource: service_net}
+        fixed_ips:
+          - subnet: { get_resource: service_subnet }
+            ip_address: {{ openshift_openstack_kuryr_service_subnet_cidr | ipaddr('-2') | ipaddr('address') }}
+        name:
+          str_replace:
+            template: openshift-ansible-cluster_id-service-subnet-router-port
+            params:
+              cluster_id: {{ openshift_openstack_stack_name }}
+
+  service_subnet_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      port: { get_resource: service_router_port }
+{% endif %}
+
 {% endif %}
 
 #  keypair:
@@ -155,6 +288,25 @@ resources:
           protocol: icmp
           remote_ip_prefix: {{ openshift_openstack_ssh_ingress_cidr }}
 
+{% if openshift_use_kuryr|default(false)|bool %}
+  pod_access_sg:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-pod-service-secgrp
+          params:
+            cluster_id: {{ openshift_openstack_stack_name }}
+      description: Give services and nodes access to the pods
+      rules:
+      - ethertype: IPv4
+        remote_ip_prefix: {{ openshift_openstack_kuryr_service_subnet_cidr }}
+      - ethertype: IPv4
+        remote_ip_prefix: {{ openshift_openstack_subnet_cidr }}
+      - ethertype: IPv4
+        remote_mode: remote_group_id
+{% endif %}
+
 {% if openshift_openstack_flat_secgrp|default(False)|bool %}
   flat-secgrp:
     type: OS::Neutron::SecurityGroup
@@ -638,6 +790,9 @@ resources:
           data_subnet: { get_resource: data_subnet }
 {% endif %}
 {% endif %}
+{% if openshift_use_kuryr|default(false)|bool %}
+          api_lb_pool: { get_resource: api_lb_pool }
+{% endif %}
           secgrp:
 {% if openshift_openstack_flat_secgrp|default(False)|bool %}
             - { get_resource: flat-secgrp }

roles/openshift_openstack/templates/heat_stack_server.yaml.j2

@@ -95,6 +95,14 @@ parameters:
     label: Security groups
     description: Security group resources
 
+{% if openshift_use_kuryr|default(false)|bool %}
+  api_lb_pool:
+    default: ''
+    type: string
+    label: API LoadBalancer pool ID
+    description: API Loadbalancer pool resource
+{% endif %}
+
   attach_float_net:
     type: boolean
     default: true
@@ -270,3 +278,18 @@ resources:
       volume_id: { get_resource: cinder_volume }
       instance_uuid: { get_resource: server }
 {% endif %}
+
+
+{% if openshift_use_kuryr|default(false)|bool %}
+  lb_member:
+    type: OS::Neutron::LBaaS::PoolMember
+    condition:
+      equals:
+        - get_param: type
+        - master
+    properties:
+      pool: { get_param: api_lb_pool }
+      protocol_port: {{ openshift_master_api_port|default(8443) }}
+      address: { get_attr: [server, first_address]}
+      subnet: { get_param: subnet }
+{% endif %}