Add support to pre-create subports at each trunk

It adds support to precreate subports at each trunk when kuryr is
used, i.e., it allows creating ports and attaching them to the
worker VMs node as subports of their trunk.

It made use of a new variable to decide on creating or not the
subports and how many to create per trunk. The variable is named
openshift_kuryr_precreate_subports. It is false by default.
If included, its value should be equal to the number of subports
to create for each worker VM node.

playbooks/openstack/configuration.md

@@ -183,6 +183,19 @@ the Kuryr options. The following parameters should be uncommented in
 * `openshift_hosted_manage_registry`
 * `kuryr_openstack_public_subnet_id` Set to uuid of the public subnet.
 
+And if you also want to enable the ports pooling options, as well as
+preloading them, you should uncomment and set the next:
+
+* `kuryr_openstack_enable_pools: True`
+* `kuryr_openstack_pool_max: 0`
+* `kuryr_openstack_pool_min: 1`
+* `kuryr_openstack_pool_batch: 5`
+* `kuryr_openstack_pool_update_frequency: 60`
+* `openshift_kuryr_precreate_subports: 5`
+
+Note in the last variable you specify the number of subports that will
+be created per trunk port, i.e., per pool.
+
 
 ## Multi-Master Configuration
 

playbooks/openstack/sample-inventory/group_vars/all.yml

@@ -24,14 +24,6 @@ openshift_openstack_external_network_name: "public"
 # TODO: Allow the user to specify pre-existing subnets for pod and services
 #openshift_openstack_kuryr_service_subnet_cidr: "172.30.0.0/16"
 
-#
-## You can alter the port pooling defaults here
-#kuryr_openstack_enable_pools: True
-#kuryr_openstack_pool_max: 0
-#kuryr_openstack_pool_min: 1
-#kuryr_openstack_pool_batch: 5
-#kuryr_openstack_pool_update_frequency: 20
-#
 ## You should set the following if you want to use Kuryr/Neutron as your SDN
 #openshift_use_kuryr: True
 #openshift_use_openshift_sdn: False
@@ -63,6 +55,23 @@ openshift_openstack_external_network_name: "public"
 # override this
 #kuryr_cni_link_interface: eth0
 
+## To enable the ports pooling you need to enable the next
+#kuryr_openstack_enable_pools: True
+#
+## You can alter the port pooling defaults here
+#kuryr_openstack_pool_max: 0
+#kuryr_openstack_pool_min: 1
+#kuryr_openstack_pool_batch: 5
+#kuryr_openstack_pool_update_frequency: 20
+#
+## You can pre-populate the pools with subports by uncommenting the next line
+## and specifying the amount of subports you want to pre-create per
+## origin-node, e.g.: openshift_kuryr_precreate_subports: 5
+#openshift_kuryr_precreate_subports: false
+#
+## You can also change the default device owner for the precreated subports
+#openshift_kuryr_device_owner: compute:kuryr
+
 
 ## If you want to use a provider network, set its name here.
 ## NOTE: the `openshift_openstack_external_network_name` and

roles/kuryr/README.md

@@ -36,6 +36,8 @@ pods. This allows to have interconnectivity between pods and OpenStack VMs.
 * ``kuryr_openstack_pool_min=1``
 * ``kuryr_openstack_pool_batch=5``
 * ``kuryr_openstack_pool_update_frequency=20``
+* ``openshift_kuryr_precreate_subports=5``
+* ``openshift_kuryr_device_owner=compute:kuryr``
 
 ## Kuryr resources
 

roles/openshift_openstack/tasks/generate-templates.yml

@@ -27,3 +27,13 @@
   template:
     src: user_data.j2
     dest: "{{ user_data_template_path }}"
+
+- name: generate HOT nested subports template from jinja2 template
+  template:
+    src: heat_stack_subports.yaml.j2
+    dest: "{{ stack_template_pre.path }}/subports.yaml"
+
+- name: generate HOT nested segmentation ids template from jinja2 template
+  template:
+    src: heat_stack_segmentation_ids.yaml.j2
+    dest: "{{ stack_template_pre.path }}/segmentation_ids.yaml"

roles/openshift_openstack/templates/heat_stack.yaml.j2

@@ -679,6 +679,10 @@ resources:
 {% else %}
           net:         { get_resource: net }
           subnet:      { get_resource: subnet }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_net:     { get_resource: pod_net }
+          pod_subnet:  { get_resource: pod_subnet }
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -688,6 +692,10 @@ resources:
           secgrp:
             - { get_resource: {% if openshift_openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}etcd-secgrp{% endif %} }
             - { get_resource: common-secgrp }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_secgrp:
+            - { get_resource: pod_access_sg }
+{% endif %}
           floating_network:
             if:
               - no_floating
@@ -748,6 +756,10 @@ resources:
 {% else %}
           net:         { get_resource: net }
           subnet:      { get_resource: subnet }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_net:     { get_resource: pod_net }
+          pod_subnet:  { get_resource: pod_subnet }
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -757,6 +769,10 @@ resources:
           secgrp:
             - { get_resource: lb-secgrp }
             - { get_resource: common-secgrp }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_secgrp:
+            - { get_resource: pod_access_sg }
+{% endif %}
           floating_network:
             if:
               - no_floating
@@ -803,6 +819,10 @@ resources:
 {% else %}
           net:         { get_resource: net }
           subnet:      { get_resource: subnet }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_net:     { get_resource: pod_net }
+          pod_subnet:  { get_resource: pod_subnet }
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -826,8 +846,12 @@ resources:
 {% if openshift_openstack_num_etcd|int == 0 %}
             - { get_resource: etcd-secgrp }
 {% endif %}
-{% endif %}
             - { get_resource: common-secgrp }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_secgrp:
+            - { get_resource: pod_access_sg }
+{% endif %}
+{% endif %}
           floating_network:
             if:
               - no_floating
@@ -884,6 +908,10 @@ resources:
 {% else %}
           net:         { get_resource: net }
           subnet:      { get_resource: subnet }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_net:     { get_resource: pod_net }
+          pod_subnet:  { get_resource: pod_subnet }
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -898,6 +926,10 @@ resources:
           secgrp:
             - { get_resource: {% if openshift_openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}node-secgrp{% endif %} }
             - { get_resource: common-secgrp }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_secgrp:
+            - { get_resource: pod_access_sg }
+{% endif %}
           floating_network:
             if:
               - no_floating
@@ -948,6 +980,10 @@ resources:
 {% else %}
           net:         { get_resource: net }
           subnet:      { get_resource: subnet }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_net:     { get_resource: pod_net }
+          pod_subnet:  { get_resource: pod_subnet }
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -965,9 +1001,13 @@ resources:
             - { get_resource: flat-secgrp }
 {% else %}
             - { get_resource: node-secgrp }
-{% endif %}
             - { get_resource: infra-secgrp }
             - { get_resource: common-secgrp }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_secgrp:
+            - { get_resource: pod_access_sg }
+{% endif %}
+{% endif %}
           floating_network:
             if:
               - no_floating
@@ -1017,6 +1057,10 @@ resources:
 {% else %}
           net:         { get_resource: net }
           subnet:      { get_resource: subnet }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_net:     { get_resource: pod_net }
+          pod_subnet:  { get_resource: pod_subnet }
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -1033,10 +1077,18 @@ resources:
             - { get_resource: flat-secgrp }
 {% else %}
             - { get_resource: node-secgrp }
-{% endif %}
             - { get_resource: cns-secgrp }
             - { get_resource: common-secgrp }
+{% if openshift_use_kuryr|default(false)|bool %}
+          pod_secgrp:
+            - { get_resource: pod_access_sg }
+{% endif %}
+{% endif %}
 {% if not openshift_openstack_provider_network_name %}
           floating_network: {{ openshift_openstack_external_network_name }}
 {% endif %}
           volume_size: {{ openshift_openstack_cns_volume_size }}
+{% if not openshift_openstack_provider_network_name %}
+    depends_on:
+      - interface
+{% endif %}

roles/openshift_openstack/templates/heat_stack_segmentation_ids.yaml.j2

@@ -0,0 +1,25 @@
+heat_template_version: pike
+
+description: Simple template to generate the needed vlan ids
+
+parameters:
+  number_of_vlan_ids:
+    type: number
+    description: Number of vlan segmentation IDs
+    default: 1
+
+resources:
+  segmentation_ids:
+    type: OS::Heat::ResourceGroup
+    properties:
+      count: { get_param: number_of_vlan_ids }
+      removal_policies: [ { "resource_list": ["0"] } ]
+      resource_def:
+        type: OS::Heat::Value
+        properties:
+          type: string
+          value: "%index%"
+
+outputs:
+  vlan_ids:
+    value: { get_attr: [segmentation_ids, value] }

roles/openshift_openstack/templates/heat_stack_server.yaml.j2

@@ -1,4 +1,4 @@
-heat_template_version: 2016-10-14
+heat_template_version: pike
 
 description: OpenShift cluster server
 
@@ -68,6 +68,20 @@ parameters:
     description: Subnet resource
 {% endif %}
 
+{% if openshift_use_kuryr|default(false)|bool %}
+  pod_net:
+    type: string
+    label: Pod Net ID
+    description: Net resource
+    default: ''
+
+  pod_subnet:
+    type: string
+    label: Pod Subnet ID
+    description: Subnet resource
+    default: ''
+{% endif %}
+
 {% if openshift_use_flannel|default(False)|bool %}
   attach_data_net:
     type: boolean
@@ -103,6 +117,14 @@ parameters:
     description: API Loadbalancer pool resource
 {% endif %}
 
+{% if openshift_use_kuryr|default(false)|bool %}
+  pod_secgrp:
+    type: comma_delimited_list
+    label: Subports Security groups
+    description: Security group resources for subports
+    default: ''
+{% endif %}
+
   attach_float_net:
     type: boolean
     default: true
@@ -188,21 +210,13 @@ resources:
 {% if openshift_use_flannel|default(False)|bool %}
         if:
           - no_data_subnet
-{% if use_trunk_ports|default(false)|bool %}
-          - - port:  { get_attr: [trunk-port, port_id] }
-{% else %}
           - - port:  { get_resource: port }
-{% endif %}
-{% if use_trunk_ports|default(false)|bool %}
-          - - port:  { get_attr: [trunk-port, port_id] }
-{% else %}
           - - port:  { get_resource: port }
             - port:  { get_resource: data_port }
-{% endif %}
 
 {% else %}
-{% if use_trunk_ports|default(false)|bool %}
-        - port:  { get_attr: [trunk-port, port_id] }
+{% if use_trunk_ports|default(False)|bool %}
+        - port:  { get_attr: [trunk_port, port_id]}
 {% else %}
         - port:  { get_resource: port }
 {% endif %}
@@ -224,11 +238,39 @@ resources:
       scheduler_hints: { get_param: scheduler_hints }
 
 {% if use_trunk_ports|default(false)|bool %}
-  trunk-port:
+  trunk_port:
     type: OS::Neutron::Trunk
     properties:
       name: { get_param: name }
       port: { get_resource: port }
+{% if openshift_kuryr_precreate_subports|default(false) %}
+      sub_ports:
+        repeat:
+          for_each:
+            <%port%>: { get_attr: [subports, subport_id] }
+            <%vlan_id%>: { get_attr: [segmentation, vlan_ids] }
+          template:
+            port: <%port%>
+            segmentation_type: vlan
+            segmentation_id: <%vlan_id%>
+          permutations: false
+
+  segmentation:
+    type: segmentation_ids.yaml
+    properties:
+      number_of_vlan_ids: {{ openshift_kuryr_precreate_subports }}
+
+  subports:
+    type: OS::Heat::ResourceGroup
+    properties:
+      count: {{ openshift_kuryr_precreate_subports }}
+      resource_def:
+        type: subports.yaml
+        properties:
+          network: { get_param: pod_net }
+          subnet: { get_param: pod_subnet }
+          pod_access_sg: { get_param: pod_secgrp }
+{% endif %}
 {% endif %}
 
   port:

roles/openshift_openstack/templates/heat_stack_subports.yaml.j2

@@ -0,0 +1,28 @@
+heat_template_version: pike
+
+description: Template to create subports
+
+parameters:
+  network:
+    type: string
+    label: Subport Network
+  subnet:
+    type: string
+    label: Subport Subnet
+  pod_access_sg:
+    type: comma_delimited_list
+    label: Pod access sg
+
+resources:
+  subports:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_param: network }
+      security_groups: { get_param: pod_access_sg }
+      fixed_ips:
+        - subnet: { get_param: subnet }
+      device_owner: {{ openshift_kuryr_device_owner|default('compute:kuryr') }}
+
+outputs:
+  subport_id:
+    value: { get_resource: subports }