Support separate data network for Flannel SDN (#757)

* Support separate data network for Flannel SDN

Document the use case for a separate flannel data network.
Allow Nova servers for openshift cluster to be provisioned
with that isolated data network created and connected to
masters, computes and infra nodes. Do not configure dns
nameservers and router for that network.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

* Fix flannel use cases with provider network

Provider network cannot be used with flannel SDN
as the latter requires a separate isolated network,
while the provider network is an externally managed
single network.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

* Drop unused data_net_name

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

playbooks/provisioning/openstack/README.md

@@ -250,6 +250,9 @@ right after provisioning will fail (unless you're using an external DNS server
 your provider network knows about). You must make sure your nodes are able to
 resolve each other by name.
 
+**NOTE**: Flannel SDN requires a dedicated containers data network and cannot
+work over a single provider network.
+
 #### Security notes
 
 Configure required `*_ingress_cidr` variables to restrict public access
@@ -267,6 +270,10 @@ be the case for development environments. When turned off, the servers will
 be provisioned omitting the ``yum update`` command. This brings security
 implications though, and is not recommended for production deployments.
 
+Flannel network used for user applications and workloads data should be
+isolated from other networks as it has Neutron ports security disabled.
+Openshift master, compute and infra nodes will be connected to that network.
+
 ##### DNS servers security options
 
 Aside from `node_ingress_cidr` restricting public access to in-stack DNS
@@ -646,7 +653,7 @@ The `increment_by` variable is used to specify by how much the deployment should
 be scaled up (if none exists, it serves as a target number of application nodes).
 The path to `openshift-ansible` directory can be customised by the `openshift_ansible_dir`
 variable. Its value must be an absolute path to `openshift-ansible` and it cannot
-contain the '/' symbol at the end. 
+contain the '/' symbol at the end.
 
 Usage:
 

playbooks/provisioning/openstack/net_vars_check.yaml

@@ -0,0 +1,14 @@
+---
+- name: Check the provider network configuration
+  fail:
+    msg: "Flannel SDN requires a dedicated containers data network and can not work over a provider network"
+  when:
+    - openstack_provider_network_name is defined
+    - openstack_private_data_network_name is defined
+
+- name: Check the flannel network configuration
+  fail:
+    msg: "A dedicated containers data network is only supported with Flannel SDN"
+  when:
+    - openstack_private_data_network_name is defined
+    - not openshift_use_flannel|default(False)|bool

playbooks/provisioning/openstack/prerequisites.yml

@@ -2,6 +2,9 @@
 - hosts: localhost
   tasks:
 
+  # Sanity check of inventory variables
+  - include: net_vars_check.yaml
+
   # Check ansible
   - name: Check Ansible version
     assert:

playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml

@@ -51,3 +51,7 @@ openshift_override_hostname_check: true
 # NOTE(shadower): Always switch to root on the OSEv3 nodes.
 # openshift-ansible requires an explicit `become`.
 ansible_become: true
+
+# # Flannel networking
+#openshift_use_openshift_sdn: false
+#openshift_use_flannel: true

playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml

@@ -15,6 +15,10 @@ public_dns_nameservers: []
 openstack_ssh_public_key: "openshift"
 openstack_external_network_name: "public"
 #openstack_private_network_name:  "openshift-ansible-{{ stack_name }}-net"
+# # A dedicated Neutron network name for containers data network
+# # Configures the data network to be separated from openstack_private_network_name
+# # NOTE: this is only supported with Flannel SDN yet
+#openstack_private_data_network_name: "openshift-ansible-{{ stack_name }}-data-net"
 
 ## If you want to use a provider network, set its name here.
 ## NOTE: the `openstack_external_network_name` and

roles/openstack-stack/templates/heat_stack.yaml.j2

@@ -113,6 +113,22 @@ resources:
         - {{ nameserver }}
 {% endfor %}
 
+{% if openshift_use_flannel|default(False)|bool %}
+  data_net:
+    type: OS::Neutron::Net
+    properties:
+      name: openshift-ansible-{{ stack_name }}-data-net
+      port_security_enabled: false
+
+  data_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      name: openshift-ansible-{{ stack_name }}-data-subnet
+      network: { get_resource: data_net }
+      cidr: {{ osm_cluster_network_cidr|default('10.128.0.0/14') }}
+      gateway_ip: null
+{% endif %}
+
   router:
     type: OS::Neutron::Router
     properties:
@@ -641,6 +657,11 @@ resources:
               template: openshift-ansible-cluster_id-net
               params:
                 cluster_id: {{ stack_name }}
+{% if openshift_use_flannel|default(False)|bool %}
+          attach_data_net: true
+          data_net:    { get_resource: data_net }
+          data_subnet: { get_resource: data_subnet }
+{% endif %}
 {% endif %}
           secgrp:
 {% if openstack_flat_secgrp|default(False)|bool %}
@@ -713,6 +734,11 @@ resources:
               template: openshift-ansible-cluster_id-net
               params:
                 cluster_id: {{ stack_name }}
+{% if openshift_use_flannel|default(False)|bool %}
+          attach_data_net: true
+          data_net:    { get_resource: data_net }
+          data_subnet: { get_resource: data_subnet }
+{% endif %}
 {% endif %}
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}node-secgrp{% endif %} }
@@ -767,6 +793,11 @@ resources:
               template: openshift-ansible-cluster_id-net
               params:
                 cluster_id: {{ stack_name }}
+{% if openshift_use_flannel|default(False)|bool %}
+          attach_data_net: true
+          data_net:    { get_resource: data_net }
+          data_subnet: { get_resource: data_subnet }
+{% endif %}
 {% endif %}
           secgrp:
 # TODO(bogdando) filter only required node rules into infra-secgrp

roles/openstack-stack/templates/heat_stack_server.yaml.j2

@@ -68,6 +68,28 @@ parameters:
     description: Subnet resource
 {% endif %}
 
+{% if openshift_use_flannel|default(False)|bool %}
+  attach_data_net:
+    type: boolean
+    default: false
+    label: Attach-data-net
+    description: A switch for data port connection
+
+  data_net:
+    type: string
+    default: ''
+    label: Net ID
+    description: Net resource
+
+{% if not provider_network %}
+  data_subnet:
+    type: string
+    default: ''
+    label: Subnet ID
+    description: Subnet resource
+{% endif %}
+{% endif %}
+
   secgrp:
     type: comma_delimited_list
     label: Security groups
@@ -133,6 +155,11 @@ outputs:
 {% endif %}
         - addr
 
+{% if openshift_use_flannel|default(False)|bool %}
+conditions:
+  no_data_subnet: {not: { get_param: attach_data_net} }
+{% endif %}
+
 resources:
 
   server:
@@ -143,11 +170,28 @@ resources:
       image:     { get_param: image }
       flavor:    { get_param: flavor }
       networks:
+{% if openshift_use_flannel|default(False)|bool %}
+        if:
+          - no_data_subnet
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+{% endif %}
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+            - port:  { get_resource: data_port }
+{% endif %}
+
+{% else %}
 {% if use_trunk_ports|default(false)|bool %}
         - port:  { get_attr: [trunk-port, port_id] }
 {% else %}
         - port:  { get_resource: port }
 {% endif %}
+{% endif %}
       user_data:
         get_file: user-data
       user_data_format: RAW
@@ -179,6 +223,19 @@ resources:
 {% endif %}
       security_groups: { get_param: secgrp }
 
+{% if openshift_use_flannel|default(False)|bool %}
+  data_port:
+    type: OS::Neutron::Port
+    condition: { not: no_data_subnet }
+    properties:
+      network: { get_param: data_net }
+      port_security_enabled: false
+{% if not provider_network %}
+      fixed_ips:
+        - subnet: { get_param: data_subnet }
+{% endif %}
+{% endif %}
+
 {% if not provider_network %}
   floating-ip:
     type: OS::Neutron::FloatingIP

roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2

@@ -66,6 +66,26 @@ parameters:
     label: Subnet ID
     description: Subnet resource
 
+{% if openshift_use_flannel|default(False)|bool %}
+  attach_data_net:
+    type: boolean
+    default: false
+    label: Attach-data-net
+    description: A switch for data port connection
+
+  data_net:
+    type: string
+    default: ''
+    label: Net ID
+    description: Net resource
+
+  data_subnet:
+    type: string
+    default: ''
+    label: Subnet ID
+    description: Subnet resource
+{% endif %}
+
   secgrp:
     type: comma_delimited_list
     label: Security groups
@@ -110,6 +130,11 @@ outputs:
         - 0
         - addr
 
+{% if openshift_use_flannel|default(False)|bool %}
+conditions:
+  no_data_subnet: {not: { get_param: attach_data_net} }
+{% endif %}
+
 resources:
 
   server_nofloating:
@@ -120,11 +145,28 @@ resources:
       image:     { get_param: image }
       flavor:    { get_param: flavor }
       networks:
+{% if openshift_use_flannel|default(False)|bool %}
+        if:
+          - no_data_subnet
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+{% endif %}
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+            - port:  { get_resource: data_port }
+{% endif %}
+
+{% else %}
 {% if use_trunk_ports|default(false)|bool %}
         - port:  { get_attr: [trunk-port, port_id] }
 {% else %}
         - port:  { get_resource: port }
 {% endif %}
+{% endif %}
       user_data:
         get_file: user-data
       user_data_format: RAW
@@ -154,6 +196,19 @@ resources:
         - subnet: { get_param: subnet }
       security_groups: { get_param: secgrp }
 
+{% if openshift_use_flannel|default(False)|bool %}
+  data_port:
+    type: OS::Neutron::Port
+    condition: { not: no_data_subnet }
+    properties:
+      network: { get_param: data_net }
+      port_security_enabled: false
+{% if not provider_network %}
+      fixed_ips:
+        - subnet: { get_param: data_subnet }
+{% endif %}
+{% endif %}
+
 {% if not ephemeral_volumes|default(false)|bool %}
   cinder_volume:
     type: OS::Cinder::Volume