Merge pull request #9459 from vareti/update-ansible

Nuage Specific Changes for eVDF and some minor changes

playbooks/openshift-master/private/config.yml

@@ -93,10 +93,10 @@
 
   - role: openshift_control_plane
   - role: tuned
-  - role: nuage_ca
-    when: openshift_use_nuage | default(false) | bool
   - role: nuage_common
     when: openshift_use_nuage | default(false) | bool
+  - role: nuage_ca
+    when: openshift_use_nuage | default(false) | bool
   - role: nuage_master
     when: openshift_use_nuage | default(false) | bool
   - role: calico_master

roles/nuage_ca/tasks/main.yaml

@@ -40,7 +40,7 @@
   delegate_to: "{{ nuage_ca_master }}"
 
 - name: Create CA crt
-  command: openssl req -new -x509 -key "{{ nuage_ca_key }}" -out "{{ nuage_ca_crt }}" -subj "/CN=nuage-signer"
+  command: openssl req -new -x509 -key "{{ nuage_ca_key }}" -out "{{ nuage_ca_crt }}" -subj "/CN=nuage-signer" -days {{ nuage_mon_cert_validity_period }}
   run_once: true
   delegate_to: "{{ nuage_ca_master }}"
   when: nuage_ca_crt_check.stat.exists is defined and nuage_ca_crt_check.stat.exists == False

roles/nuage_master/defaults/main.yml

@@ -2,7 +2,13 @@
 r_nuage_master_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
 r_nuage_master_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
 
-nuage_mon_rest_server_port: '9443'
+nuage_mon_rest_server_port: "{{ nuage_openshift_monitor_rest_server_port | default('9443') }}"
+
+nuage_mon_image_name: "{{ nuage_mon_image_path | default('nuage/master') }}"
+nuage_vrs_image_name: "{{ nuage_vrs_image_path | default('nuage/vrs') }}"
+nuage_cni_image_name: "{{ nuage_cni_image_path | default('nuage/cni') }}"
+nuage_infra_image_name: "{{ nuage_infra_image_path | default('nuage/infra') }}"
+nuage_sswan_image_name: "{{ nuage_sswan_image_path | default('nuage/strongswan') }}"
 
 r_nuage_master_os_firewall_deny: []
 r_nuage_master_os_firewall_allow:

roles/nuage_master/tasks/main.yaml

@@ -37,6 +37,11 @@
     nuage_cni_bin_dsets_mount_dir: /var/opt/cni/bin
   when: openshift_is_atomic | bool
 
+- name: Set the Nuage VRS mount dir for daemon sets install
+  set_fact:
+    nuage_vrs_mount_dir: /etc/default
+  when: nuage_personality == "evdf"
+
 - name: Create directory /usr/share/nuage-openshift-monitor
   become: yes
   file: path=/usr/share/nuage-openshift-monitor state=directory
@@ -104,6 +109,11 @@
   become: yes
   template: src=nuage-infra-pod-config-daemonset.j2 dest=/etc/nuage-infra-pod-config-daemonset.yaml owner=root mode=0644
 
+- name: Create Nuage strongswan Pod daemon set yaml file for EVDF platform
+  become: yes
+  template: src=nuage-strongswan-pod-config-daemonset.j2 dest=/etc/nuage-strongswan-pod-config-daemonset.yaml owner=root mode=0644
+  when: nuage_personality == "evdf"
+
 - name: Add the service account to the privileged scc to have root permissions for kube-system
   shell: oc adm policy add-scc-to-user privileged system:serviceaccount:kube-system:daemon-set-controller
   ignore_errors: true
@@ -129,6 +139,11 @@
   ignore_errors: true
   when: inventory_hostname == groups.oo_first_master.0
 
+- name: Spawn strongswan daemon sets pod for EVDF platform
+  shell: oc create -f /etc/nuage-strongswan-pod-config-daemonset.yaml
+  ignore_errors: true
+  when: inventory_hostname == groups.oo_first_master.0 and nuage_personality == "evdf"
+
 - name: Restart daemons
   command: /bin/true
   notify:

roles/nuage_master/templates/nuage-infra-pod-config-daemonset.j2

@@ -1,5 +1,24 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: nuage-infra-config
+  namespace: kube-system
+data:
+  # Name of the enterprise in which pods will reside
+  enterprise_name: {{ enterprise }}
+  # Name of the domain in which pods will reside
+  domain_name: {{ domain }}
+  # Name of the VSD user in admin group
+  vsd_user: {{ vsd_user }}
+  # Name of the VSD user in admin group
+  pod_network_cidr: {{ nuage_pod_cidr | default('70.70.0.0/16') }}
+  # Infra pod functionality depends on personality type
+  personality: {{ nuage_personality | default('vrs') }}
+
+---
+
 # This manifest installs Nuage Infra pod on
-# each worker node in an Openshift cluster.
+# each worker node in a Kubernetes cluster.
 kind: DaemonSet
 apiVersion: extensions/v1beta1
 metadata:
@@ -18,6 +37,9 @@ spec:
       labels:
         k8s-app: nuage-infra-ds
     spec:
+{% if nuage_personality == 'evdf' %}
+      hostNetwork: true
+{% endif %}
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
@@ -26,14 +48,49 @@ spec:
         # This container spawns a Nuage Infra pod
         # on each worker node
         - name: install-nuage-infra
-          image: nuage/infra:{{ nuage_infra_container_image_version }}
-          command: ["/install-nuage-infra-pod.sh"]
+          image: {{ nuage_infra_image_name }}:{{ nuage_infra_container_image_version }}
+          command: ["/usr/bin/nuage-k8s-infra-pod.sh"]
           securityContext:
             privileged: true
           volumeMounts:
             - mountPath: /var/log
               name: log-dir
+            - mountPath: /var/run
+              name: openvswitch-dir
+          env:
+            - name: VSP_ENTERPRISE
+              valueFrom:
+                configMapKeyRef:
+                  name: nuage-infra-config
+                  key: enterprise_name
+            - name: VSP_DOMAIN
+              valueFrom:
+                configMapKeyRef:
+                  name: nuage-infra-config
+                  key: domain_name
+            - name: VSP_USER
+              valueFrom:
+                configMapKeyRef:
+                  name: nuage-infra-config
+                  key: vsd_user
+            - name: POD_NETWORK_CIDR
+              valueFrom:
+                configMapKeyRef:
+                  name: nuage-infra-config
+                  key: pod_network_cidr
+            - name: PERSONALITY
+              valueFrom:
+                configMapKeyRef:
+                  name: nuage-infra-config
+                  key: personality
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/usr/bin/nuage-k8s-infra-pod.sh", "-c"]
       volumes:
         - name: log-dir
           hostPath:
             path: /var/log
+        - name: openvswitch-dir
+          hostPath:
+            path: /var/run

roles/nuage_master/templates/nuage-master-config-daemonset.j2

@@ -23,6 +23,12 @@ data:
       enterpriseName: {{ enterprise }}
       # Name of the domain in which pods will reside
       domainName: {{ domain }}
+      # Enable/Disable encryption flags on VSD
+      encryptionEnabled: {{ '1' if nuage_personality == 'evdf' else '0' }}
+      #Enable Underlay Support for this domain on VSD. 1 => enabled, 0 => disabled(default)
+      underlaySupport: {{ enable_underlay_support | default(0) }}
+      #Enable Stats logging for this domain on VSD. 1 => enabled, 0 => disabled(default)
+      statsLogging: {{ enable_stats_logging | default(0) }}
       # VSD generated user certificate file location on master node
       userCertificateFile: {{ nuage_master_crt_dir }}/{{ vsd_user }}.pem
       # VSD generated user key file location on master node
@@ -35,7 +41,7 @@ data:
       logLevel: 0
       # Parameters related to the nuage monitor REST server
       nuageMonServer:
-          URL: 0.0.0.0:9443
+          URL: 0.0.0.0:{{ nuage_mon_rest_server_port }}
           certificateDirectory: {{ nuage_master_crt_dir }}
           clientCA: ""
           serverCertificate: ""
@@ -49,6 +55,10 @@ data:
       {% for etcd_url in openshift_master_etcd_urls %}
               - {{ etcd_url }}
       {% endfor %}
+      # auto scale subnets feature
+      # 0 => disabled(default)
+      # 1 => enabled
+      autoScaleSubnets: {{ auto_scale_subnets | default(0) }}
 
 ---
 
@@ -78,7 +88,7 @@ spec:
       containers:
         # This container configures Nuage Master node
         - name: install-nuage-master-config
-          image: nuage/master:{{ nuage_monitor_container_image_version }}
+          image: {{nuage_mon_image_name }}:{{ nuage_monitor_container_image_version }}
           ports:
             - containerPort: 9443
               hostPort: 9443

roles/nuage_master/templates/nuage-node-config-daemonset.j2

@@ -42,9 +42,12 @@ data:
       loglevel: "info"
       portresolvetimer: 60
       logfilesize: 1
+      logfilemaxage: 2
       vrsconnectionchecktimer: 180
-      mtu: 1450
+      mtu: {{ nuage_vport_mtu }}
       staleentrytimeout: 600
+      nuagesiteid: {{ nuage_site_id | default(-1) }}
+      platform: {{ nuage_personality | default('vrs') }}
 
 ---
 
@@ -73,7 +76,7 @@ spec:
         # This container installs Nuage CNI binaries
         # and CNI network config file on each node.
         - name: install-nuage-cni
-          image: nuage/cni:{{ nuage_cni_container_image_version }}
+          image: {{ nuage_cni_image_name }}:{{ nuage_cni_container_image_version }}
           command: ["/install-cni.sh"]
           args: ["nuage-cni-openshift", "{{ slave_host_type }}"]
           securityContext:
@@ -159,26 +162,34 @@ spec:
         # This container installs Nuage VRS running as a
         # container on each worker node
         - name: install-nuage-vrs
-          image: nuage/vrs:{{ nuage_vrs_container_image_version }}
+          image: {{ nuage_vrs_image_name }}:{{ nuage_vrs_container_image_version }}
           securityContext:
             privileged: true
           env:
             # Configure parameters for VRS openvswitch file
             - name: NUAGE_ACTIVE_CONTROLLER
-              value: "{{ vsc_active_ip }}"
+              value: "{{ nuage_vsc_active_ip }}"
             - name: NUAGE_STANDBY_CONTROLLER
-              value: "{{ vsc_standby_ip }}"
+              value: "{{ nuage_vsc_standby_ip }}"
             - name: NUAGE_PLATFORM
               value: '"kvm, k8s"'
             - name: NUAGE_K8S_SERVICE_IPV4_SUBNET
-              value: '172.30.0.0\/16'
+              value: '{{ svc_ipv4_subnet }}\/{{ svc_ipv4_subnet_mask }}'
             - name: NUAGE_NETWORK_UPLINK_INTF
-              value: "eth0"
+              value: "{{ nw_uplink_intf }}"
+            - name: NUAGE_BRIDGE_MTU
+              value: "{{ vrs_bridge_mtu_config }}"
+{% if nuage_bgp_config == 'enable' %}
+            - name: ENABLE_BGP
+              value: "yes"
+{% endif %}
           volumeMounts:
             - mountPath: /var/run
               name: vrs-run-dir
             - mountPath: /var/log
               name: vrs-log-dir
+            - mountPath: {{ nuage_vrs_mount_dir }}
+              name: vrs-mnt-dir
             - mountPath: /sys/module
               name: sys-mod-dir
               readOnly: true
@@ -192,6 +203,9 @@ spec:
         - name: vrs-log-dir
           hostPath:
             path: /var/log
+        - name: vrs-mnt-dir
+          hostPath:
+            path: {{ nuage_vrs_mount_dir }}
         - name: sys-mod-dir
           hostPath:
             path: /sys/module

roles/nuage_master/templates/nuage-strongswan-pod-config-daemonset.j2

@@ -0,0 +1,45 @@
+# This manifest installs StrongSwan on
+# each worker node on a EVDF OSE cluster.
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: nuage-strongswan-ds
+  namespace: kube-system
+  labels:
+    k8s-app: nuage-strongswan-ds
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nuage-strongswan-ds
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: nuage-strongswan-ds
+    spec:
+      hostNetwork: true
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+          operator: Exists
+      containers:
+        # This container installs strongswan running as a
+        # container on each worker node
+        - name: install-strongswan
+          image: {{ nuage_sswan_image_name }}:{{ nuage_strongswan_container_image_version }}
+          securityContext:
+            privileged: true
+          ports:
+            - name: ipsec1
+              protocol: UDP
+              hostPort: 4500
+              containerPort: 4500
+            - name: ipsec2
+              protocol: UDP
+              hostPort: 500
+              containerPort: 500
+            - name: ipsec3
+              protocol: UDP
+              hostPort: 68
+              containerPort: 68

roles/nuage_master/vars/main.yaml

@@ -25,14 +25,22 @@ nuage_master_config_dsets_mount_dir: /usr/share/
 nuage_node_config_dsets_mount_dir: /usr/share/
 nuage_cni_bin_dsets_mount_dir: /opt/cni/bin
 nuage_cni_netconf_dsets_mount_dir: /etc/cni/net.d
-nuage_monitor_container_image_version: "{{ nuage_monitor_image_version | default('v5.2.1') }}"
-nuage_vrs_container_image_version: "{{ nuage_vrs_image_version | default('v5.2.1') }}"
-nuage_cni_container_image_version: "{{ nuage_cni_image_version | default('v5.2.1') }}"
-nuage_infra_container_image_version: "{{ nuage_infra_image_version | default('v5.2.1') }}"
+nuage_vrs_mount_dir: /tmp
+nuage_monitor_container_image_version: "{{ nuage_monitor_image_version | default('latest') }}"
+nuage_vrs_container_image_version: "{{ nuage_vrs_image_version | default('latest') }}"
+nuage_cni_container_image_version: "{{ nuage_cni_image_version | default('latest') }}"
+nuage_infra_container_image_version: "{{ nuage_infra_image_version | default('latest') }}"
+nuage_strongswan_container_image_version: "{{ nuage_strongswan_image_version | default('latest') }}"
 api_server_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
 nuage_vport_mtu: "{{ nuage_interface_mtu | default('1460') }}"
 master_host_type: "{{ master_base_host_type | default('is_rhel_server') }}"
 slave_host_type: "{{ slave_base_host_type | default('is_rhel_server') }}"
+svc_ipv4_subnet: "{{ nuage_svc_ipv4_subnet | default('172.30.0.0') }}"
+svc_ipv4_subnet_mask: "{{ nuage_svc_ipv4_subnet_mask | default('16') }}"
+nw_uplink_intf: "{{ uplink_interface | default('eth0') }}"
+nuage_vsc_active_ip: "{{ vsc_active_ip | default('') }}"
+nuage_vsc_standby_ip: "{{ vsc_standby_ip | default('') }}"
+nuage_bgp_config: "{{ nuage_bgp | default('disable') }}"
 
 nuage_tasks:
 - resource_kind: cluster-role

roles/nuage_node/defaults/main.yml

@@ -2,7 +2,7 @@
 r_nuage_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
 r_nuage_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
 
-nuage_mon_rest_server_port: '9443'
+nuage_mon_rest_server_port: "{{ nuage_openshift_monitor_rest_server_port | default('9443') }}"
 
 r_nuage_node_os_firewall_deny: []
 r_nuage_node_os_firewall_allow:

roles/nuage_node/tasks/iptables.yml

@@ -15,9 +15,3 @@
   when: "'nuage-underlay-overlay' not in iptablesrules.stdout"
   notify:
     - save iptable rules
-
-- name: Allow docker daemon traffic from underlay to overlay
-  command: /sbin/iptables -t nat -A POSTROUTING ! -s {{ openshift_cluster_network_cidr }} -o svc-pat-tap -j MASQUERADE -m comment --comment "nuage-docker-underlay-overlay"
-  when: "'nuage-docker-underlay-overlay' not in iptablesrules.stdout"
-  notify:
-    - save iptable rules

roles/nuage_node/vars/main.yaml

@@ -12,7 +12,6 @@ docker_bridge: "{{ nuage_docker_bridge | default('docker0') }}"
 rest_client_cert: "{{ vsp_openshift_dir }}/nuageMonClient.crt"
 rest_client_key: "{{ vsp_openshift_dir }}/nuageMonClient.key"
 rest_server_ca_cert: "{{ vsp_openshift_dir }}/nuageMonCA.crt"
-vport_mtu: "{{ nuage_interface_mtu | default('1460') }}"
 plugin_log_level: "{{ nuage_plugin_log_level | default('err') }}"
 
 nuage_plugin_rest_client_crt_dir: "{{ nuage_ca_master_crt_dir }}/{{ ansible_nodename }}"