Add Nuage support to openshift ansible

Added variables

Made changes for node configuration

Add service restart logic to node

Fixed ansible syntax errors

Add cert and key info for nodes

Added active and standby controller ip configuration information

Uncommented the nuage sdn check

Changed ca_crt -> ca_cert

Added restarting of atomic openshift master

Removed service account dependencies

Fixes

Fixed the api server url

Removed redundant restart of atomic openshift master

Configure nuagekubemon on all of the master nodes

Restart master api and controllers as well on nuagekubemon installation

Converted plugin config into template

Add template for nuagekubemon

Removed uplink interface from vars

Able to copy cert keys

Uninstall default ovs

Add the kubemon template

Do not install rdo sdn rpms in case of nuage

Addressed latest review comments

Set the networkPluginName for nuage

playbooks/common/openshift-master/config.yml

@@ -336,6 +336,8 @@
   - role: nickhammond.logrotate
   - role: fluentd_master
     when: openshift.common.use_fluentd | bool
+  - role: nuage_master
+    when: openshift.common.use_nuage | bool
   post_tasks:
   - name: Create group for deployment type
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}

playbooks/common/openshift-node/config.yml

@@ -180,6 +180,8 @@
   - openshift_node
   - role: flannel
     when: openshift.common.use_flannel | bool
+  - role: nuage_node
+    when: openshift.common.use_nuage | bool
   - role: nickhammond.logrotate
   - role: fluentd_node
     when: openshift.common.use_fluentd | bool

roles/nuage_master/README.md

@@ -0,0 +1,8 @@
+Nuage Master
+============
+Setup Nuage Kubernetes Monitor on the Master node
+
+
+Requirements
+------------
+This role assumes it has been deployed on RHEL/Fedora

roles/nuage_master/files/serviceaccount.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Parse CLI options
+for i in "$@"; do
+    case $i in
+        --master-cert-dir=*)
+            MASTER_DIR="${i#*=}"
+            CA_CERT=${MASTER_DIR}/ca.crt
+            CA_KEY=${MASTER_DIR}/ca.key
+            CA_SERIAL=${MASTER_DIR}/ca.serial.txt
+            ADMIN_FILE=${MASTER_DIR}/admin.kubeconfig
+        ;;
+        --server=*)
+            SERVER="${i#*=}"
+        ;;
+        --output-cert-dir=*)
+            OUTDIR="${i#*=}"
+            CONFIG_FILE=${OUTDIR}/nuage.kubeconfig
+        ;;
+    esac
+done
+
+# If any are missing, print the usage and exit
+if [ -z $SERVER ] || [ -z $OUTDIR ] || [ -z $MASTER_DIR ]; then
+    echo "Invalid syntax: $@"
+    echo "Usage:"
+    echo "  $0 --server=<address>:<port> --output-cert-dir=/path/to/output/dir/ --master-cert-dir=/path/to/master/"
+    echo "--master-cert-dir:  Directory where the master's configuration is held"
+    echo "--server:           Address of Kubernetes API server (default port is 8443)"
+    echo "--output-cert-dir:  Directory to put artifacts in"
+    echo ""
+    echo "All options are required"
+    exit 1
+fi
+
+# Login as admin so that we can create the service account
+oc login -u system:admin --config=$ADMIN_FILE || exit 1
+oc project default --config=$ADMIN_FILE
+
+ACCOUNT_CONFIG='
+{
+  "apiVersion": "v1",
+  "kind": "ServiceAccount",
+  "metadata": {
+    "name": "nuage"
+  }
+}
+'
+
+# Create the account with the included info
+echo $ACCOUNT_CONFIG|oc create --config=$ADMIN_FILE -f -
+
+# Add the cluser-reader role, which allows this service account read access to
+# everything in the cluster except secrets
+oadm policy add-cluster-role-to-user cluster-reader system:serviceaccounts:default:nuage --config=$ADMIN_FILE
+
+# Generate certificates and a kubeconfig for the service account
+oadm create-api-client-config --certificate-authority=${CA_CERT} --client-dir=${OUTDIR} --signer-cert=${CA_CERT} --signer-key=${CA_KEY} --signer-serial=${CA_SERIAL} --user=system:serviceaccounts:default:nuage --master=${SERVER} --public-master=${SERVER} --basename='nuage'
+
+# Verify the finalized kubeconfig
+if ! [ $(oc whoami --config=$CONFIG_FILE) == 'system:serviceaccounts:default:nuage' ]; then
+    echo "Service account creation failed!"
+    exit 1
+fi

roles/nuage_master/handlers/main.yaml

@@ -0,0 +1,18 @@
+---
+- name: restart nuagekubemon
+  sudo: true
+  service: name=nuagekubemon state=restarted
+
+- name: restart master
+  service: name={{ openshift.common.service_type }}-master state=restarted
+  when: (not openshift_master_ha | bool) and (not master_service_status_changed | default(false))
+
+- name: restart master api
+  service: name={{ openshift.common.service_type }}-master-api state=restarted
+  when: (openshift_master_ha | bool) and (not master_api_service_status_changed | default(false)) and openshift.master.cluster_method == 'native'
+
+# TODO: need to fix up ignore_errors here
+- name: restart master controllers
+  service: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  when: (openshift_master_ha | bool) and (not master_controllers_service_status_changed | default(false)) and openshift.master.cluster_method == 'native'
+  ignore_errors: yes

roles/nuage_master/tasks/main.yaml

@@ -0,0 +1,34 @@
+---
+- name: Create directory /usr/share/nuagekubemon
+  sudo: true
+  file: path=/usr/share/nuagekubemon state=directory  
+
+- name: Create the log directory
+  sudo: true
+  file: path={{ nuagekubemon_log_dir }} state=directory
+
+- name: Install Nuage Kubemon
+  sudo: true
+  yum: name={{ nuage_kubemon_rpm }} state=present
+
+- name: Run the service account creation script
+  sudo: true
+  script: serviceaccount.sh --server={{ openshift.master.api_url }} --output-cert-dir={{ cert_output_dir }} --master-cert-dir={{ openshift_master_config_dir }}
+
+- name: Download the certs and keys
+  sudo: true
+  fetch: src={{ cert_output_dir }}/{{ item }} dest=/tmp/{{ item }} flat=yes
+  with_items:
+        - ca.crt
+        - nuage.crt
+        - nuage.key
+        - nuage.kubeconfig 
+ 
+- name: Create nuagekubemon.yaml
+  sudo: true
+  template: src=nuagekubemon.j2 dest=/usr/share/nuagekubemon/nuagekubemon.yaml owner=root mode=0644
+  notify:
+    - restart master
+    - restart master api
+    - restart master controllers
+    - restart nuagekubemon

roles/nuage_master/templates/nuagekubemon.j2

@@ -0,0 +1,19 @@
+# .kubeconfig that includes the nuage service account
+kubeConfig: {{ kube_config }}
+# name of the nuage service account, or another account with 'cluster-reader'
+# permissions
+# Openshift master config file
+openshiftMasterConfig: {{ master_config_yaml }} 
+# URL of the VSD Architect
+vsdApiUrl: {{ vsd_api_url }} 
+# API version to query against.  Usually "v3_2"
+vspVersion: {{ vsp_version }} 
+# File containing a VSP license to install.  Only necessary if no license has
+# been installed on the VSD Architect before, only valid for standalone vsd install
+# licenseFile: "/path/to/base_vsp_license.txt"
+# Name of the enterprise in which pods will reside
+enterpriseName: {{ enterprise }} 
+# Name of the domain in which pods will reside
+domainName: {{ domain }}
+# Location where logs should be saved
+log_dir: {{ nuagekubemon_log_dir }} 

roles/nuage_master/vars/main.yaml

@@ -0,0 +1,7 @@
+openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
+ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
+admin_config: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+cert_output_dir: /usr/share/nuagekubemon
+kube_config: /usr/share/nuagekubemon/nuage.kubeconfig
+kubemon_yaml: /usr/share/nuagekubemon/nuagekubemon.yaml 
+master_config_yaml: "{{ openshift_master_config_dir }}/master-config.yaml" 

roles/nuage_node/README.md

@@ -0,0 +1,9 @@
+Nuage Node
+==========
+
+Setup Nuage VRS (Virtual Routing Switching) on the Openshift Node
+
+Requirements
+------------
+
+This role assumes it has been deployed on RHEL/Fedora

roles/nuage_node/handlers/main.yaml

@@ -0,0 +1,4 @@
+---
+- name: restart vrs
+  sudo: true
+  service: name=openvswitch state=restarted

roles/nuage_node/tasks/main.yaml

@@ -0,0 +1,37 @@
+---
+- name: Install Nuage VRS
+  sudo: true
+  yum: name={{ vrs_rpm }} state=present
+  
+- name: Set the uplink interface 
+  sudo: true
+  lineinfile: dest={{ vrs_config }} regexp=^NETWORK_UPLINK_INTF line='NETWORK_UPLINK_INTF={{ uplink_interface }}'
+
+- name: Set the Active Controller 
+  sudo: true
+  lineinfile: dest={{ vrs_config }} regexp=^ACTIVE_CONTROLLER line='ACTIVE_CONTROLLER={{ vsc_active_ip }}'
+
+- name: Set the Standby Controller 
+  sudo: true
+  lineinfile: dest={{ vrs_config }} regexp=^STANDBY_CONTROLLER line='STANDBY_CONTROLLER={{ vsc_standby_ip }}'
+  when: vsc_standby_ip is defined
+
+- name: Install plugin rpm
+  sudo: true
+  yum: name={{ plugin_rpm }} state=present
+
+- name: Copy the certificates and keys
+  sudo: true
+  copy: src="/tmp/{{ item }}" dest="{{ vsp_k8s_dir }}/{{ item }}"
+  with_items:
+        - ca.crt
+        - nuage.crt
+        - nuage.key
+        - nuage.kubeconfig 
+
+- name: Set the vsp-k8s.yaml 
+  sudo: true
+  template: src=vsp-k8s.j2 dest={{ vsp_k8s_yaml }} owner=root mode=0644 
+  notify:
+    - restart vrs
+    - restart node 

roles/nuage_node/templates/vsp-k8s.j2

@@ -0,0 +1,14 @@
+clientCert: {{ client_cert }} 
+# The key to the certificate in clientCert above
+clientKey: {{ client_key }}
+# The certificate authority's certificate for the local kubelet.  Usually the
+# same as the CA cert used to create the client Cert/Key pair.
+CACert: {{ ca_cert }} 
+# Name of the enterprise in which pods will reside
+enterpriseName: {{ enterprise }} 
+# Name of the domain in which pods will reside
+domainName: {{ domain }}
+# IP address and port number of master API server
+masterApiServer: {{ api_server }}
+# Bridge name for the docker bridge
+dockerBridgeName: {{ docker_bridge }}

roles/nuage_node/vars/main.yaml

@@ -0,0 +1,9 @@
+---
+vrs_config: /etc/default/openvswitch
+vsp_k8s_dir: /usr/share/vsp-k8s
+vsp_k8s_yaml: "{{ vsp_k8s_dir }}/vsp-k8s.yaml"
+client_cert: "{{ vsp_k8s_dir }}/nuage.crt"
+client_key: "{{ vsp_k8s_dir }}/nuage.key"
+ca_cert: "{{ vsp_k8s_dir }}/ca.crt"
+api_server: "{{ openshift_node_master_api_url }}"
+docker_bridge: "docker0"

roles/openshift_common/tasks/main.yml

@@ -4,6 +4,14 @@
   when: openshift_use_openshift_sdn | default(false) | bool and openshift_use_flannel | default(false) | bool
 
 - fail:
+   msg: Nuage sdn can not be used with openshift sdn
+  when: openshift_use_openshift_sdn | default(false) | bool and openshift_use_nuage | default(false) | bool
+
+- fail:
+   msg: Nuage sdn can not be used with flannel 
+  when: openshift_use_flannel | default(false) | bool and openshift_use_nuage | default(false) | bool
+
+- fail:
     msg: openshift_hostname must be 64 characters or less
   when: openshift_hostname is defined and openshift_hostname | length > 64
 
@@ -22,6 +30,7 @@
       deployment_type: "{{ openshift_deployment_type }}"
       use_fluentd: "{{ openshift_use_fluentd | default(None) }}"
       use_flannel: "{{ openshift_use_flannel | default(None) }}"
+      use_nuage: "{{ openshift_use_nuage | default(None) }}"
       use_manageiq: "{{ openshift_use_manageiq | default(None) }}"
 
   # For enterprise versions < 3.1 and origin versions < 1.1 we want to set the

roles/openshift_facts/library/openshift_facts.py

@@ -341,6 +341,23 @@ def set_flannel_facts_if_unset(facts):
             facts['common']['use_flannel'] = use_flannel
     return facts
 
+def set_nuage_facts_if_unset(facts):
+    """ Set nuage facts if not already present in facts dict
+            dict: the facts dict updated with the nuage facts if
+            missing
+        Args:
+            facts (dict): existing facts
+        Returns:
+            dict: the facts dict updated with the nuage
+            facts if they were not already present
+
+    """
+    if 'common' in facts:
+        if 'use_nuage' not in facts['common']:
+            use_nuage = False
+            facts['common']['use_nuage'] = use_nuage
+    return facts
+
 def set_node_schedulability(facts):
     """ Set schedulable facts if not already present in facts dict
         Args:
@@ -1022,6 +1039,7 @@ class OpenShiftFacts(object):
         facts = set_project_cfg_facts_if_unset(facts)
         facts = set_fluentd_facts_if_unset(facts)
         facts = set_flannel_facts_if_unset(facts)
+        facts = set_nuage_facts_if_unset(facts)
         facts = set_node_schedulability(facts)
         facts = set_master_selectors(facts)
         facts = set_metrics_facts_if_unset(facts)

roles/openshift_master/templates/master.yaml.v1.j2

@@ -102,7 +102,7 @@ masterPublicURL: {{ openshift.master.public_api_url }}
 networkConfig:
   clusterNetworkCIDR: {{ openshift.master.sdn_cluster_network_cidr }}
   hostSubnetLength: {{ openshift.master.sdn_host_subnet_length }}
-{% if openshift.common.use_openshift_sdn %}
+{% if openshift.common.use_openshift_sdn or openshift.common.use_nuage %}
   networkPluginName: {{ openshift.common.sdn_network_plugin_name }}
 {% endif %}
 # serviceNetworkCIDR must match kubernetesMasterConfig.servicesSubnet

roles/openshift_node/templates/node.yaml.v1.j2

@@ -22,7 +22,7 @@ networkPluginName: {{ openshift.common.sdn_network_plugin_name }}
 # deprecates networkPluginName above. The two should match.
 networkConfig:
    mtu: {{ openshift.node.sdn_mtu }}
-{% if openshift.common.use_openshift_sdn %}
+{% if openshift.common.use_openshift_sdn or openshift.common.use_nuage %}
    networkPluginName: {{ openshift.common.sdn_network_plugin_name }}
 {% endif %}
 {% if openshift.node.set_node_ip | bool %}