Merge pull request #698 from lebauce/flannel

Add support for flannel

playbooks/common/openshift-master/config.yml

@@ -338,6 +338,14 @@
     when: ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
       (osm_use_cockpit | bool or osm_use_cockpit is undefined )
 
+- name: Configure flannel
+  hosts: oo_first_master
+  vars:
+    etcd_urls: "{{ openshift.master.etcd_urls }}"
+  roles:
+  - role: flannel_register
+    when: openshift.common.use_flannel | bool
+
 # Additional instance config for online deployments
 - name: Additional instance config
   hosts: oo_masters_deployment_type_online

playbooks/common/openshift-node/config.yml

@@ -38,6 +38,21 @@
       node_subdir: node-{{ openshift.common.hostname }}
       config_dir: "{{ openshift.common.config_base }}/generated-configs/node-{{ openshift.common.hostname }}"
       node_cert_dir: "{{ openshift.common.config_base }}/node"
+  - name: Check status of flannel external etcd certificates
+    stat:
+      path: "{{ openshift.common.config_base }}/node/{{ item }}"
+    with_items:
+    - node.etcd-client.crt
+    - node.etcd-ca.crt
+    register: g_external_etcd_flannel_cert_stat_result
+  - set_fact:
+      etcd_client_flannel_certs_missing: "{{ g_external_etcd_flannel_cert_stat_result.results
+                                             | map(attribute='stat.exists')
+                                             | list | intersect([false])}}"
+      etcd_cert_subdir: openshift-node-{{ openshift.common.hostname }}
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/node"
+      etcd_cert_prefix: node.etcd-
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config and (openshift.common.use_flannel | bool)
 
 - name: Create temp directory for syncing certs
   hosts: localhost
@@ -50,6 +65,60 @@
     register: mktemp
     changed_when: False
 
+- name: Configure flannel etcd certificates
+  hosts: oo_first_etcd
+  vars:
+    etcd_generated_certs_dir: /etc/etcd/generated_certs
+    etcd_needing_client_certs: "{{ hostvars
+                                   | oo_select_keys(groups['oo_nodes_to_config'])
+                                   | oo_filter_list(filter_attr='etcd_client_flannel_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
+  pre_tasks:
+  roles:
+  - role: etcd_certificates
+  post_tasks:
+  - name: Create a tarball of the etcd flannel certs
+    command: >
+      tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz
+        -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} .
+    args:
+      creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+    with_items: etcd_needing_client_certs
+  - name: Retrieve the etcd cert tarballs
+    fetch:
+      src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: etcd_needing_client_certs
+
+- name: Copy the external etcd flannel certs to the nodes
+  hosts: oo_nodes_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
+  tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: "{{ openshift.common.config_base }}/node"
+      state: directory
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz"
+      dest: "{{ etcd_cert_config_dir }}"
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
+  - file:
+      path: "{{ etcd_cert_config_dir }}/{{ item }}"
+      owner: root
+      group: root
+      mode: 0600
+    with_items:
+    - node.etcd-client.crt
+    - node.etcd-client.key
+    - node.etcd-ca.crt
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
+
 - name: Create node certificates
   hosts: oo_first_master
   vars:
@@ -84,6 +153,8 @@
   vars:
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
     openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
+    etcd_urls: "{{ hostvars[groups.oo_first_master.0].openshift.master.etcd_urls }}"
+    embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -100,6 +171,8 @@
     when: certs_missing
   roles:
   - openshift_node
+  - role: flannel
+    when: openshift.common.use_flannel | bool
   - role: nickhammond.logrotate
   - role: fluentd_node
     when: openshift.common.use_fluentd | bool

roles/flannel/README.md

@@ -0,0 +1,45 @@
+Role Name
+=========
+
+Configure flannel on openshift nodes
+
+Requirements
+------------
+
+This role assumes it's being deployed on a RHEL/Fedora based host with package
+named 'flannel' available via yum, in version superior to 0.3.
+
+Role Variables
+--------------
+
+| Name                | Default value                           | Description                                   |
+|---------------------|-----------------------------------------|-----------------------------------------------|
+| flannel_interface   | ansible_default_ipv4.interface          | interface to use for inter-host communication |
+| flannel_etcd_key    | /openshift.com/network                  | etcd prefix                                   |
+| etcd_hosts          | etcd_urls                               | a list of etcd endpoints                      |
+| etcd_conf_dir       | {{ openshift.common.config_base }}/node | SSL certificates directory                    |
+| etcd_peer_ca_file   | {{ etcd_conf_dir }}/ca.crt              | SSL CA to use for etcd                        |
+| etcd_peer_cert_file | Openshift SSL cert                      | SSL cert to use for etcd                      |
+| etcd_peer_key_file  | Openshift SSL key                       | SSL key to use for etcd                       |
+
+Dependencies
+------------
+
+openshift_facts
+
+Example Playbook
+----------------
+
+    - hosts: openshift_node
+      roles:
+        - { role: flannel, etcd_urls: ['https://127.0.0.1:2379'] }
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Sylvain Baubeau <sbaubeau@redhat.com>

roles/flannel/defaults/main.yaml

@@ -0,0 +1,8 @@
+---
+flannel_interface: "{{ ansible_default_ipv4.interface }}"
+flannel_etcd_key: /openshift.com/network
+etcd_hosts: "{{ etcd_urls }}"
+etcd_conf_dir: "{{ openshift.common.config_base }}/node"
+etcd_peer_ca_file: "{{ etcd_conf_dir }}/{{ 'ca' if (embedded_etcd | bool) else 'node.etcd-ca' }}.crt"
+etcd_peer_cert_file: "{{ etcd_conf_dir }}/{{ 'system:node:' + openshift.common.hostname if (embedded_etcd | bool) else 'node.etcd-client' }}.crt"
+etcd_peer_key_file: "{{ etcd_conf_dir }}/{{ 'system:node:' + openshift.common.hostname if (embedded_etcd | bool) else 'node.etcd-client' }}.key"

roles/flannel/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: restart flanneld
+  sudo: true
+  service: name=flanneld state=restarted
+
+- name: restart docker
+  sudo: true
+  service: name=docker state=restarted

roles/flannel/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Sylvain
+  description: flannel management
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.2
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }

roles/flannel/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+- name: Install flannel
+  sudo: true
+  yum: pkg=flannel state=present
+
+- name: Set flannel etcd url
+  sudo: true
+  lineinfile:
+    dest: /etc/sysconfig/flanneld
+    backrefs: yes
+    regexp: "^(FLANNEL_ETCD=)"
+    line: '\1{{ etcd_hosts|join(",") }}'
+
+- name: Set flannel etcd key
+  sudo: true
+  lineinfile:
+    dest: /etc/sysconfig/flanneld
+    backrefs: yes
+    regexp: "^(FLANNEL_ETCD_KEY=)"
+    line: '\1{{ flannel_etcd_key }}'
+
+- name: Set flannel options
+  sudo: true
+  lineinfile:
+    dest: /etc/sysconfig/flanneld
+    backrefs: yes
+    regexp: "^#?(FLANNEL_OPTIONS=)"
+    line: '\1--iface {{ flannel_interface }} --etcd-cafile={{ etcd_peer_ca_file }} --etcd-keyfile={{ etcd_peer_key_file }} --etcd-certfile={{ etcd_peer_cert_file }}'
+
+- name: Enable flanneld
+  sudo: true
+  service:
+    name: flanneld
+    state: started
+    enabled: yes
+  register: start_result
+
+- name: Remove docker bridge ip
+  sudo: true
+  shell: ip a del `ip a show docker0 | grep "inet[[:space:]]" | awk '{print $2}'` dev docker0
+  notify:
+    - restart docker
+    - restart node

roles/flannel_register/README.md

@@ -0,0 +1,47 @@
+Role Name
+=========
+
+Register flannel configuration into etcd
+
+Requirements
+------------
+
+This role assumes it's being deployed on a RHEL/Fedora based host with package
+named 'flannel' available via yum, in version superior to 0.3.
+
+Role Variables
+--------------
+
+| Name                | Default value                                      | Description                                     |
+|---------------------|----------------------------------------------------|-------------------------------------------------|
+| flannel_network     | {{ openshift.master.portal_net }} or 172.16.1.1/16 | interface to use for inter-host communication   |
+| flannel_min_network | {{ min_network }} or 172.16.5.0                    | beginning of IP range for the subnet allocation |
+| flannel_subnet_len  | /openshift.com/network                             | size of the subnet allocated to each host       |
+| flannel_etcd_key    | /openshift.com/network                             | etcd prefix                                     |
+| etcd_hosts          | etcd_urls                                          | a list of etcd endpoints                        |
+| etcd_conf_dir       | {{ openshift.common.config_base }}/master          | SSL certificates directory                      |
+| etcd_peer_ca_file   | {{ etcd_conf_dir }}/ca.crt                         | SSL CA to use for etcd                          |
+| etcd_peer_cert_file | {{ etcd_conf_dir }}/master.etcd-client.crt         | SSL cert to use for etcd                        |
+| etcd_peer_key_file  | {{ etcd_conf_dir }}/master.etcd-client.key         | SSL key to use for etcd                         |
+
+Dependencies
+------------
+
+openshift_facts
+
+Example Playbook
+----------------
+
+    - hosts: openshift_master
+      roles:
+         - { flannel_register }
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Sylvain Baubeau <sbaubeau@redhat.com>

roles/flannel_register/defaults/main.yaml

@@ -0,0 +1,11 @@
+---
+flannel_network: "{{ openshift.master.portal_net | default('172.30.0.0/16', true) }}"
+flannel_min_network: 172.30.5.0
+flannel_subnet_len: 24
+flannel_etcd_key: /openshift.com/network
+etcd_hosts: "{{ etcd_urls }}"
+etcd_conf_dir: "{{ openshift.common.config_base }}/master"
+etcd_peer_ca_file: "{{ etcd_conf_dir + '/ca.crt' if (openshift.master.embedded_etcd | bool) else etcd_conf_dir + '/master.etcd-ca.crt' }}"
+etcd_peer_cert_file: "{{ etcd_conf_dir }}/master.etcd-client.crt"
+etcd_peer_key_file: "{{ etcd_conf_dir }}/master.etcd-client.key"
+

roles/flannel_register/meta/main.yml

@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Sylvain
+  description: register flannel configuration into etcd
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.2
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }

roles/flannel_register/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+- name: Assures /etc/flannel dir exists
+  sudo: true
+  file: path=/etc/flannel state=directory
+
+- name: Generate etcd configuration for etcd
+  sudo: true
+  template:
+    src: "flannel-config.json"
+    dest: "/etc/flannel/config.json"
+
+- name: Insert flannel configuration into etcd
+  sudo: true
+  command: 'curl -L --cacert "{{ etcd_peer_ca_file }}" --cert "{{ etcd_peer_cert_file }}" --key "{{ etcd_peer_key_file }}" "{{ etcd_hosts[0] }}/v2/keys{{ flannel_etcd_key }}/config" -XPUT --data-urlencode value@/etc/flannel/config.json'

roles/flannel_register/templates/flannel-config.json

@@ -0,0 +1,8 @@
+{
+    "Network": "{{ flannel_network }}",
+    "SubnetLen": {{ flannel_subnet_len }},
+    "SubnetMin": "{{ flannel_min_network }}",
+    "Backend": {
+        "Type": "host-gw"
+     }
+}

roles/openshift_common/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+- fail:
+    msg: Flannel can not be used with openshift sdn
+  when: openshift_use_openshift_sdn | default(false) | bool and openshift_use_flannel | default(false) | bool
+
 - name: Set common Cluster facts
   openshift_facts:
     role: common
@@ -13,6 +17,7 @@
       sdn_network_plugin_name: "{{ os_sdn_network_plugin_name | default(None) }}"
       deployment_type: "{{ openshift_deployment_type }}"
       use_fluentd: "{{ openshift_use_fluentd | default(None) }}"
+      use_flannel: "{{ openshift_use_flannel | default(None) }}"
 
 - name: Set hostname
   hostname: name={{ openshift.common.hostname }}

roles/openshift_facts/library/openshift_facts.py

@@ -307,6 +307,23 @@ def set_fluentd_facts_if_unset(facts):
             facts['common']['use_fluentd'] = use_fluentd
     return facts
 
+def set_flannel_facts_if_unset(facts):
+    """ Set flannel facts if not already present in facts dict
+            dict: the facts dict updated with the flannel facts if
+            missing
+        Args:
+            facts (dict): existing facts
+        Returns:
+            dict: the facts dict updated with the flannel
+            facts if they were not already present
+
+    """
+    if 'common' in facts:
+        if 'use_flannel' not in facts['common']:
+            use_flannel = False
+            facts['common']['use_flannel'] = use_flannel
+    return facts
+
 def set_node_schedulability(facts):
     """ Set schedulable facts if not already present in facts dict
         Args:
@@ -911,6 +928,7 @@ class OpenShiftFacts(object):
         facts = set_url_facts_if_unset(facts)
         facts = set_project_cfg_facts_if_unset(facts)
         facts = set_fluentd_facts_if_unset(facts)
+        facts = set_flannel_facts_if_unset(facts)
         facts = set_node_schedulability(facts)
         facts = set_master_selectors(facts)
         facts = set_metrics_facts_if_unset(facts)