Merge pull request #951 from abutcher/nfs

NFS for registry storage

inventory/byo/hosts.origin.example

@@ -6,6 +6,7 @@ masters
 nodes
 etcd
 lb
+nfs
 
 # Set variables common for all OSEv3 hosts
 [OSEv3:vars]
@@ -175,6 +176,11 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # Configure dnsIP in the node config
 #openshift_dns_ip=172.30.0.1
 
+# NFS Options
+#openshift_nfs_exports_dir=/var/export
+#openshift_nfs_registry_volume=regvol
+#openshift_nfs_export_options='*(rw,sync,all_squash)'
+
 # host group for masters
 [masters]
 ose3-master[1:3]-ansible.test.example.com

playbooks/aws/openshift-cluster/config.yml

@@ -15,6 +15,7 @@
     g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([]))     | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_master_hosts: "{{ (groups['tag_host-type_master']|default([])) | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
+    g_nfs_hosts:   "{{ (groups['tag_host-type_nfs']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_node_hosts:   "{{ (groups['tag_host-type_node']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
     g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"

playbooks/byo/openshift-cluster/config.yml

@@ -3,6 +3,7 @@
   vars:
     g_etcd_hosts: "{{ groups.etcd | default([]) }}"
     g_master_hosts: "{{ groups.masters | default([]) }}"
+    g_nfs_hosts: "{{ groups.nfs | default([]) }}"
     g_node_hosts: "{{ groups.nodes | default([]) }}"
     g_lb_hosts: "{{ groups.lb | default([]) }}"
     openshift_cluster_id: "{{ cluster_id | default('default') }}"

playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml

@@ -3,6 +3,7 @@
   vars:
     g_etcd_hosts: "{{ groups.etcd | default([]) }}"
     g_master_hosts: "{{ groups.masters | default([]) }}"
+    g_nfs_hosts: "{{ groups.nfs | default([]) }}"
     g_node_hosts: "{{ groups.nodes | default([]) }}"
     g_lb_hosts: "{{ groups.lb | default([]) }}"
     openshift_cluster_id: "{{ cluster_id | default('default') }}"

playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml

@@ -3,6 +3,7 @@
   vars:
     g_etcd_hosts: "{{ groups.etcd | default([]) }}"
     g_master_hosts: "{{ groups.masters | default([]) }}"
+    g_nfs_hosts: "{{ groups.nfs | default([]) }}"
     g_node_hosts: "{{ groups.nodes | default([]) }}"
     g_lb_hosts: "{{ groups.lb | default([]) }}"
     openshift_cluster_id: "{{ cluster_id | default('default') }}"

playbooks/common/openshift-cluster/config.yml

@@ -5,6 +5,8 @@
 
 - include: ../openshift-etcd/config.yml
 
+- include: ../openshift-nfs/config.yml
+
 - include: ../openshift-master/config.yml
 
 - include: ../openshift-node/config.yml

playbooks/common/openshift-cluster/evaluate_groups.yml

@@ -21,6 +21,14 @@
       msg: This playbook requires g_lb_hosts to be set
     when: g_lb_hosts is not defined
 
+  - fail:
+      msg: This playbook requires g_nfs_hosts to be set
+    when: g_nfs_hosts is not defined
+
+  - fail:
+      msg: The nfs group must be limited to one host
+    when: (groups[g_nfs_hosts] | default([])) | length > 1
+
   - name: Evaluate oo_etcd_to_config
     add_host:
       name: "{{ item }}"
@@ -81,3 +89,11 @@
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
     with_items: "{{ g_lb_hosts | default([]) }}"
+
+  - name: Evaluate oo_nfs_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_nfs_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ g_nfs_hosts | default([]) }}"

playbooks/common/openshift-master/config.yml

@@ -397,7 +397,15 @@
 
 - name: Create services
   hosts: oo_first_master
+  vars:
+    attach_registry_volume: "{{ groups.oo_nfs_to_config | length > 0 }}"
+  pre_tasks:
+  - set_fact:
+      nfs_host: "{{ groups.oo_nfs_to_config.0 }}"
+      registry_volume_path: "{{ hostvars[groups.oo_nfs_to_config.0].openshift.nfs.exports_dir + '/' + hostvars[groups.oo_nfs_to_config.0].openshift.nfs.registry_volume }}"
+    when: attach_registry_volume | bool
   roles:
   - role: openshift_router
     when: openshift.master.infra_nodes is defined
-  #- role: openshift_registry
+  - role: openshift_registry
+    when: openshift.master.infra_nodes is defined and attach_registry_volume | bool

playbooks/common/openshift-nfs/config.yml

@@ -0,0 +1,5 @@
+---
+- name: Configure nfs hosts
+  hosts: oo_nfs_to_config
+  roles:
+  - role: openshift_storage_nfs

playbooks/common/openshift-nfs/filter_plugins

@@ -0,0 +1 @@
+../../../filter_plugins

playbooks/common/openshift-nfs/lookup_plugins

@@ -0,0 +1 @@
+../../../lookup_plugins

playbooks/common/openshift-nfs/roles

@@ -0,0 +1 @@
+../../../roles/

playbooks/common/openshift-nfs/service.yml

@@ -0,0 +1,18 @@
+---
+- name: Populate g_service_nfs host group if needed
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - fail: msg="new_cluster_state is required to be injected in this playbook"
+    when: new_cluster_state is not defined
+
+  - name: Evaluate g_service_nfs
+    add_host: name={{ item }} groups=g_service_nfs
+    with_items: oo_host_group_exp | default([])
+
+- name: Change state on nfs instance(s)
+  hosts: g_service_nfs
+  connection: ssh
+  gather_facts: no
+  tasks:
+    - service: name=nfs-server state="{{ new_cluster_state }}"

playbooks/gce/openshift-cluster/config.yml

@@ -20,6 +20,7 @@
     g_etcd_hosts:   "{{ (groups['tag_host-type-etcd']|default([]))   | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_lb_hosts:     "{{ (groups['tag_host-type-lb']|default([]))     | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_master_hosts: "{{ (groups['tag_host-type-master']|default([])) | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
+    g_nfs_hosts:   "{{ (groups['tag_host-type-nfs']|default([]))   | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_node_hosts:   "{{ (groups['tag_host-type-node']|default([]))   | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
     g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"

playbooks/libvirt/openshift-cluster/config.yml

@@ -19,6 +19,7 @@
     g_etcd_hosts:   "{{ (groups['tag_host-type-etcd']|default([]))   | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_lb_hosts:     "{{ (groups['tag_host-type-lb']|default([]))     | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_master_hosts: "{{ (groups['tag_host-type-master']|default([])) | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
+    g_nfs_hosts:   "{{ (groups['tag_host-type-node']|default([]))   | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_node_hosts:   "{{ (groups['tag_host-type-node']|default([]))   | intersect((groups['tag_env-' ~ cluster_id]|default([]))) }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
     g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"

playbooks/openstack/openshift-cluster/config.yml

@@ -14,6 +14,7 @@
     g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([]))     | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_master_hosts: "{{ (groups['tag_host-type_master']|default([])) | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
+    g_nfs_hosts:   "{{ (groups['tag_host-type_nfs']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_node_hosts:   "{{ (groups['tag_host-type_node']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
     g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"

roles/openshift_facts/library/openshift_facts.py

@@ -1052,7 +1052,7 @@ class OpenShiftFacts(object):
         Raises:
             OpenShiftFactsUnsupportedRoleError:
     """
-    known_roles = ['common', 'master', 'node', 'master_sdn', 'node_sdn', 'etcd']
+    known_roles = ['common', 'master', 'node', 'master_sdn', 'node_sdn', 'etcd', 'nfs']
 
     def __init__(self, role, filename, local_facts, additive_facts_to_overwrite=False):
         self.changed = False
@@ -1146,6 +1146,12 @@ class OpenShiftFacts(object):
             node = dict(labels={}, annotations={}, portal_net='172.30.0.0/16',
                         iptables_sync_period='5s', set_node_ip=False)
             defaults['node'] = node
+
+        if 'nfs' in roles:
+            nfs = dict(exports_dir='/var/export', registry_volume='regvol',
+                       export_options='*(rw,sync,all_squash)')
+            defaults['nfs'] = nfs
+
         return defaults
 
     def guess_host_provider(self):

roles/openshift_registry/tasks/main.yml

@@ -1,6 +1,4 @@
 ---
-# This role is unused until we add options for configuring the backend storage
-
 - set_fact: _oreg_images="--images='{{ openshift.master.registry_url }}'"
 
 - set_fact: _oreg_selector="--selector='{{ openshift.master.registry_selector }}'"
@@ -12,3 +10,19 @@
     --credentials={{ openshift_master_config_dir }}/openshift-registry.kubeconfig {{ _oreg_images }}
   register: _oreg_results
   changed_when: "'service exists' not in _oreg_results.stdout"
+
+- name: Determine if nfs volume is already attached
+  command: "{{ openshift.common.client_binary }} get -o template dc/docker-registry --template=\\{\\{.spec.template.spec.volumes\\}\\}"
+  register: registry_volumes_output
+  when: attach_registry_volume | bool
+
+- set_fact:
+    volume_already_attached: "{{ 'server:' + nfs_host in registry_volumes_output.stdout and 'path:' + registry_volume_path in registry_volumes_output.stdout }}"
+  when: attach_registry_volume | bool
+
+- name: Add nfs volume to dc/docker-registry
+  command: >
+    {{ openshift.common.client_binary }} volume dc/docker-registry
+    --add --overwrite --name=registry-storage --mount-path=/registry
+    --source='{"nfs": {"server": "{{ nfs_host }}", "path": "{{ registry_volume_path }}"}}'
+  when: attach_registry_volume | bool and not volume_already_attached | bool

roles/openshift_storage_nfs/README.md

@@ -0,0 +1,52 @@
+OpenShift NFS Server
+====================
+
+OpenShift NFS Server Installation
+
+Requirements
+------------
+
+This role is intended to be applied to the [nfs] host group which is
+separate from OpenShift infrastructure components.
+
+Requires access to the 'nfs-utils' package.
+
+Role Variables
+--------------
+
+From this role:
+| Name                          | Default value         |                                                  |
+|-------------------------------|-----------------------|--------------------------------------------------|
+| openshift_nfs_exports_dir     | /var/export           | Root export directory.                           |
+| openshift_nfs_registry_volume | regvol                | Registry volume within openshift_nfs_exports_dir |
+| openshift_nfs_export_options  | *(rw,sync,all_squash) | NFS options for configured exports.              |
+
+
+From openshift_common:
+| Name                          | Default Value  |                                        |
+|-------------------------------|----------------|----------------------------------------|
+| openshift_debug_level         | 2              | Global openshift debug log verbosity   |
+
+
+Dependencies
+------------
+
+
+
+Example Playbook
+----------------
+
+- name: Configure nfs hosts
+  hosts: oo_nfs_to_config
+  roles:
+  - role: openshift_storage_nfs
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Andrew Butcher (abutcher@redhat.com)

roles/openshift_storage_nfs/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+exports_dir: /var/export
+registry_volume: regvol
+export_options: '*(rw,sync,all_squash)'
+os_firewall_use_firewalld: False
+os_firewall_allow:
+- service: nfs
+  port: "2049/tcp"

roles/openshift_storage_nfs/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: restart nfs-server
+  service:
+    name: nfs-server
+    state: restarted
+  when: not (nfs_service_status_changed | default(false))

roles/openshift_storage_nfs/meta/main.yml

@@ -0,0 +1,15 @@
+---
+galaxy_info:
+  author: Andrew Butcher
+  description: OpenShift NFS Server
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 7
+dependencies:
+- { role: os_firewall }
+- { role: openshift_common }
+- { role: openshift_repos }

roles/openshift_storage_nfs/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+- name: Set nfs facts
+  openshift_facts:
+    role: nfs
+    local_facts:
+      exports_dir: "{{ openshift_nfs_exports_dir | default(None) }}"
+      export_options: "{{ openshift_nfs_export_options | default(None) }}"
+      registry_volume: "{{ openshift_nfs_registry_volume | default(None) }}"
+
+- name: Install nfs-utils
+  yum:
+    pkg: nfs-utils
+    state: present
+
+- name: Ensure exports directory exists
+  file:
+    path: "{{ openshift.nfs.exports_dir }}"
+    state: directory
+
+- name: Ensure export directories exist
+  file:
+    path: "{{ openshift.nfs.exports_dir }}/{{ item }}"
+    state: directory
+    mode: 0777
+    owner: nfsnobody
+    group: nfsnobody
+  with_items:
+  - "{{ openshift.nfs.registry_volume }}"
+
+- name: Configure exports
+  template:
+    dest: /etc/exports
+    src: exports.j2
+  notify:
+  - restart nfs-server
+
+- name: Enable and start services
+  service:
+    name: "{{ item }}"
+    state: started
+    enabled: yes
+  register: start_result
+  with_items:
+  - nfs-server
+
+- set_fact:
+    nfs_service_status_changed: "{{ True in (start_result.results
+                                    | map(attribute='changed')
+                                    | list) }}"

roles/openshift_storage_nfs/templates/exports.j2

@@ -0,0 +1 @@
+{{ openshift.nfs.exports_dir }}/{{ openshift.nfs.registry_volume }} {{ openshift.nfs.export_options }}