GlusterFS: Allow cleaner separation of multiple clusters

Signed-off-by: Jose A. Rivera <jarrpa@redhat.com>

roles/openshift_storage_glusterfs/README.md

@@ -73,7 +73,8 @@ GlusterFS cluster into a new or existing OpenShift cluster:
 | openshift_storage_glusterfs_timeout              | 300                     | Seconds to wait for pods to become ready
 | openshift_storage_glusterfs_namespace            | 'default'               | Namespace in which to create GlusterFS resources
 | openshift_storage_glusterfs_is_native            | True                    | GlusterFS should be containerized
-| openshift_storage_glusterfs_nodeselector         | 'storagenode=glusterfs' | Selector to determine which nodes will host GlusterFS pods in native mode
+| openshift_storage_glusterfs_name                 | 'storage'               | A name to identify the GlusterFS cluster, which will be used in resource names
+| openshift_storage_glusterfs_nodeselector         | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name
 | openshift_storage_glusterfs_image                | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7'
 | openshift_storage_glusterfs_version              | 'latest'                | Container image version to use for GlusterFS pods
 | openshift_storage_glusterfs_wipe                 | False                   | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.**
@@ -96,7 +97,7 @@ are an exception:
 | Name                                              | Default value         | Description                             |
 |---------------------------------------------------|-----------------------|-----------------------------------------|
 | openshift_storage_glusterfs_registry_namespace    | registry namespace    | Default is to use the hosted registry's namespace, otherwise 'default'
-| openshift_storage_glusterfs_registry_nodeselector | 'storagenode=registry'| This allows for the logical separation of the registry GlusterFS cluster from any regular-use GlusterFS clusters
+| openshift_storage_glusterfs_registry_name         | 'registry'            | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters
 
 Additionally, this role's behavior responds to the following registry-specific
 variables:

roles/openshift_storage_glusterfs/defaults/main.yml

@@ -2,7 +2,8 @@
 openshift_storage_glusterfs_timeout: 300
 openshift_storage_glusterfs_namespace: 'default'
 openshift_storage_glusterfs_is_native: True
-openshift_storage_glusterfs_nodeselector: 'storagenode=glusterfs'
+openshift_storage_glusterfs_name: 'storage'
+openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host"
 openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}"
 openshift_storage_glusterfs_version: 'latest'
 openshift_storage_glusterfs_wipe: False
@@ -20,7 +21,8 @@ openshift_storage_glusterfs_heketi_url: "{{ omit }}"
 openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
 openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
 openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}"
-openshift_storage_glusterfs_registry_nodeselector: 'storagenode=registry'
+openshift_storage_glusterfs_registry_name: 'registry'
+openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host"
 openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}"
 openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}"
 openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}"

roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml

@@ -13,43 +13,43 @@ objects:
 - kind: Service
   apiVersion: v1
   metadata:
-    name: deploy-heketi
+    name: deploy-heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: deploy-heketi-service
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-service
       deploy-heketi: support
     annotations:
       description: Exposes Heketi service
   spec:
     ports:
-    - name: deploy-heketi
+    - name: deploy-heketi-${CLUSTER_NAME}
       port: 8080
       targetPort: 8080
     selector:
-      name: deploy-heketi
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
 - kind: Route
   apiVersion: v1
   metadata:
-    name: deploy-heketi
+    name: deploy-heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: deploy-heketi-route
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-route
       deploy-heketi: support
   spec:
     to:
       kind: Service
-      name: deploy-heketi
+      name: deploy-heketi-${CLUSTER_NAME}
 - kind: DeploymentConfig
   apiVersion: v1
   metadata:
-    name: deploy-heketi
+    name: deploy-heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: deploy-heketi-dc
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-dc
       deploy-heketi: support
     annotations:
       description: Defines how to deploy Heketi
   spec:
     replicas: 1
     selector:
-      name: deploy-heketi
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
     triggers:
     - type: ConfigChange
     strategy:
@@ -58,11 +58,10 @@ objects:
       metadata:
         name: deploy-heketi
         labels:
-          name: deploy-heketi
-          glusterfs: deploy-heketi-pod
+          glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
           deploy-heketi: support
       spec:
-        serviceAccountName: heketi-service-account
+        serviceAccountName: heketi-${CLUSTER_NAME}-service-account
         containers:
         - name: heketi
           image: ${IMAGE_NAME}:${IMAGE_VERSION}
@@ -117,3 +116,6 @@ parameters:
 - name: IMAGE_VERSION
   displayName: heketi container versiona
   required: True
+- name: CLUSTER_NAME
+  displayName: GlusterFS cluster name
+  value: glusterfs

roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml

@@ -12,20 +12,21 @@ objects:
 - kind: DaemonSet
   apiVersion: extensions/v1beta1
   metadata:
-    name: glusterfs
+    name: glusterfs-${CLUSTER_NAME}
     labels:
-      glusterfs: daemonset
+      glusterfs: ${CLUSTER_NAME}-daemonset
     annotations:
       description: GlusterFS DaemonSet
       tags: glusterfs
   spec:
     selector:
       matchLabels:
-        glusterfs-node: pod
+        glusterfs: ${CLUSTER_NAME}-pod
     template:
       metadata:
-        name: glusterfs
+        name: glusterfs-${CLUSTER_NAME}
         labels:
+          glusterfs: ${CLUSTER_NAME}-pod
           glusterfs-node: pod
       spec:
         nodeSelector: "${{NODE_LABELS}}"
@@ -121,11 +122,14 @@ objects:
 parameters:
 - name: NODE_LABELS
   displayName: Daemonset Node Labels
-  description: Labels which define the daemonset node selector.
-  value: '{ "storagenode": "glusterfs" }'
+  description: Labels which define the daemonset node selector. Must contain at least one label of the format \'glusterfs=<CLUSTER_NAME>-host\'
+  value: '{ "glusterfs": "storage-host" }'
 - name: IMAGE_NAME
   displayName: GlusterFS container name
   required: True
 - name: IMAGE_VERSION
   displayName: GlusterFS container versiona
   required: True
+- name: CLUSTER_NAME
+  displayName: GlusterFS cluster name
+  value: storage

roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml

@@ -12,9 +12,9 @@ objects:
 - kind: Service
   apiVersion: v1
   metadata:
-    name: heketi
+    name: heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: heketi-service
+      glusterfs: heketi-${CLUSTER_NAME}-service
     annotations:
       description: Exposes Heketi service
   spec:
@@ -23,40 +23,40 @@ objects:
       port: 8080
       targetPort: 8080
     selector:
-      glusterfs: heketi-pod
+      glusterfs: heketi-${CLUSTER_NAME}-pod
 - kind: Route
   apiVersion: v1
   metadata:
-    name: heketi
+    name: heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: heketi-route
+      glusterfs: heketi-${CLUSTER_NAME}-route
   spec:
     to:
       kind: Service
-      name: heketi
+      name: heketi-${CLUSTER_NAME}
 - kind: DeploymentConfig
   apiVersion: v1
   metadata:
-    name: heketi
+    name: heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: heketi-dc
+      glusterfs: heketi-${CLUSTER_NAME}-dc
     annotations:
       description: Defines how to deploy Heketi
   spec:
     replicas: 1
     selector:
-      glusterfs: heketi-pod
+      glusterfs: heketi-${CLUSTER_NAME}-pod
     triggers:
     - type: ConfigChange
     strategy:
       type: Recreate
     template:
       metadata:
-        name: heketi
+        name: heketi-${CLUSTER_NAME}
         labels:
-          glusterfs: heketi-pod
+          glusterfs: heketi-${CLUSTER_NAME}-pod
       spec:
-        serviceAccountName: heketi-service-account
+        serviceAccountName: heketi-${CLUSTER_NAME}-service-account
         containers:
         - name: heketi
           image: ${IMAGE_NAME}:${IMAGE_VERSION}
@@ -96,7 +96,7 @@ objects:
         volumes:
         - name: db
           glusterfs:
-            endpoints: heketi-storage-endpoints
+            endpoints: heketi-db-${CLUSTER_NAME}-endpoints
             path: heketidbstorage
 parameters:
 - name: HEKETI_USER_KEY
@@ -115,3 +115,6 @@ parameters:
 - name: IMAGE_VERSION
   displayName: heketi container versiona
   required: True
+- name: CLUSTER_NAME
+  displayName: GlusterFS cluster name
+  value: glusterfs

roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml

@@ -21,12 +21,16 @@
   with_items:
   - kind: "template,route,service,dc,jobs,secret"
     selector: "deploy-heketi"
-  - kind: "template,route,service,dc"
-    name: "heketi"
-  - kind: "svc,ep"
+  - kind: "svc"
     name: "heketi-storage-endpoints"
+  - kind: "template,route,service,dc"
+    name: "heketi-{{ glusterfs_name }}"
+  - kind: "svc"
+    name: "heketi-db-{{ glusterfs_name }}-endpoints"
   - kind: "sa"
-    name: "heketi-service-account"
+    name: "heketi-{{ glusterfs_name }}-service-account"
+  - kind: "secret"
+    name: "heketi-{{ glusterfs_name }}-secret"
   failed_when: False
   when: glusterfs_heketi_wipe
 
@@ -35,7 +39,7 @@
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=deploy-heketi-pod"
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until: "heketi_pod.results.results[0]['items'] | count == 0"
   delay: 10
@@ -47,7 +51,7 @@
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=heketi-pod"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until: "heketi_pod.results.results[0]['items'] | count == 0"
   delay: 10
@@ -57,13 +61,13 @@
 - name: Create heketi service account
   oc_serviceaccount:
     namespace: "{{ glusterfs_namespace }}"
-    name: heketi-service-account
+    name: "heketi-{{ glusterfs_name }}-service-account"
     state: present
   when: glusterfs_heketi_is_native
 
 - name: Add heketi service account to privileged SCC
   oc_adm_policy_user:
-    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account"
+    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account"
     resource_kind: scc
     resource_name: privileged
     state: present
@@ -71,7 +75,7 @@
 
 - name: Allow heketi service account to view/edit pods
   oc_adm_policy_user:
-    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account"
+    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account"
     resource_kind: role
     resource_name: edit
     state: present
@@ -82,7 +86,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: list
     kind: pod
-    selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   when: glusterfs_heketi_is_native
 
@@ -100,7 +104,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: list
     kind: pod
-    selector: "glusterfs=heketi-pod"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   when: glusterfs_heketi_is_native
 
@@ -124,7 +128,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: list
     kind: ep
-    selector: "glusterfs in (deploy-heketi-service, heketi-service)"
+    selector: "glusterfs in (deploy-heketi-{{ glusterfs_name }}-service, heketi-{{ glusterfs_name }}-service)"
   register: heketi_url
   until:
   - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"

roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml

@@ -3,7 +3,8 @@
     glusterfs_timeout: "{{ openshift_storage_glusterfs_timeout }}"
     glusterfs_namespace: "{{ openshift_storage_glusterfs_namespace }}"
     glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native }}"
-    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | map_from_pairs }}"
+    glusterfs_name: "{{ openshift_storage_glusterfs_name }}"
+    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}"
     glusterfs_image: "{{ openshift_storage_glusterfs_image }}"
     glusterfs_version: "{{ openshift_storage_glusterfs_version }}"
     glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe }}"

roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml

@@ -6,9 +6,14 @@
 - name: Delete pre-existing GlusterFS resources
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
-    kind: "template,daemonset"
-    name: glusterfs
+    kind: "{{ item.kind }}"
+    name: "{{ item.name }}"
     state: absent
+  with_items:
+  - kind: template
+    name: glusterfs
+  - kind: daemonset
+    name: "glusterfs-{{ glusterfs_name }}"
   when: glusterfs_wipe
 
 - name: Unlabel any existing GlusterFS nodes
@@ -72,7 +77,7 @@
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: template
-    name: glusterfs
+    name: "glusterfs"
     state: present
     files:
     - "{{ mktemp.stdout }}/glusterfs-template.yml"
@@ -86,13 +91,14 @@
       IMAGE_NAME: "{{ glusterfs_image }}"
       IMAGE_VERSION: "{{ glusterfs_version }}"
       NODE_LABELS: "{{ glusterfs_nodeselector }}"
+      CLUSTER_NAME: "{{ glusterfs_name }}"
 
 - name: Wait for GlusterFS pods
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs-node=pod"
+    selector: "glusterfs={{ glusterfs_name }}-pod"
   register: glusterfs_pods
   until:
   - "glusterfs_pods.results.results[0]['items'] | count > 0"

roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml

@@ -3,7 +3,8 @@
     glusterfs_timeout: "{{ openshift_storage_glusterfs_registry_timeout }}"
     glusterfs_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}"
     glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}"
-    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | map_from_pairs }}"
+    glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}"
+    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}"
     glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}"
     glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}"
     glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe }}"
@@ -31,8 +32,8 @@
     name: "{{ item.name }}"
     state: absent
   with_items:
-  - kind: "svc,ep"
-    name: "glusterfs-registry-endpoints"
+  - kind: "svc"
+    name: "glusterfs-{{ glusterfs_name }}-endpoints"
   failed_when: False
 
 - name: Generate GlusterFS registry endpoints
@@ -41,8 +42,8 @@
     dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
 
 - name: Copy GlusterFS registry service
-  copy:
-    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml"
+  template:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2"
     dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
 
 - name: Create GlusterFS registry endpoints
@@ -50,7 +51,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: present
     kind: endpoints
-    name: glusterfs-registry-endpoints
+    name: "glusterfs-{{ glusterfs_name }}-endpoints"
     files:
     - "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
 
@@ -59,7 +60,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: present
     kind: service
-    name: glusterfs-registry-endpoints
+    name: "glusterfs-{{ glusterfs_name }}-endpoints"
     files:
     - "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
 

roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml

@@ -26,13 +26,14 @@
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
+      CLUSTER_NAME: "{{ glusterfs_name }}"
 
 - name: Wait for deploy-heketi pod
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until:
   - "heketi_pod.results.results[0]['items'] | count > 0"

roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml

@@ -46,6 +46,36 @@
   with_items:
   - kind: "template,route,service,jobs,dc,secret"
     selector: "deploy-heketi"
+  - kind: "svc"
+    name: "heketi-storage-endpoints"
+
+- name: Generate heketi endpoints
+  template:
+    src: "{{ openshift.common.examples_content_version }}/heketi-endpoints.yml.j2"
+    dest: "{{ mktemp.stdout }}/heketi-endpoints.yml"
+
+- name: Generate heketi service
+  template:
+    src: "{{ openshift.common.examples_content_version }}/heketi-service.yml.j2"
+    dest: "{{ mktemp.stdout }}/heketi-service.yml"
+
+- name: Create heketi endpoints
+  oc_obj:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    kind: endpoints
+    name: "heketi-db-{{ glusterfs_name }}-endpoints"
+    files:
+    - "{{ mktemp.stdout }}/heketi-endpoints.yml"
+
+- name: Create heketi service
+  oc_obj:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    kind: service
+    name: "heketi-db-{{ glusterfs_name }}-endpoints"
+    files:
+    - "{{ mktemp.stdout }}/heketi-service.yml"
 
 - name: Copy heketi template
   copy:
@@ -72,13 +102,14 @@
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
+      CLUSTER_NAME: "{{ glusterfs_name }}"
 
 - name: Wait for heketi pod
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=heketi-pod"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until:
   - "heketi_pod.results.results[0]['items'] | count > 0"
@@ -92,7 +123,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: list
     kind: ep
-    selector: "glusterfs=heketi-service"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-service"
   register: heketi_url
   until:
   - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"

roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2

@@ -1,7 +1,8 @@
+---
 apiVersion: v1
 kind: Endpoints
 metadata:
-  name: glusterfs-registry-endpoints
+  name: glusterfs-{{ glusterfs_name }}-endpoints
 subsets:
 - addresses:
 {% for node in glusterfs_nodes %}

roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: glusterfs-registry-endpoints
+  name: glusterfs-{{ glusterfs_name }}-endpoints
 spec:
   ports:
   - port: 1

roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: heketi-db-{{ glusterfs_name }}-endpoints
+subsets:
+- addresses:
+{% for node in glusterfs_nodes %}
+  - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }}
+{% endfor %}
+  ports:
+  - port: 1

roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: heketi-db-{{ glusterfs_name }}-endpoints
+spec:
+  ports:
+  - port: 1
+status:
+  loadBalancer: {}