Merge pull request #4245 from jarrpa/glusterfs-fixes

Merged by openshift-bot

inventory/byo/hosts.byo.native-glusterfs.example

@@ -24,7 +24,7 @@ glusterfs
 
 [OSEv3:vars]
 ansible_ssh_user=root
-deployment_type=origin
+openshift_deployment_type=origin
 # Specify that we want to use GlusterFS storage for a hosted registry
 openshift_hosted_registry_storage_kind=glusterfs
 

roles/openshift_storage_glusterfs/README.md

@@ -1,7 +1,31 @@
 OpenShift GlusterFS Cluster
 ===========================
 
-OpenShift GlusterFS Cluster Installation
+OpenShift GlusterFS Cluster Configuration
+
+This role handles the configuration of GlusterFS clusters. It can handle
+two primary configuration scenarios:
+
+* Configuring a new, natively-hosted GlusterFS cluster. In this scenario,
+  GlusterFS pods are deployed on nodes in the OpenShift cluster which are
+  configured to provide storage.
+* Configuring a new, external GlusterFS cluster. In this scenario, the
+  cluster nodes have the GlusterFS software pre-installed but have not
+  been configured yet. The installer will take care of configuring the
+  cluster(s) for use by OpenShift applications.
+* Using existing GlusterFS clusters. In this scenario, one or more
+  GlusterFS clusters are assumed to be already setup. These clusters can
+  be either natively-hosted or external, but must be managed by a
+  [heketi service](https://github.com/heketi/heketi).
+
+As part of the configuration, a particular GlusterFS cluster may be
+specified to provide backend storage for a natively-hosted Docker
+registry.
+
+Unless configured otherwise, a StorageClass will be automatically
+created for each non-registry GlusterFS cluster. This will allow
+applications which can mount PersistentVolumes to request
+dynamically-provisioned GlusterFS volumes.
 
 Requirements
 ------------
@@ -21,26 +45,50 @@ hosted Docker registry:
 
 * `[glusterfs_registry]`
 
+Host Variables
+--------------
+
+For configuring new clusters, the following role variables are available.
+
+Each host in either of the above groups must have the following variable
+defined:
+
+| Name              | Default value | Description                             |
+|-------------------|---------------|-----------------------------------------|
+| glusterfs_devices | None          | A list of block devices that will be completely managed as part of a GlusterFS cluster. There must be at least one device listed. Each device must be bare, e.g. no partitions or LVM PVs. **Example:** '[ "/dev/sdb" ]'
+
+In addition, each host may specify the following variables to further control
+their configuration as GlusterFS nodes:
+
+| Name               | Default value             | Description                             |
+|--------------------|---------------------------|-----------------------------------------|
+| glusterfs_cluster  | 1                         | The ID of the cluster this node should belong to. This is useful when a single heketi service is expected to manage multiple distinct clusters. **NOTE:** For natively-hosted clusters, all pods will be in the same OpenShift namespace
+| glusterfs_hostname | openshift.common.hostname | A hostname (or IP address) that will be used for internal GlusterFS communication
+| glusterfs_ip       | openshift.common.ip       | An IP address that will be used by pods to communicate with the GlusterFS node
+| glusterfs_zone     | 1                         | A zone number for the node. Zones are used within the cluster for determining how to distribute the bricks of GlusterFS volumes. heketi will try to spread each volumes' bricks as evenly as possible across all zones
+
 Role Variables
 --------------
 
 This role has the following variables that control the integration of a
 GlusterFS cluster into a new or existing OpenShift cluster:
 
-| Name                                             | Default value           |                                         |
+| Name                                             | Default value           | Description                             |
 |--------------------------------------------------|-------------------------|-----------------------------------------|
 | openshift_storage_glusterfs_timeout              | 300                     | Seconds to wait for pods to become ready
 | openshift_storage_glusterfs_namespace            | 'default'               | Namespace in which to create GlusterFS resources
 | openshift_storage_glusterfs_is_native            | True                    | GlusterFS should be containerized
-| openshift_storage_glusterfs_nodeselector         | 'storagenode=glusterfs' | Selector to determine which nodes will host GlusterFS pods in native mode
+| openshift_storage_glusterfs_name                 | 'storage'               | A name to identify the GlusterFS cluster, which will be used in resource names
+| openshift_storage_glusterfs_nodeselector         | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name
+| openshift_storage_glusterfs_storageclass         | True                    | Automatically create a StorageClass for each GlusterFS cluster
 | openshift_storage_glusterfs_image                | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7'
 | openshift_storage_glusterfs_version              | 'latest'                | Container image version to use for GlusterFS pods
 | openshift_storage_glusterfs_wipe                 | False                   | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.**
 | openshift_storage_glusterfs_heketi_is_native     | True                    | heketi should be containerized
 | openshift_storage_glusterfs_heketi_image         | 'heketi/heketi'         | Container image to use for heketi pods, enterprise default is 'rhgs3/rhgs-volmanager-rhel7'
 | openshift_storage_glusterfs_heketi_version       | 'latest'                | Container image version to use for heketi pods
-| openshift_storage_glusterfs_heketi_admin_key     | ''                      | String to use as secret key for performing heketi commands as admin
-| openshift_storage_glusterfs_heketi_user_key      | ''                      | String to use as secret key for performing heketi commands as user that can only view or modify volumes
+| openshift_storage_glusterfs_heketi_admin_key     | auto-generated          | String to use as secret key for performing heketi commands as admin
+| openshift_storage_glusterfs_heketi_user_key      | auto-generated          | String to use as secret key for performing heketi commands as user that can only view or modify volumes
 | openshift_storage_glusterfs_heketi_topology_load | True                    | Load the GlusterFS topology information into heketi
 | openshift_storage_glusterfs_heketi_url           | Undefined               | URL for the heketi REST API, dynamically determined in native mode
 | openshift_storage_glusterfs_heketi_wipe          | False                   | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe`
@@ -52,17 +100,24 @@ registry. These variables start with the prefix
 values in their corresponding non-registry variables. The following variables
 are an exception:
 
-| Name                                              | Default value         |                                         |
-|---------------------------------------------------|-----------------------|-----------------------------------------|
-| openshift_storage_glusterfs_registry_namespace    | registry namespace    | Default is to use the hosted registry's namespace, otherwise 'default'
-| openshift_storage_glusterfs_registry_nodeselector | 'storagenode=registry'| This allows for the logical separation of the registry GlusterFS cluster from any regular-use GlusterFS clusters
+| Name                                                  | Default value         | Description                             |
+|-------------------------------------------------------|-----------------------|-----------------------------------------|
+| openshift_storage_glusterfs_registry_namespace        | registry namespace    | Default is to use the hosted registry's namespace, otherwise 'default'
+| openshift_storage_glusterfs_registry_name             | 'registry'            | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters
+| openshift_storage_glusterfs_registry_storageclass     | False                 | It is recommended to not create a StorageClass for GlusterFS clusters serving registry storage, so as to avoid performance penalties
+| openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated        | Separate from the above
+| openshift_storage_glusterfs_registry_heketi_user_key  | auto-generated        | Separate from the above
 
 Additionally, this role's behavior responds to the following registry-specific
-variable:
-
-| Name                                         | Default value | Description                                                                  |
-|----------------------------------------------|---------------|------------------------------------------------------------------------------|
-| openshift_hosted_registry_glusterfs_swap     | False         | Whether to swap an existing registry's storage volume for a GlusterFS volume |
+variables:
+
+| Name                                          | Default value                | Description                             |
+|-----------------------------------------------|------------------------------|-----------------------------------------|
+| openshift_hosted_registry_glusterfs_endpoints | glusterfs-registry-endpoints | The name for the Endpoints resource that will point the registry to the GlusterFS nodes
+| openshift_hosted_registry_glusterfs_path      | glusterfs-registry-volume    | The name for the GlusterFS volume that will provide registry storage
+| openshift_hosted_registry_glusterfs_readonly  | False                        | Whether the GlusterFS volume should be read-only
+| openshift_hosted_registry_glusterfs_swap      | False                        | Whether to swap an existing registry's storage volume for a GlusterFS volume
+| openshift_hosted_registry_glusterfs_swapcopy  | True                         | If swapping, copy the contents of the pre-existing registry storage to the new GlusterFS volume
 
 Dependencies
 ------------

roles/openshift_storage_glusterfs/defaults/main.yml

@@ -2,7 +2,9 @@
 openshift_storage_glusterfs_timeout: 300
 openshift_storage_glusterfs_namespace: 'default'
 openshift_storage_glusterfs_is_native: True
-openshift_storage_glusterfs_nodeselector: 'storagenode=glusterfs'
+openshift_storage_glusterfs_name: 'storage'
+openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host"
+openshift_storage_glusterfs_storageclass: True
 openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}"
 openshift_storage_glusterfs_version: 'latest'
 openshift_storage_glusterfs_wipe: False
@@ -11,8 +13,8 @@ openshift_storage_glusterfs_heketi_is_missing: True
 openshift_storage_glusterfs_heketi_deploy_is_missing: True
 openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}"
 openshift_storage_glusterfs_heketi_version: 'latest'
-openshift_storage_glusterfs_heketi_admin_key: ''
-openshift_storage_glusterfs_heketi_user_key: ''
+openshift_storage_glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
+openshift_storage_glusterfs_heketi_user_key: "{{ 32 | oo_generate_secret }}"
 openshift_storage_glusterfs_heketi_topology_load: True
 openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}"
 openshift_storage_glusterfs_heketi_url: "{{ omit }}"
@@ -20,7 +22,9 @@ openshift_storage_glusterfs_heketi_url: "{{ omit }}"
 openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
 openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
 openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}"
-openshift_storage_glusterfs_registry_nodeselector: 'storagenode=registry'
+openshift_storage_glusterfs_registry_name: 'registry'
+openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host"
+openshift_storage_glusterfs_registry_storageclass: False
 openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}"
 openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}"
 openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}"
@@ -29,8 +33,8 @@ openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_gl
 openshift_storage_glusterfs_registry_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}"
 openshift_storage_glusterfs_registry_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}"
 openshift_storage_glusterfs_registry_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}"
-openshift_storage_glusterfs_registry_heketi_admin_key: "{{ openshift_storage_glusterfs_heketi_admin_key }}"
-openshift_storage_glusterfs_registry_heketi_user_key: "{{ openshift_storage_glusterfs_heketi_user_key }}"
+openshift_storage_glusterfs_registry_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
+openshift_storage_glusterfs_registry_heketi_user_key: "{{ 32 | oo_generate_secret }}"
 openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
 openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
 openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}"

roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml

@@ -9,49 +9,47 @@ metadata:
   annotations:
     description: Bootstrap Heketi installation
     tags: glusterfs,heketi,installation
-labels:
-  template: deploy-heketi
 objects:
 - kind: Service
   apiVersion: v1
   metadata:
-    name: deploy-heketi
+    name: deploy-heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: deploy-heketi-service
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-service
       deploy-heketi: support
     annotations:
       description: Exposes Heketi service
   spec:
     ports:
-    - name: deploy-heketi
+    - name: deploy-heketi-${CLUSTER_NAME}
       port: 8080
       targetPort: 8080
     selector:
-      name: deploy-heketi
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
 - kind: Route
   apiVersion: v1
   metadata:
-    name: deploy-heketi
+    name: deploy-heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: deploy-heketi-route
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-route
       deploy-heketi: support
   spec:
     to:
       kind: Service
-      name: deploy-heketi
+      name: deploy-heketi-${CLUSTER_NAME}
 - kind: DeploymentConfig
   apiVersion: v1
   metadata:
-    name: deploy-heketi
+    name: deploy-heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: deploy-heketi-dc
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-dc
       deploy-heketi: support
     annotations:
       description: Defines how to deploy Heketi
   spec:
     replicas: 1
     selector:
-      name: deploy-heketi
+      glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
     triggers:
     - type: ConfigChange
     strategy:
@@ -60,13 +58,12 @@ objects:
       metadata:
         name: deploy-heketi
         labels:
-          name: deploy-heketi
-          glusterfs: deploy-heketi-pod
+          glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
           deploy-heketi: support
       spec:
-        serviceAccountName: heketi-service-account
+        serviceAccountName: heketi-${CLUSTER_NAME}-service-account
         containers:
-        - name: deploy-heketi
+        - name: heketi
           image: ${IMAGE_NAME}:${IMAGE_VERSION}
           env:
           - name: HEKETI_USER_KEY
@@ -81,11 +78,15 @@ objects:
             value: '14'
           - name: HEKETI_KUBE_GLUSTER_DAEMONSET
             value: '1'
+          - name: HEKETI_KUBE_NAMESPACE
+            value: ${HEKETI_KUBE_NAMESPACE}
           ports:
           - containerPort: 8080
           volumeMounts:
           - name: db
             mountPath: /var/lib/heketi
+          - name: topology
+            mountPath: ${TOPOLOGY_PATH}
           readinessProbe:
             timeoutSeconds: 3
             initialDelaySeconds: 3
@@ -100,6 +101,9 @@ objects:
               port: 8080
         volumes:
         - name: db
+        - name: topology
+          secret:
+            secretName: heketi-${CLUSTER_NAME}-topology-secret
 parameters:
 - name: HEKETI_USER_KEY
   displayName: Heketi User Secret
@@ -107,9 +111,19 @@ parameters:
 - name: HEKETI_ADMIN_KEY
   displayName: Heketi Administrator Secret
   description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_KUBE_NAMESPACE
+  displayName: Namespace
+  description: Set the namespace where the GlusterFS pods reside
+  value: default
 - name: IMAGE_NAME
-  displayName: GlusterFS container name
+  displayName: heketi container name
   required: True
 - name: IMAGE_VERSION
-  displayName: GlusterFS container versiona
+  displayName: heketi container versiona
+  required: True
+- name: CLUSTER_NAME
+  displayName: GlusterFS cluster name
+  value: glusterfs
+- name: TOPOLOGY_PATH
+  displayName: heketi topology file location
   required: True

roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml

@@ -12,24 +12,24 @@ objects:
 - kind: DaemonSet
   apiVersion: extensions/v1beta1
   metadata:
-    name: glusterfs
+    name: glusterfs-${CLUSTER_NAME}
     labels:
-      glusterfs: daemonset
+      glusterfs: ${CLUSTER_NAME}-daemonset
     annotations:
       description: GlusterFS DaemonSet
       tags: glusterfs
   spec:
     selector:
       matchLabels:
-        glusterfs-node: pod
+        glusterfs: ${CLUSTER_NAME}-pod
     template:
       metadata:
-        name: glusterfs
+        name: glusterfs-${CLUSTER_NAME}
         labels:
+          glusterfs: ${CLUSTER_NAME}-pod
           glusterfs-node: pod
       spec:
-        nodeSelector:
-          storagenode: glusterfs
+        nodeSelector: "${{NODE_LABELS}}"
         hostNetwork: true
         containers:
         - name: glusterfs
@@ -63,26 +63,26 @@ objects:
             privileged: true
           readinessProbe:
             timeoutSeconds: 3
-            initialDelaySeconds: 100
+            initialDelaySeconds: 40
             exec:
               command:
               - "/bin/bash"
               - "-c"
               - systemctl status glusterd.service
-            periodSeconds: 10
+            periodSeconds: 25
             successThreshold: 1
-            failureThreshold: 3
+            failureThreshold: 15
           livenessProbe:
             timeoutSeconds: 3
-            initialDelaySeconds: 100
+            initialDelaySeconds: 40
             exec:
               command:
               - "/bin/bash"
               - "-c"
               - systemctl status glusterd.service
-            periodSeconds: 10
+            periodSeconds: 25
             successThreshold: 1
-            failureThreshold: 3
+            failureThreshold: 15
           resources: {}
           terminationMessagePath: "/dev/termination-log"
         volumes:
@@ -120,9 +120,16 @@ objects:
         dnsPolicy: ClusterFirst
         securityContext: {}
 parameters:
+- name: NODE_LABELS
+  displayName: Daemonset Node Labels
+  description: Labels which define the daemonset node selector. Must contain at least one label of the format \'glusterfs=<CLUSTER_NAME>-host\'
+  value: '{ "glusterfs": "storage-host" }'
 - name: IMAGE_NAME
   displayName: GlusterFS container name
   required: True
 - name: IMAGE_VERSION
   displayName: GlusterFS container versiona
   required: True
+- name: CLUSTER_NAME
+  displayName: GlusterFS cluster name
+  value: storage

roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml

@@ -8,15 +8,13 @@ metadata:
   annotations:
     description: Heketi service deployment template
     tags: glusterfs,heketi
-labels:
-  template: heketi
 objects:
 - kind: Service
   apiVersion: v1
   metadata:
-    name: heketi
+    name: heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: heketi-service
+      glusterfs: heketi-${CLUSTER_NAME}-service
     annotations:
       description: Exposes Heketi service
   spec:
@@ -25,40 +23,40 @@ objects:
       port: 8080
       targetPort: 8080
     selector:
-      glusterfs: heketi-pod
+      glusterfs: heketi-${CLUSTER_NAME}-pod
 - kind: Route
   apiVersion: v1
   metadata:
-    name: heketi
+    name: heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: heketi-route
+      glusterfs: heketi-${CLUSTER_NAME}-route
   spec:
     to:
       kind: Service
-      name: heketi
+      name: heketi-${CLUSTER_NAME}
 - kind: DeploymentConfig
   apiVersion: v1
   metadata:
-    name: heketi
+    name: heketi-${CLUSTER_NAME}
     labels:
-      glusterfs: heketi-dc
+      glusterfs: heketi-${CLUSTER_NAME}-dc
     annotations:
       description: Defines how to deploy Heketi
   spec:
     replicas: 1
     selector:
-      glusterfs: heketi-pod
+      glusterfs: heketi-${CLUSTER_NAME}-pod
     triggers:
     - type: ConfigChange
     strategy:
       type: Recreate
     template:
       metadata:
-        name: heketi
+        name: heketi-${CLUSTER_NAME}
         labels:
-          glusterfs: heketi-pod
+          glusterfs: heketi-${CLUSTER_NAME}-pod
       spec:
-        serviceAccountName: heketi-service-account
+        serviceAccountName: heketi-${CLUSTER_NAME}-service-account
         containers:
         - name: heketi
           image: ${IMAGE_NAME}:${IMAGE_VERSION}
@@ -76,6 +74,8 @@ objects:
             value: '14'
           - name: HEKETI_KUBE_GLUSTER_DAEMONSET
             value: '1'
+          - name: HEKETI_KUBE_NAMESPACE
+            value: ${HEKETI_KUBE_NAMESPACE}
           ports:
           - containerPort: 8080
           volumeMounts:
@@ -96,7 +96,7 @@ objects:
         volumes:
         - name: db
           glusterfs:
-            endpoints: heketi-storage-endpoints
+            endpoints: heketi-db-${CLUSTER_NAME}-endpoints
             path: heketidbstorage
 parameters:
 - name: HEKETI_USER_KEY
@@ -105,9 +105,16 @@ parameters:
 - name: HEKETI_ADMIN_KEY
   displayName: Heketi Administrator Secret
   description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_KUBE_NAMESPACE
+  displayName: Namespace
+  description: Set the namespace where the GlusterFS pods reside
+  value: default
 - name: IMAGE_NAME
-  displayName: GlusterFS container name
+  displayName: heketi container name
   required: True
 - name: IMAGE_VERSION
-  displayName: GlusterFS container versiona
+  displayName: heketi container versiona
   required: True
+- name: CLUSTER_NAME
+  displayName: GlusterFS cluster name
+  value: glusterfs

roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml

@@ -5,12 +5,6 @@
     name: "{{ glusterfs_namespace }}"
   when: glusterfs_is_native or glusterfs_heketi_is_native
 
-- include: glusterfs_deploy.yml
-  when: glusterfs_is_native
-
-- name: Make sure heketi-client is installed
-  package: name=heketi-client state=present
-
 - name: Delete pre-existing heketi resources
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
@@ -21,12 +15,18 @@
   with_items:
   - kind: "template,route,service,dc,jobs,secret"
     selector: "deploy-heketi"
-  - kind: "template,route,service,dc"
-    name: "heketi"
-  - kind: "svc,ep"
+  - kind: "svc"
     name: "heketi-storage-endpoints"
+  - kind: "secret"
+    name: "heketi-{{ glusterfs_name }}-topology-secret"
+  - kind: "template,route,service,dc"
+    name: "heketi-{{ glusterfs_name }}"
+  - kind: "svc"
+    name: "heketi-db-{{ glusterfs_name }}-endpoints"
   - kind: "sa"
-    name: "heketi-service-account"
+    name: "heketi-{{ glusterfs_name }}-service-account"
+  - kind: "secret"
+    name: "heketi-{{ glusterfs_name }}-user-secret"
   failed_when: False
   when: glusterfs_heketi_wipe
 
@@ -35,11 +35,11 @@
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=deploy-heketi-pod"
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until: "heketi_pod.results.results[0]['items'] | count == 0"
   delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"
   when: glusterfs_heketi_wipe
 
 - name: Wait for heketi pods to terminate
@@ -47,23 +47,26 @@
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=heketi-pod"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until: "heketi_pod.results.results[0]['items'] | count == 0"
   delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"
   when: glusterfs_heketi_wipe
 
+- include: glusterfs_deploy.yml
+  when: glusterfs_is_native
+
 - name: Create heketi service account
   oc_serviceaccount:
     namespace: "{{ glusterfs_namespace }}"
-    name: heketi-service-account
+    name: "heketi-{{ glusterfs_name }}-service-account"
     state: present
   when: glusterfs_heketi_is_native
 
 - name: Add heketi service account to privileged SCC
   oc_adm_policy_user:
-    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account"
+    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account"
     resource_kind: scc
     resource_name: privileged
     state: present
@@ -71,7 +74,7 @@
 
 - name: Allow heketi service account to view/edit pods
   oc_adm_policy_user:
-    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account"
+    user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account"
     resource_kind: role
     resource_name: edit
     state: present
@@ -82,7 +85,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: list
     kind: pod
-    selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   when: glusterfs_heketi_is_native
 
@@ -100,7 +103,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: list
     kind: pod
-    selector: "glusterfs=heketi-pod"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   when: glusterfs_heketi_is_native
 
@@ -113,48 +116,35 @@
   # heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
   - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
 
+- name: Generate topology file
+  template:
+    src: "{{ openshift.common.examples_content_version }}/topology.json.j2"
+    dest: "{{ mktemp.stdout }}/topology.json"
+  when:
+  - glusterfs_heketi_topology_load
+
 - include: heketi_deploy_part1.yml
   when:
   - glusterfs_heketi_is_native
   - glusterfs_heketi_deploy_is_missing
   - glusterfs_heketi_is_missing
 
-- name: Determine heketi URL
-  oc_obj:
-    namespace: "{{ glusterfs_namespace }}"
-    state: list
-    kind: ep
-    selector: "glusterfs in (deploy-heketi-service, heketi-service)"
-  register: heketi_url
-  until:
-  - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"
-  - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''"
-  delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
-  when:
-  - glusterfs_heketi_is_native
-  - glusterfs_heketi_url is undefined
-
 - name: Set heketi URL
   set_fact:
-    glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}"
+    glusterfs_heketi_url: "localhost:8080"
   when:
   - glusterfs_heketi_is_native
-  - glusterfs_heketi_url is undefined
+
+- name: Set heketi-cli command
+  set_fact:
+    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
 
 - name: Verify heketi service
-  command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' cluster list"
+  command: "{{ glusterfs_heketi_client }} cluster list"
   changed_when: False
 
-- name: Generate topology file
-  template:
-    src: "{{ openshift.common.examples_content_version }}/topology.json.j2"
-    dest: "{{ mktemp.stdout }}/topology.json"
-  when:
-  - glusterfs_heketi_topology_load
-
 - name: Load heketi topology
-  command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' topology load --json={{ mktemp.stdout }}/topology.json 2>&1"
+  command: "{{ glusterfs_heketi_client }} topology load --json={{ mktemp.stdout }}/topology.json 2>&1"
   register: topology_load
   failed_when: "topology_load.rc != 0 or 'Unable' in topology_load.stdout"
   when:
@@ -164,3 +154,29 @@
   when:
   - glusterfs_heketi_is_native
   - glusterfs_heketi_is_missing
+
+- name: Create heketi user secret
+  oc_secret:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    name: "heketi-{{ glusterfs_name }}-user-secret"
+    type: "kubernetes.io/glusterfs"
+    force: True
+    contents:
+    - path: key
+      data: "{{ glusterfs_heketi_user_key }}"
+
+- name: Generate GlusterFS StorageClass file
+  template:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2"
+    dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml"
+
+- name: Create GlusterFS StorageClass
+  oc_obj:
+    state: present
+    kind: storageclass
+    name: "glusterfs-{{ glusterfs_name }}"
+    files:
+    - "{{ mktemp.stdout }}/glusterfs-storageclass.yml"
+  when:
+  - glusterfs_storageclass

roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml

@@ -3,7 +3,9 @@
     glusterfs_timeout: "{{ openshift_storage_glusterfs_timeout }}"
     glusterfs_namespace: "{{ openshift_storage_glusterfs_namespace }}"
     glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native }}"
-    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | map_from_pairs }}"
+    glusterfs_name: "{{ openshift_storage_glusterfs_name }}"
+    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}"
+    glusterfs_storageclass: "{{ openshift_storage_glusterfs_storageclass }}"
     glusterfs_image: "{{ openshift_storage_glusterfs_image }}"
     glusterfs_version: "{{ openshift_storage_glusterfs_version }}"
     glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe }}"
@@ -17,6 +19,6 @@
     glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}"
-    glusterfs_nodes: "{{ g_glusterfs_hosts }}"
+    glusterfs_nodes: "{{ groups.glusterfs }}"
 
 - include: glusterfs_common.yml

roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml

@@ -1,23 +1,24 @@
 ---
 - assert:
-    that: "glusterfs_nodeselector.keys() | count == 1"
-    msg: Only one GlusterFS nodeselector key pair should be provided
-
-- assert:
     that: "glusterfs_nodes | count >= 3"
     msg: There must be at least three GlusterFS nodes specified
 
 - name: Delete pre-existing GlusterFS resources
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
-    kind: "template,daemonset"
-    name: glusterfs
+    kind: "{{ item.kind }}"
+    name: "{{ item.name }}"
     state: absent
+  with_items:
+  - kind: template
+    name: glusterfs
+  - kind: daemonset
+    name: "glusterfs-{{ glusterfs_name }}"
   when: glusterfs_wipe
 
 - name: Unlabel any existing GlusterFS nodes
   oc_label:
-    name: "{{ item }}"
+    name: "{{ hostvars[item].openshift.common.hostname }}"
     kind: node
     state: absent
     labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}"
@@ -40,11 +41,16 @@
   failed_when: False
   when: glusterfs_wipe
 
-  # Runs "vgremove -fy <vg>; pvremove -fy <pv>" for every device found to be a physical volume.
+  # Runs "lvremove -ff <vg>; vgremove -fy <vg>; pvremove -fy <pv>" for every device found to be a physical volume.
 - name: Clear GlusterFS storage device contents
-  shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}"
+  shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}lvremove -ff {{ fields[1] }}; vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}"
   delegate_to: "{{ item.item }}"
   with_items: "{{ devices_info.results }}"
+  register: clear_devices
+  until:
+  - "'contains a filesystem in use' not in clear_devices.stderr"
+  delay: 1
+  retries: 30
   when:
   - glusterfs_wipe
   - item.stdout_lines | count > 0
@@ -61,13 +67,11 @@
 
 - name: Label GlusterFS nodes
   oc_label:
-    name: "{{ glusterfs_host }}"
+    name: "{{ hostvars[item].openshift.common.hostname }}"
     kind: node
     state: add
     labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}"
   with_items: "{{ glusterfs_nodes | default([]) }}"
-  loop_control:
-    loop_var: glusterfs_host
 
 - name: Copy GlusterFS DaemonSet template
   copy:
@@ -78,7 +82,7 @@
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: template
-    name: glusterfs
+    name: "glusterfs"
     state: present
     files:
     - "{{ mktemp.stdout }}/glusterfs-template.yml"
@@ -91,17 +95,19 @@
     params:
       IMAGE_NAME: "{{ glusterfs_image }}"
       IMAGE_VERSION: "{{ glusterfs_version }}"
+      NODE_LABELS: "{{ glusterfs_nodeselector }}"
+      CLUSTER_NAME: "{{ glusterfs_name }}"
 
 - name: Wait for GlusterFS pods
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs-node=pod"
+    selector: "glusterfs={{ glusterfs_name }}-pod"
   register: glusterfs_pods
   until:
   - "glusterfs_pods.results.results[0]['items'] | count > 0"
   # There must be as many pods with 'Ready' staus  True as there are nodes expecting those pods
   - "glusterfs_pods.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == glusterfs_nodes | count"
   delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"

roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml

@@ -3,7 +3,9 @@
     glusterfs_timeout: "{{ openshift_storage_glusterfs_registry_timeout }}"
     glusterfs_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}"
     glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}"
-    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | map_from_pairs }}"
+    glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}"
+    glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}"
+    glusterfs_storageclass: "{{ openshift_storage_glusterfs_registry_storageclass }}"
     glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}"
     glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}"
     glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe }}"
@@ -17,21 +19,22 @@
     glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_registry_heketi_topology_load }}"
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}"
-    glusterfs_nodes: "{{ g_glusterfs_registry_hosts }}"
+    glusterfs_nodes: "{{ groups.glusterfs_registry }}"
 
 - include: glusterfs_common.yml
-  when: g_glusterfs_registry_hosts != g_glusterfs_hosts
+  when:
+  - groups.glusterfs_registry | default([]) | count > 0
+  - "'glusterfs' not in groups or groups.glusterfs_registry != groups.glusterfs"
 
 - name: Delete pre-existing GlusterFS registry resources
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: "{{ item.kind }}"
-    name: "{{ item.name | default(omit) }}"
-    selector: "{{ item.selector | default(omit) }}"
+    name: "{{ item.name }}"
     state: absent
   with_items:
-  - kind: "svc,ep"
-    name: "glusterfs-registry-endpoints"
+  - kind: "svc"
+    name: "glusterfs-{{ glusterfs_name }}-endpoints"
   failed_when: False
 
 - name: Generate GlusterFS registry endpoints
@@ -40,8 +43,8 @@
     dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
 
 - name: Copy GlusterFS registry service
-  copy:
-    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml"
+  template:
+    src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2"
     dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
 
 - name: Create GlusterFS registry endpoints
@@ -49,7 +52,7 @@
     namespace: "{{ glusterfs_namespace }}"
     state: present
     kind: endpoints
-    name: glusterfs-registry-endpoints
+    name: "glusterfs-{{ glusterfs_name }}-endpoints"
     files:
     - "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
 
@@ -58,14 +61,14 @@
     namespace: "{{ glusterfs_namespace }}"
     state: present
     kind: service
-    name: glusterfs-registry-endpoints
+    name: "glusterfs-{{ glusterfs_name }}-endpoints"
     files:
     - "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
 
 - name: Check if GlusterFS registry volume exists
-  command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' volume list"
+  command: "{{ glusterfs_heketi_client }} volume list"
   register: registry_volume
 
 - name: Create GlusterFS registry volume
-  command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}"
+  command: "{{ glusterfs_heketi_client }} volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}"
   when: "openshift.hosted.registry.storage.glusterfs.path not in registry_volume.stdout"

roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml

@@ -6,11 +6,21 @@
   with_items:
   - "deploy-heketi-template.yml"
 
-- name: Create deploy-heketi resources
+- name: Create heketi topology secret
+  oc_secret:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    name: "heketi-{{ glusterfs_name }}-topology-secret"
+    force: True
+    files:
+    - name: topology.json
+      path: "{{ mktemp.stdout }}/topology.json"
+
+- name: Create deploy-heketi template
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: template
-    name: deploy-heketi
+    name: "deploy-heketi"
     state: present
     files:
     - "{{ mktemp.stdout }}/deploy-heketi-template.yml"
@@ -25,17 +35,20 @@
       IMAGE_VERSION: "{{ glusterfs_heketi_version }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+      HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
+      CLUSTER_NAME: "{{ glusterfs_name }}"
+      TOPOLOGY_PATH: "{{ mktemp.stdout }}"
 
 - name: Wait for deploy-heketi pod
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until:
   - "heketi_pod.results.results[0]['items'] | count > 0"
   # Pod's 'Ready' status must be True
   - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
   delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"

roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml

@@ -1,8 +1,10 @@
 ---
 - name: Create heketi DB volume
-  command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' setup-openshift-heketi-storage --listfile {{ mktemp.stdout }}/heketi-storage.json"
+  command: "{{ glusterfs_heketi_client }} setup-openshift-heketi-storage --listfile /tmp/heketi-storage.json"
   register: setup_storage
-  failed_when: False
+
+- name: Copy heketi-storage list
+  shell: "{{ openshift.common.client_binary }} rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json"
 
 # This is used in the subsequent task
 - name: Copy the admin client config
@@ -28,7 +30,7 @@
   # Pod's 'Complete' status must be True
   - "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Complete'}) | map('bool') | select | list | count == 1"
   delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"
   failed_when:
   - "'results' in heketi_job.results"
   - "heketi_job.results.results | count > 0"
@@ -46,14 +48,45 @@
   with_items:
   - kind: "template,route,service,jobs,dc,secret"
     selector: "deploy-heketi"
-  failed_when: False
+  - kind: "svc"
+    name: "heketi-storage-endpoints"
+  - kind: "secret"
+    name: "heketi-{{ glusterfs_name }}-topology-secret"
+
+- name: Generate heketi endpoints
+  template:
+    src: "{{ openshift.common.examples_content_version }}/heketi-endpoints.yml.j2"
+    dest: "{{ mktemp.stdout }}/heketi-endpoints.yml"
+
+- name: Generate heketi service
+  template:
+    src: "{{ openshift.common.examples_content_version }}/heketi-service.yml.j2"
+    dest: "{{ mktemp.stdout }}/heketi-service.yml"
+
+- name: Create heketi endpoints
+  oc_obj:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    kind: endpoints
+    name: "heketi-db-{{ glusterfs_name }}-endpoints"
+    files:
+    - "{{ mktemp.stdout }}/heketi-endpoints.yml"
+
+- name: Create heketi service
+  oc_obj:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    kind: service
+    name: "heketi-db-{{ glusterfs_name }}-endpoints"
+    files:
+    - "{{ mktemp.stdout }}/heketi-service.yml"
 
 - name: Copy heketi template
   copy:
     src: "{{ openshift.common.examples_content_version }}/heketi-template.yml"
     dest: "{{ mktemp.stdout }}/heketi-template.yml"
 
-- name: Create heketi resources
+- name: Create heketi template
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: template
@@ -72,38 +105,27 @@
       IMAGE_VERSION: "{{ glusterfs_heketi_version }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+      HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
+      CLUSTER_NAME: "{{ glusterfs_name }}"
 
 - name: Wait for heketi pod
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
     kind: pod
     state: list
-    selector: "glusterfs=heketi-pod"
+    selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod
   until:
   - "heketi_pod.results.results[0]['items'] | count > 0"
   # Pod's 'Ready' status must be True
   - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
   delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
-
-- name: Determine heketi URL
-  oc_obj:
-    namespace: "{{ glusterfs_namespace }}"
-    state: list
-    kind: ep
-    selector: "glusterfs=heketi-service"
-  register: heketi_url
-  until:
-  - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"
-  - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''"
-  delay: 10
-  retries: "{{ (glusterfs_timeout / 10) | int }}"
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"
 
-- name: Set heketi URL
+- name: Set heketi-cli command
   set_fact:
-    glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}"
+    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
 
 - name: Verify heketi service
-  command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' cluster list"
+  command: "{{ glusterfs_heketi_client }} cluster list"
   changed_when: False

roles/openshift_storage_glusterfs/tasks/main.yml

@@ -7,12 +7,11 @@
 
 - include: glusterfs_config.yml
   when:
-  - g_glusterfs_hosts | default([]) | count > 0
+  - groups.glusterfs | default([]) | count > 0
 
 - include: glusterfs_registry.yml
   when:
-  - g_glusterfs_registry_hosts | default([]) | count > 0
-  - "openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap"
+  - "groups.glusterfs_registry | default([]) | count > 0 or openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap"
 
 - name: Delete temp directory
   file:

roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2

@@ -1,7 +1,8 @@
+---
 apiVersion: v1
 kind: Endpoints
 metadata:
-  name: glusterfs-registry-endpoints
+  name: glusterfs-{{ glusterfs_name }}-endpoints
 subsets:
 - addresses:
 {% for node in glusterfs_nodes %}

roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: glusterfs-registry-endpoints
+  name: glusterfs-{{ glusterfs_name }}-endpoints
 spec:
   ports:
   - port: 1

roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: glusterfs-{{ glusterfs_name }}
+provisioner: kubernetes.io/glusterfs
+parameters:
+  resturl: "http://{{ glusterfs_heketi_url }}:8081"
+  secretNamespace: "{{ glusterfs_namespace }}"
+  secretName: "heketi-{{ glusterfs_name }}-user-secret"

roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: heketi-db-{{ glusterfs_name }}-endpoints
+subsets:
+- addresses:
+{% for node in glusterfs_nodes %}
+  - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }}
+{% endfor %}
+  ports:
+  - port: 1

roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: heketi-db-{{ glusterfs_name }}-endpoints
+spec:
+  ports:
+  - port: 1
+status:
+  loadBalancer: {}