GlusterFS: Fix SSH-based heketi configuration

Signed-off-by: Jose A. Rivera <jarrpa@redhat.com>

roles/openshift_storage_glusterfs/README.md

@@ -92,6 +92,11 @@ GlusterFS cluster into a new or existing OpenShift cluster:
 | openshift_storage_glusterfs_heketi_topology_load | True                    | Load the GlusterFS topology information into heketi
 | openshift_storage_glusterfs_heketi_url           | Undefined               | When heketi is native, this sets the hostname portion of the final heketi route URL. When heketi is external, this is the full URL to the heketi service.
 | openshift_storage_glusterfs_heketi_port          | 8080                    | TCP port for external heketi service **NOTE:** This has no effect in native mode
+| openshift_storage_glusterfs_heketi_executor      | 'kubernetes'            | Selects how a native heketi service will manage GlusterFS nodes: 'kubernetes' for native nodes, 'ssh' for external nodes
+| openshift_storage_glusterfs_heketi_ssh_port      | 22                      | SSH port for external GlusterFS nodes via native heketi
+| openshift_storage_glusterfs_heketi_ssh_user      | 'root'                  | SSH user for external GlusterFS nodes via native heketi
+| openshift_storage_glusterfs_heketi_ssh_sudo      | False                   | Whether to sudo (if non-root user) for SSH to external GlusterFS nodes via native heketi
+| openshift_storage_glusterfs_heketi_ssh_keyfile   | '/dev/null'             | Path to a private key file for use with SSH connections to external GlusterFS nodes via native heketi **NOTE:** This must be an absolute path
 | openshift_storage_glusterfs_heketi_wipe          | False                   | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe`
 
 Each role variable also has a corresponding variable to optionally configure a

roles/openshift_storage_glusterfs/defaults/main.yml

@@ -19,6 +19,11 @@ openshift_storage_glusterfs_heketi_topology_load: True
 openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}"
 openshift_storage_glusterfs_heketi_url: "{{ omit }}"
 openshift_storage_glusterfs_heketi_port: 8080
+openshift_storage_glusterfs_heketi_executor: 'kubernetes'
+openshift_storage_glusterfs_heketi_ssh_port: 22
+openshift_storage_glusterfs_heketi_ssh_user: 'root'
+openshift_storage_glusterfs_heketi_ssh_sudo: False
+openshift_storage_glusterfs_heketi_ssh_keyfile: '/dev/null'
 
 openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
 openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
@@ -40,3 +45,8 @@ openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage
 openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
 openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}"
 openshift_storage_glusterfs_registry_heketi_port: 8080
+openshift_storage_glusterfs_registry_heketi_executor: 'kubernetes'
+openshift_storage_glusterfs_registry_heketi_ssh_port: 22
+openshift_storage_glusterfs_registry_heketi_ssh_user: 'root'
+openshift_storage_glusterfs_registry_heketi_ssh_sudo: False
+openshift_storage_glusterfs_registry_heketi_ssh_keyfile: '/dev/null'

roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml

@@ -71,7 +71,7 @@ objects:
           - name: HEKETI_ADMIN_KEY
             value: ${HEKETI_ADMIN_KEY}
           - name: HEKETI_EXECUTOR
-            value: kubernetes
+            value: ${HEKETI_EXECUTOR}
           - name: HEKETI_FSTAB
             value: /var/lib/heketi/fstab
           - name: HEKETI_SNAPSHOT_LIMIT
@@ -87,6 +87,8 @@ objects:
             mountPath: /var/lib/heketi
           - name: topology
             mountPath: ${TOPOLOGY_PATH}
+          - name: config
+            mountPath: /etc/heketi
           readinessProbe:
             timeoutSeconds: 3
             initialDelaySeconds: 3
@@ -104,6 +106,9 @@ objects:
         - name: topology
           secret:
             secretName: heketi-${CLUSTER_NAME}-topology-secret
+        - name: config
+          secret:
+            secretName: heketi-${CLUSTER_NAME}-config-secret
 parameters:
 - name: HEKETI_USER_KEY
   displayName: Heketi User Secret
@@ -111,6 +116,10 @@ parameters:
 - name: HEKETI_ADMIN_KEY
   displayName: Heketi Administrator Secret
   description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_EXECUTOR
+  displayName: heketi executor type
+  description: Set the executor type, kubernetes or ssh
+  value: kubernetes
 - name: HEKETI_KUBE_NAMESPACE
   displayName: Namespace
   description: Set the namespace where the GlusterFS pods reside

roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml

@@ -67,7 +67,7 @@ objects:
           - name: HEKETI_ADMIN_KEY
             value: ${HEKETI_ADMIN_KEY}
           - name: HEKETI_EXECUTOR
-            value: kubernetes
+            value: ${HEKETI_EXECUTOR}
           - name: HEKETI_FSTAB
             value: /var/lib/heketi/fstab
           - name: HEKETI_SNAPSHOT_LIMIT
@@ -81,6 +81,8 @@ objects:
           volumeMounts:
           - name: db
             mountPath: /var/lib/heketi
+          - name: config
+            mountPath: /etc/heketi
           readinessProbe:
             timeoutSeconds: 3
             initialDelaySeconds: 3
@@ -98,6 +100,9 @@ objects:
           glusterfs:
             endpoints: heketi-db-${CLUSTER_NAME}-endpoints
             path: heketidbstorage
+        - name: config
+          secret:
+            secretName: heketi-${CLUSTER_NAME}-config-secret
 parameters:
 - name: HEKETI_USER_KEY
   displayName: Heketi User Secret
@@ -105,6 +110,10 @@ parameters:
 - name: HEKETI_ADMIN_KEY
   displayName: Heketi Administrator Secret
   description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_EXECUTOR
+  displayName: heketi executor type
+  description: Set the executor type, kubernetes or ssh
+  value: kubernetes
 - name: HEKETI_KUBE_NAMESPACE
   displayName: Namespace
   description: Set the namespace where the GlusterFS pods reside

roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml

@@ -19,6 +19,8 @@
     name: "heketi-storage-endpoints"
   - kind: "secret"
     name: "heketi-{{ glusterfs_name }}-topology-secret"
+  - kind: "secret"
+    name: "heketi-{{ glusterfs_name }}-config-secret"
   - kind: "template,route,service,dc"
     name: "heketi-{{ glusterfs_name }}"
   - kind: "svc"
@@ -125,6 +127,13 @@
   when:
   - glusterfs_heketi_topology_load
 
+- name: Generate heketi config file
+  template:
+    src: "{{ openshift.common.examples_content_version }}/heketi.json.j2"
+    dest: "{{ mktemp.stdout }}/heketi.json"
+  when:
+  - glusterfs_heketi_is_native
+
 - name: Generate heketi admin key
   set_fact:
     glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
@@ -142,6 +151,20 @@
   - glusterfs_heketi_is_native
   - glusterfs_heketi_user_key is undefined
 
+- name: Create heketi config secret
+  oc_secret:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    name: "heketi-{{ glusterfs_name }}-config-secret"
+    force: True
+    files:
+    - name: heketi.json
+      path: "{{ mktemp.stdout }}/heketi.json"
+    - name: private_key
+      path: "{{ glusterfs_heketi_ssh_keyfile }}"
+  when:
+  - glusterfs_heketi_is_native
+
 - include: heketi_deploy_part1.yml
   when:
   - glusterfs_heketi_is_native

roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml

@@ -20,6 +20,11 @@
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}"
     glusterfs_heketi_port: "{{ openshift_storage_glusterfs_heketi_port }}"
+    glusterfs_heketi_executor: "{{ openshift_storage_glusterfs_heketi_executor }}"
+    glusterfs_heketi_ssh_port: "{{ openshift_storage_glusterfs_heketi_ssh_port }}"
+    glusterfs_heketi_ssh_user: "{{ openshift_storage_glusterfs_heketi_ssh_user }}"
+    glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_heketi_ssh_sudo }}"
+    glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_heketi_ssh_keyfile }}"
     glusterfs_nodes: "{{ groups.glusterfs }}"
 
 - include: glusterfs_common.yml

roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml

@@ -20,6 +20,11 @@
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}"
     glusterfs_heketi_port: "{{ openshift_storage_glusterfs_registry_heketi_port }}"
+    glusterfs_heketi_executor: "{{ openshift_storage_glusterfs_registry_heketi_executor }}"
+    glusterfs_heketi_ssh_port: "{{ openshift_storage_glusterfs_registry_heketi_ssh_port }}"
+    glusterfs_heketi_ssh_user: "{{ openshift_storage_glusterfs_registry_heketi_ssh_user }}"
+    glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_registry_heketi_ssh_sudo }}"
+    glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_registry_heketi_ssh_keyfile }}"
     glusterfs_nodes: "{{ groups.glusterfs_registry | default(groups.glusterfs) }}"
 
 - include: glusterfs_common.yml

roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml

@@ -36,6 +36,7 @@
       HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+      HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
       CLUSTER_NAME: "{{ glusterfs_name }}"
       TOPOLOGY_PATH: "{{ mktemp.stdout }}"

roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml

@@ -106,6 +106,7 @@
       HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+      HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
       CLUSTER_NAME: "{{ glusterfs_name }}"
 

roles/openshift_storage_glusterfs/templates/v3.6/heketi.json.j2

@@ -0,0 +1,36 @@
+{
+	"_port_comment": "Heketi Server Port Number",
+	"port" : "8080",
+
+	"_use_auth": "Enable JWT authorization. Please enable for deployment",
+	"use_auth" : false,
+
+	"_jwt" : "Private keys for access",
+	"jwt" : {
+		"_admin" : "Admin has access to all APIs",
+		"admin" : {
+			"key" : "My Secret"
+		},
+		"_user" : "User only has access to /volumes endpoint",
+		"user" : {
+			"key" : "My Secret"
+		}
+	},
+
+	"_glusterfs_comment": "GlusterFS Configuration",
+	"glusterfs" : {
+
+		"_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
+		"executor" : "{{ glusterfs_heketi_executor }}",
+
+		"_db_comment": "Database file name",
+		"db" : "/var/lib/heketi/heketi.db",
+
+		"sshexec" : {
+			"keyfile" : "/etc/heketi/private_key",
+			"port" : "{{ glusterfs_heketi_ssh_port }}",
+			"user" : "{{ glusterfs_heketi_ssh_user }}",
+			"sudo" : {{ glusterfs_heketi_ssh_sudo | lower }}
+		}
+	}
+}