GlusterFS: Fix heketi_pod check

The previous solution for tracking which heketi pods are active was broken
with the merging of bb5888854. In a two-cluster deployment of GlusterFS
(`glusterfs` and `glusterfs_registry` groups defined) the second cluster
would erroneously receive information for which heketi pods were running from
the first deployment, due to problems with how set_fact was used. This commit
overhauls that solution to fix the problem as well as a few more that arose in
development.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1623777
Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1626751

Signed-off-by: Jose A. Rivera <jarrpa@redhat.com>

roles/openshift_storage_glusterfs/tasks/heketi_deploy.yml

@@ -69,18 +69,10 @@
   delay: 10
   retries: "{{ (glusterfs_timeout | int / 10) | int }}"
 
-- name: Update heketi pod result
-  set_fact:
+- import_tasks: heketi_set_cli.yml
+  vars:
     heketi_pod: "{{ heketi_pod_wait.results.results[0]['items'][0] }}"
 
-- name: Set heketi-cli command
-  set_fact:
-    glusterfs_heketi_client: "{{ openshift_client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.metadata.name }} {{ glusterfs_heketi_cli }} -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
-
-- name: Verify heketi service
-  command: "{{ glusterfs_heketi_client }} cluster list"
-  changed_when: False
-
 - name: Set heketi deployed fact
   set_fact:
     glusterfs_heketi_is_missing: False

roles/openshift_storage_glusterfs/tasks/heketi_init_db.yml

@@ -4,7 +4,7 @@
   register: setup_storage
 
 - name: Copy heketi-storage list
-  shell: "{{ openshift_client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig rsh --namespace={{ glusterfs_namespace }} {{ deploy_heketi_pod.metadata.name }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json"
+  shell: "{{ openshift_client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod_name }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json"
 
 # Need `command` here because heketi-storage.json contains multiple objects.
 - name: Copy heketi DB to GlusterFS volume
@@ -45,3 +45,7 @@
     name: "heketi-storage-endpoints"
   - kind: "secret"
     name: "heketi-{{ glusterfs_name | default }}-topology-secret"
+
+- name: Set deploy-heketi deployed fact
+  set_fact:
+    glusterfs_heketi_deploy_is_missing: True

roles/openshift_storage_glusterfs/tasks/heketi_init_deploy.yml

@@ -43,9 +43,9 @@
   delay: 10
   retries: "{{ (glusterfs_timeout | int / 10) | int }}"
 
-- name: Update deploy-heketi pod result
-  set_fact:
-    deploy_heketi_pod: "{{ deploy_heketi_pod_wait.results.results[0]['items'][0] }}"
+- import_tasks: heketi_set_cli.yml
+  vars:
+    heketi_pod: "{{ deploy_heketi_pod_wait.results.results[0]['items'][0] }}"
 
 - name: Set deploy-heketi deployed fact
   set_fact:

roles/openshift_storage_glusterfs/tasks/heketi_pod_check.yml

@@ -7,14 +7,14 @@
     selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
   register: deploy_heketi_pod_check
 
+# deploy-heketi pod is not missing when there are one or more pods with matching labels whose 'Ready' status is True
 - name: Check if need to deploy deploy-heketi
   set_fact:
-    glusterfs_heketi_deploy_is_missing: False
-    deploy_heketi_pod: "{{ deploy_heketi_pod_check.results.results[0]['items'][0] }}"
-  when:
-  - "deploy_heketi_pod_check.results.results[0]['items'] | count > 0"
-  # deploy-heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
-  - "deploy_heketi_pod_check.results.results[0]['items'] | lib_utils_oo_collect(attribute='status.conditions') | lib_utils_oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
+    glusterfs_heketi_deploy_is_missing: "{{ heketi_pod_results_count | int == 0 or heketi_pod_results_ready_count | int == 0 }}"
+  vars:
+    heketi_pod_results: "{{ deploy_heketi_pod_check.results.results[0]['items'] }}"
+    heketi_pod_results_count: "{{ heketi_pod_results | count }}"
+    heketi_pod_results_ready_count: "{{ heketi_pod_results | lib_utils_oo_collect(attribute='status.conditions') | lib_utils_oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count }}"
 
 - name: Check for existing heketi pod
   oc_obj:
@@ -24,11 +24,17 @@
     selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
   register: heketi_pod_check
 
+# heketi pod is not missing when there are one or more pods with matching labels whose 'Ready' status is True
 - name: Check if need to deploy heketi
   set_fact:
-    glusterfs_heketi_is_missing: False
-    heketi_pod: "{{ heketi_pod_check.results.results[0]['items'][0] }}"
-  when:
-  - "heketi_pod_check.results.results[0]['items'] | count > 0"
-  # heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
-  - "heketi_pod_check.results.results[0]['items'] | lib_utils_oo_collect(attribute='status.conditions') | lib_utils_oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
+    glusterfs_heketi_is_missing: "{{ heketi_pod_results_count | int == 0 or heketi_pod_results_ready_count | int == 0 }}"
+  vars:
+    heketi_pod_results: "{{ heketi_pod_check.results.results[0]['items'] }}"
+    heketi_pod_results_count: "{{ heketi_pod_results | count }}"
+    heketi_pod_results_ready_count: "{{ heketi_pod_results | lib_utils_oo_collect(attribute='status.conditions') | lib_utils_oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count }}"
+
+- import_tasks: heketi_set_cli.yml
+  vars:
+    deploy_heketi_pod_results: "{{ deploy_heketi_pod_check.results.results[0]['items'] }}"
+    heketi_pod_results: "{{ heketi_pod_check.results.results[0]['items'] }}"
+    heketi_pod: "{{ heketi_pod_results | default(deploy_heketi_pod_results, true) | default([''], true) | first }}"

roles/openshift_storage_glusterfs/tasks/heketi_set_cli.yml

@@ -0,0 +1,18 @@
+---
+- name: Set heketi_pod_name
+  set_fact:
+    heketi_pod_name: "{{ heketi_pod.metadata.name if glusterfs_heketi_is_native and heketi_pod is defined and 'metadata' in heketi_pod else '' }}"
+
+- name: Set heketi-cli command
+  set_fact:
+    glusterfs_heketi_client: "{{ heketi_pod_rsh }}{{ glusterfs_heketi_cli }} -s http://{{ heketi_server }} --user admin {{ heketi_admin_key }}"
+  vars:
+    heketi_pod_rsh: "{% if glusterfs_heketi_is_native %}{{ openshift_client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod_name }} {% endif %}"
+    heketi_server: "{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %}"
+    heketi_admin_key: " {% if glusterfs_heketi_admin_key is defined %}--secret '{{ glusterfs_heketi_admin_key }}'{% endif %}"
+
+- name: Verify heketi service
+  command: "{{ glusterfs_heketi_client }} cluster list"
+  changed_when: False
+  when:
+  - heketi_pod_name != '' or not glusterfs_heketi_is_native