Add containerized glusterfs cluster health check

This commit adds a custom module to perform glusterfs
volume health check.

This check is executed before and during openshift upgrades
to ensure gluster services are not disturbed on more than
one host at a time.

playbooks/common/openshift-cluster/upgrades/pre/verify_cluster.yml

@@ -41,6 +41,13 @@
     - openshift_release is defined
     - not (openshift_release is version_compare(openshift_upgrade_target ,'='))
 
+  # Ensure glusterfs clusters are healthy before starting an upgrade.
+  - import_role:
+      name: openshift_storage_glusterfs
+      tasks_from: check_cluster_health.yml
+    when: >
+          "'glusterfs' in groups" and "groups['glusterfs'] | length > 0"
+          or "'glusterfs_registry' in groups" and "groups['glusterfs_registry'] | length > 0"
 - name: Verify master processes
   hosts: oo_masters_to_config
   roles:

playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml

@@ -78,6 +78,15 @@
   - include_tasks: "{{ openshift_node_upgrade_post_hook }}"
     when: openshift_node_upgrade_post_hook is defined
 
+  # If the host is a gluster host, need to verify gluster is healthy before
+  # moving to the next host.
+  - import_role:
+      name: openshift_storage_glusterfs
+      tasks_from: check_cluster_health.yml
+    when: >
+          inventory_hostname in groups['glusterfs']
+          or inventory_hostname in groups['glusterfs_registry']
+
 - name: Re-enable excluders
   hosts: oo_nodes_to_upgrade:!oo_masters_to_config
   tasks:

playbooks/common/openshift-cluster/upgrades/v3_11/upgrade_control_plane.yml

@@ -102,6 +102,12 @@
   - import_role:
       name: openshift_node
       tasks_from: upgrade
+  - import_role:
+      name: openshift_storage_glusterfs
+      tasks_from: check_cluster_health.yml
+    when: >
+          inventory_hostname in groups['glusterfs']
+          or inventory_hostname in groups['glusterfs_registry']
 
 - import_playbook: ../upgrade_control_plane.yml
   vars:

playbooks/openshift-glusterfs/cluster_health_check.yml

@@ -0,0 +1,8 @@
+---
+- import_playbook: ../init/main.yml
+  vars:
+    l_init_fact_hosts: "oo_masters_to_config"
+    l_openshift_version_set_hosts: "oo_masters_to_config:!oo_first_master"
+    l_sanity_check_hosts: "{{ groups['oo_masters_to_config'] }}"
+
+- import_playbook: private/cluster_health_check.yml

playbooks/openshift-glusterfs/private/cluster_health_check.yml

@@ -0,0 +1,8 @@
+---
+- name: Verify Gluster Health
+  hosts: oo_first_master
+  tasks:
+  - name: Run glusterfs health check
+    import_role:
+      name: openshift_storage_glusterfs
+      tasks_from: check_cluster_health.yml

playbooks/openshift-master/private/upgrade.yml

@@ -185,8 +185,8 @@
 - name: Reconcile Cluster Roles and Cluster Role Bindings and Security Context Constraints
   hosts: oo_masters_to_config
   roles:
-  - { role: openshift_cli }
-  - { role: openshift_facts }
+  - openshift_cli
+  - openshift_facts
   vars:
     __master_shared_resource_viewer_file: "shared_resource_viewer_role.yaml"
   tasks:
@@ -243,16 +243,10 @@
   # and is evaluated early. Values such as "20%" can also be used.
   serial: "{{ openshift_upgrade_control_plane_nodes_serial | default(1) }}"
   max_fail_percentage: "{{ openshift_upgrade_control_plane_nodes_max_fail_percentage | default(0) }}"
-
-  pre_tasks:
-  - name: Load lib_openshift modules
-    import_role:
-      name: lib_openshift
-
   roles:
+  - lib_openshift
   - openshift_facts
-
-  post_tasks:
+  tasks:
   - import_role:
       name: openshift_manage_node
       tasks_from: config.yml

roles/lib_utils/library/glusterfs_check_containerized.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+"""glusterfs_check_containerized module"""
+# Copyright 2018 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+DOCUMENTATION = '''
+---
+module: glusterfs_check_containerized
+
+short_description: Check health of each volume in glusterfs on openshift.
+
+version_added: "2.6"
+
+description:
+    - This module attempts to ensure all volumes are in healthy state
+      in a glusterfs cluster.  The module is meant to be failure-prone, retries
+      should be executed at the ansible level, they are not implemented in
+      this module.
+      This module by executing the following (roughly):
+      oc exec --namespace=<namespace> <podname> -- gluster volume list
+      for volume in <volume list>:
+        gluster volume heal <volume> info
+
+author:
+    - "Michael Gugino <mgugino@redhat.com>"
+'''
+
+EXAMPLES = '''
+- name: glusterfs volumes check
+  glusterfs_check_containerized
+    oc_bin: "/usr/bin/oc"
+    oc_conf: "/etc/origin/master/admin.kubeconfig"
+    oc_namespace: "glusterfs"
+    cluster_name: "glusterfs"
+'''
+
+
+def fail(module, err):
+    """Fail on error"""
+    result = {'failed': True,
+              'changed': False,
+              'msg': err,
+              'state': 'unknown'}
+    module.fail_json(**result)
+
+
+def call_or_fail(module, call_args):
+    """Call subprocess.check_output and return utf-8 decoded stdout or fail"""
+    try:
+        # Must decode as utf-8 for python3 compatibility
+        res = subprocess.check_output(call_args).decode('utf-8')
+    except subprocess.CalledProcessError as err:
+        fail(module, str(err))
+    return res
+
+
+def get_valid_nodes(module, oc_exec, exclude_node):
+    """Return a list of nodes that will be used to filter running pods"""
+    call_args = oc_exec + ['get', 'nodes']
+    res = call_or_fail(module, call_args)
+    valid_nodes = []
+    for line in res.split('\n'):
+        fields = line.split()
+        if not fields:
+            continue
+        if fields[0] != exclude_node and fields[1] == "Ready":
+            valid_nodes.append(fields[0])
+    if not valid_nodes:
+        fail(module,
+             'Unable to find suitable node in get nodes output: {}'.format(res))
+    return valid_nodes
+
+
+def select_pod(module, oc_exec, cluster_name, valid_nodes):
+    """Select a pod to attempt to run gluster commands on"""
+    call_args = oc_exec + ['get', 'pods', '-owide']
+    res = call_or_fail(module, call_args)
+    # res is returned as a tab/space-separated list with headers.
+    res_lines = res.split('\n')
+    pod_name = None
+    name_search = 'glusterfs-{}'.format(cluster_name)
+    res_lines = list(filter(None, res.split('\n')))
+
+    for line in res_lines[1:]:
+        fields = line.split()
+        if not fields:
+            continue
+        if name_search in fields[0]:
+            if fields[2] == "Running" and fields[6] in valid_nodes:
+                pod_name = fields[0]
+                break
+
+    if pod_name is None:
+        fail(module,
+             "Unable to find suitable pod in get pods output: {}".format(res))
+    else:
+        return pod_name
+
+
+def get_volume_list(module, oc_exec, pod_name):
+    """Retrieve list of active volumes from gluster cluster"""
+    call_args = oc_exec + ['exec', pod_name, '--', 'gluster', 'volume', 'list']
+    res = call_or_fail(module, call_args)
+    # This should always at least return heketidbstorage, so no need to check
+    # for empty string.
+    return list(filter(None, res.split('\n')))
+
+
+def check_volume_health_info(module, oc_exec, pod_name, volume):
+    """Check health info of gluster volume"""
+    call_args = oc_exec + ['exec', pod_name, '--', 'gluster', 'volume', 'heal',
+                           volume, 'info']
+    res = call_or_fail(module, call_args)
+    # Output is not easily parsed
+    for line in res.split('\n'):
+        if line.startswith('Number of entries:'):
+            cols = line.split(':')
+            if cols[1].strip() != '0':
+                fail(module, 'volume {} is not ready'.format(volume))
+
+
+def check_volumes(module, oc_exec, pod_name):
+    """Check status of all volumes on cluster"""
+    volume_list = get_volume_list(module, oc_exec, pod_name)
+    for volume in volume_list:
+        check_volume_health_info(module, oc_exec, pod_name, volume)
+
+
+def run_module():
+    '''Run this module'''
+    module_args = dict(
+        oc_bin=dict(type='path', required=True),
+        oc_conf=dict(type='path', required=True),
+        oc_namespace=dict(type='str', required=True),
+        cluster_name=dict(type='str', required=True),
+        exclude_node=dict(type='str', required=True),
+    )
+    module = AnsibleModule(
+        supports_check_mode=False,
+        argument_spec=module_args
+    )
+    oc_bin = module.params['oc_bin']
+    oc_conf = '--config={}'.format(module.params['oc_conf'])
+    oc_namespace = '--namespace={}'.format(module.params['oc_namespace'])
+    cluster_name = module.params['cluster_name']
+    exclude_node = module.params['exclude_node']
+
+    oc_exec = [oc_bin, oc_conf, oc_namespace]
+
+    # create a nodes to find a pod on; We don't want to try to execute on a
+    # pod running on a "NotReady" node or the inventory_hostname node because
+    # the pods might not actually be alive.
+    valid_nodes = get_valid_nodes(module, [oc_bin, oc_conf], exclude_node)
+
+    # Need to find an alive pod to run gluster commands in.
+    pod_name = select_pod(module, oc_exec, cluster_name, valid_nodes)
+
+    check_volumes(module, oc_exec, pod_name)
+
+    result = {'changed': False}
+    module.exit_json(**result)
+
+
+def main():
+    """main"""
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

roles/lib_utils/test/test_glusterfs_check_containerized.py

@@ -0,0 +1,157 @@
+import os
+import sys
+
+import pytest
+
+try:
+    # python3, mock is built in.
+    from unittest.mock import patch
+except ImportError:
+    # In python2, mock is installed via pip.
+    from mock import patch
+
+MODULE_PATH = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'library'))
+sys.path.insert(1, MODULE_PATH)
+
+import glusterfs_check_containerized  # noqa
+
+
+NODE_LIST_STD_OUT_1 = ("""
+NAME                       STATUS    ROLES                  AGE       VERSION
+fedora1.openshift.io   Ready     compute,infra,master   1d        v1.11.0+d4cacc0
+fedora2.openshift.io   Ready     infra                  1d        v1.11.0+d4cacc0
+fedora3.openshift.io   Ready     infra                  1d        v1.11.0+d4cacc0
+""")
+
+NODE_LIST_STD_OUT_2 = ("""
+NAME                       STATUS    ROLES                  AGE       VERSION
+fedora1.openshift.io   Ready     compute,infra,master   1d        v1.11.0+d4cacc0
+fedora2.openshift.io   NotReady     infra                  1d        v1.11.0+d4cacc0
+fedora3.openshift.io   Ready     infra                  1d        v1.11.0+d4cacc0
+""")
+
+NODE_LIST_STD_OUT_3 = ("""
+NAME                       STATUS    ROLES                  AGE       VERSION
+fedora1.openshift.io   Ready     compute,infra,master   1d        v1.11.0+d4cacc0
+fedora2.openshift.io   NotReady     infra                  1d        v1.11.0+d4cacc0
+fedora3.openshift.io   Invalid     infra                  1d        v1.11.0+d4cacc0
+""")
+
+POD_SELECT_STD_OUT = ("""NAME                                          READY     STATUS    RESTARTS   AGE       IP                NODE
+glusterblock-storage-provisioner-dc-1-ks5zt   1/1       Running   0          1d        10.130.0.5        fedora3.openshift.io
+glusterfs-storage-fzdn2                       1/1       Running   0          1d        192.168.124.175   fedora1.openshift.io
+glusterfs-storage-mp9nk                       1/1       Running   4          1d        192.168.124.233   fedora2.openshift.io
+glusterfs-storage-t9c6d                       1/1       Running   0          1d        192.168.124.50    fedora3.openshift.io
+heketi-storage-1-rgj8b                        1/1       Running   0          1d        10.130.0.4        fedora3.openshift.io""")
+
+# Need to ensure we have extra empty lines in this output;
+# thus the quotes are one line above and below the text.
+VOLUME_LIST_STDOUT = ("""
+heketidbstorage
+volume1
+""")
+
+VOLUME_HEAL_INFO_GOOD = ("""
+Brick 192.168.124.233:/var/lib/heketi/mounts/vg_936ddf24061d55788f50496757d2f3b2/brick_9df1b6229025ea45521ab1b370d24a06/brick
+Status: Connected
+Number of entries: 0
+
+Brick 192.168.124.175:/var/lib/heketi/mounts/vg_95975e77a6dc7a8e45586eac556b0f24/brick_172b6be6704a3d9f706535038f7f2e52/brick
+Status: Connected
+Number of entries: 0
+
+Brick 192.168.124.50:/var/lib/heketi/mounts/vg_6523756fe1becfefd3224d3082373344/brick_359e4cf44cd1b82674f7d931cb5c481e/brick
+Status: Connected
+Number of entries: 0
+""")
+
+VOLUME_HEAL_INFO_BAD = ("""
+Brick 192.168.124.233:/var/lib/heketi/mounts/vg_936ddf24061d55788f50496757d2f3b2/brick_9df1b6229025ea45521ab1b370d24a06/brick
+Status: Connected
+Number of entries: 0
+
+Brick 192.168.124.175:/var/lib/heketi/mounts/vg_95975e77a6dc7a8e45586eac556b0f24/brick_172b6be6704a3d9f706535038f7f2e52/brick
+Status: Connected
+Number of entries: 0
+
+Brick 192.168.124.50:/var/lib/heketi/mounts/vg_6523756fe1becfefd3224d3082373344/brick_359e4cf44cd1b82674f7d931cb5c481e/brick
+Status: Connected
+Number of entries: -
+""")
+
+
+class DummyModule(object):
+    def exit_json(*args, **kwargs):
+        return 0
+
+    def fail_json(*args, **kwargs):
+        raise Exception(kwargs['msg'])
+
+
+def test_get_valid_nodes():
+    with patch('glusterfs_check_containerized.call_or_fail') as call_mock:
+        module = DummyModule()
+        oc_exec = []
+        exclude_node = "fedora1.openshift.io"
+
+        call_mock.return_value = NODE_LIST_STD_OUT_1
+        valid_nodes = glusterfs_check_containerized.get_valid_nodes(module, oc_exec, exclude_node)
+        assert valid_nodes == ['fedora2.openshift.io', 'fedora3.openshift.io']
+
+        call_mock.return_value = NODE_LIST_STD_OUT_2
+        valid_nodes = glusterfs_check_containerized.get_valid_nodes(module, oc_exec, exclude_node)
+        assert valid_nodes == ['fedora3.openshift.io']
+
+        call_mock.return_value = NODE_LIST_STD_OUT_3
+        with pytest.raises(Exception) as err:
+            valid_nodes = glusterfs_check_containerized.get_valid_nodes(module, oc_exec, exclude_node)
+        assert 'Exception: Unable to find suitable node in get nodes output' in str(err)
+
+
+def test_select_pod():
+    with patch('glusterfs_check_containerized.call_or_fail') as call_mock:
+        module = DummyModule()
+        oc_exec = []
+        cluster_name = "storage"
+        valid_nodes = ["fedora2.openshift.io", "fedora3.openshift.io"]
+        call_mock.return_value = POD_SELECT_STD_OUT
+        # Should select first valid podname in call_or_fail output.
+        pod_name = glusterfs_check_containerized.select_pod(module, oc_exec, cluster_name, valid_nodes)
+        assert pod_name == 'glusterfs-storage-mp9nk'
+        with pytest.raises(Exception) as err:
+            pod_name = glusterfs_check_containerized.select_pod(module, oc_exec, "does not exist", valid_nodes)
+        assert 'Exception: Unable to find suitable pod in get pods output' in str(err)
+
+
+def test_get_volume_list():
+    with patch('glusterfs_check_containerized.call_or_fail') as call_mock:
+        module = DummyModule()
+        oc_exec = []
+        pod_name = ''
+        call_mock.return_value = VOLUME_LIST_STDOUT
+        volume_list = glusterfs_check_containerized.get_volume_list(module, oc_exec, pod_name)
+        assert volume_list == ['heketidbstorage', 'volume1']
+
+
+def test_check_volume_health_info():
+    with patch('glusterfs_check_containerized.call_or_fail') as call_mock:
+        module = DummyModule()
+        oc_exec = []
+        pod_name = ''
+        volume = 'somevolume'
+        call_mock.return_value = VOLUME_HEAL_INFO_GOOD
+        # this should just complete quietly.
+        glusterfs_check_containerized.check_volume_health_info(module, oc_exec, pod_name, volume)
+
+        call_mock.return_value = VOLUME_HEAL_INFO_BAD
+        expected_error = 'volume {} is not ready'.format(volume)
+        with pytest.raises(Exception) as err:
+            glusterfs_check_containerized.check_volume_health_info(module, oc_exec, pod_name, volume)
+        assert expected_error in str(err)
+
+
+if __name__ == '__main__':
+    test_get_valid_nodes()
+    test_select_pod()
+    test_get_volume_list()
+    test_check_volume_health_info()

roles/openshift_storage_glusterfs/defaults/main.yml

@@ -125,3 +125,6 @@ r_openshift_storage_glusterfs_os_firewall_allow:
   port: "3260/tcp"
 - service: rpcbind
   port: "111/tcp"
+
+# One retry every 10 seconds.
+openshift_glusterfs_cluster_health_check_retries: 120

roles/openshift_storage_glusterfs/tasks/check_cluster_health.yml

@@ -0,0 +1,36 @@
+---
+# glusterfs_check_containerized is a custom module defined at
+# lib_utils/library/glusterfs_check_containerized.py
+- name: Check for cluster health of glusterfs
+  glusterfs_check_containerized:
+    oc_bin: "{{ first_master_client_binary }}"
+    oc_conf: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+    oc_namespace: "{{ openshift_storage_glusterfs_namespace }}"
+    cluster_name: "{{ openshift_storage_glusterfs_name }}"
+    exclude_node: "{{ openshift.common.hostname }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
+  retries: "{{ openshift_glusterfs_cluster_health_check_retries | int}}"
+  delay: 10
+  register: glusterfs_check_containerized_res
+  until: glusterfs_check_containerized_res is succeeded
+  when:
+  - openshift_storage_glusterfs_is_native | bool
+  - "'glusterfs' in groups"
+  - "groups['glusterfs'] | length > 0"
+
+- name: Check for cluster health of glusterfs (registry)
+  glusterfs_check_containerized:
+    oc_bin: "{{ first_master_client_binary }}"
+    oc_conf: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+    oc_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}"
+    cluster_name: "{{ openshift_storage_glusterfs_registry_name }}"
+    exclude_node: "{{ openshift.common.hostname }}"
+  delegate_to: "{{ groups.oo_first_master.0 }}"
+  retries: "{{ openshift_glusterfs_cluster_health_check_retries | int}}"
+  delay: 10
+  register: glusterfs_check_containerized_reg_res
+  until: glusterfs_check_containerized_reg_res is succeeded
+  when:
+  - openshift_storage_glusterfs_registry_is_native | bool
+  - "'glusterfs_registry' in groups"
+  - "groups['glusterfs_registry'] | length > 0"

roles/openshift_storage_glusterfs/tasks/uninstall.yml

@@ -3,10 +3,14 @@
   block:
     - include_tasks: glusterfs_config_facts.yml
     - include_tasks: glusterfs_uninstall.yml
-  when: "'glusterfs' in groups and groups['glusterfs'] | length > 0"
+  when:
+    - "'glusterfs' in groups"
+    - "groups['glusterfs'] | length > 0"
 
 - name: uninstall glusterfs registry
   block:
     - include_tasks: glusterfs_registry_facts.yml
     - include_tasks: glusterfs_uninstall.yml
-  when: "'glusterfs_registry' in groups and groups['glusterfs_registry'] | length > 0"
+  when:
+    - "'glusterfs_registry' in groups"
+    - "groups['glusterfs_registry'] | length > 0"