Updating logging_facts to be able to pull values from config maps yaml files, use diffs to keep custom changes, white list certain settings when creating diffs

roles/openshift_sanitize_inventory/library/conditional_set_fact.py

@@ -29,6 +29,10 @@ EXAMPLES = '''
     fact1: not_defined_variable
     fact2: defined_variable
 
+- name: Conditionally set fact falling back on default
+  conditional_set_fact:
+    fact1: not_defined_var | defined_variable
+
 '''
 
 
@@ -48,12 +52,14 @@ def run_module():
     is_changed = False
 
     for param in module.params['vars']:
-        other_var = module.params['vars'][param]
-
-        if other_var in module.params['facts']:
-            local_facts[param] = module.params['facts'][other_var]
-            if not is_changed:
-                is_changed = True
+        other_vars = module.params['vars'][param].replace(" ", "")
+
+        for other_var in other_vars.split('|'):
+            if other_var in module.params['facts']:
+                local_facts[param] = module.params['facts'][other_var]
+                if not is_changed:
+                    is_changed = True
+                break
 
     return module.exit_json(changed=is_changed,  # noqa: F405
                             ansible_facts=local_facts)

roles/openshift_logging/filter_plugins/openshift_logging.py

@@ -102,6 +102,28 @@ def serviceaccount_namespace(qualified_sa, default=None):
     return seg[-1]
 
 
+def flatten_dict(data, parent_key=None):
+    """ This filter plugin will flatten a dict and its sublists into a single dict
+    """
+    if not isinstance(data, dict):
+        raise RuntimeError("flatten_dict failed, expects to flatten a dict")
+
+    merged = dict()
+
+    for key in data:
+        if parent_key is not None:
+            insert_key = '.'.join((parent_key, key))
+        else:
+            insert_key = key
+
+        if isinstance(data[key], dict):
+            merged.update(flatten_dict(data[key], insert_key))
+        else:
+            merged[insert_key] = data[key]
+
+    return merged
+
+
 # pylint: disable=too-few-public-methods
 class FilterModule(object):
     ''' OpenShift Logging Filters '''
@@ -117,5 +139,6 @@ class FilterModule(object):
             'es_storage': es_storage,
             'serviceaccount_name': serviceaccount_name,
             'serviceaccount_namespace': serviceaccount_namespace,
-            'walk': walk
+            'walk': walk,
+            "flatten_dict": flatten_dict
         }

roles/openshift_logging/library/logging_patch.py

@@ -0,0 +1,112 @@
+#!/usr/bin/python
+
+""" Ansible module to help with creating context patch file with whitelisting for logging """
+
+import difflib
+import re
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+DOCUMENTATION = '''
+---
+module: logging_patch
+
+short_description: This will create a context patch file while giving ability
+  to whitelist some lines (excluding them from comparison)
+
+description:
+    - "To create configmap patches for logging"
+
+author:
+    - Eric Wolinetz ewolinet@redhat.com
+'''
+
+
+EXAMPLES = '''
+- logging_patch:
+    original_file: "{{ tempdir }}/current.yml"
+    new_file: "{{ configmap_new_file }}"
+    whitelist: "{{ configmap_protected_lines | default([]) }}"
+
+'''
+
+
+def account_for_whitelist(file_contents, white_list=None):
+    """ This method will remove lines that contain whitelist values from the content
+        of the file so that we aren't build a patch based on that line
+
+        Usage:
+
+          for file_contents:
+
+            index:
+              number_of_shards: {{ es_number_of_shards | default ('1') }}
+              number_of_replicas: {{ es_number_of_replicas | default ('0') }}
+              unassigned.node_left.delayed_timeout: 2m
+              translog:
+                flush_threshold_size: 256mb
+                flush_threshold_period: 5m
+
+
+          and white_list:
+
+            ['number_of_shards', 'number_of_replicas']
+
+
+        We would end up with:
+
+            index:
+              unassigned.node_left.delayed_timeout: 2m
+              translog:
+                flush_threshold_size: 256mb
+                flush_threshold_period: 5m
+
+    """
+
+    for line in white_list:
+        file_contents = re.sub(r".*%s:.*\n" % line, "", file_contents)
+
+    return file_contents
+
+
+def run_module():
+    """ The body of the module, we check if the variable name specified as the value
+        for the key is defined. If it is then we use that value as for the original key """
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            original_file=dict(type='str', required=True),
+            new_file=dict(type='str', required=True),
+            whitelist=dict(required=False, type='list', default=[])
+        ),
+        supports_check_mode=True
+    )
+
+    original_fh = open(module.params['original_file'], "r")
+    original_contents = original_fh.read()
+    original_fh.close()
+
+    original_contents = account_for_whitelist(original_contents, module.params['whitelist'])
+
+    new_fh = open(module.params['new_file'], "r")
+    new_contents = new_fh.read()
+    new_fh.close()
+
+    new_contents = account_for_whitelist(new_contents, module.params['whitelist'])
+
+    uni_diff = difflib.unified_diff(new_contents.splitlines(),
+                                    original_contents.splitlines(),
+                                    lineterm='')
+
+    return module.exit_json(changed=False,  # noqa: F405
+                            raw_patch="\n".join(uni_diff))
+
+
+def main():
+    """ main """
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

roles/openshift_logging/library/openshift_logging_facts.py

@@ -204,6 +204,14 @@ class OpenshiftLoggingFacts(OCBaseCommand):
             if comp is not None:
                 self.add_facts_for(comp, "services", name, dict())
 
+    # pylint: disable=too-many-arguments
+    def facts_from_configmap(self, comp, kind, name, config_key, yaml_file=None):
+        '''Extracts facts in logging namespace from configmap'''
+        if yaml_file is not None:
+            config_facts = yaml.load(yaml_file)
+            self.facts[comp][kind][name][config_key] = config_facts
+            self.facts[comp][kind][name]["raw"] = yaml_file
+
     def facts_for_configmaps(self, namespace):
         ''' Gathers facts for configmaps in logging namespace '''
         self.default_keys_for("configmaps")
@@ -214,7 +222,10 @@ class OpenshiftLoggingFacts(OCBaseCommand):
             name = item["metadata"]["name"]
             comp = self.comp(name)
             if comp is not None:
-                self.add_facts_for(comp, "configmaps", name, item["data"])
+                self.add_facts_for(comp, "configmaps", name, dict(item["data"]))
+                if comp in ["elasticsearch", "elasticsearch_ops"]:
+                    for config_key in item["data"]:
+                        self.facts_from_configmap(comp, "configmaps", name, config_key, item["data"][config_key])
 
     def facts_for_oauthclients(self, namespace):
         ''' Gathers facts for oauthclients used with logging '''

roles/openshift_logging/tasks/install_logging.yaml

@@ -4,6 +4,9 @@
     oc_bin: "{{openshift_client_binary}}"
     openshift_logging_namespace: "{{openshift_logging_namespace}}"
 
+## This is include vs import because we need access to group/inventory variables
+- include_tasks: set_defaults_from_current.yml
+
 - name: Set logging project
   oc_project:
     state: present

roles/openshift_logging/tasks/patch_configmap_file.yaml

@@ -0,0 +1,35 @@
+---
+## The purpose of this task file is to get a patch that is based on the diff
+##  between configmap_current_file and configmap_new_file. The module
+##  logging_patch takes the paths of two files to compare and also a list of
+##  variables whose line we exclude from the diffs.
+##  We then patch the new configmap file so that we can build a configmap
+##  using that file later. We then use oc apply to idempotenly modify any
+##  existing configmap.
+
+## The following variables are expected to be provided when including this task:
+# __configmap_output         -- This is provided to us from patch_configmap_files.yaml
+#                                it is a dict of the configmap where configmap_current_file exists
+# configmap_current_file     -- The name of the data file in the __configmap_output
+# configmap_new_file         -- The path to the file that we intend to oc apply later
+#                                we apply our generated patch to this file.
+# configmap_protected_lines  -- The list of variables to exclude from the diff
+
+- copy:
+    content: "{{ __configmap_output.results.results[0]['data'][configmap_current_file] }}"
+    dest: "{{ tempdir }}/current.yml"
+
+- logging_patch:
+    original_file: "{{ tempdir }}/current.yml"
+    new_file: "{{ configmap_new_file }}"
+    whitelist: "{{ configmap_protected_lines | default([]) }}"
+  register: patch_output
+
+- copy:
+    content: "{{ patch_output.raw_patch }}\n"
+    dest: "{{ tempdir }}/patch.patch"
+  when: patch_output.raw_patch | length > 0
+
+- command: >
+    patch --force --quiet -u "{{ configmap_new_file }}" "{{ tempdir }}/patch.patch"
+  when: patch_output.raw_patch | length > 0

roles/openshift_logging/tasks/patch_configmap_files.yaml

@@ -0,0 +1,31 @@
+---
+## The purpose of this task file is to take in a list of configmap files provided
+##  in the variable configmap_file_names, which correspond to the data sections
+##  within a configmap. We iterate over each of these files and create a patch
+##  from the diff between current_file and new_file to try to maintain any custom
+##  changes that a user may have made to a currently deployed configmap while
+##  trying to idempotently update with any role provided files.
+
+## The following variables are expected to be provided when including this task:
+# configmap_name        -- This is the name of the configmap that the files exist in
+# configmap_namespace   -- The namespace that the configmap lives in
+# configmap_file_names  -- This is expected to be passed in as a dict
+#   current_file        -- The name of the data entry within the configmap
+#   new_file            -- The file path to the file we are comparing to current_file
+#   protected_lines     -- List of variables whose line will be excluded when creating a diff
+
+- oc_configmap:
+    name: "{{ configmap_name }}"
+    state: list
+    namespace: "{{ configmap_namespace }}"
+  register: __configmap_output
+
+- when: __configmap_output.results.stderr is undefined
+  include_tasks: patch_configmap_file.yaml
+  vars:
+    configmap_current_file: "{{ configmap_files.current_file }}"
+    configmap_new_file: "{{ configmap_files.new_file }}"
+    configmap_protected_lines: "{{ configmap_files.protected_lines | default([]) }}"
+  with_items: "{{ configmap_file_names }}"
+  loop_control:
+    loop_var: configmap_files

roles/openshift_logging/tasks/set_defaults_from_current.yml

@@ -0,0 +1,34 @@
+---
+
+## We are pulling default values from configmaps if they exist already
+## Using conditional_set_fact allows us to set the value of a variable based on
+##  the value of another one, if it is already defined. Else we don't set the
+##  left hand side (it stays undefined as well).
+
+## conditional_set_fact allows us to specify a fact source, so first we try to
+##  set variables in the logging-elasticsearch & logging-elasticsearch-ops configmaps
+##  afterwards we set the value of the variable based on the value in the inventory
+##  but fall back to using the value from a configmap as a default. If neither is set
+##  then the variable remains undefined and the role default will be used.
+
+- conditional_set_fact:
+    facts: "{{ openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch']['elasticsearch.yml'] | flatten_dict }}"
+    vars:
+      __openshift_logging_es_number_of_shards: index.number_of_shards
+      __openshift_logging_es_number_of_replicas: index.number_of_replicas
+  when: openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch'] is defined
+
+- conditional_set_fact:
+    facts: "{{ openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops']['elasticsearch.yml'] | flatten_dict }}"
+    vars:
+      __openshift_logging_es_ops_number_of_shards: index.number_of_shards
+      __openshift_logging_es_ops_number_of_replicas: index.number_of_replicas
+  when: openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops'] is defined
+
+- conditional_set_fact:
+    facts: "{{ hostvars[inventory_hostname] }}"
+    vars:
+      openshift_logging_es_number_of_shards: openshift_logging_es_number_of_shards | __openshift_logging_es_number_of_shards
+      openshift_logging_es_number_of_replicas: openshift_logging_es_number_of_replicas | __openshift_logging_es_number_of_replicas
+      openshift_logging_es_ops_number_of_shards: openshift_logging_es_ops_number_of_shards | __openshift_logging_es_ops_number_of_shards
+      openshift_logging_es_ops_number_of_replicas: openshift_logging_es_ops_number_of_replicas | __openshift_logging_es_ops_number_of_replicas

roles/openshift_logging_curator/tasks/main.yaml

@@ -54,14 +54,17 @@
 - copy:
     src: curator.yml
     dest: "{{ tempdir }}/curator.yml"
-  when: curator_config_contents is undefined
   changed_when: no
 
-- copy:
-    content: "{{ curator_config_contents }}"
-    dest: "{{ tempdir }}/curator.yml"
-  when: curator_config_contents is defined
-  changed_when: no
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-curator"
+    configmap_namespace: "logging"
+    configmap_file_names:
+      - current_file: "config.yaml"
+        new_file: "{{ tempdir }}/curator.yml"
 
 - name: Set Curator configmap
   oc_configmap:

roles/openshift_logging_elasticsearch/tasks/main.yaml

@@ -168,33 +168,31 @@
   when: es_logging_contents is undefined
   changed_when: no
 
-- set_fact:
-    __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}"
-    __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}"
-
 - template:
     src: elasticsearch.yml.j2
     dest: "{{ tempdir }}/elasticsearch.yml"
   vars:
     allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}"
-    es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}"
-    es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}"
+    es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}"
+    es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}"
     es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}"
 
   when: es_config_contents is undefined
   changed_when: no
 
-- copy:
-    content: "{{ es_logging_contents }}"
-    dest: "{{ tempdir }}/elasticsearch-logging.yml"
-  when: es_logging_contents is defined
-  changed_when: no
-
-- copy:
-    content: "{{ es_config_contents }}"
-    dest: "{{ tempdir }}/elasticsearch.yml"
-  when: es_config_contents is defined
-  changed_when: no
+# create diff between current configmap files and our current files
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-elasticsearch"
+    configmap_namespace: "logging"
+    configmap_file_names:
+      - current_file: "elasticsearch.yml"
+        new_file: "{{ tempdir }}/elasticsearch.yml"
+        protected_lines: ["number_of_shards", "number_of_replicas"]
+      - current_file: "logging.yml"
+        new_file: "{{ tempdir }}/elasticsearch-logging.yml"
 
 - name: Set ES configmap
   oc_configmap:

roles/openshift_logging_fluentd/tasks/main.yaml

@@ -108,38 +108,28 @@
     dest: "{{ tempdir }}/fluent.conf"
   vars:
     deploy_type: "{{ openshift_logging_fluentd_deployment_type }}"
-  when: fluentd_config_contents is undefined
-  changed_when: no
 
 - copy:
     src: fluentd-throttle-config.yaml
     dest: "{{ tempdir }}/fluentd-throttle-config.yaml"
-  when: fluentd_throttle_contents is undefined
-  changed_when: no
 
 - copy:
     src: secure-forward.conf
     dest: "{{ tempdir }}/secure-forward.conf"
-  when: fluentd_secureforward_contents is undefined
-  changed_when: no
-
-- copy:
-    content: "{{ fluentd_config_contents }}"
-    dest: "{{ tempdir }}/fluent.conf"
-  when: fluentd_config_contents is defined
-  changed_when: no
 
-- copy:
-    content: "{{ fluentd_throttle_contents }}"
-    dest: "{{ tempdir }}/fluentd-throttle-config.yaml"
-  when: fluentd_throttle_contents is defined
-  changed_when: no
-
-- copy:
-    content: "{{ fluentd_secureforward_contents }}"
-    dest: "{{ tempdir }}/secure-forward.conf"
-  when: fluentd_secureforward_contents is defined
-  changed_when: no
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-fluentd"
+    configmap_namespace: "logging"
+    configmap_file_names:
+      - current_file: "fluent.conf"
+        new_file: "{{ tempdir }}/fluent.conf"
+      - current_file: "throttle-config.yaml"
+        new_file: "{{ tempdir }}/fluentd-throttle-config.yaml"
+      - current_file: "secure-forward.conf"
+        new_file: "{{ tempdir }}/secure-forward.conf"
 
 - name: Set Fluentd configmap
   oc_configmap:

roles/openshift_logging_mux/tasks/main.yaml

@@ -88,26 +88,24 @@
 - copy:
     src: fluent.conf
     dest: "{{mktemp.stdout}}/fluent-mux.conf"
-  when: fluentd_mux_config_contents is undefined
   changed_when: no
 
 - copy:
     src: secure-forward.conf
     dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
-  when: fluentd_mux_securefoward_contents is undefined
   changed_when: no
 
-- copy:
-    content: "{{fluentd_mux_config_contents}}"
-    dest: "{{mktemp.stdout}}/fluent-mux.conf"
-  when: fluentd_mux_config_contents is defined
-  changed_when: no
-
-- copy:
-    content: "{{fluentd_mux_secureforward_contents}}"
-    dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
-  when: fluentd_mux_secureforward_contents is defined
-  changed_when: no
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-mux"
+    configmap_namespace: "{{ openshift_logging_mux_namespace }}"
+    configmap_file_names:
+      - current_file: "fluent.conf"
+        new_file: "{{ tempdir }}/fluent-mux.conf"
+      - current_file: "secure-forward.conf"
+        new_file: "{{ tempdir }}/secure-forward-mux.conf"
 
 - name: Set Mux configmap
   oc_configmap:

roles/openshift_sanitize_inventory/meta/main.yml

@@ -14,3 +14,4 @@ galaxy_info:
   - system
 dependencies:
 - role: lib_utils
+- role: lib_openshift