Update curator to use k8s cronjob

roles/openshift_logging/README.md

@@ -42,6 +42,7 @@ When `openshift_logging_install_logging` is set to `False` the `openshift_loggin
 - `openshift_logging_curator_run_hour`: The hour of the day that Curator will run at. Defaults to '0'.
 - `openshift_logging_curator_run_minute`: The minute of the hour that Curator will run at. Defaults to '0'.
 - `openshift_logging_curator_run_timezone`: The timezone that Curator uses for figuring out its run time. Defaults to 'UTC'.
+- `openshift_logging_curator_timeout`: The timeout for each Curator operation. Defaults to 300.
 - `openshift_logging_curator_script_log_level`: The script log level for Curator. Defaults to 'INFO'.
 - `openshift_logging_curator_log_level`: The log level for the Curator process. Defaults to 'ERROR'.
 - `openshift_logging_curator_cpu_request`: The minimum amount of CPU to allocate to Curator. Default is '100m'.

roles/openshift_logging/defaults/main.yml

@@ -18,6 +18,7 @@ openshift_logging_curator_default_days: 30
 openshift_logging_curator_run_hour: 0
 openshift_logging_curator_run_minute: 0
 openshift_logging_curator_run_timezone: UTC
+openshift_logging_curator_timeout: 300
 openshift_logging_curator_script_log_level: INFO
 openshift_logging_curator_log_level: ERROR
 openshift_logging_curator_cpu_limit: null

roles/openshift_logging/tasks/delete_logging.yaml

@@ -13,6 +13,7 @@
     - routes
     - templates
     - ds
+    - cronjobs
 
 # return all persistent volume claims as well if purge is set
 - name: delete logging pvc objects

roles/openshift_logging_curator/defaults/main.yml

@@ -27,6 +27,7 @@ openshift_logging_curator_run_minute: 0
 openshift_logging_curator_run_timezone: UTC
 openshift_logging_curator_script_log_level: INFO
 openshift_logging_curator_log_level: ERROR
+openshift_logging_curator_timeout: 300
 
 # following can be uncommented to provide values for configmaps -- take care when providing file contents as it may cause your cluster to not operate correctly
 #curator_config_contents:

roles/openshift_logging_curator/files/5.x/actions.yaml

@@ -0,0 +1,38 @@
+# ---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+# actions:
+#   1:
+#     action: delete_indices
+#     description: >-
+#       Delete .operations indices older than 30 days.
+#       Ignore the error if the filter does not
+#       result in an actionable list of indices (ignore_empty_list).
+#       See https://www.elastic.co/guide/en/elasticsearch/client/curator/5.2/ex_delete_indices.html
+#     options:
+#       # Swallow curator.exception.NoIndices exception
+#       ignore_empty_list: True
+#       # In seconds, default is 300
+#       timeout_override: ${CURATOR_TIMEOUT}
+#       # Don't swallow any other exceptions
+#       continue_if_exception: False
+#       # Optionally disable action, useful for debugging
+#       disable_action: False
+#     # All filters are bound by logical AND
+#     filters:
+#     - filtertype: pattern
+#       kind: regex
+#       value: '^\.operations\..*$'
+#       exclude: False
+#     - filtertype: age
+#       # Parse timestamp from index name
+#       source: name
+#       direction: older
+#       timestring: '%Y.%m.%d'
+#       unit: days
+#       unit_count: 30
+#       exclude: False

roles/openshift_logging_curator/files/5.x/config.yaml

@@ -0,0 +1,17 @@
+---
+client:
+  hosts:
+  - ${ES_HOST}
+  port: ${ES_PORT}
+  use_ssl: True
+  certificate: ${ES_CA}
+  client_cert: ${ES_CLIENT_CERT}
+  client_key: ${ES_CLIENT_KEY}
+  ssl_no_validate: False
+  timeout: ${CURATOR_TIMEOUT}
+  master_only: False
+logging:
+  loglevel: ${CURATOR_LOG_LEVEL}
+  logformat: default
+  blacklist: ['elasticsearch', 'urllib3'] 
+  

roles/openshift_logging_curator/tasks/main.yaml

@@ -12,6 +12,10 @@
     openshift_logging_curator_image_prefix: "{{ openshift_logging_curator_image_prefix | default(__openshift_logging_curator_image_prefix) }}"
     openshift_logging_curator_image_version: "{{ openshift_logging_curator_image_version | default(__openshift_logging_curator_image_version) }}"
 
+- name: Detect running upgrade
+  set_fact:
+    is_upgrade: "{{ openshift_logging_facts['curator']['configmaps']['logging-curator'] is defined }}"
+
 - name: Ensure that Logging Curator has nodes to run on
   import_role:
     name: openshift_master
@@ -56,30 +60,6 @@
   when:
     - openshift_logging_image_pull_secret == ''
 
-# configmap
-- copy:
-    src: "{{ __base_file_dir }}/curator.yml"
-    dest: "{{ tempdir }}/curator.yml"
-  changed_when: no
-
-- import_role:
-    name: openshift_logging
-    tasks_from: patch_configmap_files.yaml
-  vars:
-    configmap_name: "logging-curator"
-    configmap_namespace: "logging"
-    configmap_file_names:
-      - current_file: "config.yaml"
-        new_file: "{{ tempdir }}/curator.yml"
-
-- name: Set Curator configmap
-  oc_configmap:
-    state: present
-    name: "logging-curator"
-    namespace: "{{ openshift_logging_namespace }}"
-    from_file:
-      config.yaml: "{{ tempdir }}/curator.yml"
-
 # secret
 - name: Set Curator secret
   oc_secret:
@@ -98,36 +78,137 @@
     curator_name: "{{ 'logging-curator' ~ ( (openshift_logging_curator_ops_deployment | default(false) | bool) | ternary('-ops', '') ) }}"
     curator_component: "{{ 'curator' ~ ( (openshift_logging_curator_ops_deployment | default(false) | bool) | ternary('-ops', '') ) }}"
 
-# DC
-# TODO: scale should not exceed 1
-- name: Generate Curator deploymentconfig
-  template:
-    src: "{{ __base_file_dir }}/curator.j2"
-    dest: "{{ tempdir }}/templates/curator-dc.yaml"
-  vars:
-    component: "{{ curator_component }}"
-    logging_component: curator
-    deploy_name: "{{ curator_name }}"
-    image: "{{openshift_logging_curator_image_prefix}}logging-curator:{{openshift_logging_curator_image_version}}"
-    es_host: "{{ openshift_logging_curator_es_host }}"
-    es_port: "{{ openshift_logging_curator_es_port }}"
-    curator_cpu_limit: "{{ openshift_logging_curator_cpu_limit }}"
-    curator_cpu_request: "{{ openshift_logging_curator_cpu_request | min_cpu(openshift_logging_curator_cpu_limit | default(none)) }}"
-    curator_memory_limit: "{{ openshift_logging_curator_memory_limit }}"
-    curator_replicas: "{{ openshift_logging_curator_replicas | default (1) }}"
-    curator_node_selector: "{{openshift_logging_curator_nodeselector | default({})}}"
-  check_mode: no
-  changed_when: no
-
-- name: Set Curator DC
-  oc_obj:
-    state: present
-    name: "{{ curator_name }}"
-    namespace: "{{ openshift_logging_namespace }}"
-    kind: dc
-    files:
-      - "{{ tempdir }}/templates/curator-dc.yaml"
-    delete_after: true
+# Deployment Config - v2.x
+- when: not openshift_logging_es5_techpreview
+  block:
+    - name: Generate Curator deploymentconfig
+      template:
+        src: "{{ __base_file_dir }}/curator.j2"
+        dest: "{{ tempdir }}/templates/curator-dc.yaml"
+      vars:
+        component: "{{ curator_component }}"
+        logging_component: curator
+        deploy_name: "{{ curator_name }}"
+        image: "{{openshift_logging_curator_image_prefix}}logging-curator:{{openshift_logging_curator_image_version}}"
+        es_host: "{{ openshift_logging_curator_es_host }}"
+        es_port: "{{ openshift_logging_curator_es_port }}"
+        curator_cpu_limit: "{{ openshift_logging_curator_cpu_limit }}"
+        curator_cpu_request: "{{ openshift_logging_curator_cpu_request | min_cpu(openshift_logging_curator_cpu_limit | default(none)) }}"
+        curator_memory_limit: "{{ openshift_logging_curator_memory_limit }}"
+        curator_replicas: "{{ openshift_logging_curator_replicas | default (1) }}"
+        curator_node_selector: "{{openshift_logging_curator_nodeselector | default({})}}"
+      check_mode: no
+      changed_when: no
+
+    - copy:
+        src: "{{ __base_file_dir }}/curator.yml"
+        dest: "{{ tempdir }}/curator.yml"
+      changed_when: no
+
+    - import_role:
+        name: openshift_logging
+        tasks_from: patch_configmap_files.yaml
+      vars:
+        configmap_name: "logging-curator"
+        configmap_namespace: "logging"
+        configmap_file_names:
+          - current_file: "config.yaml"
+            new_file: "{{ tempdir }}/curator.yml"
+
+    - name: Set Curator configmap
+      oc_configmap:
+        state: present
+        name: "logging-curator"
+        namespace: "{{ openshift_logging_namespace }}"
+        from_file:
+          config.yaml: "{{ tempdir }}/curator.yml"
+
+    - name: Set Curator DC
+      oc_obj:
+        state: present
+        name: "{{ curator_name }}"
+        namespace: "{{ openshift_logging_namespace }}"
+        kind: dc
+        files:
+          - "{{ tempdir }}/templates/curator-dc.yaml"
+        delete_after: true
+
+# Cron Job - v5.x
+- when: openshift_logging_es5_techpreview | bool
+  block:
+    # Keep the old DC around
+    - name: Scale the old DC to 0
+      oc_scale:
+        name: "{{ curator_name }}"
+        namespace: "{{ openshift_logging_namespace }}"
+        kind: dc
+        replicas: 0
+      when: is_upgrade | bool
+
+    - name: Generate Curator cronjob
+      template:
+        src: "{{ __base_file_dir }}/curator-cj.j2"
+        dest: "{{ tempdir }}/templates/curator-cj.yaml"
+      vars:
+        component: "{{ curator_component }}"
+        logging_component: curator
+        deploy_name: "{{ curator_name }}"
+        image: "{{openshift_logging_curator_image_prefix}}logging-curator:{{openshift_logging_curator_image_version}}"
+        es_host: "{{ openshift_logging_curator_es_host }}"
+        es_port: "{{ openshift_logging_curator_es_port }}"
+        curator_cpu_limit: "{{ openshift_logging_curator_cpu_limit }}"
+        curator_cpu_request: "{{ openshift_logging_curator_cpu_request | min_cpu(openshift_logging_curator_cpu_limit | default(none)) }}"
+        curator_memory_limit: "{{ openshift_logging_curator_memory_limit }}"
+        curator_node_selector: "{{openshift_logging_curator_nodeselector | default({})}}"
+        cron_job_schedule: "{{ openshift_logging_curator_run_minute | default(0) }} {{ openshift_logging_curator_run_hour | default(0) }} * * *"
+      check_mode: no
+      changed_when: no
+
+    # Copy config files
+    - copy:
+        src: "{{ __base_file_dir }}/{{ item }}"
+        dest: "{{ tempdir }}/{{ item }}"
+      with_items:
+        - "actions.yaml"
+        - "config.yaml"
+        - "curator.yml"
+
+    # Patch existing configuration, if present
+    - import_role:
+        name: openshift_logging
+        tasks_from: patch_configmap_files.yaml
+      vars:
+        configmap_name: "logging-curator"
+        configmap_namespace: "logging"
+        configmap_file_names:
+          - current_file: "actions.yaml"
+            new_file: "{{ tempdir }}/actions.yaml"
+          - current_file: "curator5.yaml"
+            new_file: "{{ tempdir }}/config.yaml"
+          - current_file: "config.yaml"
+            new_file: "{{ tempdir }}/curator.yml"
+
+    # Create cronjob
+    - name: Set Curator Cronjob
+      oc_obj:
+        state: present
+        name: "{{ curator_name }}"
+        namespace: "{{ openshift_logging_namespace }}"
+        kind: cronjob
+        files:
+          - "{{ tempdir }}/templates/curator-cj.yaml"
+        delete_after: true
+
+    # Create config map
+    - name: Set Curator configmap
+      oc_configmap:
+        state: present
+        name: "logging-curator"
+        namespace: "{{ openshift_logging_namespace }}"
+        from_file:
+          actions.yaml: "{{ tempdir }}/actions.yaml"
+          curator5.yaml: "{{ tempdir }}/config.yaml"
+          config.yaml: "{{ tempdir }}/curator.yml"
 
 - name: Delete temp directory
   file:

roles/openshift_logging_curator/templates/5.x/curator-cj.j2

@@ -0,0 +1,107 @@
+apiVersion: "batch/v1beta1"
+kind: "CronJob"
+metadata:
+  name: "{{deploy_name}}"
+  labels:
+    provider: openshift
+    component: "{{component}}"
+    logging-infra: "{{logging_component}}"
+spec:
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  schedule: "{{ cron_job_schedule }}"
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      concurrencyPolicy: Forbid
+      replicas: 1
+      strategy:
+        type: Recreate
+      template:
+        metadata:
+          name: "{{deploy_name}}"
+          labels:
+            logging-infra: "{{logging_component}}"
+            provider: openshift
+            component: "{{component}}"
+        spec:
+          restartPolicy: Never
+          terminationGracePeriod: 600
+          serviceAccountName: aggregated-logging-curator
+{% if curator_node_selector is iterable and curator_node_selector | length > 0 %}
+          nodeSelector:
+{% for key, value in curator_node_selector.iteritems() %}
+            {{key}}: "{{value}}"
+{% endfor %}
+{% endif %}
+          containers:
+            -
+              name: "curator"
+              image: {{image}}
+              imagePullPolicy: IfNotPresent
+{% if (curator_memory_limit is defined and curator_memory_limit) or (curator_cpu_limit is defined and curator_cpu_limit) or (curator_cpu_request is defined and curator_cpu_request) %}
+              resources:
+{%   if (curator_memory_limit is defined and curator_memory_limit) or (curator_cpu_limit is defined and curator_cpu_limit) %}
+                limits:
+{%     if curator_cpu_limit is defined and curator_cpu_limit %}
+                  cpu: "{{curator_cpu_limit}}"
+{%     endif %}
+{%     if curator_memory_limit is defined and curator_memory_limit %}
+                  memory: "{{curator_memory_limit}}"
+{%     endif %}
+{%   endif %}
+{%   if (curator_memory_limit is defined and curator_memory_limit) or (curator_cpu_request is defined and curator_cpu_request) %}
+                requests:
+{%     if curator_cpu_request is defined and curator_cpu_request %}
+                  cpu: "{{curator_cpu_request}}"
+{%     endif %}
+{%     if curator_memory_limit is defined and curator_memory_limit %}
+                  memory: "{{curator_memory_limit}}"
+{%     endif %}
+{%   endif %}
+{% endif %}
+              env:
+                -
+                  name: "K8S_HOST_URL"
+                  value: "{{openshift_logging_curator_master_url}}"
+                -
+                  name: "ES_HOST"
+                  value: "{{es_host}}"
+                -
+                  name: "ES_PORT"
+                  value: "{{es_port}}"
+                -
+                  name: "ES_CLIENT_CERT"
+                  value: "/etc/curator/keys/cert"
+                -
+                  name: "ES_CLIENT_KEY"
+                  value: "/etc/curator/keys/key"
+                -
+                  name: "ES_CA"
+                  value: "/etc/curator/keys/ca"
+                -
+                  name: "CURATOR_DEFAULT_DAYS"
+                  value: "{{openshift_logging_curator_default_days}}"
+                -
+                  name: "CURATOR_SCRIPT_LOG_LEVEL"
+                  value: "{{openshift_logging_curator_script_log_level}}"
+                -
+                  name: "CURATOR_LOG_LEVEL"
+                  value: "{{openshift_logging_curator_log_level}}"
+                -
+                  name: "CURATOR_TIMEOUT"
+                  value: "{{openshift_logging_curator_timeout}}"
+              volumeMounts:
+                - name: certs
+                  mountPath: /etc/curator/keys
+                  readOnly: true
+                - name: config
+                  mountPath: /etc/curator/settings
+                  readOnly: true
+          volumes:
+            - name: certs
+              secret:
+                secretName: logging-curator
+            - name: config
+              configMap:
+                name: logging-curator

roles/openshift_logging_curator/templates/5.x/curator.j2

@@ -1,113 +0,0 @@
-apiVersion: "v1"
-kind: "DeploymentConfig"
-metadata:
-  name: "{{deploy_name}}"
-  labels:
-    provider: openshift
-    component: "{{component}}"
-    logging-infra: "{{logging_component}}"
-spec:
-  replicas: {{curator_replicas|default(1)}}
-  selector:
-    provider: openshift
-    component: "{{component}}"
-    logging-infra: "{{logging_component}}"
-  strategy:
-    rollingParams:
-      intervalSeconds: 1
-      timeoutSeconds: 600
-      updatePeriodSeconds: 1
-    type: Recreate
-  template:
-    metadata:
-      name: "{{deploy_name}}"
-      labels:
-        logging-infra: "{{logging_component}}"
-        provider: openshift
-        component: "{{component}}"
-    spec:
-      terminationGracePeriod: 600
-      serviceAccountName: aggregated-logging-curator
-{% if curator_node_selector is iterable and curator_node_selector | length > 0 %}
-      nodeSelector:
-{% for key, value in curator_node_selector.items() %}
-        {{key}}: "{{value}}"
-{% endfor %}
-{% endif %}
-      containers:
-        -
-          name: "curator"
-          image: {{image}}
-          imagePullPolicy: IfNotPresent
-{% if (curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "") or (curator_cpu_limit is defined and curator_cpu_limit is not none and curator_cpu_limit != "") or (curator_cpu_request is defined and curator_cpu_request is not none and curator_cpu_request != "") %}
-          resources:
-{%   if (curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "") or (curator_cpu_limit is defined and curator_cpu_limit is not none and curator_cpu_limit != "") %}
-            limits:
-{%     if curator_cpu_limit is defined and curator_cpu_limit is not none and curator_cpu_limit != "" %}
-              cpu: "{{curator_cpu_limit}}"
-{%     endif %}
-{%     if curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "" %}
-              memory: "{{curator_memory_limit}}"
-{%     endif %}
-{%   endif %}
-{%   if (curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "") or (curator_cpu_request is defined and curator_cpu_request is not none and curator_cpu_request != "") %}
-            requests:
-{%     if curator_cpu_request is defined and curator_cpu_request is not none and curator_cpu_request != "" %}
-              cpu: "{{curator_cpu_request}}"
-{%     endif %}
-{%     if curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "" %}
-              memory: "{{curator_memory_limit}}"
-{%     endif %}
-{%   endif %}
-{% endif %}
-          env:
-            -
-              name: "K8S_HOST_URL"
-              value: "{{openshift_logging_curator_master_url}}"
-            -
-              name: "ES_HOST"
-              value: "{{es_host}}"
-            -
-              name: "ES_PORT"
-              value: "{{es_port}}"
-            -
-              name: "ES_CLIENT_CERT"
-              value: "/etc/curator/keys/cert"
-            -
-              name: "ES_CLIENT_KEY"
-              value: "/etc/curator/keys/key"
-            -
-              name: "ES_CA"
-              value: "/etc/curator/keys/ca"
-            -
-              name: "CURATOR_DEFAULT_DAYS"
-              value: "{{openshift_logging_curator_default_days}}"
-            -
-              name: "CURATOR_RUN_HOUR"
-              value: "{{openshift_logging_curator_run_hour}}"
-            -
-              name: "CURATOR_RUN_MINUTE"
-              value: "{{openshift_logging_curator_run_minute}}"
-            -
-              name: "CURATOR_RUN_TIMEZONE"
-              value: "{{openshift_logging_curator_run_timezone}}"
-            -
-              name: "CURATOR_SCRIPT_LOG_LEVEL"
-              value: "{{openshift_logging_curator_script_log_level}}"
-            -
-              name: "CURATOR_LOG_LEVEL"
-              value: "{{openshift_logging_curator_log_level}}"
-          volumeMounts:
-            - name: certs
-              mountPath: /etc/curator/keys
-              readOnly: true
-            - name: config
-              mountPath: /etc/curator/settings
-              readOnly: true
-      volumes:
-        - name: certs
-          secret:
-            secretName: logging-curator
-        - name: config
-          configMap:
-            name: logging-curator