Merge pull request #7062 from aweiteka/prometheus-liveness-reload

Automatic merge from submit-queue.

Prometheus config updates

Add a liveness probe to auto-reload config when hash changes
Increase retention to 3d matching production default

roles/openshift_prometheus/defaults/main.yaml

@@ -31,7 +31,7 @@ l_openshift_prometheus_serviceaccount_annotations:
 openshift_prometheus_additional_rules_file: null
 
 #prometheus application arguments
-openshift_prometheus_args: ['--storage.tsdb.retention=6h']
+openshift_prometheus_args: ['--storage.tsdb.retention=3d']
 
 # storage
 # One of ['emptydir', 'pvc']

roles/openshift_prometheus/templates/prometheus.j2

@@ -82,6 +82,20 @@ spec:
         - --web.listen-address=localhost:9090
         image: "{{ l_openshift_prometheus_image_prefix }}prometheus:{{ l_openshift_prometheus_image_version }}"
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -c
+            - |-
+              set -euo pipefail;
+              touch /tmp/prometheusconfig.hash;
+              if [[ $(find /etc/prometheus -type f | sort | xargs md5sum | md5sum) != $(cat /tmp/prometheusconfig.hash) ]]; then
+                find /etc/prometheus -type f | sort | xargs md5sum | md5sum > /tmp/prometheusconfig.hash;
+                kill -HUP 1;
+              fi
+          initialDelaySeconds: 60
+          periodSeconds: 60
         resources:
           requests:
 {% if openshift_prometheus_memory_requests is defined and openshift_prometheus_memory_requests is not none %}