Updating upgrade_logging to be more idempotent

roles/openshift_logging/README.md

@@ -31,7 +31,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log
 - `openshift_logging_curator_script_log_level`: The script log level for Curator. Defaults to 'INFO'.
 - `openshift_logging_curator_log_level`: The log level for the Curator process. Defaults to 'ERROR'.
 - `openshift_logging_curator_cpu_limit`: The amount of CPU to allocate to Curator. Default is '100m'.
-- `openshift_logging_curator_memory_limit`: The amount of memor to allocate to Curator. Unset if not specified.
+- `openshift_logging_curator_memory_limit`: The amount of memory to allocate to Curator. Unset if not specified.
 
 - `openshift_logging_kibana_hostname`: The Kibana hostname. Defaults to 'kibana.example.com'.
 - `openshift_logging_kibana_cpu_limit`: The amount of CPU to allocate to Kibana or unset if not specified.
@@ -39,6 +39,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log
 - `openshift_logging_kibana_proxy_debug`: When "True", set the Kibana Proxy log level to DEBUG. Defaults to 'false'.
 - `openshift_logging_kibana_proxy_cpu_limit`: The amount of CPU to allocate to Kibana proxy or unset if not specified.
 - `openshift_logging_kibana_proxy_memory_limit`: The amount of memory to allocate to Kibana proxy or unset if not specified.
+- `openshift_logging_kibana_replica_count`: The number of replicas Kibana should be scaled up to. Defaults to 1.
 
 - `openshift_logging_fluentd_nodeselector`: The node selector that the Fluentd daemonset uses to determine where to deploy to. Defaults to '"logging-infra-fluentd": "true"'.
 - `openshift_logging_fluentd_cpu_limit`: The CPU limit for Fluentd pods. Defaults to '100m'.
@@ -84,3 +85,4 @@ same as above for their non-ops counterparts, but apply to the OPS cluster insta
 - `openshift_logging_kibana_ops_memory_limit`: The amount of memory to allocate to Kibana or unset if not specified.
 - `openshift_logging_kibana_ops_proxy_cpu_limit`: The amount of CPU to allocate to Kibana proxy or unset if not specified.
 - `openshift_logging_kibana_ops_proxy_memory_limit`: The amount of memory to allocate to Kibana proxy or unset if not specified.
+- `openshift_logging_kibana_ops_replica_count`: The number of replicas Kibana ops should be scaled up to. Defaults to 1.

roles/openshift_logging/defaults/main.yml

@@ -25,6 +25,7 @@ openshift_logging_kibana_memory_limit: null
 openshift_logging_kibana_proxy_debug: false
 openshift_logging_kibana_proxy_cpu_limit: null
 openshift_logging_kibana_proxy_memory_limit: null
+openshift_logging_kibana_replica_count: 1
 
 openshift_logging_kibana_ops_hostname: "kibana-ops.{{openshift.common.dns_domain}}"
 openshift_logging_kibana_ops_cpu_limit: null
@@ -32,6 +33,7 @@ openshift_logging_kibana_ops_memory_limit: null
 openshift_logging_kibana_ops_proxy_debug: false
 openshift_logging_kibana_ops_proxy_cpu_limit: null
 openshift_logging_kibana_ops_proxy_memory_limit: null
+openshift_logging_kibana_ops_replica_count: 1
 
 openshift_logging_fluentd_nodeselector: {'logging-infra-fluentd': 'true'}
 openshift_logging_fluentd_cpu_limit: 100m

roles/openshift_logging/files/es_migration.sh

@@ -1,5 +1,3 @@
-#! bin/bash
-
 CA=${1:-/etc/openshift/logging/ca.crt}
 KEY=${2:-/etc/openshift/logging/system.admin.key}
 CERT=${3:-/etc/openshift/logging/system.admin.crt}

roles/openshift_logging/tasks/install_curator.yaml

@@ -1,5 +1,6 @@
 ---
-- command: >
+- name: Check Curator current replica count
+  command: >
     {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-curator
     -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}}
   register: curator_replica_count
@@ -7,7 +8,8 @@
   ignore_errors: yes
   changed_when: no
 
-- command: >
+- name: Check Curator ops current replica count
+  command: >
     {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-curator-ops
     -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}}
   register: curator_ops_replica_count
@@ -28,7 +30,7 @@
     es_port: "{{openshift_logging_es_port}}"
     curator_cpu_limit: "{{openshift_logging_curator_cpu_limit }}"
     curator_memory_limit: "{{openshift_logging_curator_memory_limit }}"
-    replicas: "{{curator_replica_count.stdout | default (1)}}"
+    replicas: "{{curator_replica_count.stdout | default (0)}}"
   check_mode: no
   changed_when: no
 
@@ -43,7 +45,7 @@
     es_port: "{{openshift_logging_es_ops_port}}"
     curator_cpu_limit: "{{openshift_logging_curator_ops_cpu_limit }}"
     curator_memory_limit: "{{openshift_logging_curator_ops_memory_limit }}"
-    replicas: "{{curator_ops_replica_count.stdout | default (1)}}"
+    replicas: "{{curator_ops_replica_count.stdout | default (0)}}"
   when: openshift_logging_use_ops
   check_mode: no
   changed_when: no

roles/openshift_logging/tasks/install_kibana.yaml

@@ -1,5 +1,6 @@
 ---
-- command: >
+- name: Check Kibana current replica count
+  command: >
     {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-kibana
     -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}}
   register: kibana_replica_count
@@ -7,7 +8,8 @@
   ignore_errors: yes
   changed_when: no
 
-- command: >
+- name: Check Kibana ops current replica count
+  command: >
     {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-kibana-ops
     -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}}
   register: kibana_ops_replica_count

roles/openshift_logging/tasks/start_cluster.yaml

@@ -39,7 +39,7 @@
 - name: start kibana
   include: scale.yaml
   vars:
-    desired: 1
+    desired: "{{ openshift_logging_kibana_replica_count | default (1) }}"
   with_items: "{{kibana_dc.stdout_lines}}"
   loop_control:
     loop_var: object
@@ -82,7 +82,7 @@
 - name: start kibana-ops
   include: scale.yaml
   vars:
-    desired: 1
+    desired: "{{ openshift_logging_kibana_ops_replica_count | default (1) }}"
   with_items: "{{kibana_dc.stdout_lines}}"
   loop_control:
     loop_var: object

roles/openshift_logging/tasks/upgrade_logging.yaml

@@ -7,7 +7,7 @@
   vars:
     start_cluster: False
 
-# ensure that ES is running
+# start ES so that we can run migrate script
 - command: >
     {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get dc -l component=es -o name -n {{openshift_logging_namespace}}
   register: es_dc
@@ -21,13 +21,21 @@
   loop_control:
     loop_var: object
 
-- copy:
-    src: es_migration.sh
-    dest: {{mktemp.stdout}}/es_migration.sh
+- command: >
+    {{ openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get pods -n {{openshift_logging_namespace}} -l component=es -o jsonpath='{.items[?(@.status.phase == "Running")].metadata.name}'
+  register: running_pod
+  until: running_pod.stdout != ''
+  retries: 30
+  delay: 10
+  changed_when: no
+  check_mode: no
 
-- name: Run upgrade scripts
-  command: >
-    sh {{mktemp.stdout}}/es_migration.sh {{openshift.common.config_base}}/logging/ca.crt {{openshift.common.config_base}}/logging/system.admin.key {{openshift.common.config_base}}/logging/system.admin.crt {{openshift_logging_es_host}} {{openshift_logging_es_port}} {{openshift_logging_namespace}}
+- name: Run upgrade script
+  script: es_migration.sh {{openshift.common.config_base}}/logging/ca.crt {{openshift.common.config_base}}/logging/system.admin.key {{openshift.common.config_base}}/logging/system.admin.crt {{openshift_logging_es_host}} {{openshift_logging_es_port}} {{openshift_logging_namespace}}
+  register: script_output
+  changed_when:
+    - script_output.rc == 0
+    - script_output.stdout.find("skipping update_for_uuid") == -1 or script_output.stdout.find("skipping update_for_common_data_model") == -1
 
 - name: Start up rest of cluster
   include: start_cluster.yaml