migrate embedded etcd to external etcd

playbooks/byo/openshift-etcd/embedded2external.yml

@@ -0,0 +1,6 @@
+---
+- include: ../openshift-cluster/initialize_groups.yml
+
+- include: ../../common/openshift-cluster/std_include.yml
+
+- include: ../../common/openshift-etcd/embedded2external.yml

playbooks/common/openshift-etcd/certificates.yml

@@ -1,29 +1,4 @@
 ---
-- name: Create etcd server certificates for etcd hosts
-  hosts: oo_etcd_to_config
-  any_errors_fatal: true
-  roles:
-    - role: openshift_etcd_facts
-  post_tasks:
-    - include_role:
-        name: etcd
-        tasks_from: server_certificates
-      vars:
-        etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
-        etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
-        etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
-        r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+- include: server_certificates.yml
 
-- name: Create etcd client certificates for master hosts
-  hosts: oo_masters_to_config
-  any_errors_fatal: true
-  roles:
-    - role: openshift_etcd_facts
-    - role: openshift_etcd_client_certificates
-      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
-      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
-      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
-      etcd_cert_prefix: "master.etcd-"
-      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
-      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
-      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+- include: master_etcd_certificates.yml

playbooks/common/openshift-etcd/embedded2external.yml

@@ -0,0 +1,188 @@
+---
+- name: Pre-migrate checks
+  hosts: localhost
+  tasks:
+  # Check there is only one etcd host
+  - assert:
+      that: groups.oo_etcd_to_config | default([]) | length == 1
+      msg: "[etcd] group must contain only one host"
+  # Check there is only one master
+  - assert:
+      that: groups.oo_masters_to_config | default([]) | length == 1
+      msg: "[master] group must contain only one host"
+
+# Check the etcd cluster and master are running and healthy
+- name: Check master and embedded etcd are healthy
+  hosts: oo_first_master
+  tasks:
+  - name: Check the embedded etcd is healthy
+    include_role:
+      name: etcd
+      tasks_from: check_cluster_health
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+      etcd_peer: "{{ openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+  - name: Check the master API is ready
+    include_role:
+      name: openshift_master
+      tasks_from: check_master_api_is_ready
+
+# 1. stop a master
+- name: Prepare masters for etcd data migration
+  hosts: oo_first_master
+  roles:
+  - role: openshift_facts
+  tasks:
+  - set_fact:
+      master_service: "{{ openshift.common.service_type + '-master' }}"
+      embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+  - debug:
+      msg: "master service name: {{ master_service }}"
+  - name: Stop master
+    service:
+      name: "{{ master_service }}"
+      state: stopped
+  # 2. backup embedded etcd
+  # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285
+  - include_role:
+      name: etcd
+      tasks_from: backup
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.archive
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}"
+
+# 3. deploy certificates (for etcd and master)
+- include: ca.yml
+
+- include: server_certificates.yml
+
+- name: Backup etcd client certificates for master host
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup_master_etcd_certificates
+
+- name: Redeploy master etcd certificates
+  include: master_etcd_certificates.yml
+  vars:
+    etcd_certificates_redeploy: "{{ true }}"
+
+# 4. deploy external etcd
+- include: ../openshift-etcd/config.yml
+
+# 5. stop external etcd
+- name: Cleanse etcd
+  hosts: oo_etcd_to_config[0]
+  gather_facts: no
+  pre_tasks:
+  - include_role:
+      name: etcd
+      tasks_from: disable_etcd
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+  - include_role:
+      name: etcd
+      tasks_from: clean_data
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+
+# 6. copy the embedded etcd backup to the external host
+# TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory
+- name: Copy embedded etcd backup to the external host
+  hosts: localhost
+  tasks:
+  - name: Create local temp directory for syncing etcd backup
+    local_action: command mktemp -d /tmp/etcd_backup-XXXXXXX
+    register: g_etcd_client_mktemp
+    changed_when: False
+    become: no
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.fetch
+    vars:
+      r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_first_master.0].openshift.common.etcd_runtime }}"
+      etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+    delegate_to: "{{ groups.oo_first_master[0] }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.copy
+    vars:
+      r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.etcd_runtime }}"
+      etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+    delegate_to: "{{ groups.oo_etcd_to_config[0] }}"
+
+  - debug:
+      msg: "etcd_backup_dest_directory: {{ g_etcd_client_mktemp.stdout }}"
+
+  - name: Delete temporary directory
+    local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent
+    changed_when: False
+    become: no
+
+# 7. force new cluster from the backup
+- name: Force new etcd cluster
+  hosts: oo_etcd_to_config[0]
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup.unarchive
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.force_new_cluster
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+      etcd_peer: "{{ openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+
+# 8. re-configure master to use the external etcd
+- name: Configure master to use external etcd
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: openshift_master
+      tasks_from: configure_external_etcd
+    vars:
+      etcd_peer_url_scheme: "https"
+      etcd_ip: "{{ openshift.common.ip }}"
+      etcd_peer_port: 2379
+
+  # 9. start the master
+  - name: Start master
+    service:
+      name: "{{ master_service }}"
+      state: started
+    register: service_status
+    until: service_status.state is defined and service_status.state == "started"
+    retries: 5
+    delay: 10

playbooks/common/openshift-etcd/master_etcd_certificates.yml

@@ -0,0 +1,14 @@
+---
+- name: Create etcd client certificates for master hosts
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+    - role: openshift_etcd_client_certificates
+      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+      etcd_cert_prefix: "master.etcd-"
+      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config

playbooks/common/openshift-etcd/server_certificates.yml

@@ -0,0 +1,15 @@
+---
+- name: Create etcd server certificates for etcd hosts
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+  post_tasks:
+    - include_role:
+        name: etcd
+        tasks_from: server_certificates
+      vars:
+        etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+        etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"

roles/etcd/defaults/main.yaml

@@ -70,7 +70,8 @@ etcd_listen_peer_urls: "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_
 etcd_advertise_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}"
 etcd_listen_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_port }}"
 
-etcd_peer: 127.0.0.1
+# required role variable
+#etcd_peer: 127.0.0.1
 etcdctlv2: "etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://{{ etcd_peer }}:{{ etcd_client_port }}"
 
 etcd_service: "{{ 'etcd_container' if r_etcd_common_etcd_runtime == 'docker' else 'etcd' }}"

roles/etcd/tasks/auxiliary/clean_data.yml

@@ -1,5 +1,5 @@
 ---
 - name: Remove member data
   file:
-    path: /var/lib/etcd/member
+    path: "{{ etcd_data_dir }}/member"
     state: absent

roles/etcd/tasks/auxiliary/disable_etcd.yml

@@ -0,0 +1,5 @@
+---
+- name: Disable etcd members
+  service:
+    name: "{{ etcd_service }}"
+    state: stopped

roles/etcd/tasks/auxiliary/force_new_cluster.yml

@@ -0,0 +1,31 @@
+---
+- name: Set ETCD_FORCE_NEW_CLUSTER=true on first etcd host
+  lineinfile:
+    line: "ETCD_FORCE_NEW_CLUSTER=true"
+    dest: /etc/etcd/etcd.conf
+    backup: true
+
+- name: Start etcd
+  systemd:
+    name: "{{ etcd_service }}"
+    state: started
+
+- name: Wait for cluster to become healthy after bringing up first member
+  command: >
+    etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health
+  register: l_etcd_migrate_health
+  until: l_etcd_migrate_health.rc == 0
+  retries: 3
+  delay: 30
+
+- name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host
+  lineinfile:
+    line: "ETCD_FORCE_NEW_CLUSTER=true"
+    dest: /etc/etcd/etcd.conf
+    state: absent
+    backup: true
+
+- name: Restart first etcd host
+  systemd:
+    name: "{{ etcd_service }}"
+    state: restarted

roles/etcd/tasks/backup.archive.yml

@@ -0,0 +1,3 @@
+---
+- include: backup/vars.yml
+- include: backup/archive.yml

roles/etcd/tasks/backup.copy.yml

@@ -0,0 +1,3 @@
+---
+- include: backup/vars.yml
+- include: backup/copy.yml

roles/etcd/tasks/backup.fetch.yml

@@ -0,0 +1,3 @@
+---
+- include: backup/vars.yml
+- include: backup/fetch.yml

roles/etcd/tasks/backup.force_new_cluster.yml

@@ -0,0 +1,12 @@
+---
+- include: backup/vars.yml
+
+- name: Move content of etcd backup under the etcd data directory
+  command: >
+    mv "{{ l_etcd_backup_dir }}/member" "{{ l_etcd_data_dir }}"
+
+- name: Set etcd group for the etcd data directory
+  command: >
+    chown -R etcd:etcd "{{ l_etcd_data_dir }}"
+
+- include: auxiliary/force_new_cluster.yml

roles/etcd/tasks/backup.unarchive.yml

@@ -0,0 +1,3 @@
+---
+- include: backup/vars.yml
+- include: backup/unarchive.yml

roles/etcd/tasks/backup/archive.yml

@@ -0,0 +1,5 @@
+---
+- name: Archive backup
+  archive:
+    path: "{{ l_etcd_backup_dir }}"
+    dest: "{{ l_etcd_backup_dir }}.tgz"

roles/etcd/tasks/backup/backup.yml

@@ -1,21 +1,5 @@
 ---
-# set the etcd backup directory name here in case the tag or sufix consists of dynamic value that changes over time
-# e.g. openshift-backup-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }} value will change every second so if the date changes
-# right after setting l_etcd_incontainer_backup_dir and before l_etcd_backup_dir facts, the backup directory name is different
-- set_fact:
-    l_backup_dir_name: "openshift-backup-{{ r_etcd_common_backup_tag }}{{ r_etcd_common_backup_sufix_name }}"
-
-- set_fact:
-    l_etcd_data_dir: "{{ etcd_data_dir }}{{ '/etcd.etcd' if r_etcd_common_etcd_runtime == 'runc' else '' }}"
-
-- set_fact:
-    l_etcd_incontainer_data_dir: "{{ etcd_data_dir }}"
-
-- set_fact:
-    l_etcd_incontainer_backup_dir: "{{ l_etcd_incontainer_data_dir }}/{{ l_backup_dir_name }}"
-
-- set_fact:
-    l_etcd_backup_dir: "{{ l_etcd_data_dir }}/{{ l_backup_dir_name }}"
+- include: vars.yml
 
 # TODO: replace shell module with command and update later checks
 - name: Check available disk space for etcd backup

roles/etcd/tasks/backup/copy.yml

@@ -0,0 +1,5 @@
+---
+- name: Copy etcd backup
+  copy:
+    src: "{{ etcd_backup_sync_directory }}/{{ l_backup_dir_name }}.tgz"
+    dest: "{{ l_etcd_data_dir }}"

roles/etcd/tasks/backup/fetch.yml

@@ -0,0 +1,8 @@
+---
+- name: Fetch etcd backup
+  fetch:
+    src: "{{ l_etcd_backup_dir }}.tgz"
+    dest: "{{ etcd_backup_sync_directory }}/"
+    flat: yes
+    fail_on_missing: yes
+    validate_checksum: yes

roles/etcd/tasks/backup/unarchive.yml

@@ -0,0 +1,14 @@
+---
+- shell: ls /var/lib/etcd
+  register: output
+
+- debug:
+    msg: "output: {{ output }}"
+
+- name: Unarchive backup
+  # can't use unarchive https://github.com/ansible/ansible/issues/30821
+  # unarchive:
+  #   src: "{{ l_etcd_backup_dir }}.tgz"
+  #   dest: "{{ l_etcd_backup_dir }}"
+  command: >
+    tar -xf "{{ l_etcd_backup_dir }}.tgz" -C "{{ l_etcd_data_dir }}"

roles/etcd/tasks/backup/vars.yml

@@ -0,0 +1,18 @@
+---
+# set the etcd backup directory name here in case the tag or sufix consists of dynamic value that changes over time
+# e.g. openshift-backup-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }} value will change every second so if the date changes
+# right after setting l_etcd_incontainer_backup_dir and before l_etcd_backup_dir facts, the backup directory name is different
+- set_fact:
+    l_backup_dir_name: "openshift-backup-{{ r_etcd_common_backup_tag }}{{ r_etcd_common_backup_sufix_name }}"
+
+- set_fact:
+    l_etcd_data_dir: "{{ etcd_data_dir }}{{ '/etcd.etcd' if r_etcd_common_etcd_runtime == 'runc' else '' }}"
+
+- set_fact:
+    l_etcd_incontainer_data_dir: "{{ etcd_data_dir }}"
+
+- set_fact:
+    l_etcd_incontainer_backup_dir: "{{ l_etcd_incontainer_data_dir }}/{{ l_backup_dir_name }}"
+
+- set_fact:
+    l_etcd_backup_dir: "{{ l_etcd_data_dir }}/{{ l_backup_dir_name }}"

roles/etcd/tasks/backup_master_etcd_certificates.yml

@@ -0,0 +1,2 @@
+---
+- include: certificates/backup_master_etcd_certificates.yml

roles/etcd/tasks/certificates/backup_master_etcd_certificates.yml

@@ -0,0 +1,7 @@
+---
+- name: Backup master etcd certificates
+  shell: >
+    tar -czvf /etc/origin/master/master-etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz
+    /etc/origin/master/master.etcd-*
+  args:
+    warn: no

roles/etcd/tasks/check_cluster_health.yml

@@ -0,0 +1,2 @@
+---
+- include: migration/check_cluster_health.yml

roles/etcd/tasks/disable_etcd.yml

@@ -0,0 +1,2 @@
+---
+- include: auxiliary/disable_etcd.yml

roles/etcd/tasks/fetch_backup.yml

@@ -0,0 +1,8 @@
+---
+- include: backup/vars.yml
+
+- include: backup/archive.yml
+
+- include: backup/sync_backup.yml
+
+- include: backup/

roles/openshift_master/tasks/check_master_api_is_ready.yml

@@ -0,0 +1,14 @@
+---
+- name: Wait for API to become available
+  # Using curl here since the uri module requires python-httplib2 and
+  # wait_for port doesn't provide health information.
+  command: >
+    curl --silent --tlsv1.2
+    --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt
+    {{ openshift.master.api_url }}/healthz/ready
+  register: l_api_available_output
+  until: l_api_available_output.stdout == 'ok'
+  retries: 120
+  delay: 1
+  run_once: true
+  changed_when: false

roles/openshift_master/tasks/configure_external_etcd.yml

@@ -0,0 +1,17 @@
+---
+- name: Remove etcdConfig section
+  yedit:
+    src: /etc/origin/master/master-config.yaml
+    key: "etcdConfig"
+    state: absent
+- name: Set etcdClientInfo.ca to master.etcd-ca.crt
+  yedit:
+    src: /etc/origin/master/master-config.yaml
+    key: etcdClientInfo.ca
+    value: master.etcd-ca.crt
+- name: Set etcdClientInfo.urls to the external etcd
+  yedit:
+    src: /etc/origin/master/master-config.yaml
+    key: etcdClientInfo.urls
+    value:
+      - "{{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}"

roles/openshift_master/tasks/main.yml

@@ -311,23 +311,7 @@
 
 # A separate wait is required here for native HA since notifies will
 # be resolved after all tasks in the role.
-- name: Wait for API to become available
-  # Using curl here since the uri module requires python-httplib2 and
-  # wait_for port doesn't provide health information.
-  command: >
-    curl --silent --tlsv1.2
-    {% if openshift.common.version_gte_3_2_or_1_2 | bool %}
-    --cacert {{ openshift.common.config_base }}/master/ca-bundle.crt
-    {% else %}
-    --cacert {{ openshift.common.config_base }}/master/ca.crt
-    {% endif %}
-    {{ openshift.master.api_url }}/healthz/ready
-  register: l_api_available_output
-  until: l_api_available_output.stdout == 'ok'
-  retries: 120
-  delay: 1
-  run_once: true
-  changed_when: false
+- include: check_master_api_is_ready.yml
   when:
   - openshift.master.cluster_method == 'native'
   - master_api_service_status_changed | bool