etcd: use as system container

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

playbooks/adhoc/uninstall.yml

@@ -164,9 +164,12 @@
     - atomic-enterprise
     - origin
 
-  - shell: atomic uninstall openvswitch
+  - shell: atomic uninstall "{{ item }}"
     changed_when: False
     failed_when: False
+    with_items:
+    - etcd
+    - openvswitch
 
   - shell: find /var/lib/origin/openshift.local.volumes -type d -exec umount {} \; 2>/dev/null || true
     changed_when: False

playbooks/common/openshift-cluster/upgrades/etcd/backup.yml

@@ -4,7 +4,7 @@
   vars:
     embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
     timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
-    etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' }}"
+    etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' if not openshift.common.is_etcd_system_container else 'runc exec etcd etcdctl' }}"
   roles:
   - openshift_facts
   tasks:

playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml

@@ -14,6 +14,16 @@
     register: etcd_container_version
     failed_when: false
     when: openshift.common.is_containerized | bool
+  - name: Record containerized etcd version
+    command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\*
+    register: etcd_container_version
+    failed_when: false
+    when: openshift.common.is_containerized | bool and not openshift.common.is_etcd_system_container | bool
+  - name: Record containerized etcd version
+    command: runc exec etcd_container rpm -qa --qf '%{version}' etcd\*
+    register: etcd_container_version
+    failed_when: false
+    when: openshift.common.is_containerized | bool and openshift.common.is_etcd_system_container | bool
 
 # I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
 # through hosts, then loop through tasks only when appropriate

roles/etcd/defaults/main.yaml

@@ -1,5 +1,5 @@
 ---
-etcd_service: "{{ 'etcd' if not etcd_is_containerized | bool else 'etcd_container' }}"
+etcd_service: "{{ 'etcd' if openshift.common.is_etcd_system_container | bool or not etcd_is_containerized | bool else 'etcd_container' }}"
 etcd_client_port: 2379
 etcd_peer_port: 2380
 etcd_url_scheme: http

roles/etcd/tasks/main.yml

@@ -14,13 +14,17 @@
   command: docker pull {{ openshift.etcd.etcd_image }}
   register: pull_result
   changed_when: "'Downloaded newer image' in pull_result.stdout"
-  when: etcd_is_containerized | bool
+  when:
+  - etcd_is_containerized | bool
+  - not openshift.common.is_etcd_system_container | bool
 
 - name: Install etcd container service file
   template:
     dest: "/etc/systemd/system/etcd_container.service"
     src: etcd.docker.service
-  when: etcd_is_containerized | bool
+  when:
+  - etcd_is_containerized | bool
+  - not openshift.common.is_etcd_system_container | bool
 
 - name: Ensure etcd datadir exists when containerized
   file:
@@ -36,10 +40,22 @@
     enabled: no
     masked: yes
     daemon_reload: yes
-  when: etcd_is_containerized | bool
+  when:
+  - etcd_is_containerized | bool
+  - not openshift.common.is_etcd_system_container | bool
   register: task_result
   failed_when: "task_result|failed and 'could not' not in task_result.msg|lower"
 
+- name: Install etcd container service file
+  template:
+    dest: "/etc/systemd/system/etcd_container.service"
+    src: etcd.docker.service
+  when: etcd_is_containerized | bool and not openshift.common.is_etcd_system_container | bool
+
+- name: Install Etcd system container
+  include: system_container.yml
+  when: etcd_is_containerized | bool and openshift.common.is_etcd_system_container | bool
+
 - name: Validate permissions on the config dir
   file:
     path: "{{ etcd_conf_dir }}"
@@ -54,7 +70,7 @@
     dest: /etc/etcd/etcd.conf
     backup: true
   notify:
-    - restart etcd
+  - restart etcd
 
 - name: Enable etcd
   systemd:

roles/etcd/tasks/system_container.yml

@@ -0,0 +1,63 @@
+---
+- name: Pull etcd system container
+  command: atomic pull --storage=ostree {{ openshift.etcd.etcd_image }}
+  register: pull_result
+  changed_when: "'Pulling layer' in pull_result.stdout"
+
+- name: Check etcd system container package
+  command: >
+    atomic containers list --no-trunc -a -f container=etcd
+  register: result
+
+- name: Set initial Etcd cluster
+  set_fact:
+    etcd_initial_cluster: >
+      {% for host in etcd_peers | default([]) -%}
+      {% if loop.last -%}
+      {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }}
+      {%- else -%}
+      {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }},
+      {%- endif -%}
+      {% endfor -%}
+
+- name: Update Etcd system container package
+  command: >
+    atomic containers update
+    --set ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }}
+    --set ETCD_NAME={{ etcd_hostname }}
+    --set ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster | replace('\n', '') }}
+    --set ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }}
+    --set ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }}
+    --set ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }}
+    --set ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }}
+    --set ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }}
+    --set ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt
+    --set ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key
+    --set ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt
+    --set ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key
+    etcd
+  when:
+  - ("etcd" in result.stdout)
+
+- name: Install Etcd system container package
+  command: >
+    atomic install --system --name=etcd
+    --set ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }}
+    --set ETCD_NAME={{ etcd_hostname }}
+    --set ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster | replace('\n', '') }}
+    --set ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }}
+    --set ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }}
+    --set ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }}
+    --set ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }}
+    --set ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }}
+    --set ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt
+    --set ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key
+    --set ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt
+    --set ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key
+    {{ openshift.etcd.etcd_image }}
+  when:
+  - ("etcd" not in result.stdout)

roles/etcd_common/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 # etcd server vars
-etcd_conf_dir: /etc/etcd
+etcd_conf_dir: "{{ '/etc/etcd' if not openshift.common.is_etcd_system_container else '/var/lib/etcd/etcd.etcd/etc'  }}"
+etcd_system_container_conf_dir: /var/lib/etcd/etc
 etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt"
 etcd_cert_file: "{{ etcd_conf_dir }}/server.crt"
 etcd_key_file: "{{ etcd_conf_dir }}/server.key"

roles/openshift_etcd_facts/vars/main.yml

@@ -5,6 +5,6 @@ etcd_hostname: "{{ openshift.common.hostname }}"
 etcd_ip: "{{ openshift.common.ip }}"
 etcd_cert_subdir: "etcd-{{ openshift.common.hostname }}"
 etcd_cert_prefix:
-etcd_cert_config_dir: /etc/etcd
+etcd_cert_config_dir: "{{ '/etc/etcd' if not openshift.common.is_etcd_system_container | bool else '/var/lib/etcd/etcd.etcd/etc' }}"
 etcd_peer_url_scheme: https
 etcd_url_scheme: https

roles/openshift_facts/tasks/main.yml

@@ -12,6 +12,7 @@
     l_is_openvswitch_system_container: "{{ (use_openvswitch_system_container | default(use_system_containers) | bool) }}"
     l_is_node_system_container: "{{ (use_node_system_container | default(use_system_containers) | bool) }}"
     l_is_master_system_container: "{{ (use_master_system_container | default(use_system_containers) | bool) }}"
+    l_is_etcd_system_container: "{{ (use_etcd_system_container | default(use_system_containers) | bool) }}"
 
 - name: Ensure various deps are installed
   package: name={{ item }} state=present
@@ -33,6 +34,7 @@
       is_openvswitch_system_container: "{{ l_is_openvswitch_system_container | default(false) }}"
       is_node_system_container: "{{ l_is_node_system_container | default(false) }}"
       is_master_system_container: "{{ l_is_master_system_container | default(false) }}"
+      is_etcd_system_container: "{{ l_is_etcd_system_container | default(false) }}"
       system_images_registry: "{{ system_images_registry | default('') }}"
       public_hostname: "{{ openshift_public_hostname | default(None) }}"
       public_ip: "{{ openshift_public_ip | default(None) }}"