Use an etcd static pod when master bootstrapping is set

roles/etcd/defaults/main.yaml

@@ -80,7 +80,7 @@ etcd_listen_client_urls: "{{ etcd_url_scheme }}://{{ etcd_ip }}:{{ etcd_client_p
 #etcd_peer: 127.0.0.1
 etcdctlv2: "{{ r_etcd_common_etcdctl_command }} --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://{{ etcd_peer }}:{{ etcd_client_port }}"
 
-etcd_service: "{{ 'etcd_container' if r_etcd_common_etcd_runtime == 'docker' else 'etcd' }}"
+etcd_service: etcd
 # Location of the service file is fixed and not meant to be changed
 etcd_service_file: "/etc/systemd/system/{{ etcd_service }}.service"
 

roles/etcd/files/etcd.yaml

@@ -0,0 +1,40 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: master-etcd
+  namespace: kube-system
+  labels:
+    openshift.io/control-plane: "true"
+    openshift.io/component: etcd
+spec:
+  restartPolicy: Always
+  hostNetwork: true
+  containers:
+  - name: etcd
+    image: quay.io/coreos/etcd:v3.3
+    workingDir: /var/lib/etcd
+    command: ["/bin/sh", "-c"]
+    args:
+    - |
+      #!/bin/sh
+      set -o allexport
+      source /etc/etcd/etcd.conf
+      exec etcd
+    securityContext:
+      privileged: true
+    volumeMounts:
+     - mountPath: /etc/etcd/
+       name: master-config
+       readOnly: true
+     - mountPath: /var/lib/etcd/
+       name: master-data
+    livenessProbe:
+      tcpSocket:
+        port: 2379
+  volumes:
+  - name: master-config
+    hostPath:
+      path: /etc/etcd/
+  - name: master-data
+    hostPath:
+      path: /var/lib/etcd

roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml

@@ -3,7 +3,9 @@
   package:
     name: "etcd{{ '-' + etcd_version if etcd_version is defined else '' }}"
     state: present
-  when: not etcd_is_containerized | bool
+  when: not etcd_is_atomic | bool
+  delegate_to: "{{ etcd_ca_host }}"
+  run_once: true
   register: result
   until: result is succeeded
 
@@ -178,8 +180,8 @@
   file:
     path: "{{ item }}"
     mode: 0600
-    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    owner: "etcd"
+    group: "etcd"
   when: etcd_url_scheme == 'https'
   with_items:
   - "{{ etcd_ca_file }}"
@@ -190,8 +192,8 @@
   file:
     path: "{{ item }}"
     mode: 0600
-    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    owner: "etcd"
+    group: "etcd"
   when: etcd_peer_url_scheme == 'https'
   with_items:
   - "{{ etcd_peer_ca_file }}"
@@ -202,6 +204,6 @@
   file:
     path: "{{ etcd_conf_dir }}"
     state: directory
-    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
+    owner: "etcd"
+    group: "etcd"
     mode: 0700

roles/etcd/tasks/main.yml

@@ -1,136 +1,8 @@
 ---
-- name: Set hostname and ip facts
-  set_fact:
-    # Store etcd_hostname and etcd_ip such that they will be available
-    # in hostvars. Defaults for these variables are set in etcd_common.
-    etcd_hostname: "{{ etcd_hostname }}"
-    etcd_ip: "{{ etcd_ip }}"
+- name: Configure etcd with static pods
+  import_tasks: static.yml
+  when: openshift_master_bootstrap_enabled | default(False) | bool
 
-- name: setup firewall
-  import_tasks: firewall.yml
-
-- name: Install etcd
-  package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present
-  when: not etcd_is_containerized | bool
-  register: result
-  until: result is succeeded
-
-- include_tasks: drop_etcdctl.yml
-  when:
-  - openshift_etcd_etcdctl_profile | default(true) | bool
-
-- block:
-  - name: Pull etcd container
-    command: docker pull {{ etcd_image }}
-    register: pull_result
-    changed_when: "'Downloaded newer image' in pull_result.stdout"
-
-  - name: Install etcd container service file
-    template:
-      dest: "/etc/systemd/system/etcd_container.service"
-      src: etcd.docker.service
-  when:
-  - etcd_is_containerized | bool
-  - not l_is_etcd_system_container | bool
-
-# Start secondary etcd instance for third party integrations
-# TODO: Determine an alternative to using thirdparty variable
-- block:
-  - name: Create configuration directory
-    file:
-      path: "{{ etcd_conf_dir }}"
-      state: directory
-      mode: 0700
-
-  # TODO: retest with symlink to confirm it does or does not function
-  - name: Copy service file for etcd instance
-    copy:
-      src: /usr/lib/systemd/system/etcd.service
-      dest: "/etc/systemd/system/{{ etcd_service }}.service"
-      remote_src: True
-
-  - name: Create third party etcd service.d directory exists
-    file:
-      path: "{{ etcd_systemd_dir }}"
-      state: directory
-
-  - name: Configure third part etcd service unit file
-    template:
-      dest: "{{ etcd_systemd_dir }}/custom.conf"
-      src: custom.conf.j2
-  when: etcd_is_thirdparty
-
-  # TODO: this task may not be needed with Validate permissions
-- name: Ensure etcd datadir exists
-  file:
-    path: "{{ etcd_data_dir }}"
-    state: directory
-    mode: 0700
-  when: etcd_is_containerized | bool
-
-- name: Ensure etcd datadir ownership for thirdparty datadir
-  file:
-    path: "{{ etcd_data_dir }}"
-    state: directory
-    mode: 0700
-    owner: etcd
-    group: etcd
-    recurse: True
-  when: etcd_is_thirdparty | bool
-
-  # TODO: Determine if the below reload would work here, for now just reload
-- name:
-  command: systemctl daemon-reload
-  when: etcd_is_thirdparty | bool
-
-- block:
-  - name: Disable system etcd when containerized
-    systemd:
-      name: etcd
-      state: stopped
-      enabled: no
-      masked: yes
-      daemon_reload: yes
-    when: not l_is_etcd_system_container | bool
-    register: task_result
-    failed_when:
-    - task_result is failed
-    - ('could not' not in task_result.msg|lower)
-
-  - name: Install etcd container service file
-    template:
-      dest: "/etc/systemd/system/etcd_container.service"
-      src: etcd.docker.service
-    when: not l_is_etcd_system_container | bool
-
-  - name: Install Etcd system container
-    include_tasks: system_container.yml
-    when: l_is_etcd_system_container | bool
-  when: etcd_is_containerized | bool
-
-- name: Validate permissions on the config dir
-  file:
-    path: "{{ etcd_conf_dir }}"
-    state: directory
-    owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}"
-    mode: 0700
-
-- name: Write etcd global config file
-  template:
-    src: etcd.conf.j2
-    dest: "{{ etcd_conf_file }}"
-    backup: true
-  notify:
-  - restart etcd
-
-- name: Enable etcd
-  systemd:
-    name: "{{ etcd_service }}"
-    state: started
-    enabled: yes
-  register: start_result
-
-- name: Set fact etcd_service_status_changed
-  set_fact:
-    etcd_service_status_changed: "{{ start_result is changed }}"
+- name: Configure etcd with RPMs
+  import_tasks: rpm.yml
+  when: not (openshift_master_bootstrap_enabled | default(False) | bool)

roles/etcd/tasks/rpm.yml

@@ -0,0 +1,84 @@
+---
+- name: Set hostname and ip facts
+  set_fact:
+    # Store etcd_hostname and etcd_ip such that they will be available
+    # in hostvars. Defaults for these variables are set in etcd_common.
+    etcd_hostname: "{{ etcd_hostname }}"
+    etcd_ip: "{{ etcd_ip }}"
+
+- name: setup firewall
+  import_tasks: firewall.yml
+
+- name: Install etcd
+  package: name=etcd{{ '-' + etcd_version if etcd_version is defined else '' }} state=present
+  register: result
+  until: result is succeeded
+
+- include_tasks: drop_etcdctl.yml
+  when:
+  - openshift_etcd_etcdctl_profile | default(true) | bool
+
+  # Start secondary etcd instance for third party integrations
+# TODO: Determine an alternative to using thirdparty variable
+- block:
+  - name: Create configuration directory
+    file:
+      path: "{{ etcd_conf_dir }}"
+      state: directory
+      mode: 0700
+
+  # TODO: retest with symlink to confirm it does or does not function
+  - name: Copy service file for etcd instance
+    copy:
+      src: /usr/lib/systemd/system/etcd.service
+      dest: "/etc/systemd/system/{{ etcd_service }}.service"
+      remote_src: True
+
+  - name: Create third party etcd service.d directory exists
+    file:
+      path: "{{ etcd_systemd_dir }}"
+      state: directory
+
+  - name: Configure third part etcd service unit file
+    template:
+      dest: "{{ etcd_systemd_dir }}/custom.conf"
+      src: custom.conf.j2
+  when: etcd_is_thirdparty
+
+- name: Ensure etcd datadir ownership for thirdparty datadir
+  file:
+    path: "{{ etcd_data_dir }}"
+    state: directory
+    mode: 0700
+    owner: etcd
+    group: etcd
+    recurse: True
+  when: etcd_is_thirdparty | bool
+
+- name: Validate permissions on the config dir
+  file:
+    path: "{{ etcd_conf_dir }}"
+    state: directory
+    owner: "etcd"
+    group: "etcd"
+    mode: 0700
+
+- name: Write etcd global config file
+  template:
+    src: etcd.conf.j2
+    dest: "{{ etcd_conf_file }}"
+    backup: true
+  notify:
+  - restart etcd
+
+- name: Enable etcd
+  systemd:
+    name: "{{ etcd_service }}"
+    state: started
+    enabled: yes
+    daemon_reload: yes
+  register: start_result
+
+- name: Set fact etcd_service_status_changed
+  set_fact:
+    etcd_service_status_changed: "{{ start_result is changed }}"

roles/etcd/tasks/static.yml

@@ -0,0 +1,76 @@
+---
+- name: Set hostname and ip facts
+  set_fact:
+    # Store etcd_hostname and etcd_ip such that they will be available
+    # in hostvars. Defaults for these variables are set in etcd_common.
+    etcd_hostname: "{{ etcd_hostname }}"
+    etcd_ip: "{{ etcd_ip }}"
+
+- name: setup firewall
+  import_tasks: firewall.yml
+
+  # TODO: this task may not be needed with Validate permissions
+- name: Ensure etcd datadir exists
+  file:
+    path: "{{ etcd_data_dir }}"
+    state: directory
+    mode: 0700
+
+- name: Validate permissions on the config dir
+  file:
+    path: "{{ etcd_conf_dir }}"
+    state: directory
+    owner: "etcd"
+    group: "etcd"
+    mode: 0700
+
+- name: Validate permissions on the static pods dir
+  file:
+    path: "/etc/origin/node/pods/"
+    state: directory
+    owner: "root"
+    group: "root"
+    mode: 0700
+
+- name: Write etcd global config file
+  template:
+    src: etcd.conf.j2
+    dest: "{{ etcd_conf_file }}"
+    backup: true
+
+- name: Create temp directory for static pods
+  command: mktemp -d /tmp/openshift-ansible-XXXXXX
+  register: mktemp
+  changed_when: false
+
+- name: Prepare etcd static pod
+  copy:
+    src: "{{ item }}"
+    dest: "{{ mktemp.stdout }}"
+    mode: 0600
+  with_items:
+  - etcd.yaml
+
+- name: Update etcd static pod
+  yedit:
+    src: "{{ mktemp.stdout }}/{{ item }}"
+    edits:
+    - key: spec.containers[0].image
+      value: "{{ etcd_image }}"
+  with_items:
+  - etcd.yaml
+
+- name: Deploy etcd static pod
+  copy:
+    remote_src: true
+    src: "{{ mktemp.stdout }}/{{ item }}"
+    dest: "/etc/origin/node/pods/"
+    mode: 0600
+  with_items:
+  - etcd.yaml
+
+- name: Remove temp directory
+  file:
+    state: absent
+    name: "{{ mktemp.stdout }}"
+  changed_when: False

roles/etcd/tasks/system_container.yml

@@ -1,94 +0,0 @@
----
-- name: Pull etcd system container
-  command: atomic pull --storage=ostree {{ etcd_image }}
-  register: pull_result
-  changed_when: "'Pulling layer' in pull_result.stdout"
-
-- name: Set initial Etcd cluster
-  set_fact:
-    etcd_initial_cluster: >-
-      {% for host in etcd_peers | default([]) -%}
-      {% if loop.last -%}
-      {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }}
-      {%- else -%}
-      {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }},
-      {%- endif -%}
-      {% endfor -%}
-  when: etcd_initial_cluster is undefined
-
-- name: Check etcd system container package
-  command: >
-    atomic containers list --no-trunc -a -f container=etcd -f backend=ostree
-  register: etcd_result
-
-- name: Unmask etcd service
-  systemd:
-    name: etcd
-    state: stopped
-    enabled: no
-    masked: no
-    daemon_reload: yes
-  register: task_result
-  failed_when:
-    - task_result is failed
-    - ('could not' not in task_result.msg|lower)
-  when: "'etcd' not in etcd_result.stdout"
-
-- name: Disable etcd_container
-  systemd:
-    name: etcd_container
-    state: stopped
-    enabled: no
-    daemon_reload: yes
-  register: task_result
-  failed_when:
-    - task_result is failed
-    - ('could not' not in task_result.msg|lower)
-
-- name: Remove etcd_container.service
-  file:
-    path: /etc/systemd/system/etcd_container.service
-    state: absent
-
-- name: Systemd reload configuration
-  systemd: name=etcd_container daemon_reload=yes
-
-- name: Install or Update Etcd system container package
-  oc_atomic_container:
-    name: etcd
-    image: "{{ etcd_image }}"
-    state: latest
-    values:
-      - ETCD_DATA_DIR=/var/lib/etcd
-      - ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }}
-      - ETCD_NAME={{ etcd_hostname }}
-      - ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster }}
-      - ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }}
-      - ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }}
-      - ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }}
-      - ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }}
-      - ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }}
-      - ETCD_CA_FILE={{ etcd_ca_file }}
-      - ETCD_CERT_FILE={{ etcd_cert_file }}
-      - ETCD_KEY_FILE={{ etcd_key_file }}
-      - ETCD_PEER_CA_FILE={{ etcd_peer_ca_file }}
-      - ETCD_PEER_CERT_FILE={{ etcd_peer_cert_file }}
-      - ETCD_PEER_KEY_FILE={{ etcd_peer_key_file }}
-      - ETCD_TRUSTED_CA_FILE={{ etcd_ca_file }}
-      - ETCD_PEER_TRUSTED_CA_FILE={{ etcd_peer_ca_file }}
-      - 'ADDTL_MOUNTS=,{"type":"bind","source":"/etc/","destination":"/etc/","options":["rbind","rw","rslave"]},{"type":"bind","source":"/var/lib/etcd","destination":"/var/lib/etcd/","options":["rbind","rw","rslave"]}'
-
-- name: Ensure etcd datadir ownership for the system container
-  file:
-    path: "{{ etcd_data_dir }}"
-    state: directory
-    mode: 0700
-    owner: root
-    group: root
-    recurse: True
-
-- name: Ensure correct permissions are set for etcd_data_dir
-  template:
-    src: etcd-dir.conf.j2
-    dest: "/etc/tmpfiles.d/etcd-dir.conf"
-    backup: true