Merge pull request #3109 from giuseppe/system-containers

System containers

playbooks/adhoc/uninstall.yml

@@ -148,6 +148,29 @@
       - vovsbr
     when: "{{ openshift_remove_all | default(true) | bool }}"
 
+  - shell: atomic uninstall "{{ item }}"-master
+    changed_when: False
+    failed_when: False
+    with_items:
+    - openshift-enterprise
+    - atomic-enterprise
+    - origin
+
+  - shell: atomic uninstall "{{ item }}"-node
+    changed_when: False
+    failed_when: False
+    with_items:
+    - openshift-enterprise
+    - atomic-enterprise
+    - origin
+
+  - shell: atomic uninstall "{{ item }}"
+    changed_when: False
+    failed_when: False
+    with_items:
+    - etcd
+    - openvswitch
+
   - shell: find /var/lib/origin/openshift.local.volumes -type d -exec umount {} \; 2>/dev/null || true
     changed_when: False
 
@@ -263,6 +286,9 @@
     - /var/lib/atomic-enterprise
     - /var/lib/openshift
 
+  - shell: systemctl daemon-reload
+    changed_when: False
+
   - name: restart docker
     service: name=docker state=restarted
 

playbooks/common/openshift-cluster/upgrades/etcd/backup.yml

@@ -4,7 +4,7 @@
   vars:
     embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
     timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
-    etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' }}"
+    etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' if not openshift.common.is_etcd_system_container else 'runc exec etcd etcdctl' }}"
   roles:
   - openshift_facts
   tasks:

playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml

@@ -14,6 +14,16 @@
     register: etcd_container_version
     failed_when: false
     when: openshift.common.is_containerized | bool
+  - name: Record containerized etcd version
+    command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\*
+    register: etcd_container_version
+    failed_when: false
+    when: openshift.common.is_containerized | bool and not openshift.common.is_etcd_system_container | bool
+  - name: Record containerized etcd version
+    command: runc exec etcd_container rpm -qa --qf '%{version}' etcd\*
+    register: etcd_container_version
+    failed_when: false
+    when: openshift.common.is_containerized | bool and openshift.common.is_etcd_system_container | bool
 
 # I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
 # through hosts, then loop through tasks only when appropriate

roles/etcd/defaults/main.yaml

@@ -1,5 +1,5 @@
 ---
-etcd_service: "{{ 'etcd' if not etcd_is_containerized | bool else 'etcd_container' }}"
+etcd_service: "{{ 'etcd' if openshift.common.is_etcd_system_container | bool or not etcd_is_containerized | bool else 'etcd_container' }}"
 etcd_client_port: 2379
 etcd_peer_port: 2380
 etcd_url_scheme: http

roles/etcd/tasks/main.yml

@@ -14,13 +14,17 @@
   command: docker pull {{ openshift.etcd.etcd_image }}
   register: pull_result
   changed_when: "'Downloaded newer image' in pull_result.stdout"
-  when: etcd_is_containerized | bool
+  when:
+  - etcd_is_containerized | bool
+  - not openshift.common.is_etcd_system_container | bool
 
 - name: Install etcd container service file
   template:
     dest: "/etc/systemd/system/etcd_container.service"
     src: etcd.docker.service
-  when: etcd_is_containerized | bool
+  when:
+  - etcd_is_containerized | bool
+  - not openshift.common.is_etcd_system_container | bool
 
 - name: Ensure etcd datadir exists when containerized
   file:
@@ -36,10 +40,22 @@
     enabled: no
     masked: yes
     daemon_reload: yes
-  when: etcd_is_containerized | bool
+  when:
+  - etcd_is_containerized | bool
+  - not openshift.common.is_etcd_system_container | bool
   register: task_result
   failed_when: "task_result|failed and 'could not' not in task_result.msg|lower"
 
+- name: Install etcd container service file
+  template:
+    dest: "/etc/systemd/system/etcd_container.service"
+    src: etcd.docker.service
+  when: etcd_is_containerized | bool and not openshift.common.is_etcd_system_container | bool
+
+- name: Install Etcd system container
+  include: system_container.yml
+  when: etcd_is_containerized | bool and openshift.common.is_etcd_system_container | bool
+
 - name: Validate permissions on the config dir
   file:
     path: "{{ etcd_conf_dir }}"
@@ -54,7 +70,7 @@
     dest: /etc/etcd/etcd.conf
     backup: true
   notify:
-    - restart etcd
+  - restart etcd
 
 - name: Enable etcd
   systemd:

roles/etcd/tasks/system_container.yml

@@ -0,0 +1,63 @@
+---
+- name: Pull etcd system container
+  command: atomic pull --storage=ostree {{ openshift.etcd.etcd_image }}
+  register: pull_result
+  changed_when: "'Pulling layer' in pull_result.stdout"
+
+- name: Check etcd system container package
+  command: >
+    atomic containers list --no-trunc -a -f container=etcd
+  register: result
+
+- name: Set initial Etcd cluster
+  set_fact:
+    etcd_initial_cluster: >
+      {% for host in etcd_peers | default([]) -%}
+      {% if loop.last -%}
+      {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }}
+      {%- else -%}
+      {{ hostvars[host].etcd_hostname }}={{ etcd_peer_url_scheme }}://{{ hostvars[host].etcd_ip }}:{{ etcd_peer_port }},
+      {%- endif -%}
+      {% endfor -%}
+
+- name: Update Etcd system container package
+  command: >
+    atomic containers update
+    --set ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }}
+    --set ETCD_NAME={{ etcd_hostname }}
+    --set ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster | replace('\n', '') }}
+    --set ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }}
+    --set ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }}
+    --set ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }}
+    --set ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }}
+    --set ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }}
+    --set ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt
+    --set ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key
+    --set ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt
+    --set ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key
+    etcd
+  when:
+  - ("etcd" in result.stdout)
+
+- name: Install Etcd system container package
+  command: >
+    atomic install --system --name=etcd
+    --set ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }}
+    --set ETCD_NAME={{ etcd_hostname }}
+    --set ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster | replace('\n', '') }}
+    --set ETCD_LISTEN_CLIENT_URLS={{ etcd_listen_client_urls }}
+    --set ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_initial_advertise_peer_urls }}
+    --set ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }}
+    --set ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }}
+    --set ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }}
+    --set ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt
+    --set ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key
+    --set ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt
+    --set ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt
+    --set ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key
+    {{ openshift.etcd.etcd_image }}
+  when:
+  - ("etcd" not in result.stdout)

roles/etcd_common/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 # etcd server vars
-etcd_conf_dir: /etc/etcd
+etcd_conf_dir: "{{ '/etc/etcd' if not openshift.common.is_etcd_system_container else '/var/lib/etcd/etcd.etcd/etc'  }}"
+etcd_system_container_conf_dir: /var/lib/etcd/etc
 etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt"
 etcd_cert_file: "{{ etcd_conf_dir }}/server.crt"
 etcd_key_file: "{{ etcd_conf_dir }}/server.key"

roles/openshift_etcd_facts/vars/main.yml

@@ -5,6 +5,6 @@ etcd_hostname: "{{ openshift.common.hostname }}"
 etcd_ip: "{{ openshift.common.ip }}"
 etcd_cert_subdir: "etcd-{{ openshift.common.hostname }}"
 etcd_cert_prefix:
-etcd_cert_config_dir: /etc/etcd
+etcd_cert_config_dir: "{{ '/etc/etcd' if not openshift.common.is_etcd_system_container | bool else '/var/lib/etcd/etcd.etcd/etc' }}"
 etcd_peer_url_scheme: https
 etcd_url_scheme: https

roles/openshift_facts/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+use_system_containers: false

roles/openshift_facts/library/openshift_facts.py

@@ -1785,11 +1785,14 @@ def set_container_facts_if_unset(facts):
         facts['etcd']['etcd_image'] = etcd_image
     if 'master' in facts and 'master_image' not in facts['master']:
         facts['master']['master_image'] = master_image
+        facts['master']['master_system_image'] = master_image
     if 'node' in facts:
         if 'node_image' not in facts['node']:
             facts['node']['node_image'] = node_image
+            facts['node']['node_system_image'] = node_image
         if 'ovs_image' not in facts['node']:
             facts['node']['ovs_image'] = ovs_image
+            facts['node']['ovs_system_image'] = ovs_image
 
     if safe_get_bool(facts['common']['is_containerized']):
         facts['common']['admin_binary'] = '/usr/local/bin/oadm'

roles/openshift_facts/tasks/main.yml

@@ -9,6 +9,10 @@
     l_is_atomic: "{{ ostree_booted.stat.exists }}"
 - set_fact:
     l_is_containerized: "{{ (l_is_atomic | bool) or (containerized | default(false) | bool) }}"
+    l_is_openvswitch_system_container: "{{ (use_openvswitch_system_container | default(use_system_containers) | bool) }}"
+    l_is_node_system_container: "{{ (use_node_system_container | default(use_system_containers) | bool) }}"
+    l_is_master_system_container: "{{ (use_master_system_container | default(use_system_containers) | bool) }}"
+    l_is_etcd_system_container: "{{ (use_etcd_system_container | default(use_system_containers) | bool) }}"
 
 - name: Ensure various deps are installed
   package: name={{ item }} state=present
@@ -27,6 +31,11 @@
       hostname: "{{ openshift_hostname | default(None) }}"
       ip: "{{ openshift_ip | default(None) }}"
       is_containerized: "{{ l_is_containerized | default(None) }}"
+      is_openvswitch_system_container: "{{ l_is_openvswitch_system_container | default(false) }}"
+      is_node_system_container: "{{ l_is_node_system_container | default(false) }}"
+      is_master_system_container: "{{ l_is_master_system_container | default(false) }}"
+      is_etcd_system_container: "{{ l_is_etcd_system_container | default(false) }}"
+      system_images_registry: "{{ system_images_registry | default('') }}"
       public_hostname: "{{ openshift_public_hostname | default(None) }}"
       public_ip: "{{ openshift_public_ip | default(None) }}"
       portal_net: "{{ openshift_portal_net | default(openshift_master_portal_net) | default(None) }}"

roles/openshift_master/tasks/main.yml

@@ -131,6 +131,10 @@
 - name: Install the systemd units
   include: systemd_units.yml
 
+- name: Install Master system container
+  include: system_container.yml
+  when: openshift.common.is_containerized | bool and openshift.common.is_master_system_container | bool
+
 - name: Create session secrets file
   template:
     dest: "{{ openshift.master.session_secrets_file }}"

roles/openshift_master/tasks/system_container.yml

@@ -0,0 +1,36 @@
+---
+- name: Pre-pull master system container image
+  command: >
+    atomic pull --storage=ostree {{ openshift.common.system_images_registry }}/{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}
+  register: pull_result
+  changed_when: "'Pulling layer' in pull_result.stdout"
+
+- name: Check Master system container package
+  command: >
+    atomic containers list --no-trunc -a -f container={{ openshift.common.service_type }}-master
+  register: result
+
+- name: Update Master system container package
+  command: >
+    atomic containers update {{ openshift.common.service_type }}-master
+  register: update_result
+  changed_when: "'Extracting' in update_result.stdout"
+  when:
+    - ("master" in result.stdout)
+    - (openshift.common.version is defined) and (openshift.common.version == openshift_version) | bool
+
+- name: Uninstall Master system container package
+  command: >
+    atomic uninstall {{ openshift.common.service_type }}-master
+  failed_when: False
+  when:
+    - ("master" in result.stdout)
+    - (openshift.common.version is not defined) or (openshift.common.version != openshift_version) | bool
+
+- name: Install Master system container package
+  command: >
+    atomic install --system --name={{ openshift.common.service_type }}-master {{ openshift.common.system_images_registry }}/{{ openshift.master.master_system_image }}:{{ openshift_image_tag }}
+  when:
+    - (openshift.common.version is not defined) or (openshift.common.version != openshift_version) or ("master" not in result.stdout) | bool
+  notify:
+    - restart master

roles/openshift_master/tasks/systemd_units.yml

@@ -20,14 +20,14 @@
     docker pull {{ openshift.master.master_image }}:{{ openshift_image_tag }}
   register: pull_result
   changed_when: "'Downloaded newer image' in pull_result.stdout"
-  when: openshift.common.is_containerized | bool
+  when: openshift.common.is_containerized | bool and not openshift.common.is_master_system_container | bool
 
 # workaround for missing systemd unit files
 - name: Create the systemd unit files
   template:
     src: "master_docker/master.docker.service.j2"
     dest: "{{ containerized_svc_dir }}/{{ openshift.common.service_type }}-master.service"
-  when: openshift.common.is_containerized | bool and (openshift.master.ha is not defined or not openshift.master.ha | bool)
+  when: openshift.common.is_containerized | bool and (openshift.master.ha is not defined or not openshift.master.ha | bool and not openshift.common.is_master_system_container | bool)
   register: create_master_unit_file
 
 - command: systemctl daemon-reload
@@ -132,7 +132,7 @@
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-master.service"
     src: master_docker/master.docker.service.j2
   register: install_result
-  when: openshift.common.is_containerized | bool and openshift.master.ha is defined and not openshift.master.ha | bool
+  when: openshift.common.is_containerized | bool and openshift.master.ha is defined and not openshift.master.ha | bool and not openshift.common.is_master_system_container | bool
 
 - name: Preserve Master Proxy Config options
   command: grep PROXY /etc/sysconfig/{{ openshift.common.service_type }}-master

roles/openshift_node/tasks/main.yml

@@ -69,7 +69,7 @@
 - name: Persist net.ipv4.ip_forward sysctl entry
   sysctl: name="net.ipv4.ip_forward" value=1 sysctl_set=yes state=present reload=yes
 
-- name: Start and enable openvswitch docker service
+- name: Start and enable openvswitch service
   systemd:
     name: openvswitch.service
     enabled: yes

roles/openshift_node/tasks/node_system_container.yml

@@ -0,0 +1,34 @@
+---
+- name: Pre-pull node system container image
+  command: >
+    atomic pull --storage=ostree {{ openshift.common.system_images_registry }}/{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}
+  register: pull_result
+  changed_when: "'Pulling layer' in pull_result.stdout"
+
+- name: Check Node system container package
+  command: >
+    atomic containers list --no-trunc -a -f container={{ openshift.common.service_type }}-node
+  register: result
+
+- name: Update Node system container package
+  command: >
+    atomic containers update {{ openshift.common.service_type }}-node
+  register: update_result
+  changed_when: "'Extracting' in update_result.stdout"
+  when:
+  - (openshift.common.version is defined) and (openshift.common.version == openshift_version) and ("node" in result.stdout) | bool
+
+- name: Uninstall Node system container package
+  command: >
+    atomic uninstall {{ openshift.common.service_type }}-node
+  failed_when: False
+  when:
+  - (openshift.common.version is not defined) or (openshift.common.version != openshift_version) and ("node" in result.stdout) | bool
+
+- name: Install Node system container package
+  command: >
+    atomic install --system --name={{ openshift.common.service_type }}-node {{ openshift.common.system_images_registry }}/{{ openshift.node.node_system_image }}:{{ openshift_image_tag }}
+  register: install_node_result
+  changed_when: "'Extracting' in pull_result.stdout"
+  when:
+  - (openshift.common.version is not defined) or (openshift.common.version != openshift_version) or ("node" not in result.stdout) | bool

roles/openshift_node/tasks/openvswitch_system_container.yml

@@ -0,0 +1,36 @@
+---
+- name: Pre-pull OpenVSwitch system container image
+  command: >
+    atomic pull --storage=ostree {{ openshift.common.system_images_registry }}/{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}
+  register: pull_result
+  changed_when: "'Pulling layer' in pull_result.stdout"
+
+- name: Check OpenvSwitch system container package
+  command: >
+    atomic containers list --no-trunc -a -f container=openvswitch
+  register: result
+  when:
+  - openshift.common.is_openvswitch_system_container | bool
+
+- name: Update OpenvSwitch system container package
+  command: >
+    atomic containers update openvswitch
+  register: update_result
+  changed_when: "'Extracting' in update_result.stdout"
+  when:
+  - (openshift.common.version is defined) and (openshift.common.version == openshift_version) and ("openvswitch" in result.stdout) | bool
+
+- name: Uninstall OpenvSwitch system container package
+  command: >
+    atomic uninstall openvswitch
+  failed_when: False
+  when:
+  - (openshift.common.version is not defined) or (openshift.common.version != openshift_version) and ("openvswitch" in result.stdout) | bool
+
+- name: Install OpenvSwitch system container package
+  command: >
+    atomic install --system --name=openvswitch {{ openshift.common.system_images_registry }}/{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }}
+  when:
+  - (openshift.common.version is not defined) or (openshift.common.version != openshift_version) or ("openvswitch" not in result.stdout) | bool
+  notify:
+  - restart docker

roles/openshift_node/tasks/systemd_units.yml

@@ -2,20 +2,6 @@
 # This file is included both in the openshift_master role and in the upgrade
 # playbooks.
 
-- name: Pre-pull node image
-  command: >
-    docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }}
-  register: pull_result
-  changed_when: "'Downloaded newer image' in pull_result.stdout"
-  when: openshift.common.is_containerized | bool
-
-- name: Pre-pull openvswitch image
-  command: >
-    docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }}
-  register: pull_result
-  changed_when: "'Downloaded newer image' in pull_result.stdout"
-  when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool
-
 - name: Install Node dependencies docker service file
   template:
     dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service"
@@ -23,12 +9,21 @@
   register: install_node_dep_result
   when: openshift.common.is_containerized | bool
 
-- name: Install Node docker service file
-  template:
-    dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
-    src: openshift.docker.node.service
-  register: install_node_result
-  when: openshift.common.is_containerized | bool
+- block:
+  - name: Pre-pull node image
+    command: >
+      docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }}
+    register: pull_result
+    changed_when: "'Downloaded newer image' in pull_result.stdout"
+
+  - name: Install Node docker service file
+    template:
+      dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
+      src: openshift.docker.node.service
+    register: install_node_result
+  when:
+  - openshift.common.is_containerized | bool
+  - not openshift.common.is_node_system_container | bool
 
 - name: Create the openvswitch service env file
   template:
@@ -39,6 +34,19 @@
   notify:
   - restart openvswitch
 
+- name: Install Node system container
+  include: node_system_container.yml
+  when:
+  - openshift.common.is_containerized | bool
+  - openshift.common.is_node_system_container | bool
+
+- name: Install OpenvSwitch system containers
+  include: openvswitch_system_container.yml
+  when:
+  - openshift.common.use_openshift_sdn | default(true) | bool
+  - openshift.common.is_containerized | bool
+  - openshift.common.is_openvswitch_system_container | bool
+
 # May be a temporary workaround.
 # https://bugzilla.redhat.com/show_bug.cgi?id=1331590
 - name: Create OpenvSwitch service.d directory
@@ -54,13 +62,23 @@
   notify:
   - restart openvswitch
 
-- name: Install OpenvSwitch docker service file
-  template:
-    dest: "/etc/systemd/system/openvswitch.service"
-    src: openvswitch.docker.service
-  when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | default(true) | bool
-  notify:
-  - restart openvswitch
+- block:
+  - name: Pre-pull openvswitch image
+    command: >
+      docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }}
+    register: pull_result
+    changed_when: "'Downloaded newer image' in pull_result.stdout"
+
+  - name: Install OpenvSwitch docker service file
+    template:
+      dest: "/etc/systemd/system/openvswitch.service"
+      src: openvswitch.docker.service
+    notify:
+    - restart openvswitch
+  when:
+  - openshift.common.is_containerized | bool
+  - openshift.common.use_openshift_sdn | default(true) | bool
+  - not openshift.common.is_openvswitch_system_container | bool
 
 - name: Configure Node settings
   lineinfile: