Merge pull request #2409 from abutcher/secure-registry

Secure registry for atomic registry deployment

playbooks/common/openshift-cluster/openshift_hosted.yml

@@ -45,4 +45,86 @@
   - role: openshift_metrics
     when: openshift.hosted.metrics.deploy | bool
   - role: cockpit-ui
-    when: ( openshift.common.deployment_subtype == 'registry' )
+    when: openshift.common.deployment_subtype == 'registry'
+
+- name: Configure CA certificate for secure registry
+  hosts: oo_nodes_to_config
+  tags:
+  - hosted
+  tasks:
+  - name: Create temp directory for kubeconfig
+    command: mktemp -d /tmp/openshift-ansible-XXXXXX
+    register: mktemp
+    when: openshift.common.deployment_subtype == 'registry'
+    changed_when: false
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    run_once: true
+  - set_fact:
+      openshift_hosted_kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig"
+    when: openshift.common.deployment_subtype == 'registry'
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    run_once: true
+  - name: Copy the admin client config(s)
+    command: >
+      cp {{ openshift.common.config_base }}/master/admin.kubeconfig {{ openshift_hosted_kubeconfig }}
+    when: openshift.common.deployment_subtype == 'registry'
+    changed_when: false
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    run_once: true
+  - name: Retrieve docker-registry route
+    command: >
+      {{ openshift.common.client_binary }} get route docker-registry
+      --template='{{ '{{' }} .spec.host {{ '}}' }}'
+      --config={{ openshift_hosted_kubeconfig }}
+      -n default
+    register: docker_registry_route
+    when: openshift.common.deployment_subtype == 'registry'
+    changed_when: false
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    run_once: true
+  - name: Retrieve registry service IP
+    command: >
+      {{ openshift.common.client_binary }} get service docker-registry
+      --template='{{ '{{' }} .spec.clusterIP {{ '}}' }}'
+      --config={{ openshift_hosted_kubeconfig }}
+      -n default      
+    register: docker_registry_service_ip
+    when: openshift.common.deployment_subtype == 'registry'
+    changed_when: false
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    run_once: true
+  - name: Create registry CA directories
+    file:
+      path: "/etc/docker/certs.d/{{ item }}"
+      state: directory
+    with_items:
+    - "{{ docker_registry_service_ip.stdout }}:5000"
+    - "{{ docker_registry_route.stdout }}"
+    - "docker-registry.default.svc.cluster.local:5000"
+    when: openshift.common.deployment_subtype == 'registry'
+  - name: Copy CA to registry CA directories
+    copy:
+      src: "{{ openshift.common.config_base }}/node/ca.crt"
+      dest: "/etc/docker/certs.d/{{ item }}"
+      remote_src: yes
+      force: yes
+    with_items:
+    - "{{ docker_registry_service_ip.stdout }}:5000"
+    - "{{ docker_registry_route.stdout }}"
+    - "docker-registry.default.svc.cluster.local:5000"
+    when: openshift.common.deployment_subtype == 'registry'
+    notify:
+    - Restart docker
+  - name: Delete temp directory
+    file:
+      name: "{{ mktemp.stdout }}"
+      state: absent
+    when: openshift.common.deployment_subtype == 'registry'
+    changed_when: False
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    run_once: true
+  handlers:
+  - name: Restart docker
+    service:
+      name: docker
+      state: restarted

roles/cockpit-ui/tasks/main.yml

@@ -1,31 +1,53 @@
 ---
-- name: Expose docker-registry
+- name: Create temp directory for kubeconfig
+  command: mktemp -d /tmp/openshift-ansible-XXXXXX
+  register: mktemp
+  changed_when: False
+
+- set_fact:
+    openshift_hosted_kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig"
+
+- name: Copy the admin client config(s)
   command: >
-    {{ openshift.common.client_binary }} expose service docker-registry -n default
-  register: expose_docker_registry
-  changed_when: "'already exists' not in expose_docker_registry.stderr"
-  failed_when: "'already exists' not in expose_docker_registry.stderr and expose_docker_registry.rc != 0"
+    cp {{ openshift_master_config_dir }}/admin.kubeconfig {{ openshift_hosted_kubeconfig }}
+  changed_when: False
+
+- name: Create passthrough route for docker-registry
+  command: >
+    {{ openshift.common.client_binary }} create route passthrough
+    --service docker-registry
+    --config={{ openshift_hosted_kubeconfig }}
+    -n default
+  register: create_docker_registry_route
+  changed_when: "'already exists' not in create_docker_registry_route.stderr"
+  failed_when: "'already exists' not in create_docker_registry_route.stderr and create_docker_registry_route.rc != 0"
 
 - name: Create passthrough route for registry-console
   command: >
     {{ openshift.common.client_binary }} create route passthrough
     --service registry-console
     --port registry-console
+    --config={{ openshift_hosted_kubeconfig }}
     -n default
   register: create_registry_console_route
   changed_when: "'already exists' not in create_registry_console_route.stderr"
   failed_when: "'already exists' not in create_registry_console_route.stderr and create_registry_console_route.rc != 0"
 
 - name: Retrieve docker-registry route
-  command: "{{ openshift.common.client_binary }} get route docker-registry -n default --template='{{ '{{' }} .spec.host {{ '}}' }}'"
+  command: >
+    {{ openshift.common.client_binary }} get route docker-registry
+    --template='{{ '{{' }} .spec.host {{ '}}' }}'
+    --config={{ openshift_hosted_kubeconfig }}
+    -n default
   register: docker_registry_route
-  failed_when: false
   changed_when: false
 
 - name: Retrieve cockpit kube url
-  command: "{{ openshift.common.client_binary }} get route registry-console -n default --template='https://{{ '{{' }} .spec.host {{ '}}' }}'"
+  command: >
+    {{ openshift.common.client_binary }} get route registry-console
+    --template='https://{{ '{{' }} .spec.host {{ '}}' }}'
+    -n default
   register: registry_console_cockpit_kube_url
-  failed_when: false
   changed_when: false
 
 - set_fact:
@@ -36,9 +58,16 @@
     {{ openshift.common.client_binary }} new-app --template=registry-console
     {{ cockpit_image_prefix }}
     -p OPENSHIFT_OAUTH_PROVIDER_URL="{{ openshift.master.public_api_url }}"
-    -p REGISTRY_HOST="{{ docker_registry_route.stdout }}:80"
+    -p REGISTRY_HOST="{{ docker_registry_route.stdout }}"
     -p COCKPIT_KUBE_URL="{{ registry_console_cockpit_kube_url.stdout }}"
+    --config={{ openshift_hosted_kubeconfig }}
     -n default
   register: deploy_registry_console
   changed_when: "'already exists' not in deploy_registry_console.stderr"
   failed_when: "'already exists' not in deploy_registry_console.stderr and deploy_registry_console.rc != 0"
+
+- name: Delete temp directory
+  file:
+    name: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False

roles/openshift_docker_facts/tasks/main.yml

@@ -13,7 +13,7 @@
       log_options: "{{ openshift_docker_log_options | default(None) }}"
       options: "{{ openshift_docker_options | default(None) }}"
       disable_push_dockerhub: "{{ openshift_disable_push_dockerhub | default(None) }}"
-      hosted_registry_insecure: "{{ openshift_docker_hosted_registry_insecure | default(None) }}"
+      hosted_registry_insecure: "{{ openshift_docker_hosted_registry_insecure | default(openshift.common.deployment_subtype != 'registry') }}"
       hosted_registry_network: "{{ openshift_docker_hosted_registry_network | default(None) }}"
 
 - set_fact:

roles/openshift_hosted/tasks/registry/registry.yml

@@ -18,7 +18,9 @@
     l_default_replicas: 0
   when: l_node_count | int == 0
 
-# If registry nodes are defined and the registry storage kind is defined, default should be the number of registry nodes, otherwise just 1:
+# If registry nodes are defined and the registry storage kind is
+# defined, default should be the number of registry nodes, otherwise
+# just 1:
 - set_fact:
     l_default_replicas: "{{ l_node_count if openshift.hosted.registry.storage.kind | default(none) is not none else 1 }}"
   when: l_node_count | int > 0
@@ -49,6 +51,10 @@
   failed_when: "openshift_hosted_registry_results.rc != 0 and 'service exists' not in openshift_hosted_registry_results.stdout and 'deployment_config' not in openshift_hosted_registry_results.stderr and 'service' not in openshift_hosted_registry_results.stderr"
   when: replicas | int > 0
 
+- include: secure.yml
+  static: no
+  when: openshift.common.deployment_subtype == 'registry'
+
 - include: storage/object_storage.yml
   static: no
   when: replicas | int > 0 and openshift.hosted.registry.storage.kind | default(none) == 'object'

roles/openshift_hosted/tasks/registry/secure.yml

@@ -0,0 +1,83 @@
+---
+- name: Determine if registry certificates must be created
+  stat:
+    path: "{{ openshift_master_config_dir }}/{{ item }}"
+  with_items:
+  - registry.crt
+  - registry.key
+  register: docker_registry_certificates_stat_result
+  changed_when: false
+  failed_when: false
+
+- name: Retrieve registry service IP
+  command: >
+    {{ openshift.common.client_binary }} get service docker-registry
+    --template='{{ '{{' }} .spec.clusterIP {{ '}}' }}'
+  register: docker_registry_service_ip
+  changed_when: false
+
+- set_fact:
+    docker_registry_route_hostname: "{{ 'docker-registry-default.' ~ (openshift.master.default_subdomain | default('router.default.svc.cluster.local', true)) }}"
+
+- name: Create registry certificates if they do not exist
+  command: >
+    {{ openshift.common.admin_binary }} ca create-server-cert
+    --signer-cert=/etc/origin/master/ca.crt
+    --signer-key=/etc/origin/master/ca.key
+    --signer-serial=/etc/origin/master/ca.serial.txt
+    --hostnames="{{ docker_registry_service_ip.stdout }},docker-registry.default.svc.cluster.local,{{ docker_registry_route_hostname }}"
+    --cert={{ openshift_master_config_dir }}/registry.crt
+    --key={{ openshift_master_config_dir }}/registry.key
+  when: False in (docker_registry_certificates_stat_result.results | default([]) | oo_collect(attribute='stat.exists') | list)
+
+- name: Create the secret for the registry certificates
+  command: >
+    {{ openshift.common.client_binary }} secrets new registry-certificates
+    {{ openshift_master_config_dir }}/registry.crt
+    {{ openshift_master_config_dir }}/registry.key
+    --config={{ openshift_hosted_kubeconfig }}
+    -n default
+  register: create_registry_certificates_secret
+  changed_when: "'already exists' not in create_registry_certificates_secret.stderr"
+  failed_when: "'already exists' not in create_registry_certificates_secret.stderr and create_registry_certificates_secret.rc != 0"
+
+- name: "Add the secret to the registry's pod service accounts"
+  command: >
+    {{ openshift.common.client_binary }} secrets link {{ item }} registry-certificates
+    --config={{ openshift_hosted_kubeconfig }}
+    -n default
+  with_items:
+  - registry
+  - default
+
+- name: Determine if registry-certificates secret volume attached
+  command: >
+    {{ openshift.common.client_binary }} get dc/docker-registry
+    --template='{{ '{{' }} range .spec.template.spec.volumes {{ '}}' }}{{ '{{' }} .secret.secretName {{ '}}' }}{{ '{{' }} end {{ '}}' }}'
+    --config={{ openshift_hosted_kubeconfig }}
+    -n default
+  register: docker_registry_volumes
+  changed_when: false
+  failed_when: false
+
+- name: Attach registry-certificates secret volume
+  command: >
+   {{ openshift.common.client_binary }} volume dc/docker-registry --add --type=secret
+   --secret-name=registry-certificates
+   -m /etc/secrets
+   --config={{ openshift_hosted_kubeconfig }}
+   -n default
+  when: "'registry-certificates' not in docker_registry_volumes.stdout"
+
+- name: Set registry environment variables for TLS certificate
+  command: >
+    {{ openshift.common.client_binary }} env dc/docker-registry
+    REGISTRY_HTTP_TLS_CERTIFICATE=/etc/secrets/registry.crt
+    REGISTRY_HTTP_TLS_KEY=/etc/secrets/registry.key
+
+# These commands are on a single line to preserve patch json.
+- name: Update registry liveness probe from HTTP to HTTPS
+  command: "{{ openshift.common.client_binary }} patch dc/docker-registry --api-version=v1 -p '{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"registry\",\"livenessProbe\":{\"httpGet\":{\"scheme\":\"HTTPS\"}}}]}}}}' --config={{ openshift_hosted_kubeconfig }} -n default"
+
+- name: Update registry readiness probe from HTTP to HTTPS
+  command: "{{ openshift.common.client_binary }} patch dc/docker-registry --api-version=v1 -p '{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"registry\",\"readinessProbe\":{\"httpGet\":{\"scheme\":\"HTTPS\"}}}]}}}}' --config={{ openshift_hosted_kubeconfig }} -n default"