Add metrics-server to openshift-metrics playbook

roles/openshift_metrics/README.md

@@ -17,6 +17,8 @@ The following variables need to be set and will be validated:
 - `openshift_metrics_project`: project (i.e. namespace) where the components will be
   deployed.
 
+- `openshift_metrics_server_project`: project (i.e. namespace) where
+  the metrics-server components will be deployed.
 
 Role Variables
 --------------

roles/openshift_metrics/defaults/main.yaml

@@ -54,6 +54,7 @@ openshift_metrics_resolution: 30s
 openshift_metrics_master_url: https://kubernetes.default.svc
 openshift_metrics_node_id: nodename
 openshift_metrics_project: openshift-infra
+openshift_metrics_server_project: openshift-metrics
 
 openshift_metrics_cassandra_pvc_prefix: metrics-cassandra
 openshift_metrics_cassandra_pvc_access: "{{ openshift_metrics_storage_access_modes | default(['ReadWriteOnce']) }}"

roles/openshift_metrics/tasks/generate_certificates.yaml

@@ -8,4 +8,5 @@
     --serial='{{ mktemp.stdout }}/ca.serial.txt'
     --name="metrics-signer@{{lookup('pipe','date +%s')}}"
 
+- include_tasks: generate_metrics_server_certificates.yaml
 - include_tasks: generate_hawkular_certificates.yaml

roles/openshift_metrics/tasks/generate_metrics_server_certificates.yaml

@@ -0,0 +1,50 @@
+---
+- name: generate metrics-server certificates
+  include_tasks: setup_certificate.yaml
+  vars:
+    component: metrics-server
+    hostnames: "metrics-server,metrics-server.{{ openshift_metrics_server_project }}.svc,metrics-server.{{ openshift_metrics_server_project }}.svc.cluster.local"
+  changed_when: no
+
+- name: read files for the metrics-server-certs secret
+  shell: >
+    printf '%s: ' '{{ item }}'
+    && base64 --wrap 0 '{{ mktemp.stdout }}/{{ item }}'
+  register: metrics_server_secrets
+  with_items:
+  - metrics-server.crt
+  - metrics-server.key
+  changed_when: false
+
+- set_fact:
+    metrics_server_secrets: |
+      {{ metrics_server_secrets.results|map(attribute='stdout')|join('
+      ')|from_yaml }}
+
+- slurp:
+    src: "{{ mktemp.stdout }}/ca.crt"
+  register: apiserver_ca
+
+- name: generate metrics-server secret template
+  template:
+    src: secret.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-certs.yaml"
+  vars:
+    name: metrics-server-certs
+    labels:
+      metrics-infra: metrics-server
+    data:
+      tls.crt: >
+        {{ metrics_server_secrets['metrics-server.crt'] }}
+      tls.key: >
+        {{ metrics_server_secrets['metrics-server.key'] }}
+  when: name not in existing_metrics_server_secrets.stdout_lines
+  changed_when: no
+
+- name: Generate metrics-server apiservice
+  template:
+    src: metrics-server-apiservice.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-apiservice.yaml"
+  vars:
+    caBundle: "{{ apiserver_ca.content }}"
+  changed_when: no

roles/openshift_metrics/tasks/generate_rolebindings.yaml

@@ -46,3 +46,27 @@
     files:
     - "{{ mktemp.stdout }}/templates/hawkular-cluster-role.yaml"
     delete_after: true
+
+- name: generate the metrics-server cluster role
+  template:
+    src: metrics-server-role.js2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-role.yaml"
+  changed_when: no
+
+- name: generate auth-delegator role binding for the metrics-server service account
+  template:
+    src: metrics-server-auth-delegator.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-auth-delegator-rolebinding.yaml"
+  changed_when: no
+
+- name: generate auth-reader role binding for the metrics-server api extension server
+  template:
+    src: metrics-server-auth-reader.j2
+    dest: "{{ mktemp.stdout }}/templates/extension-apiserver-authentication-reader-metrics-server-rolebinding.yaml"
+  changed_when: no
+
+- name: generate resource-reader role binding for the metrics-server service account
+  template:
+    src: metrics-server-resource-reader.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-resource-reader-rolebinding.yaml"
+  changed_when: no

roles/openshift_metrics/tasks/generate_serviceaccounts.yaml

@@ -25,3 +25,9 @@
   with_items:
   - hawkular
   - cassandra
+
+- name: Generating serviceaccounts for metrics-server
+  template:
+    src: metrics-server-sa.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-sa.yaml"
+  changed_when: no

roles/openshift_metrics/tasks/generate_services.yaml

@@ -31,3 +31,9 @@
   - cassandra
   - cassandra-nodes
   changed_when: no
+
+- name: Generate service for metrics-server
+  template:
+    src: metrics-server-service.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-service.yaml"
+  changed_when: no

roles/openshift_metrics/tasks/install_metrics.yaml

@@ -20,9 +20,14 @@
   include_tasks: install_hosa.yaml
   when: openshift_metrics_install_hawkular_agent | default(false) | bool
 
+- name: Generate metrics-server deployment
+  template:
+    src: metrics-server-deployment.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-server-deployment.yaml"
+
 - find:
     paths: "{{ mktemp.stdout }}/templates"
-    patterns: "^(?!metrics-hawkular-openshift-agent).*.yaml"
+    patterns: "^(?!metrics-hawkular-openshift-agent|metrics-server-|extension-apiserver-authentication-reader-metrics-server-).*.yaml"
     use_regex: true
   register: object_def_files
   changed_when: no
@@ -67,6 +72,50 @@
   with_items: "{{ hawkular_agent_object_defs.results }}"
   when: openshift_metrics_install_hawkular_agent | bool
 
+- find:
+    paths: "{{ mktemp.stdout }}/templates"
+    patterns: "^metrics-server-.*.yaml"
+    use_regex: true
+  register: metrics_server_object_def_files
+  changed_when: no
+
+- slurp:
+    src: "{{item.path}}"
+  register: metrics_server_object_defs
+  with_items: "{{ metrics_server_object_def_files.files }}"
+  changed_when: no
+
+- name: Create Metrics Server objects
+  include_tasks: oc_apply.yaml
+  vars:
+    kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig"
+    namespace: "{{ openshift_metrics_server_project }}"
+    file_name: "{{ item.source }}"
+    file_content: "{{ item.content | b64decode | from_yaml }}"
+  with_items: "{{ metrics_server_object_defs.results }}"
+
+- find:
+    paths: "{{ mktemp.stdout }}/templates"
+    patterns: "^extension-apiserver-authentication-reader-metrics-server-rolebinding.yaml"
+    use_regex: true
+  register: apiextension_object_def_files
+  changed_when: no
+
+- slurp:
+    src: "{{item.path}}"
+  register: apiextension_object_defs
+  with_items: "{{ apiextension_object_def_files.files }}"
+  changed_when: no
+
+- name: Create Metrics Server kube-system objects
+  include_tasks: oc_apply.yaml
+  vars:
+    kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig"
+    namespace: kube-system
+    file_name: "{{ item.source }}"
+    file_content: "{{ item.content | b64decode | from_yaml }}"
+  with_items: "{{ apiextension_object_defs.results }}"
+
 - include_tasks: update_master_config.yaml
   when: not openshift_version_gte_3_9
 

roles/openshift_metrics/tasks/install_support.yaml

@@ -19,6 +19,12 @@
 - fail: msg="'keytool' is unavailable. Please install java-1.8.0-openjdk-headless on the control node"
   when: keytool_check.rc  == 1
 
+- name: Set metrics server namespace
+  oc_project:
+    state: present
+    name: "{{ openshift_metrics_server_project }}"
+    node_selector: ""
+
 - include_tasks: generate_certificates.yaml
 - include_tasks: generate_serviceaccounts.yaml
 - include_tasks: generate_services.yaml

roles/openshift_metrics/tasks/pre_install.yaml

@@ -19,3 +19,11 @@
     get secrets -o name
   register: metrics_secrets
   changed_when: false
+
+- name: list existing metrics server secrets
+  command: >
+    {{ openshift_client_binary }} -n {{ openshift_metrics_server_project }}
+    --config={{ mktemp.stdout }}/admin.kubeconfig
+    get secrets -o name
+  register: existing_metrics_server_secrets
+  changed_when: false

roles/openshift_metrics/tasks/uninstall_metrics.yaml

@@ -31,3 +31,19 @@
   when:
     - openshift_web_console_install | default(true) | bool
     - openshift_version_gte_3_9
+
+- name: remove metrics server components
+  command: >
+    {{ openshift_client_binary }} -n {{ openshift_metrics_server_project }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    delete --ignore-not-found --selector=metrics-infra
+    all,sa,secrets,rolebindings,clusterrolebindings,clusterrole,apiservice,deployment
+  register: delete_metrics
+  changed_when: delete_metrics.stdout != 'No resources found'
+
+- name: remove metrics server system components
+  command: >
+    {{ openshift_client_binary }} -n kube-system --config={{ mktemp.stdout }}/admin.kubeconfig
+    delete --ignore-not-found --selector=metrics-infra
+    rolebindings
+  register: delete_metrics
+  changed_when: delete_metrics.stdout != 'No resources found'

roles/openshift_metrics/templates/metrics-server-apiservice.j2

@@ -0,0 +1,17 @@
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+  labels:
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: support
+spec:
+  service:
+    name: metrics-server
+    namespace: "{{ openshift_metrics_server_project }}"
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: false
+  groupPriorityMinimum: 100
+  versionPriority: 100
+  caBundle: "{{ caBundle }}"

roles/openshift_metrics/templates/metrics-server-auth-delegator.j2

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+  labels:
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: support
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: "{{ openshift_metrics_server_project }}"

roles/openshift_metrics/templates/metrics-server-auth-reader.j2

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: support
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: "{{ openshift_metrics_server_project }}"

roles/openshift_metrics/templates/metrics-server-deployment.j2

@@ -0,0 +1,45 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: "{{ openshift_metrics_server_project }}"
+  labels:
+    k8s-app: metrics-server
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: metrics-server
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: metrics-server
+      containers:
+      - name: metrics-server
+        image: {{openshift_metrics_image_prefix}}metrics-server:{{openshift_metrics_image_version}}
+        command:
+          - /usr/bin/metrics-server
+          - "--source=kubernetes.summary_api:?useServiceAccount=true&kubeletHttps=true&kubeletPort=10250"
+          - "--tls-cert-file=/certs/tls.crt"
+          - "--tls-private-key-file=/certs/tls.key"
+          - --secure-port=8443
+        ports:
+        - containerPort: 8443
+          name: https
+          protocol: TCP
+        volumeMounts:
+        - name: metrics-server-certs
+          mountPath: /certs
+          readOnly: true
+      volumes:
+      - name: metrics-server-certs
+        secret:
+          defaultMode: 420
+          secretName: metrics-server-certs

roles/openshift_metrics/templates/metrics-server-resource-reader.j2

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+  labels:
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: support
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: "{{ openshift_metrics_server_project }}"

roles/openshift_metrics/templates/metrics-server-role.js2

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+  labels:
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: support
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch

roles/openshift_metrics/templates/metrics-server-sa.j2

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: "{{ openshift_metrics_server_project }}"
+  labels:
+    kubernetes.io/cluster-service: "true"
+    metrics-infra: support

roles/openshift_metrics/templates/metrics-server-service.j2

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: "{{ openshift_metrics_server_project }}"
+  labels:
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Metrics-server"
+    metrics-infra: metrics-server
+spec:
+  ports:
+  -
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    k8s-app: metrics-server