Support installing HOSA via ansible

roles/openshift_metrics/README.md

@@ -68,6 +68,9 @@ For default values, see [`defaults/main.yaml`](defaults/main.yaml).
 
 - `openshift_metrics_resolution`: How often metrics should be gathered.
 
+- `openshift_metrics_install_hawkular_agent`: Install the Hawkular OpenShift Agent (HOSA). HOSA can be used
+  to collect custom metrics from your pods. This component is currently in tech-preview and is not installed by default.
+
 ## Additional variables to control resource limits
 Each metrics component (hawkular, cassandra, heapster) can specify a cpu and memory limits and requests by setting
 the corresponding role variable:

roles/openshift_metrics/defaults/main.yaml

@@ -30,6 +30,14 @@ openshift_metrics_heapster_requests_memory: 0.9375G
 openshift_metrics_heapster_requests_cpu: null
 openshift_metrics_heapster_nodeselector: ""
 
+openshift_metrics_install_hawkular_agent: False
+openshift_metrics_hawkular_agent_limits_memory: null
+openshift_metrics_hawkular_agent_limits_cpu: null
+openshift_metrics_hawkular_agent_requests_memory: null
+openshift_metrics_hawkular_agent_requests_cpu: null
+openshift_metrics_hawkular_agent_nodeselector: ""
+openshift_metrics_hawkular_agent_namespace: "default"
+
 openshift_metrics_hawkular_hostname: "hawkular-metrics.{{openshift_master_default_subdomain}}"
 
 openshift_metrics_duration: 7

roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml

@@ -73,6 +73,8 @@
         {{ hawkular_secrets['hawkular-metrics.key'] }}
       tls.truststore.crt: >
         {{ hawkular_secrets['hawkular-cassandra.crt'] }}
+      ca.crt: >
+        {{ hawkular_secrets['ca.crt'] }}
   when: name not in metrics_secrets.stdout_lines
   changed_when: no
 

roles/openshift_metrics/tasks/install_hosa.yaml

@@ -0,0 +1,44 @@
+---
+- name: Generate Hawkular Agent (HOSA) Cluster Role
+  template:
+    src: hawkular_openshift_agent_role.j2
+    dest: "{{mktemp.stdout}}/templates/metrics-hawkular-openshift-agent-role.yaml"
+  changed_when: no
+
+- name: Generate Hawkular Agent (HOSA) Service Account
+  template:
+    src: hawkular_openshift_agent_sa.j2
+    dest: "{{mktemp.stdout}}/templates/metrics-hawkular-openshift-agent-sa.yaml"
+  changed_when: no
+
+- name: Generate Hawkular Agent (HOSA) Daemon Set
+  template:
+    src: hawkular_openshift_agent_ds.j2
+    dest: "{{mktemp.stdout}}/templates/metrics-hawkular-openshift-agent-ds.yaml"
+  vars:
+    node_selector: "{{openshift_metrics_hawkular_agent_nodeselector | default('') }}"
+  changed_when: no
+
+- name: Generate the Hawkular Agent (HOSA) Configmap
+  template:
+    src: hawkular_openshift_agent_cm.j2
+    dest: "{{mktemp.stdout}}/templates/metrics-hawkular-openshift-agent-cm.yaml"
+  changed_when: no
+
+- name: Generate role binding for the hawkular-openshift-agent service account
+  template:
+    src: rolebinding.j2
+    dest: "{{ mktemp.stdout }}/templates/metrics-hawkular-agent-rolebinding.yaml"
+  vars:
+    cluster: True
+    obj_name: hawkular-openshift-agent-rb
+    labels:
+      metrics-infra: hawkular-agent
+    roleRef:
+      kind: ClusterRole
+      name: hawkular-openshift-agent
+    subjects:
+      - kind: ServiceAccount
+        name: hawkular-openshift-agent
+        namespace: "{{openshift_metrics_hawkular_agent_namespace}}"
+  changed_when: no

roles/openshift_metrics/tasks/install_metrics.yaml

@@ -16,11 +16,19 @@
   include: install_heapster.yaml
   when: openshift_metrics_heapster_standalone | bool
 
-- find: paths={{ mktemp.stdout }}/templates patterns=*.yaml
+- name: Install Hawkular OpenShift Agent (HOSA)
+  include: install_hosa.yaml
+  when: openshift_metrics_install_hawkular_agent | default(false) | bool
+
+- find:
+    paths: "{{ mktemp.stdout }}/templates"
+    patterns: "^(?!metrics-hawkular-openshift-agent).*.yaml"
+    use_regex: true
   register: object_def_files
   changed_when: no
 
-- slurp: src={{item.path}}
+- slurp:
+    src: "{{item.path}}"
   register: object_defs
   with_items: "{{object_def_files.files}}"
   changed_when: no
@@ -34,6 +42,31 @@
     file_content: "{{ item.content | b64decode | from_yaml }}"
   with_items: "{{ object_defs.results }}"
 
+- find:
+    paths: "{{ mktemp.stdout }}/templates"
+    patterns: "^metrics-hawkular-openshift-agent.*.yaml"
+    use_regex: true
+  register: hawkular_agent_object_def_files
+  when: openshift_metrics_install_hawkular_agent | bool
+  changed_when: no
+
+- slurp:
+    src: "{{item.path}}"
+  register: hawkular_agent_object_defs
+  with_items: "{{ hawkular_agent_object_def_files.files }}"
+  when: openshift_metrics_install_hawkular_agent | bool
+  changed_when: no
+
+- name: Create Hawkular Agent objects
+  include: oc_apply.yaml
+  vars:
+    kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig"
+    namespace: "{{ openshift_metrics_hawkular_agent_namespace }}"
+    file_name: "{{ item.source }}"
+    file_content: "{{ item.content | b64decode | from_yaml }}"
+  with_items: "{{ hawkular_agent_object_defs.results }}"
+  when: openshift_metrics_install_hawkular_agent | bool
+
 - include: update_master_config.yaml
 
 - command: >

roles/openshift_metrics/tasks/main.yaml

@@ -44,6 +44,9 @@
 
 - include: "{{ (openshift_metrics_install_metrics | bool) | ternary('install_metrics.yaml','uninstall_metrics.yaml') }}"
 
+- include: uninstall_hosa.yaml
+  when: not openshift_metrics_install_hawkular_agent | bool
+
 - name: Delete temp directory
   local_action: file path=local_tmp.stdout state=absent
   tags: metrics_cleanup

roles/openshift_metrics/tasks/oc_apply.yaml

@@ -14,7 +14,7 @@
   command: >
     {{ openshift.common.client_binary }} --config={{ kubeconfig }}
     apply -f {{ file_name }}
-    -n {{ openshift_metrics_project }}
+    -n {{namespace}}
   register: generation_apply
   failed_when: "'error' in generation_apply.stderr"
   changed_when: no

roles/openshift_metrics/tasks/uninstall_hosa.yaml

@@ -0,0 +1,15 @@
+---
+- name: remove Hawkular Agent (HOSA) components
+  command: >
+    {{ openshift.common.client_binary }} -n {{ openshift_metrics_hawkular_agent_namespace }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    delete --ignore-not-found --selector=metrics-infra=agent
+    all,sa,secrets,templates,routes,pvc,rolebindings,clusterrolebindings
+  register: delete_metrics
+  changed_when: delete_metrics.stdout != 'No resources found'
+
+- name: remove rolebindings
+  command: >
+    {{ openshift.common.client_binary }} -n {{ openshift_metrics_hawkular_agent_namespace }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    delete --ignore-not-found
+    clusterrolebinding/hawkular-openshift-agent-rb
+  changed_when: delete_metrics.stdout != 'No resources found'

roles/openshift_metrics/templates/hawkular_openshift_agent_cm.j2

@@ -0,0 +1,54 @@
+id: hawkular-openshift-agent
+kind: ConfigMap
+apiVersion: v1
+name: Hawkular OpenShift Agent Configuration
+metadata:
+  name: hawkular-openshift-agent-configuration
+  labels:
+    metrics-infra: agent
+  namespace: {{openshift_metrics_hawkular_agent_namespace}}
+data:
+  config.yaml: |
+    kubernetes:
+      tenant: ${POD:namespace_name}
+    hawkular_server:
+      url: https://hawkular-metrics.openshift-infra.svc.cluster.local       
+      credentials:
+        username: secret:openshift-infra/hawkular-metrics-account/hawkular-metrics.username
+        password: secret:openshift-infra/hawkular-metrics-account/hawkular-metrics.password
+      ca_cert_file: secret:openshift-infra/hawkular-metrics-certs/ca.crt
+    emitter:
+      status_enabled: false
+    collector:
+      minimum_collection_interval: 10s
+      default_collection_interval: 30s
+      metric_id_prefix: pod/${POD:uid}/custom/
+      tags:
+        metric_name: ${METRIC:name}
+        description: ${METRIC:description}
+        units: ${METRIC:units}
+        namespace_id: ${POD:namespace_uid}
+        namespace_name: ${POD:namespace_name}
+        node_name: ${POD:node_name}
+        pod_id: ${POD:uid}
+        pod_ip: ${POD:ip}
+        pod_name: ${POD:name}
+        pod_namespace: ${POD:namespace_name}
+        hostname: ${POD:hostname}
+        host_ip: ${POD:host_ip}
+        labels: ${POD:labels}
+        type: pod
+        collector: hawkular_openshift_agent
+        custom_metric: true
+  hawkular-openshift-agent: |
+    endpoints:
+    - type: prometheus
+      protocol: "http"
+      port: 8080
+      path: /metrics
+      collection_interval: 30s
+      metrics:
+      - name: hawkular_openshift_agent_metric_data_points_collected_total
+      - name: hawkular_openshift_agent_monitored_endpoints
+      - name: hawkular_openshift_agent_monitored_pods
+      - name: hawkular_openshift_agent_monitored_metrics

roles/openshift_metrics/templates/hawkular_openshift_agent_ds.j2

@@ -0,0 +1,91 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: hawkular-openshift-agent
+  labels:
+    name: hawkular-openshift-agent
+    metrics-infra: agent
+  namespace: {{openshift_metrics_hawkular_agent_namespace}}
+spec:
+  selector:
+    matchLabels:
+      name: hawkular-openshift-agent
+  template:
+    metadata:
+      labels:
+        name: hawkular-openshift-agent
+        metrics-infra: agent
+    spec:
+      serviceAccount: hawkular-openshift-agent
+{% if node_selector is iterable and node_selector | length > 0 %}
+      nodeSelector:
+{% for key, value in node_selector.iteritems() %}
+        {{key}}: "{{value}}"
+{% endfor %}
+{% endif %}
+      containers:
+      - image: {{openshift_metrics_image_prefix}}metrics-hawkular-openshift-agent:{{openshift_metrics_image_version}}
+        imagePullPolicy: Always
+        name: hawkular-openshift-agent
+{% if ((openshift_metrics_hawkular_agent_limits_cpu is defined and openshift_metrics_hawkular_agent_limits_cpu is not none)
+   or (openshift_metrics_hawkular_agent_limits_memory is defined and openshift_metrics_hawkular_agent_limits_memory is not none)
+   or (openshift_metrics_hawkular_agent_requests_cpu is defined and openshift_metrics_hawkular_agent_requests_cpu is not none)
+   or (openshift_metrics_hawkular_agent_requests_memory is defined and openshift_metrics_hawkular_agent_requests_memory is not none))
+%}
+        resources:
+{% if (openshift_metrics_hawkular_agent_limits_cpu is not none
+   or openshift_metrics_hawkular_agent_limits_memory is not none)
+%}
+          limits:
+{% if openshift_metrics_hawkular_agent_limits_cpu is not none %}
+            cpu: "{{openshift_metrics_hawkular_agent_limits_cpu}}"
+{% endif %}
+{% if openshift_metrics_hawkular_agent_limits_memory is not none %}
+            memory: "{{openshift_metrics_hawkular_agent_limits_memory}}"
+{% endif %}
+{% endif %}
+{% if (openshift_metrics_hawkular_agent_requests_cpu is not none
+   or openshift_metrics_hawkular_agent_requests_memory is not none)
+%}
+          requests:
+{% if openshift_metrics_hawkular_agent_requests_cpu is not none %}
+            cpu: "{{openshift_metrics_hawkular_agent_requests_cpu}}"
+{% endif %}
+{% if openshift_metrics_hawkular_agent_requests_memory is not none %}
+            memory: "{{openshift_metrics_hawkular_agent_requests_memory}}"
+{% endif %}
+{% endif %}
+{% endif %}
+
+        livenessProbe:
+          httpGet:
+            scheme: HTTP
+            path: /health
+            port: 8080
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        command:
+          - "hawkular-openshift-agent"
+          - "-config"
+          - "/hawkular-openshift-agent-configuration/config.yaml"
+          - "-v"
+          - "3"
+        env:
+        - name: K8S_POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: K8S_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        volumeMounts:
+        - name: hawkular-openshift-agent-configuration
+          mountPath: "/hawkular-openshift-agent-configuration"
+      volumes:
+      - name: hawkular-openshift-agent-configuration
+        configMap:
+          name: hawkular-openshift-agent-configuration
+      - name: hawkular-openshift-agent
+        configMap:
+          name: hawkular-openshift-agent-configuration

roles/openshift_metrics/templates/hawkular_openshift_agent_role.j2

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ClusterRole
+metadata:
+  name: hawkular-openshift-agent
+  labels:
+    metrics-infra: agent
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - namespaces
+  - nodes
+  - pods
+  - projects
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get

roles/openshift_metrics/templates/hawkular_openshift_agent_sa.j2

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hawkular-openshift-agent
+  labels:
+    metrics-infra: agent
+  namespace: {{openshift_metrics_hawkular_agent_namespace}}