mux does not require privileged, only hostmount-anyuid

roles/openshift_logging/defaults/main.yml

@@ -119,6 +119,15 @@ openshift_logging_es_ops_number_of_replicas: 0
 # storage related defaults
 openshift_logging_storage_access_modes: "{{ openshift_hosted_logging_storage_access_modes | default(['ReadWriteOnce']) }}"
 
+# mux - secure_forward listener service
+openshift_logging_mux_allow_external: False
+openshift_logging_use_mux: "{{ openshift_logging_mux_allow_external | default(False) }}"
+# this tells the fluentd node agent to use mux instead of sending directly to Elasticsearch
+openshift_logging_use_mux_client: False
+openshift_logging_mux_hostname: "{{ 'mux.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}"
+openshift_logging_mux_port: 24284
+openshift_logging_mux_cpu_limit: 100m
+openshift_logging_mux_memory_limit: 512Mi
 
 # following can be uncommented to provide values for configmaps -- take care when providing file contents as it may cause your cluster to not operate correctly
 #es_logging_contents:
@@ -127,3 +136,5 @@ openshift_logging_storage_access_modes: "{{ openshift_hosted_logging_storage_acc
 #fluentd_config_contents:
 #fluentd_throttle_contents:
 #fluentd_secureforward_contents:
+#fluentd_mux_config_contents:
+#fluentd_mux_secureforward_contents:

roles/openshift_logging/tasks/delete_logging.yaml

@@ -44,6 +44,7 @@
     - logging-kibana
     - logging-kibana-proxy
     - logging-curator
+    - logging-mux
   ignore_errors: yes
   register: delete_result
   changed_when: delete_result.stdout.find("deleted") != -1 and delete_result.rc == 0
@@ -109,5 +110,6 @@
     - logging-curator
     - logging-elasticsearch
     - logging-fluentd
+    - logging-mux
   register: delete_result
   changed_when: delete_result.stdout.find("deleted") != -1 and delete_result.rc == 0

roles/openshift_logging/tasks/generate_certs.yaml

@@ -45,6 +45,21 @@
     - procure_component: kibana-internal
       hostnames: "kibana, kibana-ops, {{openshift_logging_kibana_hostname}}, {{openshift_logging_kibana_ops_hostname}}"
 
+- include: procure_server_certs.yaml
+  loop_control:
+    loop_var: cert_info
+  with_items:
+    - procure_component: mux
+      hostnames: "logging-mux, {{openshift_logging_mux_hostname}}"
+  when: openshift_logging_use_mux
+
+- include: procure_shared_key.yaml
+  loop_control:
+    loop_var: shared_key_info
+  with_items:
+    - procure_component: mux
+  when: openshift_logging_use_mux
+
 - name: Copy proxy TLS configuration file
   copy: src=server-tls.json dest={{generated_certs_dir}}/server-tls.json
   when: server_tls_json is undefined
@@ -85,6 +100,14 @@
   loop_control:
     loop_var: node_name
 
+- name: Generate PEM cert for mux
+  include: generate_pems.yaml component={{node_name}}
+  with_items:
+    - system.logging.mux
+  loop_control:
+    loop_var: node_name
+  when: openshift_logging_use_mux
+
 - name: Creating necessary JKS certs
   include: generate_jks.yaml
 

roles/openshift_logging/tasks/generate_configmaps.yaml

@@ -134,3 +134,43 @@
       when: fluentd_configmap.stdout is defined
       changed_when: no
   check_mode: no
+
+- block:
+    - copy:
+        src: fluent.conf
+        dest: "{{mktemp.stdout}}/fluent-mux.conf"
+      when: fluentd_mux_config_contents is undefined
+      changed_when: no
+
+    - copy:
+        src: secure-forward.conf
+        dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
+      when: fluentd_mux_securefoward_contents is undefined
+      changed_when: no
+
+    - copy:
+        content: "{{fluentd_mux_config_contents}}"
+        dest: "{{mktemp.stdout}}/fluent-mux.conf"
+      when: fluentd_mux_config_contents is defined
+      changed_when: no
+
+    - copy:
+        content: "{{fluentd_mux_secureforward_contents}}"
+        dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
+      when: fluentd_mux_secureforward_contents is defined
+      changed_when: no
+
+    - command: >
+        {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create configmap logging-mux
+        --from-file=fluent.conf={{mktemp.stdout}}/fluent-mux.conf
+        --from-file=secure-forward.conf={{mktemp.stdout}}/secure-forward-mux.conf -o yaml --dry-run
+      register: mux_configmap
+      changed_when: no
+
+    - copy:
+        content: "{{mux_configmap.stdout}}"
+        dest: "{{mktemp.stdout}}/templates/logging-mux-configmap.yaml"
+      when: mux_configmap.stdout is defined
+      changed_when: no
+  check_mode: no
+  when: openshift_logging_use_mux

roles/openshift_logging/tasks/generate_secrets.yaml

@@ -34,6 +34,36 @@
   check_mode: no
   changed_when: no
 
+- name: Retrieving the cert to use when generating secrets for mux
+  slurp: src="{{generated_certs_dir}}/{{item.file}}"
+  register: mux_key_pairs
+  with_items:
+    - { name: "ca_file", file: "ca.crt" }
+    - { name: "mux_key", file: "system.logging.mux.key"}
+    - { name: "mux_cert", file: "system.logging.mux.crt"}
+    - { name: "mux_shared_key", file: "mux_shared_key"}
+  when: openshift_logging_use_mux
+
+- name: Generating secrets for mux
+  template: src=secret.j2 dest={{mktemp.stdout}}/templates/{{secret_name}}-secret.yaml
+  vars:
+    secret_name: "logging-{{component}}"
+    secret_key_file: "{{component}}_key"
+    secret_cert_file: "{{component}}_cert"
+    secrets:
+      - {key: ca, value: "{{mux_key_pairs | entry_from_named_pair('ca_file')| b64decode }}"}
+      - {key: key, value: "{{mux_key_pairs | entry_from_named_pair(secret_key_file)| b64decode }}"}
+      - {key: cert, value: "{{mux_key_pairs | entry_from_named_pair(secret_cert_file)| b64decode }}"}
+      - {key: shared_key, value: "{{mux_key_pairs | entry_from_named_pair('mux_shared_key')| b64decode }}"}
+    secret_keys: ["ca", "cert", "key", "shared_key"]
+  with_items:
+    - mux
+  loop_control:
+    loop_var: component
+  check_mode: no
+  changed_when: no
+  when: openshift_logging_use_mux
+
 - name: Generating secrets for kibana proxy
   template: src=secret.j2 dest={{mktemp.stdout}}/templates/{{secret_name}}-secret.yaml
   vars:

roles/openshift_logging/tasks/generate_services.yaml

@@ -85,3 +85,35 @@
   when: openshift_logging_use_ops | bool
   check_mode: no
   changed_when: no
+
+- name: Generating logging-mux service for external connections
+  template: src=service.j2 dest={{mktemp.stdout}}/templates/logging-mux-svc.yaml
+  vars:
+    obj_name: logging-mux
+    ports:
+    - {port: "{{openshift_logging_mux_port}}", targetPort: mux-forward, name: mux-forward}
+    labels:
+      logging-infra: support
+    selector:
+      provider: openshift
+      component: mux
+    externalIPs:
+    - "{{ ansible_eth0.ipv4.address }}"
+  check_mode: no
+  changed_when: no
+  when: openshift_logging_mux_allow_external
+
+- name: Generating logging-mux service for intra-cluster connections
+  template: src=service.j2 dest={{mktemp.stdout}}/templates/logging-mux-svc.yaml
+  vars:
+    obj_name: logging-mux
+    ports:
+    - {port: "{{openshift_logging_mux_port}}", targetPort: mux-forward, name: mux-forward}
+    labels:
+      logging-infra: support
+    selector:
+      provider: openshift
+      component: mux
+  check_mode: no
+  changed_when: no
+  when: openshift_logging_use_mux and not openshift_logging_mux_allow_external

roles/openshift_logging/tasks/install_logging.yaml

@@ -27,6 +27,10 @@
   loop_control:
     loop_var: install_component
 
+- name: Install logging mux
+  include: "{{ role_path }}/tasks/install_mux.yaml"
+  when: openshift_logging_use_mux
+
 - find: paths={{ mktemp.stdout }}/templates patterns=*.yaml
   register: object_def_files
   changed_when: no

roles/openshift_logging/tasks/install_mux.yaml

@@ -0,0 +1,67 @@
+---
+- set_fact: mux_ops_host={{ (openshift_logging_use_ops | bool) | ternary(openshift_logging_es_ops_host, openshift_logging_es_host) }}
+  check_mode: no
+
+- set_fact: mux_ops_port={{ (openshift_logging_use_ops | bool) | ternary(openshift_logging_es_ops_port, openshift_logging_es_port) }}
+  check_mode: no
+
+- name: Check mux current replica count
+  command: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-mux
+    -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}}
+  register: mux_replica_count
+  when: not ansible_check_mode
+  ignore_errors: yes
+  changed_when: no
+
+- name: Generating mux deploymentconfig
+  template: src=mux.j2 dest={{mktemp.stdout}}/templates/logging-mux-dc.yaml
+  vars:
+    component: mux
+    logging_component: mux
+    deploy_name: "logging-{{component}}"
+    image: "{{openshift_logging_image_prefix}}logging-fluentd:{{openshift_logging_image_version}}"
+    es_host: logging-es
+    es_port: "{{openshift_logging_es_port}}"
+    ops_host: "{{ mux_ops_host }}"
+    ops_port: "{{ mux_ops_port }}"
+    mux_cpu_limit: "{{openshift_logging_mux_cpu_limit}}"
+    mux_memory_limit: "{{openshift_logging_mux_memory_limit}}"
+    replicas: "{{mux_replica_count.stdout | default (0)}}"
+    mux_node_selector: "{{openshift_logging_mux_nodeselector | default({})}}"
+  check_mode: no
+  changed_when: no
+
+- name: "Check mux hostmount-anyuid permissions"
+  command: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    get scc/hostmount-anyuid -o jsonpath='{.users}'
+  register: mux_hostmount_anyuid
+  check_mode: no
+  changed_when: no
+
+- name: "Set hostmount-anyuid permissions for mux"
+  command: >
+    {{ openshift.common.admin_binary}} --config={{ mktemp.stdout }}/admin.kubeconfig policy
+    add-scc-to-user hostmount-anyuid system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd
+  register: mux_output
+  failed_when: "mux_output.rc == 1 and 'exists' not in mux_output.stderr"
+  check_mode: no
+  when: mux_hostmount_anyuid.stdout.find("system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd") == -1
+
+- name: "Check mux cluster-reader permissions"
+  command: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    get clusterrolebinding/cluster-readers -o jsonpath='{.userNames}'
+  register: mux_cluster_reader
+  check_mode: no
+  changed_when: no
+
+- name: "Set cluster-reader permissions for mux"
+  command: >
+    {{ openshift.common.admin_binary}} --config={{ mktemp.stdout }}/admin.kubeconfig policy
+    add-cluster-role-to-user cluster-reader system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd
+  register: mux2_output
+  failed_when: "mux2_output.rc == 1 and 'exists' not in mux2_output.stderr"
+  check_mode: no
+  when: mux_cluster_reader.stdout.find("system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd") == -1

roles/openshift_logging/tasks/procure_shared_key.yaml

@@ -0,0 +1,25 @@
+---
+- name: Checking for {{ shared_key_info.procure_component }}_shared_key
+  stat: path="{{generated_certs_dir}}/{{ shared_key_info.procure_component }}_shared_key"
+  register: component_shared_key_file
+  check_mode: no
+
+- name: Trying to discover shared key variable name for {{ shared_key_info.procure_component }}
+  set_fact: procure_component_shared_key={{ lookup('env', '{{shared_key_info.procure_component}}' + '_shared_key') }}
+  when:
+  - shared_key_info[ shared_key_info.procure_component + '_shared_key' ] is defined
+  check_mode: no
+
+- name: Creating shared_key for {{ shared_key_info.procure_component }}
+  copy: content="{{'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'|random_word(64)}}"
+        dest="{{generated_certs_dir}}/{{shared_key_info.procure_component}}_shared_key"
+  check_mode: no
+  when:
+  - not component_shared_key_file.stat.exists
+
+- name: Copying shared key for {{ shared_key_info.procure_component }} to generated certs directory
+  copy: content="{{procure_component_shared_key}}" dest="{{generated_certs_dir}}/{{shared_key_info.procure_component}}_shared_key"
+  check_mode: no
+  when:
+  - shared_key_info[ shared_key_info.procure_component + '_shared_key' ] is defined
+  - not component_shared_key_file.stat.exists

roles/openshift_logging/tasks/start_cluster.yaml

@@ -21,6 +21,26 @@
   loop_control:
     loop_var: fluentd_host
 
+- name: Retrieve mux
+  oc_obj:
+    state: list
+    kind: dc
+    selector: "component=mux"
+    namespace: "{{openshift_logging_namespace}}"
+  register: mux_dc
+  when: openshift_logging_use_mux
+
+- name: start mux
+  oc_scale:
+    kind: dc
+    name: "{{ object }}"
+    namespace: "{{openshift_logging_namespace}}"
+    replicas: "{{ openshift_logging_mux_replica_count | default (1) }}"
+  with_items: "{{ mux_dc.results.results[0]['items'] | map(attribute='metadata.name') | list }}"
+  loop_control:
+    loop_var: object
+  when: openshift_logging_use_mux
+
 - name: Retrieve elasticsearch
   oc_obj:
     state: list

roles/openshift_logging/tasks/stop_cluster.yaml

@@ -21,6 +21,26 @@
   loop_control:
     loop_var: fluentd_host
 
+- name: Retrieve mux
+  oc_obj:
+    state: list
+    kind: dc
+    selector: "component=mux"
+    namespace: "{{openshift_logging_namespace}}"
+  register: mux_dc
+  when: openshift_logging_use_mux
+
+- name: stop mux
+  oc_scale:
+    kind: dc
+    name: "{{ object }}"
+    namespace: "{{openshift_logging_namespace}}"
+    replicas: 0
+  with_items: "{{ mux_dc.results.results[0]['items'] | map(attribute='metadata.name') | list }}"
+  loop_control:
+    loop_var: object
+  when: openshift_logging_use_mux
+
 - name: Retrieve elasticsearch
   oc_obj:
     state: list

roles/openshift_logging/templates/fluentd.j2

@@ -59,6 +59,11 @@ spec:
         - name: dockercfg
           mountPath: /etc/sysconfig/docker
           readOnly: true
+{% if openshift_logging_use_mux_client %}
+        - name: muxcerts
+          mountPath: /etc/fluent/muxkeys
+          readOnly: true
+{% endif %}
         env:
         - name: "K8S_HOST_URL"
           value: "{{openshift_logging_master_url}}"
@@ -122,6 +127,8 @@ spec:
           value: "{{openshift_logging_fluentd_journal_source | default('')}}"
         - name: "JOURNAL_READ_FROM_HEAD"
           value: "{{openshift_logging_fluentd_journal_read_from_head|lower}}"
+        - name: "USE_MUX_CLIENT"
+          value: "{{openshift_logging_use_mux_client| default('false')}}"
       volumes:
       - name: runlogjournal
         hostPath:
@@ -147,3 +154,8 @@ spec:
       - name: dockercfg
         hostPath:
           path: /etc/sysconfig/docker
+{% if openshift_logging_use_mux_client %}
+      - name: muxcerts
+        secret:
+          secretName: logging-mux
+{% endif %}

roles/openshift_logging/templates/mux.j2

@@ -0,0 +1,121 @@
+apiVersion: "v1"
+kind: "DeploymentConfig"
+metadata:
+  name: "{{deploy_name}}"
+  labels:
+    provider: openshift
+    component: "{{component}}"
+    logging-infra: "{{logging_component}}"
+spec:
+  replicas: {{replicas|default(0)}}
+  selector:
+    provider: openshift
+    component: "{{component}}"
+    logging-infra: "{{logging_component}}"
+  strategy:
+    rollingParams:
+      intervalSeconds: 1
+      timeoutSeconds: 600
+      updatePeriodSeconds: 1
+    type: Rolling
+  template:
+    metadata:
+      name: "{{deploy_name}}"
+      labels:
+        logging-infra: "{{logging_component}}"
+        provider: openshift
+        component: "{{component}}"
+    spec:
+      serviceAccountName: aggregated-logging-fluentd
+{% if mux_node_selector is iterable and mux_node_selector | length > 0 %}
+      nodeSelector:
+{% for key, value in mux_node_selector.iteritems() %}
+        {{key}}: "{{value}}"
+{% endfor %}
+{% endif %}
+      containers:
+      - name: "mux"
+        image: {{image}}
+        imagePullPolicy: Always
+{% if (mux_memory_limit is defined and mux_memory_limit is not none) or (mux_cpu_limit is defined and mux_cpu_limit is not none) %}
+        resources:
+          limits:
+{% if mux_cpu_limit is not none %}
+            cpu: "{{mux_cpu_limit}}"
+{% endif %}
+{% if mux_memory_limit is not none %}
+            memory: "{{mux_memory_limit}}"
+{% endif %}
+{% endif %}
+        ports:
+        - containerPort: "{{ openshift_logging_mux_port }}"
+          name: mux-forward
+        volumeMounts:
+        - name: config
+          mountPath: /etc/fluent/configs.d/user
+          readOnly: true
+        - name: certs
+          mountPath: /etc/fluent/keys
+          readOnly: true
+        - name: dockerhostname
+          mountPath: /etc/docker-hostname
+          readOnly: true
+        - name: localtime
+          mountPath: /etc/localtime
+          readOnly: true
+        - name: muxcerts
+          mountPath: /etc/fluent/muxkeys
+          readOnly: true
+        env:
+        - name: "K8S_HOST_URL"
+          value: "{{openshift_logging_master_url}}"
+        - name: "ES_HOST"
+          value: "{{openshift_logging_es_host}}"
+        - name: "ES_PORT"
+          value: "{{openshift_logging_es_port}}"
+        - name: "ES_CLIENT_CERT"
+          value: "{{openshift_logging_es_client_cert}}"
+        - name: "ES_CLIENT_KEY"
+          value: "{{openshift_logging_es_client_key}}"
+        - name: "ES_CA"
+          value: "{{openshift_logging_es_ca}}"
+        - name: "OPS_HOST"
+          value: "{{ops_host}}"
+        - name: "OPS_PORT"
+          value: "{{ops_port}}"
+        - name: "OPS_CLIENT_CERT"
+          value: "{{openshift_logging_es_ops_client_cert}}"
+        - name: "OPS_CLIENT_KEY"
+          value: "{{openshift_logging_es_ops_client_key}}"
+        - name: "OPS_CA"
+          value: "{{openshift_logging_es_ops_ca}}"
+        - name: "USE_JOURNAL"
+          value: "false"
+        - name: "JOURNAL_SOURCE"
+          value: "{{openshift_logging_fluentd_journal_source | default('')}}"
+        - name: "JOURNAL_READ_FROM_HEAD"
+          value: "{{openshift_logging_fluentd_journal_read_from_head|lower}}"
+        - name: FORWARD_LISTEN_HOST
+          value: "{{ openshift_logging_mux_hostname }}"
+        - name: FORWARD_LISTEN_PORT
+          value: "{{ openshift_logging_mux_port }}"
+        - name: USE_MUX
+          value: "true"
+        - name: MUX_ALLOW_EXTERNAL
+          value: "{{ openshift_logging_mux_allow_external| default('false') }}"
+      volumes:
+      - name: config
+        configMap:
+          name: logging-mux
+      - name: certs
+        secret:
+          secretName: logging-fluentd
+      - name: dockerhostname
+        hostPath:
+          path: /etc/hostname
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      - name: muxcerts
+        secret:
+          secretName: logging-mux

roles/openshift_logging/templates/service.j2

@@ -26,3 +26,9 @@ spec:
   {% for key, value in selector.iteritems() %}
   {{key}}: {{value}}
   {% endfor %}
+{% if externalIPs is defined -%}
+  externalIPs:
+{% for ip in externalIPs %}
+  - {{ ip }}
+{% endfor %}
+{% endif %}