Merge pull request openshift#3932 from richm/mux

Merged by openshift-bot

roles/openshift_logging/defaults/main.yml

@@ -119,6 +119,15 @@ openshift_logging_es_ops_number_of_replicas: 0
 # storage related defaults
 openshift_logging_storage_access_modes: "{{ openshift_hosted_logging_storage_access_modes | default(['ReadWriteOnce']) }}"
 
+# mux - secure_forward listener service
+openshift_logging_mux_allow_external: False
+openshift_logging_use_mux: "{{ openshift_logging_mux_allow_external | default(False) }}"
+# this tells the fluentd node agent to use mux instead of sending directly to Elasticsearch
+openshift_logging_use_mux_client: False
+openshift_logging_mux_hostname: "{{ 'mux.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}"
+openshift_logging_mux_port: 24284
+openshift_logging_mux_cpu_limit: 100m
+openshift_logging_mux_memory_limit: 512Mi
 
 # following can be uncommented to provide values for configmaps -- take care when providing file contents as it may cause your cluster to not operate correctly
 #es_logging_contents:
@@ -127,3 +136,5 @@ openshift_logging_storage_access_modes: "{{ openshift_hosted_logging_storage_acc
 #fluentd_config_contents:
 #fluentd_throttle_contents:
 #fluentd_secureforward_contents:
+#fluentd_mux_config_contents:
+#fluentd_mux_secureforward_contents:

roles/openshift_logging/tasks/delete_logging.yaml

@@ -44,6 +44,7 @@
     - logging-kibana
     - logging-kibana-proxy
     - logging-curator
+    - logging-mux
   ignore_errors: yes
   register: delete_result
   changed_when: delete_result.stdout.find("deleted") != -1 and delete_result.rc == 0
@@ -109,5 +110,6 @@
     - logging-curator
     - logging-elasticsearch
     - logging-fluentd
+    - logging-mux
   register: delete_result
   changed_when: delete_result.stdout.find("deleted") != -1 and delete_result.rc == 0

roles/openshift_logging/tasks/generate_certs.yaml

@@ -45,6 +45,21 @@
     - procure_component: kibana-internal
       hostnames: "kibana, kibana-ops, {{openshift_logging_kibana_hostname}}, {{openshift_logging_kibana_ops_hostname}}"
 
+- include: procure_server_certs.yaml
+  loop_control:
+    loop_var: cert_info
+  with_items:
+    - procure_component: mux
+      hostnames: "logging-mux, {{openshift_logging_mux_hostname}}"
+  when: openshift_logging_use_mux
+
+- include: procure_shared_key.yaml
+  loop_control:
+    loop_var: shared_key_info
+  with_items:
+    - procure_component: mux
+  when: openshift_logging_use_mux
+
 - name: Copy proxy TLS configuration file
   copy: src=server-tls.json dest={{generated_certs_dir}}/server-tls.json
   when: server_tls_json is undefined
@@ -85,6 +100,14 @@
   loop_control:
     loop_var: node_name
 
+- name: Generate PEM cert for mux
+  include: generate_pems.yaml component={{node_name}}
+  with_items:
+    - system.logging.mux
+  loop_control:
+    loop_var: node_name
+  when: openshift_logging_use_mux
+
 - name: Creating necessary JKS certs
   include: generate_jks.yaml
 

roles/openshift_logging/tasks/generate_configmaps.yaml

@@ -134,3 +134,43 @@
       when: fluentd_configmap.stdout is defined
       changed_when: no
   check_mode: no
+
+- block:
+    - copy:
+        src: fluent.conf
+        dest: "{{mktemp.stdout}}/fluent-mux.conf"
+      when: fluentd_mux_config_contents is undefined
+      changed_when: no
+
+    - copy:
+        src: secure-forward.conf
+        dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
+      when: fluentd_mux_securefoward_contents is undefined
+      changed_when: no
+
+    - copy:
+        content: "{{fluentd_mux_config_contents}}"
+        dest: "{{mktemp.stdout}}/fluent-mux.conf"
+      when: fluentd_mux_config_contents is defined
+      changed_when: no
+
+    - copy:
+        content: "{{fluentd_mux_secureforward_contents}}"
+        dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
+      when: fluentd_mux_secureforward_contents is defined
+      changed_when: no
+
+    - command: >
+        {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create configmap logging-mux
+        --from-file=fluent.conf={{mktemp.stdout}}/fluent-mux.conf
+        --from-file=secure-forward.conf={{mktemp.stdout}}/secure-forward-mux.conf -o yaml --dry-run
+      register: mux_configmap
+      changed_when: no
+
+    - copy:
+        content: "{{mux_configmap.stdout}}"
+        dest: "{{mktemp.stdout}}/templates/logging-mux-configmap.yaml"
+      when: mux_configmap.stdout is defined
+      changed_when: no
+  check_mode: no
+  when: openshift_logging_use_mux

roles/openshift_logging/tasks/generate_secrets.yaml

@@ -34,6 +34,36 @@
   check_mode: no
   changed_when: no
 
+- name: Retrieving the cert to use when generating secrets for mux
+  slurp: src="{{generated_certs_dir}}/{{item.file}}"
+  register: mux_key_pairs
+  with_items:
+    - { name: "ca_file", file: "ca.crt" }
+    - { name: "mux_key", file: "system.logging.mux.key"}
+    - { name: "mux_cert", file: "system.logging.mux.crt"}
+    - { name: "mux_shared_key", file: "mux_shared_key"}
+  when: openshift_logging_use_mux
+
+- name: Generating secrets for mux
+  template: src=secret.j2 dest={{mktemp.stdout}}/templates/{{secret_name}}-secret.yaml
+  vars:
+    secret_name: "logging-{{component}}"
+    secret_key_file: "{{component}}_key"
+    secret_cert_file: "{{component}}_cert"
+    secrets:
+      - {key: ca, value: "{{mux_key_pairs | entry_from_named_pair('ca_file')| b64decode }}"}
+      - {key: key, value: "{{mux_key_pairs | entry_from_named_pair(secret_key_file)| b64decode }}"}
+      - {key: cert, value: "{{mux_key_pairs | entry_from_named_pair(secret_cert_file)| b64decode }}"}
+      - {key: shared_key, value: "{{mux_key_pairs | entry_from_named_pair('mux_shared_key')| b64decode }}"}
+    secret_keys: ["ca", "cert", "key", "shared_key"]
+  with_items:
+    - mux
+  loop_control:
+    loop_var: component
+  check_mode: no
+  changed_when: no
+  when: openshift_logging_use_mux
+
 - name: Generating secrets for kibana proxy
   template: src=secret.j2 dest={{mktemp.stdout}}/templates/{{secret_name}}-secret.yaml
   vars:

roles/openshift_logging/tasks/generate_services.yaml

@@ -85,3 +85,35 @@
   when: openshift_logging_use_ops | bool
   check_mode: no
   changed_when: no
+
+- name: Generating logging-mux service for external connections
+  template: src=service.j2 dest={{mktemp.stdout}}/templates/logging-mux-svc.yaml
+  vars:
+    obj_name: logging-mux
+    ports:
+    - {port: "{{openshift_logging_mux_port}}", targetPort: mux-forward, name: mux-forward}
+    labels:
+      logging-infra: support
+    selector:
+      provider: openshift
+      component: mux
+    externalIPs:
+    - "{{ ansible_eth0.ipv4.address }}"
+  check_mode: no
+  changed_when: no
+  when: openshift_logging_mux_allow_external
+
+- name: Generating logging-mux service for intra-cluster connections
+  template: src=service.j2 dest={{mktemp.stdout}}/templates/logging-mux-svc.yaml
+  vars:
+    obj_name: logging-mux
+    ports:
+    - {port: "{{openshift_logging_mux_port}}", targetPort: mux-forward, name: mux-forward}
+    labels:
+      logging-infra: support
+    selector:
+      provider: openshift
+      component: mux
+  check_mode: no
+  changed_when: no
+  when: openshift_logging_use_mux and not openshift_logging_mux_allow_external

roles/openshift_logging/tasks/install_logging.yaml

@@ -27,6 +27,10 @@
   loop_control:
     loop_var: install_component
 
+- name: Install logging mux
+  include: "{{ role_path }}/tasks/install_mux.yaml"
+  when: openshift_logging_use_mux
+
 - find: paths={{ mktemp.stdout }}/templates patterns=*.yaml
   register: object_def_files
   changed_when: no

roles/openshift_logging/tasks/install_mux.yaml

@@ -0,0 +1,67 @@
+---
+- set_fact: mux_ops_host={{ (openshift_logging_use_ops | bool) | ternary(openshift_logging_es_ops_host, openshift_logging_es_host) }}
+  check_mode: no
+
+- set_fact: mux_ops_port={{ (openshift_logging_use_ops | bool) | ternary(openshift_logging_es_ops_port, openshift_logging_es_port) }}
+  check_mode: no
+
+- name: Check mux current replica count
+  command: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get dc/logging-mux
+    -o jsonpath='{.spec.replicas}' -n {{openshift_logging_namespace}}
+  register: mux_replica_count
+  when: not ansible_check_mode
+  ignore_errors: yes
+  changed_when: no
+
+- name: Generating mux deploymentconfig
+  template: src=mux.j2 dest={{mktemp.stdout}}/templates/logging-mux-dc.yaml
+  vars:
+    component: mux
+    logging_component: mux
+    deploy_name: "logging-{{component}}"
+    image: "{{openshift_logging_image_prefix}}logging-fluentd:{{openshift_logging_image_version}}"
+    es_host: logging-es
+    es_port: "{{openshift_logging_es_port}}"
+    ops_host: "{{ mux_ops_host }}"
+    ops_port: "{{ mux_ops_port }}"
+    mux_cpu_limit: "{{openshift_logging_mux_cpu_limit}}"
+    mux_memory_limit: "{{openshift_logging_mux_memory_limit}}"
+    replicas: "{{mux_replica_count.stdout | default (0)}}"
+    mux_node_selector: "{{openshift_logging_mux_nodeselector | default({})}}"
+  check_mode: no
+  changed_when: no
+
+- name: "Check mux hostmount-anyuid permissions"
+  command: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    get scc/hostmount-anyuid -o jsonpath='{.users}'
+  register: mux_hostmount_anyuid
+  check_mode: no
+  changed_when: no
+
+- name: "Set hostmount-anyuid permissions for mux"
+  command: >
+    {{ openshift.common.admin_binary}} --config={{ mktemp.stdout }}/admin.kubeconfig policy
+    add-scc-to-user hostmount-anyuid system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd
+  register: mux_output
+  failed_when: "mux_output.rc == 1 and 'exists' not in mux_output.stderr"
+  check_mode: no
+  when: mux_hostmount_anyuid.stdout.find("system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd") == -1
+
+- name: "Check mux cluster-reader permissions"
+  command: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig
+    get clusterrolebinding/cluster-readers -o jsonpath='{.userNames}'
+  register: mux_cluster_reader
+  check_mode: no
+  changed_when: no
+
+- name: "Set cluster-reader permissions for mux"
+  command: >
+    {{ openshift.common.admin_binary}} --config={{ mktemp.stdout }}/admin.kubeconfig policy
+    add-cluster-role-to-user cluster-reader system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd
+  register: mux2_output
+  failed_when: "mux2_output.rc == 1 and 'exists' not in mux2_output.stderr"
+  check_mode: no
+  when: mux_cluster_reader.stdout.find("system:serviceaccount:{{openshift_logging_namespace}}:aggregated-logging-fluentd") == -1

roles/openshift_logging/tasks/procure_shared_key.yaml

@@ -0,0 +1,25 @@
+---
+- name: Checking for {{ shared_key_info.procure_component }}_shared_key
+  stat: path="{{generated_certs_dir}}/{{ shared_key_info.procure_component }}_shared_key"
+  register: component_shared_key_file
+  check_mode: no
+
+- name: Trying to discover shared key variable name for {{ shared_key_info.procure_component }}
+  set_fact: procure_component_shared_key={{ lookup('env', '{{shared_key_info.procure_component}}' + '_shared_key') }}
+  when:
+  - shared_key_info[ shared_key_info.procure_component + '_shared_key' ] is defined
+  check_mode: no
+
+- name: Creating shared_key for {{ shared_key_info.procure_component }}
+  copy: content="{{'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'|random_word(64)}}"
+        dest="{{generated_certs_dir}}/{{shared_key_info.procure_component}}_shared_key"
+  check_mode: no
+  when:
+  - not component_shared_key_file.stat.exists
+
+- name: Copying shared key for {{ shared_key_info.procure_component }} to generated certs directory
+  copy: content="{{procure_component_shared_key}}" dest="{{generated_certs_dir}}/{{shared_key_info.procure_component}}_shared_key"
+  check_mode: no
+  when:
+  - shared_key_info[ shared_key_info.procure_component + '_shared_key' ] is defined
+  - not component_shared_key_file.stat.exists

roles/openshift_logging/tasks/start_cluster.yaml

@@ -21,6 +21,26 @@
   loop_control:
     loop_var: fluentd_host
 
+- name: Retrieve mux
+  oc_obj:
+    state: list
+    kind: dc
+    selector: "component=mux"
+    namespace: "{{openshift_logging_namespace}}"
+  register: mux_dc
+  when: openshift_logging_use_mux
+
+- name: start mux
+  oc_scale:
+    kind: dc
+    name: "{{ object }}"
+    namespace: "{{openshift_logging_namespace}}"
+    replicas: "{{ openshift_logging_mux_replica_count | default (1) }}"
+  with_items: "{{ mux_dc.results.results[0]['items'] | map(attribute='metadata.name') | list }}"
+  loop_control:
+    loop_var: object
+  when: openshift_logging_use_mux
+
 - name: Retrieve elasticsearch
   oc_obj:
     state: list

roles/openshift_logging/tasks/stop_cluster.yaml

@@ -21,6 +21,26 @@
   loop_control:
     loop_var: fluentd_host
 
+- name: Retrieve mux
+  oc_obj:
+    state: list
+    kind: dc
+    selector: "component=mux"
+    namespace: "{{openshift_logging_namespace}}"
+  register: mux_dc
+  when: openshift_logging_use_mux
+
+- name: stop mux
+  oc_scale:
+    kind: dc
+    name: "{{ object }}"
+    namespace: "{{openshift_logging_namespace}}"
+    replicas: 0
+  with_items: "{{ mux_dc.results.results[0]['items'] | map(attribute='metadata.name') | list }}"
+  loop_control:
+    loop_var: object
+  when: openshift_logging_use_mux
+
 - name: Retrieve elasticsearch
   oc_obj:
     state: list

roles/openshift_logging/templates/fluentd.j2

@@ -59,6 +59,11 @@ spec:
         - name: dockercfg
           mountPath: /etc/sysconfig/docker
           readOnly: true
+{% if openshift_logging_use_mux_client | bool %}
+        - name: muxcerts
+          mountPath: /etc/fluent/muxkeys
+          readOnly: true
+{% endif %}
         env:
         - name: "K8S_HOST_URL"
           value: "{{openshift_logging_master_url}}"
@@ -122,6 +127,8 @@ spec:
           value: "{{openshift_logging_fluentd_journal_source | default('')}}"
         - name: "JOURNAL_READ_FROM_HEAD"
           value: "{{openshift_logging_fluentd_journal_read_from_head|lower}}"
+        - name: "USE_MUX_CLIENT"
+          value: "{{openshift_logging_use_mux_client| default('false')}}"
       volumes:
       - name: runlogjournal
         hostPath:
@@ -147,3 +154,8 @@ spec:
       - name: dockercfg
         hostPath:
           path: /etc/sysconfig/docker
+{% if openshift_logging_use_mux_client | bool %}
+      - name: muxcerts
+        secret:
+          secretName: logging-mux
+{% endif %}