diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf0f8b3d..48966ca4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,20 @@
# Change Log
+## 2023-07-24
+### Added
+- Support Fluentd's [Multi Process Workers](https://docs.fluentd.org/deployment/multi-process-workers).
+- Custom Container Image for Fluentd using OL8-Slim as base Image.
+- PV, PVC Objects Collection
+### Changed
+- Instructions and dependency versions updates to custom container image for Fluentd using OL8 as base image.
+- ClusterRole updates to add read permission for `storage.k8s.io` api group to support PV, PVC Objects collection.
+### Deprecating
+- Custom Container Image for Fluentd using Debian and OL8 as base Image.
+
+## 2023-07-19
+### Added
+- Helm repo throguh Github pages.
+
## 2023-06-14
### Added
- Kubernetes Metrics Collection to OCI Monitoring using OCI Management Agent.
diff --git a/README.md b/README.md
index 006d878f..e9776848 100644
--- a/README.md
+++ b/README.md
@@ -138,7 +138,7 @@ Dashboards needs to be imported manually. Below is an example for importing Dash
1. Download and configure [OCI CLI](https://docs.oracle.com/en-us/iaas/Content/API/SDKDocs/cliinstall.htm) or open cloud-shell where OCI CLI is pre-installed. Alternative methods like REST API, SDK, Terraform etc can also be used.
2. Find the **OCID** of the compartment, where the dashboards need to be imported.
-3. Download the dashboard JSONs from [here](logan/terraform/oke/modules/dashboards/dashboards_json/) (TBD).
+3. Download the dashboard JSONs from [here](terraform/modules/dashboards/dashboards_json/).
4. **Replace** all the instances of the keyword - "`${compartment_ocid}`" in the JSONs with the **Compartment OCID** identified in previous step.
* Following command is for quick reference that can be used in a linux/cloud-shell envirnment :
@@ -153,6 +153,7 @@ Dashboards needs to be imported manually. Below is an example for importing Dash
oci management-dashboard dashboard import --from-json file://node.json
oci management-dashboard dashboard import --from-json file://workload.json
oci management-dashboard dashboard import --from-json file://pod.json
+ oci management-dashboard dashboard import --from-json file://service-type-lb.json
```
##### 4 Uninstall
diff --git a/charts/common/templates/clusterrole.yaml b/charts/common/templates/clusterrole.yaml
index 7489230e..3777c82a 100644
--- a/charts/common/templates/clusterrole.yaml
+++ b/charts/common/templates/clusterrole.yaml
@@ -23,6 +23,7 @@ rules:
- batch
- discovery.k8s.io
- metrics.k8s.io
+ - storage.k8s.io
resources:
- '*'
verbs:
diff --git a/charts/logan/Chart.yaml b/charts/logan/Chart.yaml
index ef1968b7..463a88e5 100644
--- a/charts/logan/Chart.yaml
+++ b/charts/logan/Chart.yaml
@@ -5,11 +5,11 @@ apiVersion: v2
name: oci-onm-logan
description: Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Objects information to OCI Logging Analytics.
type: application
-version: 3.0.1
+version: 3.0.2
appVersion: "3.0.0"
dependencies:
- name: oci-onm-common
- version: "3.0.1"
+ version: "3.0.2"
repository: "file://../common"
condition: oci-onm-common.enabled
diff --git a/charts/logan/templates/logs-configmap.yaml b/charts/logan/templates/logs-configmap.yaml
index 4c3adcc0..5933ef75 100644
--- a/charts/logan/templates/logs-configmap.yaml
+++ b/charts/logan/templates/logs-configmap.yaml
@@ -11,89 +11,59 @@ metadata:
data:
# file-like keys
fluent.conf: |
+
{{- $authtype := .Values.authtype | lower }}
- @include kubernetes.conf
-
- # Filter to add kubernetes metadata
-
- @type kubernetes_metadata
- @id filter_kube_metadata
- {{- if .Values.fluentd.kubernetesMetadataFilter.kubernetes_url }}
- kubernetes_url "{{ .Values.fluentd.kubernetesMetadataFilter.kubernetes_url }}"
- {{- end }}
- verify_ssl "{{ .Values.fluentd.kubernetesMetadataFilter.verify_ssl }}"
- {{- if .Values.fluentd.kubernetesMetadataFilter.ca_file }}
- ca_file "{{ .Values.fluentd.kubernetesMetadataFilter.ca_file }}"
- {{- end }}
- skip_labels "{{ .Values.fluentd.kubernetesMetadataFilter.skip_labels }}"
- skip_container_metadata "{{ .Values.fluentd.kubernetesMetadataFilter.skip_container_metadata }}"
- skip_master_url "{{ .Values.fluentd.kubernetesMetadataFilter.skip_master_url }}"
- skip_namespace_metadata "{{ .Values.fluentd.kubernetesMetadataFilter.skip_namespace_metadata }}"
- watch "{{ .Values.fluentd.kubernetesMetadataFilter.watch }}"
- de_dot false
- annotation_match [ ".*" ]
-
-
- # Match block to ensure all the logs including concat plugin timeout logs will have same label
-
- @type relabel
- @label @NORMAL
-
-
- # Match block to set info required for oci-logging-analytics fluentd output plugin
-
- # kubernetes config file data which is included in main fluentd config file.
- kubernetes.conf: |
{{- $runtime := .Values.runtime | lower }}
{{- $excludePath := join ", " .Values.fluentd.genericContainerLogs.exclude_path }}
+
+ {{- $multiWorkersEnabled := false }}
+ {{- $workers := (int .Values.fluentd.multiProcessWorkers | default 0) }}
+ {{- if gt $workers 0 }}
+ {{- $multiWorkersEnabled = true }}
+
+ workers {{ $workers }}
+
+ {{- else }}
+ {{- /* fake it to run at least one range loop if no multiProcessWorkers enabled. */}}
+ {{- $workers = 1 }}
+ {{- end }}
+
+ {{- range until $workers }}
+ {{- $currWorker := . }}
+
+ {{- if $multiWorkersEnabled }}
+
+ {{- end }}
+
# To ignore all the fluentd core generated events
-
{{- end }}
+ {{- end }}
+
diff --git a/charts/logan/values.yaml b/charts/logan/values.yaml
index 33951228..0561df70 100644
--- a/charts/logan/values.yaml
+++ b/charts/logan/values.yaml
@@ -151,6 +151,10 @@ fluentd:
file: fluent.conf
# -- Base directory on the node (with read write permission) for storing fluentd plugins related data.
baseDir: /var/log
+ # -- To enable MultiProcessWorkers, set this to (> 0) the number of required workers. Defaults to 0.
+ # -- Set corresponding workerId using worker field against each source/log. When enabled MultiProcessWorkers, the default worker would be 0 until unless set for each source/log.
+ multiProcessWorkers: 0
+
# Configuration for oci-logging-analytics fluentd output plugin
ociLoggingAnalyticsOutputPlugin:
# -- OCI API Key profile to use, if multiple profiles are found in the OCI API config file.
@@ -229,6 +233,9 @@ fluentd:
#"Third Key": "Third Value"
#ociLAEntityID:
#encoding:
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker:
+
logs:
# -- Kube Proxy logs collection configuration
kube-proxy:
@@ -245,6 +252,8 @@ fluentd:
#ociLAEntityID:
#ociLALogGroupID:
#encoding:
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker:
# -- Kube Flannel logs collection configuration
kube-flannel:
@@ -296,6 +305,16 @@ fluentd:
multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
# Configuration for Linux System specific logs like CronLogs and SecureLogs
linuxSystem:
+ # Setting the following properties will override the default/generic configuration and applies to all Kubernetes system logs
+ #ociLALogGroupID:
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLAEntityID:
+ #encoding:
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker:
logs:
# -- Linux CRON logs collection configuration
cronlog:
@@ -375,19 +394,13 @@ fluentd:
- '"/var/log/containers/csi-oci-node-*.log"'
- '"/var/log/containers/proxymux-client-*.log"'
- '"/var/log/containers/cluster-autoscaler-*.log"'
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker: 1
# -- Configuration for any custom logs which are not part of the default configuration defined in this file.
# All the pod/container logs will be collected as per "genericContainerLogs" section.
# Use this section to create a custom configuration for any of the container logs.
# Also, you can use this section to define configuration for any other log path existing on a Kubernetes worker node
- #custom-id1:
- #path: /var/log/containers/custom*.log
- # Logging Analytics log source to use for parsing and processing the logs:
- #ociLALogSourceName: "Custom1 Logs"
- # The regular expression pattern for the starting line in case of multi-line logs.
- #multilineStartRegExp:
- # Set isContainerLog to false if the log is not a container log (/var/log/containers/*.log). Default value is true.
- #isContainerLog: true
customLogs:
# A unique identifier to represent the configuration for a single log path
#custom-id1:
@@ -398,6 +411,16 @@ fluentd:
#multilineStartRegExp:
# Set isContainerLog to false if the log is not a container log (/var/log/containers/*.log). Default value is true.
#isContainerLog: true
+ # Setting the following properties will override the default/generic configuration and applies to all Kubernetes system logs
+ #ociLALogGroupID:
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLAEntityID:
+ #encoding:
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker:
#custom-id2:
#path: /var/log/custom/*.log
# Logging Analytics log source to use for parsing and processing the logs:
@@ -436,6 +459,10 @@ fluentd:
api_endpoint: ""
events:
api_endpoint: ""
+ persistent_volumes:
+ api_endpoint: ""
+ persistent_volume_claims:
+ api_endpoint: ""
daemon_sets:
api_endpoint: apis/apps
replica_sets:
diff --git a/charts/oci-onm/Chart.yaml b/charts/oci-onm/Chart.yaml
index 61f61f66..fed4dc14 100644
--- a/charts/oci-onm/Chart.yaml
+++ b/charts/oci-onm/Chart.yaml
@@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.0.1
+version: 3.0.2
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
@@ -28,11 +28,11 @@ appVersion: "3.0.0"
dependencies:
- name: oci-onm-common
- version: "3.0.1"
+ version: "3.0.2"
repository: "file://../common"
condition: oci-onm-common.enabled
- name: oci-onm-logan
- version: "3.0.1"
+ version: "3.0.2"
repository: "file://../logan"
condition: oci-onm-logan.enabled
- name: oci-onm-mgmt-agent
diff --git a/docs/FAQ.md b/docs/FAQ.md
index 9ed21f8e..e7dba40c 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -195,4 +195,83 @@ oci-onm-logan:
# -----BEGIN RSA PRIVATE KEY-----
# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
# -----END RSA PRIVATE KEY-----
+```
+
+### Enable Multi Process Workers
+
+**Note**: This feature will not work with custom fluentd configuration and if you set custom fluentd configuration, it would be placed under worker 0.
+
+To enable multi-process workers feature of Fluentd, set value of `multiProcessWorkers` under oci-onm-logan to the desired number of workers. By default it is set 0, indicating that the feature is disabled. To assign different workers for different logs, you may set `worker` field to the desired worker id against each of the individual logs or group of logs as supported in the confguration. The default worker id is set to 0 for all the logs when enabling `multiProcessWorkers` feature. The following are few examples,
+
+#### Example1
+
+Enable multi-process worker feature with 2 workers and set the worker id 1 to all container logs (except custom logs) and keeping the default worker id (0) for the remaining all logs.
+
```
+..
+..
+oci-onm-logan:
+ ..
+ ..
+ fluentd:
+ ...
+ ...
+ multiProcessWorkers: 2
+ ...
+ ...
+ genericContainerLogs:
+ ...
+ ...
+ worker: 1
+```
+
+#### Example2
+
+Enable multi-process worker feature with 3 workers and set the worker id 1 to all container logs (except custom logs), worker id 2 to Kube Proxy, Linux Syslog and Kubelet logs, and keeping the default worker id (0) for the remaining all logs.
+
+```
+..
+..
+oci-onm-logan:
+ ..
+ ..
+ fluentd:
+ ...
+ ...
+ multiProcessWorkers: 3
+ ...
+ ...
+ kubernetesSystem:
+ ...
+ ...
+ logs:
+ ...
+ ...
+ kube-proxy:
+ ...
+ ...
+ worker: 2
+ ...
+ ...
+ ...
+ ...
+ linuxSystem:
+ ...
+ ...
+ logs:
+ ...
+ ...
+ syslog:
+ ...
+ ...
+ worker: 2
+ ...
+ ...
+ ...
+ ...
+ genericContainerLogs:
+ ...
+ ...
+ worker: 1
+```
+
diff --git a/docs/custom-images.md b/docs/custom-images.md
index bf5da24e..97e18ab0 100644
--- a/docs/custom-images.md
+++ b/docs/custom-images.md
@@ -12,7 +12,7 @@ By default, pre-built images by Oracle are used.
##### Fluentd Container Image
- Download all the files from the below mentioned dir into a local machine having access to internet and docker installed.
- - [OL8](logan/docker-images/v1.0/oraclelinux/8/)
+ - [OL8-Slim](logan/docker-images/v1.0/oraclelinux/8-slim/)
- Run the following command to build the image.
- `docker build -t oci-la-fluentd-collector-custom -f Dockerfile .`
- The docker image built from the above step, can either be pushed to Docker Hub or OCI Container Registry (OCIR) or to a Local Docker Registry depending on the requirements.
diff --git a/logan/docker-images/v1.0/oraclelinux/8-slim/Dockerfile b/logan/docker-images/v1.0/oraclelinux/8-slim/Dockerfile
new file mode 100644
index 00000000..973b5c96
--- /dev/null
+++ b/logan/docker-images/v1.0/oraclelinux/8-slim/Dockerfile
@@ -0,0 +1,88 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+### Build the docker image using multi-stage build
+
+## To build/install all the dependencies
+
+FROM container-registry.oracle.com/os/oraclelinux:8-slim AS builder
+
+USER root
+WORKDIR /fluentd
+
+# Environment variables
+ENV PATH /fluentd/vendor/bundle/ruby/2.7.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.7.0:$GEM_PATH
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.7.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+COPY Gemfile* /fluentd/
+
+# Install ruby, ruby-libs along with rubygems and bundler.
+RUN microdnf -y module enable ruby:2.7 \
+# Install ruby (it's dependencies gdbm-libs) and ruby-libs, disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs ruby-2.7.6 ruby-libs-2.7.6 gdbm-libs \
+# Install rubygems (it's dependencies rubygem-openssl rubygem-psych), disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs rubygems-3.1.6 \
+ && gem install bundler -v 2.3.25 \
+# Install development dependent packages for gems native installation
+ && microdnf -y install --nodocs gcc make redhat-rpm-config openssl ruby-devel gcc-c++ libtool libffi-devel bzip2 git \
+# Install Fluentd, it's dependencies along with other run time dependencies for OCI Logging Analytics Solution
+ && bundle config silence_root_warning true \
+ && bundle config --local path /fluentd/vendor/bundle \
+ && bundle config --global jobs 9 \
+ && bundle install --gemfile=/fluentd/Gemfile \
+# Install tini, init for containers (from EPEL repo)
+ && microdnf -y install oracle-epel-release-el8 \
+ && microdnf -y install tini-0.19.0 \
+# Install jemalloc (custom make with no docs)
+ && cd /tmp && ls /tmp \
+ && git clone -b 5.3.0 https://github.com/jemalloc/jemalloc.git && cd jemalloc/ \
+ && ./autogen.sh && make && make install_bin install_include install_lib \
+ && mv lib/libjemalloc.so.2 /usr/lib
+
+## To build the final docker image
+
+FROM container-registry.oracle.com/os/oraclelinux:8-slim
+
+USER root
+WORKDIR /fluentd
+
+# Environment variables
+ENV PATH /fluentd/vendor/bundle/ruby/2.7.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.7.0:$GEM_PATH
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.7.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+# Install ruby, ruby-libs along with rubygems and bundler.
+RUN microdnf -y module enable ruby:2.7 \
+# Install ruby (it's dependencies gdbm-libs) and ruby-libs, disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs ruby-2.7.6 ruby-libs-2.7.6 gdbm-libs \
+# Install rubygems (it's dependencies rubygem-openssl rubygem-psych), disabling week dependencies
+ && microdnf -y install --setopt=install_weak_deps=0 --nodocs rubygems-3.1.6 \
+ && gem install bundler -v 2.3.25 \
+ && bundle config --local path /fluentd/vendor/bundle \
+# clear caches
+ && microdnf clean all \
+ && rm -rf /var/cache/dnf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+# Copy binaries (tini & jemallco) and rubygems bundler environment from build stage
+COPY --from=builder /fluentd /fluentd
+COPY --from=builder /usr/bin/tini /usr/bin/tini
+COPY --from=builder /usr/lib/libjemalloc.so.2 /usr/lib/libjemalloc.so.2
+
+RUN mkdir -p /fluentd/etc /fluentd/plugins \
+ && touch /fluentd/etc/disable.conf
+
+# Environment variables
+ENV FLUENTD_CONF="/fluentd/etc/fluent.conf"
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+COPY entrypoint.sh /fluentd/entrypoint.sh
+# Give execution permission to entrypoint.sh
+RUN ["chmod", "+x", "/fluentd/entrypoint.sh"]
+
+# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib
+ENTRYPOINT ["tini", "--", "/fluentd/entrypoint.sh"]
diff --git a/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile b/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile
new file mode 100644
index 00000000..c24ac7e1
--- /dev/null
+++ b/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile
@@ -0,0 +1,15 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+source "https://rubygems.org"
+
+gem "oj", "3.14.1"
+gem "json", "2.6.3"
+gem "ext_monitor", "0.1.2"
+gem "fluentd", "1.15.3"
+gem "fluent-plugin-oci-logging-analytics", "2.0.5"
+gem "fluent-plugin-concat", "~> 2.5.0"
+gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0"
+gem "fluent-plugin-parser-cri", "~> 0.1.1"
+gem "fluent-plugin-kubernetes_metadata_filter", "2.13.0"
+gem "fluent-plugin-kubernetes-objects", "1.2.1"
diff --git a/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile.lock b/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile.lock
new file mode 100644
index 00000000..8ce51250
--- /dev/null
+++ b/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile.lock
@@ -0,0 +1,124 @@
+GEM
+ remote: https://rubygems.org/
+ specs:
+ addressable (2.8.1)
+ public_suffix (>= 2.0.2, < 6.0)
+ concurrent-ruby (1.2.0)
+ cool.io (1.7.1)
+ domain_name (0.5.20190701)
+ unf (>= 0.0.5, < 1.0.0)
+ ext_monitor (0.1.2)
+ ffi (1.15.5)
+ ffi-compiler (1.0.1)
+ ffi (>= 1.0.0)
+ rake
+ fluent-config-regexp-type (1.0.0)
+ fluentd (> 1.0.0, < 2)
+ fluent-plugin-concat (2.5.0)
+ fluentd (>= 0.14.0, < 2)
+ fluent-plugin-kubernetes-objects (1.2.1)
+ fluentd (>= 1.9.1)
+ http_parser.rb (= 0.8.0)
+ kubeclient (~> 4.9.3)
+ fluent-plugin-kubernetes_metadata_filter (2.13.0)
+ fluentd (>= 0.14.0, < 1.16)
+ kubeclient (>= 4.0.0, < 5.0.0)
+ lru_redux
+ fluent-plugin-oci-logging-analytics (2.0.5)
+ fluentd (>= 0.14.10, < 2)
+ oci (~> 2.16)
+ prometheus-client (~> 4.0.0)
+ rubyzip (~> 2.3.2)
+ yajl-ruby (~> 1.4, >= 1.4.3)
+ fluent-plugin-parser-cri (0.1.1)
+ fluentd (>= 1)
+ fluent-plugin-rewrite-tag-filter (2.4.0)
+ fluent-config-regexp-type
+ fluentd (>= 0.14.2, < 2)
+ fluentd (1.15.3)
+ bundler
+ cool.io (>= 1.4.5, < 2.0.0)
+ http_parser.rb (>= 0.5.1, < 0.9.0)
+ msgpack (>= 1.3.1, < 2.0.0)
+ serverengine (>= 2.3.0, < 3.0.0)
+ sigdump (~> 0.2.2)
+ strptime (>= 0.2.4, < 1.0.0)
+ tzinfo (>= 1.0, < 3.0)
+ tzinfo-data (~> 1.0)
+ webrick (>= 1.4.2, < 1.8.0)
+ yajl-ruby (~> 1.0)
+ http (4.4.1)
+ addressable (~> 2.3)
+ http-cookie (~> 1.0)
+ http-form_data (~> 2.2)
+ http-parser (~> 1.2.0)
+ http-accept (1.7.0)
+ http-cookie (1.0.5)
+ domain_name (~> 0.5)
+ http-form_data (2.3.0)
+ http-parser (1.2.3)
+ ffi-compiler (>= 1.0, < 2.0)
+ http_parser.rb (0.8.0)
+ inifile (3.0.0)
+ json (2.6.3)
+ jsonpath (1.1.2)
+ multi_json
+ jwt (2.7.0)
+ kubeclient (4.9.3)
+ http (>= 3.0, < 5.0)
+ jsonpath (~> 1.0)
+ recursive-open-struct (~> 1.1, >= 1.1.1)
+ rest-client (~> 2.0)
+ lru_redux (1.1.0)
+ mime-types (3.4.1)
+ mime-types-data (~> 3.2015)
+ mime-types-data (3.2022.0105)
+ msgpack (1.7.1)
+ multi_json (1.15.0)
+ netrc (0.11.0)
+ oci (2.18.0)
+ inifile (~> 3.0, >= 3.0.0)
+ json (>= 1.4.6, < 3.0.0)
+ jwt (~> 2.1)
+ oj (3.14.1)
+ prometheus-client (4.0.0)
+ public_suffix (5.0.1)
+ rake (13.0.6)
+ recursive-open-struct (1.1.3)
+ rest-client (2.1.0)
+ http-accept (>= 1.7.0, < 2.0)
+ http-cookie (>= 1.0.2, < 2.0)
+ mime-types (>= 1.16, < 4.0)
+ netrc (~> 0.8)
+ rubyzip (2.3.2)
+ serverengine (2.3.1)
+ sigdump (~> 0.2.2)
+ sigdump (0.2.4)
+ strptime (0.2.5)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ tzinfo-data (1.2022.7)
+ tzinfo (>= 1.0.0)
+ unf (0.1.4)
+ unf_ext
+ unf_ext (0.0.8.2)
+ webrick (1.7.0)
+ yajl-ruby (1.4.3)
+
+PLATFORMS
+ x86_64-linux
+
+DEPENDENCIES
+ ext_monitor (= 0.1.2)
+ fluent-plugin-concat (~> 2.5.0)
+ fluent-plugin-kubernetes-objects (= 1.2.1)
+ fluent-plugin-kubernetes_metadata_filter (= 2.13.0)
+ fluent-plugin-oci-logging-analytics (= 2.0.5)
+ fluent-plugin-parser-cri (~> 0.1.1)
+ fluent-plugin-rewrite-tag-filter (~> 2.4.0)
+ fluentd (= 1.15.3)
+ json (= 2.6.3)
+ oj (= 3.14.1)
+
+BUNDLED WITH
+ 2.3.25
diff --git a/logan/docker-images/v1.0/oraclelinux/8-slim/entrypoint.sh b/logan/docker-images/v1.0/oraclelinux/8-slim/entrypoint.sh
new file mode 100644
index 00000000..04709ea7
--- /dev/null
+++ b/logan/docker-images/v1.0/oraclelinux/8-slim/entrypoint.sh
@@ -0,0 +1,6 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+#!/usr/bin/env sh
+
+bundle exec fluentd -c ${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}
diff --git a/logan/docker-images/v1.0/oraclelinux/8/Dockerfile b/logan/docker-images/v1.0/oraclelinux/8/Dockerfile
index fcdc7973..e83717c4 100644
--- a/logan/docker-images/v1.0/oraclelinux/8/Dockerfile
+++ b/logan/docker-images/v1.0/oraclelinux/8/Dockerfile
@@ -1,7 +1,9 @@
# Copyright (c) 2023, Oracle and/or its affiliates.
# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
-FROM container-registry.oracle.com/os/oraclelinux:8
+## To build/install all the dependencies
+
+FROM container-registry.oracle.com/os/oraclelinux:8 AS builder
USER root
WORKDIR /fluentd
@@ -14,15 +16,15 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.7.0
ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
ENV TINI_VERSION=0.19.0
-COPY Gemfile /fluentd/
+COPY Gemfile* /fluentd/
# Install ruby 2.7 along with rubygems and bundler.
RUN dnf -y module enable ruby:2.7 \
- && dnf -y install --nodocs ruby ruby-libs \
- && dnf -y install --nodocs rubygems rubygem-openssl rubygem-psych \
- && dnf -y install --nodocs rubygem-bundler rubygem-io-console \
+ && dnf -y install --setopt=install_weak_deps=False --nodocs ruby-2.7.6 ruby-libs-2.7.6 gdbm-libs \
+ && dnf -y install --setopt=install_weak_deps=False --nodocs rubygems-3.1.6 \
+ && gem install bundler -v 2.3.25 \
# Install development dependent packages for gems native installation
- && dnf -y install --nodocs gcc make redhat-rpm-config openssl ruby-devel gcc-c++ libtool libffi-devel bzip2 \
+ && dnf -y install --nodocs gcc make redhat-rpm-config openssl ruby-devel-2.7.6 gcc-c++ libtool libffi-devel bzip2 \
# Install Fluentd, it's dependencies along with other run time dependencies for OCI Logging Analytics Solution
&& bundle config silence_root_warning true \
&& bundle config --local path /fluentd/vendor/bundle \
@@ -31,20 +33,43 @@ RUN dnf -y module enable ruby:2.7 \
&& curl -L -o /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini" \
&& curl -L -o /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
&& export GNUPGHOME="$(mktemp -d)" \
- && gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
+ && gpg --batch --keyserver keyserver.ubuntu.com \
+ --recv-keys 595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
&& gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
&& chmod +x /usr/local/bin/tini \
# Install jemalloc
- && curl -L -o /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
- && cd /tmp && tar -xjf jemalloc-4.5.0.tar.bz2 && cd jemalloc-4.5.0/ \
+ && curl -L -o /tmp/jemalloc-5.3.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 \
+ && cd /tmp && tar -xjf jemalloc-5.3.0.tar.bz2 && cd jemalloc-5.3.0/ \
&& ./configure && make \
- && mv lib/libjemalloc.so.2 /usr/lib \
-# Install hostname, required by fluent-plugin-rewrite-tag-filter
- && dnf -y install hostname \
-# Remove all the development dependent packages
- && dnf -y remove gcc make redhat-rpm-config openssl ruby-devel gcc-c++ libtool libffi-devel bzip2 \
+ && mv lib/libjemalloc.so.2 /usr/lib
+
+## To build the final docker image
+
+FROM container-registry.oracle.com/os/oraclelinux:8
+
+USER root
+WORKDIR /fluentd
+
+# Environment variables
+ENV PATH /fluentd/vendor/bundle/ruby/2.7.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.7.0:$GEM_PATH
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.7.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+# Install ruby 2.7 along with rubygems and bundler.
+RUN dnf -y module enable ruby:2.7 \
+ && dnf -y install --setopt=install_weak_deps=False --nodocs ruby-2.7.6 ruby-libs-2.7.6 gdbm-libs \
+ && dnf -y install --setopt=install_weak_deps=False --nodocs rubygems-3.1.6 \
+ && gem install bundler -v 2.3.25 \
+# clear cache
&& dnf clean all \
&& rm -rf /var/cache/dnf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+# Copy binaries (tini & jemallco) and rubygems bundler environment from build stage
+COPY --from=builder /fluentd /fluentd
+COPY --from=builder /usr/local/bin/tini /usr/bin/tini
+COPY --from=builder /usr/lib/libjemalloc.so.2 /usr/lib/libjemalloc.so.2
RUN mkdir -p /fluentd/etc /fluentd/plugins \
&& touch /fluentd/etc/disable.conf
diff --git a/logan/docker-images/v1.0/oraclelinux/8/Gemfile b/logan/docker-images/v1.0/oraclelinux/8/Gemfile
index 0df012dc..c24ac7e1 100644
--- a/logan/docker-images/v1.0/oraclelinux/8/Gemfile
+++ b/logan/docker-images/v1.0/oraclelinux/8/Gemfile
@@ -3,13 +3,13 @@
source "https://rubygems.org"
-gem "oj", "3.10.18"
-gem "json", "2.4.1"
+gem "oj", "3.14.1"
+gem "json", "2.6.3"
gem "ext_monitor", "0.1.2"
-gem "fluentd", "1.14.3"
-gem "fluent-plugin-oci-logging-analytics", "2.0.3"
+gem "fluentd", "1.15.3"
+gem "fluent-plugin-oci-logging-analytics", "2.0.5"
gem "fluent-plugin-concat", "~> 2.5.0"
gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0"
gem "fluent-plugin-parser-cri", "~> 0.1.1"
-gem "fluent-plugin-kubernetes_metadata_filter", "2.9.5"
-gem "fluent-plugin-kubernetes-objects", "1.1.12"
+gem "fluent-plugin-kubernetes_metadata_filter", "2.13.0"
+gem "fluent-plugin-kubernetes-objects", "1.2.1"
diff --git a/logan/docker-images/v1.0/oraclelinux/8/Gemfile.lock b/logan/docker-images/v1.0/oraclelinux/8/Gemfile.lock
new file mode 100644
index 00000000..8ce51250
--- /dev/null
+++ b/logan/docker-images/v1.0/oraclelinux/8/Gemfile.lock
@@ -0,0 +1,124 @@
+GEM
+ remote: https://rubygems.org/
+ specs:
+ addressable (2.8.1)
+ public_suffix (>= 2.0.2, < 6.0)
+ concurrent-ruby (1.2.0)
+ cool.io (1.7.1)
+ domain_name (0.5.20190701)
+ unf (>= 0.0.5, < 1.0.0)
+ ext_monitor (0.1.2)
+ ffi (1.15.5)
+ ffi-compiler (1.0.1)
+ ffi (>= 1.0.0)
+ rake
+ fluent-config-regexp-type (1.0.0)
+ fluentd (> 1.0.0, < 2)
+ fluent-plugin-concat (2.5.0)
+ fluentd (>= 0.14.0, < 2)
+ fluent-plugin-kubernetes-objects (1.2.1)
+ fluentd (>= 1.9.1)
+ http_parser.rb (= 0.8.0)
+ kubeclient (~> 4.9.3)
+ fluent-plugin-kubernetes_metadata_filter (2.13.0)
+ fluentd (>= 0.14.0, < 1.16)
+ kubeclient (>= 4.0.0, < 5.0.0)
+ lru_redux
+ fluent-plugin-oci-logging-analytics (2.0.5)
+ fluentd (>= 0.14.10, < 2)
+ oci (~> 2.16)
+ prometheus-client (~> 4.0.0)
+ rubyzip (~> 2.3.2)
+ yajl-ruby (~> 1.4, >= 1.4.3)
+ fluent-plugin-parser-cri (0.1.1)
+ fluentd (>= 1)
+ fluent-plugin-rewrite-tag-filter (2.4.0)
+ fluent-config-regexp-type
+ fluentd (>= 0.14.2, < 2)
+ fluentd (1.15.3)
+ bundler
+ cool.io (>= 1.4.5, < 2.0.0)
+ http_parser.rb (>= 0.5.1, < 0.9.0)
+ msgpack (>= 1.3.1, < 2.0.0)
+ serverengine (>= 2.3.0, < 3.0.0)
+ sigdump (~> 0.2.2)
+ strptime (>= 0.2.4, < 1.0.0)
+ tzinfo (>= 1.0, < 3.0)
+ tzinfo-data (~> 1.0)
+ webrick (>= 1.4.2, < 1.8.0)
+ yajl-ruby (~> 1.0)
+ http (4.4.1)
+ addressable (~> 2.3)
+ http-cookie (~> 1.0)
+ http-form_data (~> 2.2)
+ http-parser (~> 1.2.0)
+ http-accept (1.7.0)
+ http-cookie (1.0.5)
+ domain_name (~> 0.5)
+ http-form_data (2.3.0)
+ http-parser (1.2.3)
+ ffi-compiler (>= 1.0, < 2.0)
+ http_parser.rb (0.8.0)
+ inifile (3.0.0)
+ json (2.6.3)
+ jsonpath (1.1.2)
+ multi_json
+ jwt (2.7.0)
+ kubeclient (4.9.3)
+ http (>= 3.0, < 5.0)
+ jsonpath (~> 1.0)
+ recursive-open-struct (~> 1.1, >= 1.1.1)
+ rest-client (~> 2.0)
+ lru_redux (1.1.0)
+ mime-types (3.4.1)
+ mime-types-data (~> 3.2015)
+ mime-types-data (3.2022.0105)
+ msgpack (1.7.1)
+ multi_json (1.15.0)
+ netrc (0.11.0)
+ oci (2.18.0)
+ inifile (~> 3.0, >= 3.0.0)
+ json (>= 1.4.6, < 3.0.0)
+ jwt (~> 2.1)
+ oj (3.14.1)
+ prometheus-client (4.0.0)
+ public_suffix (5.0.1)
+ rake (13.0.6)
+ recursive-open-struct (1.1.3)
+ rest-client (2.1.0)
+ http-accept (>= 1.7.0, < 2.0)
+ http-cookie (>= 1.0.2, < 2.0)
+ mime-types (>= 1.16, < 4.0)
+ netrc (~> 0.8)
+ rubyzip (2.3.2)
+ serverengine (2.3.1)
+ sigdump (~> 0.2.2)
+ sigdump (0.2.4)
+ strptime (0.2.5)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ tzinfo-data (1.2022.7)
+ tzinfo (>= 1.0.0)
+ unf (0.1.4)
+ unf_ext
+ unf_ext (0.0.8.2)
+ webrick (1.7.0)
+ yajl-ruby (1.4.3)
+
+PLATFORMS
+ x86_64-linux
+
+DEPENDENCIES
+ ext_monitor (= 0.1.2)
+ fluent-plugin-concat (~> 2.5.0)
+ fluent-plugin-kubernetes-objects (= 1.2.1)
+ fluent-plugin-kubernetes_metadata_filter (= 2.13.0)
+ fluent-plugin-oci-logging-analytics (= 2.0.5)
+ fluent-plugin-parser-cri (~> 0.1.1)
+ fluent-plugin-rewrite-tag-filter (~> 2.4.0)
+ fluentd (= 1.15.3)
+ json (= 2.6.3)
+ oj (= 3.14.1)
+
+BUNDLED WITH
+ 2.3.25
diff --git a/logan/docker-images/v1.0/oraclelinux/8/entrypoint.sh b/logan/docker-images/v1.0/oraclelinux/8/entrypoint.sh
index b62287b1..04709ea7 100644
--- a/logan/docker-images/v1.0/oraclelinux/8/entrypoint.sh
+++ b/logan/docker-images/v1.0/oraclelinux/8/entrypoint.sh
@@ -1,6 +1,6 @@
-#!/usr/bin/env sh
-
# Copyright (c) 2023, Oracle and/or its affiliates.
# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
-exec fluentd -c ${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}
+#!/usr/bin/env sh
+
+bundle exec fluentd -c ${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}
diff --git a/terraform/modules/dashboards/dashboards_json/pod.json b/terraform/modules/dashboards/dashboards_json/pod.json
index ed90b4f3..c1d95b51 100644
--- a/terraform/modules/dashboards/dashboards_json/pod.json
+++ b/terraform/modules/dashboards/dashboards_json/pod.json
@@ -1586,7 +1586,7 @@
}
}
},
- "queryString": "'Log Source' = 'Kubernetes Pod Object Logs' and 'Pod Phase' = pending and 'Kubernetes Cluster Name' = 'mushop app (magi)' | stats latest('Pod Phase') as Status by Pod",
+ "queryString": "'Log Source' = 'Kubernetes Pod Object Logs' and 'Pod Phase' = pending | stats latest('Pod Phase') as Status by Pod",
"scopeFilters": {
"filters": [
{
diff --git a/terraform/oke/oci_images.tf b/terraform/oke/oci_images.tf
new file mode 100644
index 00000000..52340ca2
--- /dev/null
+++ b/terraform/oke/oci_images.tf
@@ -0,0 +1,2 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
\ No newline at end of file