Skip to content

Commit 7bcd3d4

Browse files
maxsivkovmaxsivkov
committed
initial port
1 parent 621251f commit 7bcd3d4

File tree

17 files changed

+356
-42
lines changed

17 files changed

+356
-42
lines changed

‎base-image/Dockerfile.windows‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
############ escape=`
2+
3+
ARG WIN_TAG=ltsc2019
4+
5+
FROM mcr.microsoft.com/windows/servercore:$WIN_TAG as ruby-base
6+
LABEL maintainer "Fluentd developers <[email protected]>"
7+
LABEL Description="Fluentd docker image" Vendor="Fluent Organization" Version="1.13.1"
8+
9+
10+
# Do not split this into multiple RUN!
11+
# Docker creates a layer for every RUN-Statement
12+
RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
13+
14+
# Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools
15+
# NOTE: For avoiding stalling with docker build on windows, we must use latest version of msys2.
16+
RUN choco install -y ruby --version=2.7.2.1 --params "'/InstallDir:C:\ruby27'" \
17+
&& MD C:\Users\ContainerUser\.gem \
18+
&& MD C:\fluentd \
19+
&& refreshenv \
20+
&& echo gem: --user-install --no-document >> c:/Users/ContainerUser/.gemrc \
21+
&& gem update --system
22+
23+
USER ContainerAdministrator
24+
RUN setx /m PATH "C:\Users\ContainerUser\.gem\bin;%PATH%" \
25+
&& setx /m GEM_HOME c:/Users/ContainerUser/.gem
26+
USER ContainerUser
27+
28+
29+
30+
FROM ruby-base as ruby-build
31+
32+
USER ContainerAdministrator
33+
RUN choco install -y msys2 --version 20210604.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\msys64'" \
34+
&& setx /m PATH "C:\msys64\usr\bin;%PATH%" \
35+
&& setx /m MSYS winsymlinks:nativestrict \
36+
&& ridk install 3 \
37+
&& mklink /J C:\msys64\fluentd C:\fluentd
38+
USER ContainerUser
39+
40+
SHELL ["bash", "-lc"]
41+
RUN "echo 'PATH=$PATH:/mingw64/bin:/c/ruby27/bin:/c/users/containeruser/.gem/ruby/2.7.0/bin' >> /etc/profile.d/000-mingw64.sh \
42+
&& pacman -Suu --noconfirm && pacman -S --needed --noconfirm git \
43+
"
44+
45+
# RUN "pacman -Suu --noconfirm && pacman -S --needed --noconfirm git"
46+
47+
COPY basegems/Gemfile /fluentd/Gemfile.basegems
48+
COPY basegems/Gemfile.lock /fluentd/Gemfile.basegems.lock
49+
50+
COPY Gemfile /fluentd/Gemfile
51+
COPY Gemfile.lock /fluentd/Gemfile.lock
52+
53+
RUN "\
54+
gem install bundler:'>= 2.2.27' rexml:'>= 3.2.5' rdoc:'>= 6.3.2' json:'>= 2.5.1' webrick:'>= 1.7.0' bigdecimal \
55+
&& cd /fluentd \
56+
&& bundler config --global jobs 4 \
57+
&& bundler config set --global without \"development test\" \
58+
&& bundler config set --global cache_all 'false' \
59+
&& bundler install --no-cache --gemfile=Gemfile.basegems \
60+
&& bundler install --no-cache --gemfile=Gemfile \
61+
&& git clone https://github.com/Cryptophobia/fluent-plugin-google-cloud.git fluent-plugin-google-cloud \
62+
&& gem build -C fluent-plugin-google-cloud fluent-plugin-google-cloud.gemspec \
63+
&& gem install --no-document fluent-plugin-google-cloud/fluent-plugin-google-cloud-*.gem \
64+
&& git clone https://github.com/Cryptophobia/fluent-plugin-loggly.git fluent-plugin-loggly \
65+
&& gem build -C fluent-plugin-loggly fluent-plugin-loggly.gemspec \
66+
&& gem install fluent-plugin-loggly/fluent-plugin-loggly-*.gem \
67+
&& rm -rf $GEM_HOME/cache/* $GEM_HOME/doc/* \
68+
"
69+
# RUN "cd /fluentd && bundle config list && du -h --max-depth 1 $GEM_HOME"
70+
71+
FROM ruby-base as ruby-runtime
72+
COPY --from=ruby-build /Users/ContainerUser/.gem /Users/ContainerUser/.gem
73+
COPY plugins /etc/fluent-plugin
74+
75+
USER ContainerAdministrator
76+
RUN tzutil /s UTC \
77+
&& (if not exist C:\etc\fluent-conf (MD C:\etc\fluent-conf)) \
78+
&& (if not exist C:\etc\fluent-plugin (MD C:\etc\fluent-plugin)) \
79+
&& (if not exist C:\var\run\secrets\kubernetes.io\serviceaccount (MD C:\var\run\secrets\kubernetes.io\serviceaccount)) \
80+
&& (if not exist C:\var\log (MD C:\var\log)) \
81+
&& (if not exist C:\var\lib\kubelet (MD C:\var\lib\kubelet))
82+
USER ContainerUser
83+
RUN powershell -Command "Invoke-WebRequest https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/docker-image/v1.13/debian-elasticsearch7/plugins/parser_kubernetes.rb -OutFile C:\etc\fluent-plugin\parser_kubernetes.rb \
84+
; Invoke-WebRequest https://raw.githubusercontent.com/fluent/fluentd-kubernetes-daemonset/master/docker-image/v1.13/debian-elasticsearch7/plugins/parser_multiline_kubernetes.rb -OutFile C:\etc\fluent-plugin\parser_multiline_kubernetes.rb \
85+
"
86+
87+
COPY failsafe.conf /etc/fluent-conf/failsafe.conf
88+
COPY entrypoint.ps1 /entrypoint.ps1
89+
ENTRYPOINT ["powershell", "/entrypoint.ps1"]

‎base-image/Makefile‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,24 @@
11
# Copyright © 2018 VMware, Inc. All Rights Reserved.
22
# SPDX-License-Identifier: BSD-2-Clause
33

4-
IMAGE ?= vmware/base-fluentd-operator
5-
TAG ?= latest
4+
IMAGE ?= vmware/base-fluentd-operator
5+
IMG_ARCH ?= amd64
6+
WIN_TAG ?=
7+
IMG_OS ?= linux
8+
TAG ?= $(IMG_ARCH)-$(IMG_OS)$(shell v='$(WIN_TAG)'; echo "$${v:+-}$${v:-}")
69

710
.PHONY: test
11+
ttt:
12+
echo $(TAG)
813

914
build-image:
1015
DOCKER_BUILDKIT=1 docker build -t $(IMAGE):$(TAG) .
1116

17+
build-windows-image:
18+
@echo Building $(IMAGE):$(TAG)
19+
docker build --network "Default Switch" -f Dockerfile.windows --build-arg WIN_TAG=$(WIN_TAG) -t $(IMAGE):$(TAG) .
20+
21+
1222
shell:
1323
docker run --entrypoint=/bin/bash \
1424
-ti --rm -v `pwd`:/workspace --net=host \

‎base-image/entrypoint.ps1‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
$FLUENTD_CONF = if ($env:FLUENTD_CONF) { $env:FLUENTD_CONF } else { "fluent.conf" };
2+
$DEFAULT_FLUENT_CONF = "/etc/fluent-conf/${FLUENTD_CONF}";
3+
$FLUENTD_OPT = $env:FLUENTD_OPT
4+
5+
echo "Using configuration: ${DEFAULT_FLUENT_CONF}"
6+
echo "With FLUENTD_OPT: ${FLUENTD_OPT}"
7+
8+
$retries = if ($env:RETRIES) { $env:RETRIES } else { 60 };
9+
$ready = $false
10+
11+
foreach ($attempt in 1..$retries) {
12+
if (Test-Path -Path $DEFAULT_FLUENT_CONF -PathType Leaf) {
13+
$ready = $true
14+
break
15+
}
16+
echo "Waiting for config file to become available: $attempt of $retries"
17+
Start-Sleep -Seconds 10
18+
}
19+
if ($ready -ne $true ) {return 1}
20+
echo "Found configuration file: ${DEFAULT_FLUENT_CONF}"
21+
$cmdline_args = "& fluentd", "-c", ${DEFAULT_FLUENT_CONF}, "-p", "/etc/fluent-plugin", $FLUENTD_OPT
22+
$cmdline = $cmdline_args -join ' '
23+
echo "cmdline $cmdline"
24+
Invoke-Expression $cmdline
25+
#& \fluentd.cmd -c ${DEFAULT_FLUENT_CONF} -p /etc/fluent/plugins $FLUENTD_OPT

‎charts/log-router/templates/_helpers.tpl‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,115 @@ extensions/v1beta1
4242
apps/v1
4343
{{- end -}}
4444
{{- end -}}
45+
46+
47+
48+
{{- define "node.kubeletRootPath" }}
49+
{{- if eq .Values.nodeProfileName "windows" }}
50+
{{- .Values.nodeProfile.kubeletRootPath | default "C:/var/lib/kubelet" -}}
51+
{{- else }}
52+
{{- .Values.nodeProfile.kubeletRootPath | default "/var/lib/kubelet" -}}
53+
{{- end }}
54+
{{- end }}
55+
56+
57+
58+
{{- define "node.fluentConfMountPath" }}
59+
{{- if eq .Values.nodeProfileName "windows" }}
60+
{{- .Values.nodeProfile.fluentConfMountPath | default "C:/etc/fluent-conf" -}}
61+
{{- else }}
62+
{{- .Values.nodeProfile.fluentConfMountPath | default "/fluentd/etc" -}}
63+
{{- end }}
64+
{{- end }}
65+
66+
67+
{{- define "node.volumes" }}
68+
- name: fluentconf
69+
emptyDir: {}
70+
{{- if eq .Values.nodeProfileName "windows" }}
71+
- name: var
72+
hostPath:
73+
path: {{ .Values.nodeProfile.varHostPath | default "C:/var" }}
74+
- name: varlibdocker
75+
hostPath:
76+
path: {{ .Values.nodeProfile.varLibDockerContainersHostPath | default "C:/ProgramData/docker" }}
77+
{{- else }}
78+
- name: kubeletroot
79+
hostPath:
80+
path: {{ .Values.nodeProfile.kubeletRootHostPath | default "/var/lib/kubelet" }}
81+
- name: varlog
82+
hostPath:
83+
path: {{ .Values.nodeProfile.varLogHostPath | default "/var/log" }}
84+
- name: varlibdockercontainers
85+
hostPath:
86+
path: {{ .Values.nodeProfile.varLibDockerContainersHostPath | default "/var/lib/docker/containers" }}
87+
{{- end }}
88+
{{- end }}
89+
90+
91+
{{- define "node.fluendVolumeMounts" }}
92+
- name: fluentconf
93+
mountPath: {{ include "node.fluentConfMountPath" . }}
94+
{{- if eq .Values.nodeProfileName "windows" }}
95+
- name: var
96+
mountPath: {{ .Values.nodeProfile.varMountPath | default "C:/var" }}
97+
- name: varlibdocker
98+
mountPath: {{ .Values.nodeProfile.varLibDockerContainersMountPath | default "C:/ProgramData/docker" }}
99+
readOnly: true
100+
{{- else }}
101+
- name: kubeletroot
102+
mountPath: {{ include "node.kubeletRootPath" . }}
103+
readOnly: true
104+
- name: varlog
105+
mountPath: {{ .Values.nodeProfile.varLogMountPath | default "/var/log" }}
106+
- name: varlibdockercontainers
107+
mountPath: {{ .Values.nodeProfile.varLibDockerContainersMountPath | default "/var/lib/docker/containers" }}
108+
readOnly: true
109+
{{- end }}
110+
{{- end }}
111+
112+
113+
{{- define "node.operatorVolumeMounts" }}
114+
- name: fluentconf
115+
{{- if eq .Values.nodeProfileName "windows" }}
116+
mountPath: {{ .Values.nodeProfile.fluentConfMountPath | default "C:/etc/fluent-conf" }}
117+
{{- else }}
118+
mountPath: {{ .Values.nodeProfile.fluentConfMountPath | default "/fluentd/etc" }}
119+
{{- end }}
120+
{{- end }}
121+
122+
123+
{{- define "node.operatorBinary" }}
124+
{{- if eq .Values.nodeProfileName "windows" }}
125+
{{- .Values.nodeProfile.operatorBinary | default "c:/bin/config-reloader.exe" -}}
126+
{{- else }}
127+
{{- .Values.nodeProfile.operatorBinary | default "/bin/config-reloader" -}}
128+
{{- end }}
129+
{{- end }}
130+
131+
132+
{{- define "node.fluentdBinary" }}
133+
{{- if eq .Values.nodeProfileName "windows" }}
134+
{{- .Values.nodeProfile.fluentdBinary | default "fluentd -p /etc/fluent-plugin" -}}
135+
{{- else }}
136+
{{- .Values.nodeProfile.fluentdBinary | default "/usr/local/bundle/bin/fluentd -p /fluentd/plugins" -}}
137+
{{- end }}
138+
{{- end }}
139+
140+
{{- define "node.templatesDir" }}
141+
{{- if eq .Values.nodeProfileName "windows" }}
142+
{{- .Values.nodeProfile.templatesDir | default "C:/templates" -}}
143+
{{- else }}
144+
{{- .Values.nodeProfile.templatesDir | default "/templates" -}}
145+
{{- end }}
146+
{{- end }}
147+
148+
{{- define "node.useSystemd" }}
149+
{{- if .Values.nodeProfile.useSystemd }}
150+
- {{ .Values.nodeProfile.useSystemd }}
151+
{{- else }}
152+
{{- if ne .Values.nodeProfileName "windows" }}
153+
- {{ "--use-systemd" }}
154+
{{- end }}
155+
{{- end }}
156+
{{- end }}

‎charts/log-router/templates/daemonset.yaml‎

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,7 @@ spec:
6363
containerPort: 24231
6464
{{- end }}
6565
volumeMounts:
66-
- name: fluentconf
67-
mountPath: /fluentd/etc
68-
- name: varlog
69-
mountPath: /var/log
70-
- name: kubeletroot
71-
mountPath: "{{ .Values.kubeletRoot }}"
72-
readOnly: true
73-
- name: varlibdockercontainers
74-
mountPath: /var/lib/docker/containers
75-
readOnly: true
66+
{{- include "node.fluendVolumeMounts" . | indent 10 }}
7667
{{- if .Values.fluentd.extraVolumeMounts }}
7768
{{ toYaml .Values.fluentd.extraVolumeMounts | indent 10 }}
7869
{{- end }}
@@ -97,7 +88,7 @@ spec:
9788
{{- end }}
9889
{{- end }}
9990
command:
100-
- /bin/config-reloader
91+
- {{ include "node.operatorBinary" . }}
10192
- --datasource={{ .Values.datasource }}
10293
{{- if .Values.crdMigrationMode }}
10394
- --crd-migration-mode
@@ -109,17 +100,17 @@ spec:
109100
{{- if not (empty .Values.bufferMountFolder) }}
110101
- --buffer-mount-folder={{ .Values.bufferMountFolder }}
111102
{{- end }}
112-
- --output-dir=/fluentd/etc
113-
- --templates-dir=/templates
103+
- --output-dir={{ include "node.fluentConfMountPath" . }}
104+
- --templates-dir={{ include "node.templatesDir" . }}
114105
- --id={{ template "fluentd-router.fullname" . }}
115106
- --fluentd-binary
116107
{{- if .Values.logRotate.enabled }}
117-
- /usr/local/bundle/bin/fluentd -p /fluentd/plugins --log-rotate-age {{ .Values.logRotate.logRotateAge }} --log-rotate-size {{ .Values.logRotate.logRotateSize | int }}
108+
- {{ include "node.fluentdBinary" . }} --log-rotate-age {{ .Values.logRotate.logRotateAge }} --log-rotate-size {{ .Values.logRotate.logRotateSize | int }}
118109
{{ else }}
119-
- /usr/local/bundle/bin/fluentd -p /fluentd/plugins
110+
- {{ include "node.fluentdBinary" . }}
120111
{{ end }}
121112
- --kubelet-root
122-
- "{{ .Values.kubeletRoot }}"
113+
- {{ include "node.kubeletRootPath" . }}
123114
{{- if .Values.meta.key }}
124115
- --meta-key={{ .Values.meta.key }}
125116
- --meta-values={{- range $k, $v := .Values.meta.values }}{{$k}}={{$v}},
@@ -143,9 +134,21 @@ spec:
143134
{{- if .Values.adminNamespace }}
144135
- --admin-namespace={{ .Values.adminNamespace }}
145136
{{- end }}
137+
{{- if .Values.fluentdReloadPath }}
138+
- --fluentd-reload-path={{ .Values.fluentdReloadPath }}
139+
{{- end }}
140+
{{- if .Values.execTimeout }}
141+
- --exec-timeout={{ .Values.execTimeout }}
142+
{{- end }}
143+
{{- if .Values.statusAnnotation }}
144+
- --status-annotation={{ .Values.statusAnnotation }}
145+
{{- end }}
146+
{{- if .Values.configmapAnnotation }}
147+
- --annotation={{ .Values.configmapAnnotation }}
148+
{{- end }}
149+
{{ include "node.useSystemd" . | indent 10}}
146150
volumeMounts:
147-
- name: fluentconf
148-
mountPath: /fluentd/etc
151+
{{- include "node.operatorVolumeMounts" . | indent 10 }}
149152
{{- if .Values.reloader.extraVolumeMounts }}
150153
{{ toYaml .Values.reloader.extraVolumeMounts | indent 10 }}
151154
{{- end }}
@@ -160,17 +163,7 @@ spec:
160163
{{ toYaml .Values.tolerations | indent 8 }}
161164
{{- end}}
162165
volumes:
163-
- name: fluentconf
164-
emptyDir: {}
165-
- name: kubeletroot
166-
hostPath:
167-
path: "{{ .Values.kubeletRoot }}"
168-
- name: varlog
169-
hostPath:
170-
path: /var/log
171-
- name: varlibdockercontainers
172-
hostPath:
173-
path: /var/lib/docker/containers
166+
{{- include "node.volumes" . | indent 6 }}
174167
{{- if .Values.extraVolumes }}
175168
{{ toYaml .Values.extraVolumes | indent 6 }}
176169
{{- end }}

‎charts/log-router/values.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ image:
3232
logLevel: debug
3333
fluentdLogLevel: debug
3434
interval: 45
35-
kubeletRoot: /var/lib/kubelet
35+
# Profile name for operator "linux" or "windows"
36+
nodeProfileName: "linux"
37+
# Profile settings. See templates\_helpers.tpl
38+
nodeProfile: {}
3639
# bufferMountFolder -- a folder inside /var/log to write all fluentd buffers to
3740
bufferMountFolder: ""
3841

‎config-reloader/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Copyright © 2018 VMware, Inc. All Rights Reserved.
22
# SPDX-License-Identifier: BSD-2-Clause
33

4+
ARG ARCH=amd64
5+
ARG OS=linux
46
# builder image
57
FROM golang:1.16 as builder
68

@@ -14,7 +16,7 @@ RUN make in-docker-test
1416
RUN make build VERSION=$VERSION
1517

1618
# always use the un-pushable image
17-
FROM vmware/base-fluentd-operator:latest
19+
FROM vmware/base-fluentd-operator:$ARCH-$OS
1820

1921
COPY templates /templates
2022
COPY validate-from-dir.sh /bin/validate-from-dir.sh

0 commit comments

Comments
 (0)