Chart: allow passing custom env to log groomer sidecar containers (apache#46003)

Author: pgvishnuram

chart/templates/dag-processor/dag-processor-deployment.yaml

@@ -210,12 +210,15 @@ spec:
           {{- if .Values.dagProcessor.logGroomerSidecar.args }}
           args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }}
           {{- end }}
-          {{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
           env:
+          {{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
             - name: AIRFLOW__LOG_RETENTION_DAYS
               value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}"
+          {{- end }}
             - name: AIRFLOW_HOME
               value: "{{ .Values.airflowHome }}"
+          {{- if .Values.dagProcessor.logGroomerSidecar.env }}
+              {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.env) $ | nindent 12 }}
           {{- end }}
           volumeMounts:
             - name: logs

chart/templates/scheduler/scheduler-deployment.yaml

@@ -268,12 +268,15 @@ spec:
           {{- if .Values.scheduler.logGroomerSidecar.args }}
           args: {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }}
           {{- end }}
-          {{- if .Values.scheduler.logGroomerSidecar.retentionDays }}
           env:
+          {{- if .Values.scheduler.logGroomerSidecar.retentionDays }}
             - name: AIRFLOW__LOG_RETENTION_DAYS
               value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
+          {{- end }}
             - name: AIRFLOW_HOME
               value: "{{ .Values.airflowHome }}"
+          {{- if .Values.scheduler.logGroomerSidecar.env }}
+              {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.env) $ | nindent 12 }}
           {{- end }}
           volumeMounts:
             - name: logs

chart/templates/triggerer/triggerer-deployment.yaml

@@ -240,12 +240,15 @@ spec:
           {{- if .Values.triggerer.logGroomerSidecar.args }}
           args: {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }}
           {{- end }}
-          {{- if .Values.triggerer.logGroomerSidecar.retentionDays }}
           env:
+          {{- if .Values.triggerer.logGroomerSidecar.retentionDays }}
             - name: AIRFLOW__LOG_RETENTION_DAYS
               value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
+          {{- end }}
             - name: AIRFLOW_HOME
               value: "{{ .Values.airflowHome }}"
+          {{- if .Values.triggerer.logGroomerSidecar.env }}
+              {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.env) $ | nindent 12 }}
           {{- end }}
           volumeMounts:
             - name: logs

chart/templates/workers/worker-deployment.yaml

@@ -322,12 +322,15 @@ spec:
           {{- if .Values.workers.logGroomerSidecar.args }}
           args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }}
           {{- end }}
-          {{- if .Values.workers.logGroomerSidecar.retentionDays }}
           env:
+          {{- if .Values.workers.logGroomerSidecar.retentionDays }}
             - name: AIRFLOW__LOG_RETENTION_DAYS
               value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
+          {{- end }}
             - name: AIRFLOW_HOME
               value: "{{ .Values.airflowHome }}"
+          {{- if .Values.workers.logGroomerSidecar.env }}
+              {{- tpl (toYaml .Values.workers.logGroomerSidecar.env) $ | nindent 12 }}
           {{- end }}
           resources: {{- toYaml .Values.workers.logGroomerSidecar.resources | nindent 12 }}
           volumeMounts:

chart/values.schema.json

@@ -12197,6 +12197,16 @@
                     "type": "integer",
                     "default": 15
                 },
+                "env": {
+                    "description": "Add additional env vars to log groomer sidecar container (templated).",
+                    "items": {
+                        "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
+                    },
+                    "type": "array",
+                    "default": [],
+                    "x-kubernetes-patch-merge-key": "name",
+                    "x-kubernetes-patch-strategy": "merge"
+                },
                 "resources": {
                     "description": "Resources for Airflow log groomer sidecar.",
                     "type": "object",

chart/values.yaml

@@ -831,6 +831,7 @@ workers:
     # Detailed default security context for logGroomerSidecar for container level
     securityContexts:
       container: {}
+    env: []
 
   waitForMigrations:
     # Whether to create init container to wait for db migrations
@@ -1034,6 +1035,7 @@ scheduler:
       container: {}
     # container level lifecycle hooks
     containerLifecycleHooks: {}
+    env: []
 
   waitForMigrations:
     # Whether to create init container to wait for db migrations
@@ -1748,6 +1750,8 @@ triggerer:
     # container level lifecycle hooks
     containerLifecycleHooks: {}
 
+    env: []
+
   waitForMigrations:
     # Whether to create init container to wait for db migrations
     enabled: true
@@ -1928,6 +1932,8 @@ dagProcessor:
     securityContexts:
       container: {}
 
+    env: []
+
   waitForMigrations:
     # Whether to create init container to wait for db migrations
     enabled: true

tests/charts/log_groomer.py

@@ -90,6 +90,32 @@ def test_log_groomer_collector_default_retention_days(self):
         )
         assert jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0]) == "15"
 
+    def test_log_groomer_collector_custom_env(self):
+        env = [
+            {"name": "APP_RELEASE_NAME", "value": "{{ .Release.Name }}-airflow"},
+            {"name": "APP__LOG_RETENTION_DAYS", "value": "5"},
+        ]
+
+        if self.obj_name == "dag-processor":
+            values = {"dagProcessor": {"enabled": True, "logGroomerSidecar": {"env": env}}}
+        else:
+            values = {
+                "workers": {"logGroomerSidecar": {"env": env}},
+                "scheduler": {"logGroomerSidecar": {"env": env}},
+                "triggerer": {"logGroomerSidecar": {"env": env}},
+            }
+
+        docs = render_chart(
+            values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
+        )
+
+        assert {"name": "APP_RELEASE_NAME", "value": "release-name-airflow"} in jmespath.search(
+            "spec.template.spec.containers[1].env", docs[0]
+        )
+        assert {"name": "APP__LOG_RETENTION_DAYS", "value": "5"} in jmespath.search(
+            "spec.template.spec.containers[1].env", docs[0]
+        )
+
     @pytest.mark.parametrize("command", [None, ["custom", "command"]])
     @pytest.mark.parametrize("args", [None, ["custom", "args"]])
     def test_log_groomer_command_and_args_overrides(self, command, args):
@@ -163,7 +189,7 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re
                 "spec.template.spec.containers[1].env[0].value", docs[0]
             )
         else:
-            assert jmespath.search("spec.template.spec.containers[1].env", docs[0]) is None
+            assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 1
 
     def test_log_groomer_resources(self):
         if self.obj_name == "dag-processor":