Merge pull request #164 from maykinmedia/feature/148-liveness-probes

[#148] Update liveness probes
maykinmedia · Jan 14, 2025 · b451956 · b451956
2 parents 4751fe0 + e7e49ed
commit b451956
Show file tree

Hide file tree

Showing 42 changed files with 283 additions and 74 deletions.
diff --git a/charts/objecten/.helmignore b/charts/objecten/.helmignore
@@ -0,0 +1 @@
+CHANGELOG.md
diff --git a/charts/objecten/CHANGELOG.md b/charts/objecten/CHANGELOG.md
@@ -0,0 +1,12 @@
+# Changelog
+
+## 2.7.0-beta.0 (2025-10-01)
+
+- [#148] Replace the worker liveness probe with the `celery inspect active` command. This should detect when a worker is down and should not interrupt long running tasks.
+- [#119] Update the syntax of the worker liveness probe. The worker probes now can be enabled/disabled with:
+
+```yaml
+worker:
+  livenessProbe:
+    enabled: true # true | false
+```
diff --git a/charts/objecten/Chart.yaml b/charts/objecten/Chart.yaml
@@ -3,7 +3,7 @@ name: objecten
 description: API om objecten te beheren die behoren bij een bepaald objecttype
 
 type: application
-version: 2.6.0-beta.3
+version: 2.7.0-beta.0
 appVersion: latest
 
 dependencies:

diff --git a/charts/objecten/README.md b/charts/objecten/README.md
@@ -1,6 +1,6 @@
 # objecten
 
-![Version: 2.6.0-beta.1](https://img.shields.io/badge/Version-2.6.0--beta.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
+![Version: 2.7.0-beta.0](https://img.shields.io/badge/Version-2.7.0--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 API om objecten te beheren die behoren bij een bepaald objecttype
 
@@ -167,13 +167,15 @@ API om objecten te beheren die behoren bij een bepaald objecttype
 | worker.autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
 | worker.autoscaling.targetMemoryUtilizationPercentage | int | `80` |  |
 | worker.concurrency | int | `4` |  |
-| worker.livenessProbe.exec.command[0] | string | `"python"` |  |
-| worker.livenessProbe.exec.command[1] | string | `"/app/bin/check_celery_worker_liveness.py"` |  |
-| worker.livenessProbe.failureThreshold | int | `10` |  |
+| worker.livenessProbe.enabled | bool | `true` |  |
+| worker.livenessProbe.exec.command[0] | string | `"/bin/sh"` |  |
+| worker.livenessProbe.exec.command[1] | string | `"-c"` |  |
+| worker.livenessProbe.exec.command[2] | string | `"celery --workdir src --app objects.celery inspect --destination celery@${HOSTNAME} active"` |  |
+| worker.livenessProbe.failureThreshold | int | `3` |  |
 | worker.livenessProbe.initialDelaySeconds | int | `60` |  |
 | worker.livenessProbe.periodSeconds | int | `50` |  |
 | worker.livenessProbe.successThreshold | int | `1` |  |
-| worker.livenessProbe.timeoutSeconds | int | `5` |  |
+| worker.livenessProbe.timeoutSeconds | int | `10` |  |
 | worker.maxWorkerLivenessDelta | string | `""` |  |
 | worker.podLabels | object | `{}` |  |
 | worker.replicaCount | int | `1` |  |

diff --git a/charts/objecten/templates/deployment.yaml b/charts/objecten/templates/deployment.yaml
@@ -183,10 +183,16 @@ spec:
             {{- if .Values.extraEnvVars }}
             {{- include "objecten.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 12 }}
             {{- end }}
-            {{- if .Values.worker.livenessProbe }}
+          {{- if .Values.worker.livenessProbe.enabled }}
           livenessProbe:
-            {{- toYaml .Values.worker.livenessProbe | nindent 12 }}
-            {{- end }}
+            exec: 
+            {{- toYaml .Values.worker.livenessProbe.exec | nindent 14 }}
+            initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.worker.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.worker.livenessProbe.failureThreshold }}
+          {{- end }}
           resources:
             {{- toYaml .Values.worker.resources | nindent 12 }}
           command:

diff --git a/charts/objecten/values.yaml b/charts/objecten/values.yaml
@@ -293,16 +293,18 @@ worker:
   resources: {}
   # Defaults to 60s
   maxWorkerLivenessDelta: ""
-  livenessProbe:
+  livenessProbe: 
+    enabled: false
     exec:
       command:
-        - python
-        - /app/bin/check_celery_worker_liveness.py
+        - /bin/sh
+        - -c
+        - celery --workdir src --app objects.celery inspect --destination celery@${HOSTNAME} active
     initialDelaySeconds: 60
     # Periodeseconds should not exceed maxWorkerLivenessDelta
     periodSeconds: 50
-    timeoutSeconds: 5
-    failureThreshold: 10
+    timeoutSeconds: 10
+    failureThreshold: 3
     successThreshold: 1
   autoscaling:
     enabled: false

diff --git a/charts/openarchiefbeheer/.helmignore b/charts/openarchiefbeheer/.helmignore
@@ -0,0 +1 @@
+CHANGELOG.md
diff --git a/charts/openarchiefbeheer/CHANGELOG.md b/charts/openarchiefbeheer/CHANGELOG.md
@@ -0,0 +1,6 @@
+# Changelog
+
+## 1.1.1-beta.0 (2025-10-01)
+
+- [#148] Add worker liveness probe command `celery inspect active`. This should detect when a worker is down and should not interrupt long running tasks.
+- Fix missing init container to run Django setup configuration.
diff --git a/charts/openarchiefbeheer/Chart.yaml b/charts/openarchiefbeheer/Chart.yaml
@@ -3,7 +3,7 @@ name: openarchiefbeheer
 description: Opstellen, beheren en uitvoeren van vernietigingslijsten, voor gebruik met Zaakgericht werken
 
 type: application
-version: 1.1.0-beta.5
+version: 1.1.1-beta.0
 appVersion: latest
 
 dependencies:

diff --git a/charts/openarchiefbeheer/README.md b/charts/openarchiefbeheer/README.md
@@ -1,6 +1,6 @@
 # openarchiefbeheer
 
-![Version: 1.1.0-beta.1](https://img.shields.io/badge/Version-1.1.0--beta.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
+![Version: 1.1.1-beta.0](https://img.shields.io/badge/Version-1.1.1--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 Opstellen, beheren en uitvoeren van vernietigingslijsten, voor gebruik met Zaakgericht werken
 
@@ -179,9 +179,16 @@ Opstellen, beheren en uitvoeren van vernietigingslijsten, voor gebruik met Zaakg
 | worker.autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
 | worker.autoscaling.targetMemoryUtilizationPercentage | int | `80` |  |
 | worker.concurrency | int | `4` |  |
-| worker.livenessProbe | object | `{}` |  |
+| worker.livenessProbe.enabled | bool | `false` |  |
+| worker.livenessProbe.exec.command[0] | string | `"/bin/sh"` |  |
+| worker.livenessProbe.exec.command[1] | string | `"-c"` |  |
+| worker.livenessProbe.exec.command[2] | string | `"celery --workdir src --app openarchiefbeheer.celery inspect --destination celery@${HOSTNAME} active"` |  |
+| worker.livenessProbe.failureThreshold | int | `3` |  |
+| worker.livenessProbe.initialDelaySeconds | int | `60` |  |
+| worker.livenessProbe.periodSeconds | int | `30` |  |
+| worker.livenessProbe.successThreshold | int | `1` |  |
+| worker.livenessProbe.timeoutSeconds | int | `10` |  |
 | worker.podLabels | object | `{}` |  |
-| worker.readinessProbe | object | `{}` |  |
 | worker.replicaCount | int | `1` |  |
 | worker.resources | object | `{}` |  |
 
diff --git a/charts/openarchiefbeheer/templates/deployment.yaml b/charts/openarchiefbeheer/templates/deployment.yaml
@@ -34,6 +34,40 @@ spec:
       serviceAccountName: {{ include "openarchiefbeheer.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- if and .Values.global.configuration.enabled .Values.configuration.enabled .Values.configuration.initContainer.enabled }}
+      initContainers:
+        - name: {{ .Chart.Name }}-config
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 16 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          envFrom:
+            - secretRef:
+                name: {{ .Values.existingSecret | default (include "openarchiefbeheer.fullname" .) }}
+            - configMapRef:
+                name: {{ include "openarchiefbeheer.fullname" . }}
+            - secretRef:
+                name: {{ include "openarchiefbeheer.fullname" . }}-config-secrets
+          env:
+            {{- if .Values.extraEnvVars }}
+            {{- include "openarchiefbeheer.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 16 }}
+            {{- end }}
+          command:
+            - "/bin/bash"
+            - "-c"
+          args:
+          - |
+            /wait_for_db.sh
+            /app/src/manage.py migrate
+            /app/src/manage.py setup_configuration --no-selftest {{- if or .Values.global.configuration.overwrite .Values.configuration.overwrite }} --overwrite{{- end }}
+          volumeMounts:
+            - name: media
+              mountPath: /app/media
+              subPath: {{ .Values.persistence.mediaMountSubpath  | default "openarchiefbeheer/media" }}
+            {{- if .Values.extraVolumeMounts }}
+            {{- include "openarchiefbeheer.tplvalues.render" ( dict "value" .Values.extraVolumeMounts "context" $ ) | nindent 12 }}
+            {{- end }}
+      {{- end }}
       containers:
         - name: {{ include "openarchiefbeheer.fullname" . }}
           securityContext:
@@ -43,8 +77,6 @@ spec:
           envFrom:
             - secretRef:
                 name: {{ .Values.existingSecret | default (include "openarchiefbeheer.fullname" .) }}
-            - secretRef:
-                name: {{ include "openarchiefbeheer.fullname" . }}-config-secrets
             - configMapRef:
                 name: {{ include "openarchiefbeheer.fullname" . }}
           env:

diff --git a/charts/openarchiefbeheer/values.yaml b/charts/openarchiefbeheer/values.yaml
@@ -257,8 +257,13 @@ worker:
   resources: {}
   livenessProbe: 
     enabled: false
+    exec:
+      command:
+        - /bin/sh
+        - -c
+        - celery --workdir src --app openarchiefbeheer.celery inspect --destination celery@${HOSTNAME} active
     initialDelaySeconds: 60
-    periodSeconds: 10
+    periodSeconds: 30
     timeoutSeconds: 10
     failureThreshold: 3
     successThreshold: 1

diff --git a/charts/openforms/.helmignore b/charts/openforms/.helmignore
@@ -0,0 +1 @@
+CHANGELOG.md
diff --git a/charts/openforms/CHANGELOG.md b/charts/openforms/CHANGELOG.md
@@ -0,0 +1,12 @@
+# Changelog
+
+## 1.8.0-beta.0 (2025-10-01)
+
+- [#148] Replace the worker liveness probe with the `celery inspect active` command. This should detect when a worker is down and should not interrupt long running tasks.
+- [#119] Update the syntax of the worker liveness probe. The worker probes now can be enabled/disabled with:
+
+```yaml
+worker:
+  livenessProbe:
+    enabled: true # true | false
+```
diff --git a/charts/openforms/Chart.yaml b/charts/openforms/Chart.yaml
@@ -3,7 +3,7 @@ name: openforms
 description: Snel en eenvoudig slimme formulieren bouwen en publiceren
 
 type: application
-version: 1.6.0-beta.4
+version: 1.8.0-beta.0
 appVersion: latest
 icon: https://open-forms.readthedocs.io/en/stable/_static/logo.svg
 

diff --git a/charts/openforms/README.md b/charts/openforms/README.md
@@ -1,6 +1,6 @@
 # openforms
 
-![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.8.1](https://img.shields.io/badge/AppVersion-2.8.1-informational?style=flat-square)
+![Version: 1.8.0-beta.0](https://img.shields.io/badge/Version-1.8.0--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 Snel en eenvoudig slimme formulieren bouwen en publiceren
 
@@ -27,6 +27,14 @@ Snel en eenvoudig slimme formulieren bouwen en publiceren
 | beat.podLabels | object | `{}` |  |
 | beat.replicaCount | int | `1` |  |
 | beat.resources | object | `{}` |  |
+| configuration.enabled | bool | `false` |  |
+| configuration.initContainer.enabled | bool | `false` | Run the setup configuration command in a init container |
+| configuration.job.backoffLimit | int | `6` |  |
+| configuration.job.enabled | bool | `true` | Run the setup configuration command as a job |
+| configuration.job.resources | object | `{}` |  |
+| configuration.job.restartPolicy | string | `"OnFailure"` |  |
+| configuration.job.ttlSecondsAfterFinished | int | `0` | 0 Will clean the job after it is finished |
+| configuration.secrets | object | `{}` |  |
 | existingSecret | string | `nil` |  |
 | extraDeploy | list | `[]` |  |
 | extraEnvVars | list | `[]` |  |
@@ -49,6 +57,8 @@ Snel en eenvoudig slimme formulieren bouwen en publiceren
 | flower.replicaCount | int | `1` |  |
 | flower.resources | object | `{}` |  |
 | fullnameOverride | string | `""` |  |
+| global.configuration.enabled | bool | `false` |  |
+| global.configuration.secrets | object | `{}` |  |
 | global.settings.databaseHost | string | `""` | Global databasehost, overrides setting.database.host |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"openformulieren/open-forms"` |  |
@@ -198,9 +208,11 @@ Snel en eenvoudig slimme formulieren bouwen en publiceren
 | worker.autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
 | worker.autoscaling.targetMemoryUtilizationPercentage | int | `80` |  |
 | worker.concurrency | int | `4` |  |
-| worker.livenessProbe.exec.command[0] | string | `"python"` |  |
-| worker.livenessProbe.exec.command[1] | string | `"/app/bin/check_celery_worker_liveness.py"` |  |
-| worker.livenessProbe.failureThreshold | int | `10` |  |
+| worker.livenessProbe.enabled | bool | `false` |  |
+| worker.livenessProbe.exec.command[0] | string | `"/bin/sh"` |  |
+| worker.livenessProbe.exec.command[1] | string | `"-c"` |  |
+| worker.livenessProbe.exec.command[2] | string | `"celery --workdir src --app openforms.celery inspect --destination celery@${HOSTNAME} active"` |  |
+| worker.livenessProbe.failureThreshold | int | `3` |  |
 | worker.livenessProbe.initialDelaySeconds | int | `60` |  |
 | worker.livenessProbe.periodSeconds | int | `50` |  |
 | worker.livenessProbe.successThreshold | int | `1` |  |

diff --git a/charts/openforms/templates/deployment.yaml b/charts/openforms/templates/deployment.yaml
@@ -285,10 +285,16 @@ spec:
             {{- if .Values.extraEnvVars }}
             {{- include "openforms.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 12 }}
             {{- end }}
-            {{- if .Values.worker.livenessProbe }}
+          {{- if .Values.worker.livenessProbe.enabled }}
           livenessProbe:
-            {{- toYaml .Values.worker.livenessProbe | nindent 12 }}
-            {{- end }}
+            exec: 
+            {{- toYaml .Values.worker.livenessProbe.exec | nindent 14 }}
+            initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.worker.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.worker.livenessProbe.failureThreshold }}
+          {{- end }}
           resources:
             {{- toYaml .Values.worker.resources | nindent 12 }}
           command:

diff --git a/charts/openforms/values.yaml b/charts/openforms/values.yaml
@@ -443,16 +443,18 @@ worker:
   resources: {}
   # Defaults to 60s
   maxWorkerLivenessDelta: ""
-  livenessProbe:
+  livenessProbe: 
+    enabled: false
     exec:
       command:
-        - python
-        - /app/bin/check_celery_worker_liveness.py
+        - /bin/sh
+        - -c
+        - celery --workdir src --app openforms.celery inspect --destination celery@${HOSTNAME} active
     initialDelaySeconds: 60
     # Periodeseconds should not exceed maxWorkerLivenessDelta
     periodSeconds: 50
     timeoutSeconds: 15
-    failureThreshold: 10
+    failureThreshold: 3
     successThreshold: 1
   autoscaling:
     enabled: false

diff --git a/charts/openinwoner/.helmignore b/charts/openinwoner/.helmignore
@@ -0,0 +1 @@
+CHANGELOG.md
diff --git a/charts/openinwoner/CHANGELOG.md b/charts/openinwoner/CHANGELOG.md
@@ -0,0 +1,12 @@
+# Changelog
+
+## 1.7.0-beta.0 (2025-10-01)
+
+- [#148] Replace the worker liveness probe with the `celery inspect active` command. This should detect when a worker is down and should not interrupt long running tasks.
+- [#119] Update the syntax of the worker liveness probe. The worker probes now can be enabled/disabled with:
+
+```yaml
+worker:
+  livenessProbe:
+    enabled: true # true | false
+```
diff --git a/charts/openinwoner/Chart.yaml b/charts/openinwoner/Chart.yaml
@@ -3,7 +3,7 @@ name: openinwoner
 description: Platform voor gemeenten en overheden om producten inzichtelijker en toegankelijker te maken voor inwoners.
 
 type: application
-version: 1.6.1-beta.0
+version: 1.7.0-beta.0
 appVersion: latest
 icon: https://docs.openinwoner.nl/en/latest/_static/logo.png
 

diff --git a/charts/openinwoner/README.md b/charts/openinwoner/README.md
@@ -1,6 +1,6 @@
 # openinwoner
 
-![Version: 1.6.1-beta.0](https://img.shields.io/badge/Version-1.6.1--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
+![Version: 1.7.0-beta.0](https://img.shields.io/badge/Version-1.7.0--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square)
 
 Platform voor gemeenten en overheden om producten inzichtelijker en toegankelijker te maken voor inwoners.
 
@@ -194,13 +194,15 @@ Platform voor gemeenten en overheden om producten inzichtelijker en toegankelijk
 | worker.autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
 | worker.autoscaling.targetMemoryUtilizationPercentage | int | `80` |  |
 | worker.concurrency | int | `4` |  |
-| worker.livenessProbe.exec.command[0] | string | `"python"` |  |
-| worker.livenessProbe.exec.command[1] | string | `"/app/bin/check_celery_worker_liveness.py"` |  |
-| worker.livenessProbe.failureThreshold | int | `10` |  |
+| worker.livenessProbe.enabled | bool | `false` |  |
+| worker.livenessProbe.exec.command[0] | string | `"/bin/sh"` |  |
+| worker.livenessProbe.exec.command[1] | string | `"-c"` |  |
+| worker.livenessProbe.exec.command[2] | string | `"celery --workdir src --app open_inwoner.celery inspect --destination celery@${HOSTNAME} active"` |  |
+| worker.livenessProbe.failureThreshold | int | `3` |  |
 | worker.livenessProbe.initialDelaySeconds | int | `60` |  |
 | worker.livenessProbe.periodSeconds | int | `50` |  |
 | worker.livenessProbe.successThreshold | int | `1` |  |
-| worker.livenessProbe.timeoutSeconds | int | `5` |  |
+| worker.livenessProbe.timeoutSeconds | int | `10` |  |
 | worker.maxWorkerLivenessDelta | string | `""` |  |
 | worker.podLabels | object | `{}` |  |
 | worker.replicaCount | int | `1` |  |

diff --git a/charts/openinwoner/templates/deployment.yaml b/charts/openinwoner/templates/deployment.yaml
@@ -365,10 +365,16 @@ spec:
             {{- if .Values.extraEnvVars }}
             {{- include "openinwoner.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 12 }}
             {{- end }}
-            {{- if .Values.worker.livenessProbe }}
+          {{- if .Values.worker.livenessProbe.enabled }}
           livenessProbe:
-            {{- toYaml .Values.worker.livenessProbe | nindent 12 }}
-            {{- end }}
+            exec: 
+            {{- toYaml .Values.worker.livenessProbe.exec | nindent 14 }}
+            initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.worker.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.worker.livenessProbe.failureThreshold }}
+          {{- end }}
           resources:
             {{- toYaml .Values.worker.resources | nindent 12 }}
           command: