From 76eac4f54b0fb16bcb6549a249ce18cbf86b1c42 Mon Sep 17 00:00:00 2001 From: Clayton O'Neill Date: Sat, 10 Oct 2026 11:54:04 -0500 Subject: [PATCH] Keep Prometheus schedulable when memory is short On 2026-10-10 Prometheus spent about 11 hours Pending, so no metrics were collected and no alerts were sent. Its VPA had raised the memory request to 12.45 GB, no worker had that much unreserved, and every pod shared the default priority, so the scheduler found nothing it could preempt. The largest reservation in the cluster was openarchiver's Meilisearch at 14.1 GiB. Prometheus and Alertmanager now use a new monitoring PriorityClass (1000000, preempting), so the scheduler can evict lower-priority pods to make room for them instead of leaving alerting down. Nothing noticed the outage either, because the alerting path itself was down. Prometheus now has an always-firing Watchdog alert, which Alertmanager routes only to a webhook that pings a new prometheus-watchdog check on the self-hosted Healthchecks every 5 minutes. The ping URL is built from the existing Healthchecks ping key in 1Password, and OpenTofu defines the check with a 10-minute timeout and 10-minute grace, so Healthchecks emails if Prometheus or Alertmanager stops for about 20 minutes. Meilisearch memory-maps its LMDB index, so much of what VPA measures is reclaimable page cache and its recommendation runs well above what the process needs. The openarchiver namespace's VPA policy now caps the meilisearch container at 8Gi, leaving the other containers on the namespace-wide policy. Meilisearch cannot limit how much it maps, but it can cap indexing memory, which otherwise defaults to two-thirds of the node's RAM in a container without a limit. Both openarchiver and karakeep now set MEILI_MAX_INDEXING_MEMORY to 2Gb and enable MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE, trading write speed for lower memory use while indexing. --- kubernetes/alertmanager/config.yaml | 10 ++++++++++ kubernetes/alertmanager/deploy.yaml | 1 + kubernetes/alertmanager/externalsecret.yaml | 9 +++++++++ .../karakeep/helm/meilisearch/configmap.yaml | 2 ++ .../karakeep/helm/meilisearch/statefulset.yaml | 2 +- kubernetes/karakeep/values.yaml | 2 ++ .../openarchiver/meilisearch-statefulset.yaml | 4 ++++ kubernetes/openarchiver/namespace.yaml | 2 +- kubernetes/prometheus/config/rules.yml | 4 ++++ kubernetes/prometheus/deploy.yaml | 1 + kubernetes/prometheus/kustomization.yaml | 1 + kubernetes/prometheus/priorityclass.yaml | 10 ++++++++++ opentofu/healthchecks.tf | 14 ++++++++++++++ 13 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 kubernetes/prometheus/priorityclass.yaml diff --git a/kubernetes/alertmanager/config.yaml b/kubernetes/alertmanager/config.yaml index fcb5ec58a4..c434f758ab 100644 --- a/kubernetes/alertmanager/config.yaml +++ b/kubernetes/alertmanager/config.yaml @@ -27,11 +27,21 @@ route: # The child route trees. routes: + - match: + alertname: Watchdog + receiver: healthchecks-watchdog + group_wait: 0s + group_interval: 1m + repeat_interval: 5m - match: realert: daily repeat_interval: 1d receivers: +- name: healthchecks-watchdog + webhook_configs: + - url_file: /secrets/HEALTHCHECKS_WATCHDOG_URL + send_resolved: false - name: default slack_configs: - api_url_file: /secrets/SLACK_WEBHOOK_URL diff --git a/kubernetes/alertmanager/deploy.yaml b/kubernetes/alertmanager/deploy.yaml index f3631c047a..938d9ce019 100644 --- a/kubernetes/alertmanager/deploy.yaml +++ b/kubernetes/alertmanager/deploy.yaml @@ -22,6 +22,7 @@ spec: app.kubernetes.io/name: alertmanager app.kubernetes.io/instance: alertmanager spec: + priorityClassName: monitoring containers: - name: alertmanager image: prom/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a diff --git a/kubernetes/alertmanager/externalsecret.yaml b/kubernetes/alertmanager/externalsecret.yaml index dadf20ee82..a3b3515e09 100644 --- a/kubernetes/alertmanager/externalsecret.yaml +++ b/kubernetes/alertmanager/externalsecret.yaml @@ -9,6 +9,11 @@ spec: secretStoreRef: name: production kind: ClusterSecretStore + target: + template: + mergePolicy: Merge + data: + HEALTHCHECKS_WATCHDOG_URL: 'https://hc.k.oneill.net/ping/{{ .HEALTHCHECKS_PING_KEY }}/prometheus-watchdog' data: - secretKey: SMTP_PASSWORD remoteRef: @@ -18,3 +23,7 @@ spec: remoteRef: key: alertmanager property: SLACK_WEBHOOK_URL + - secretKey: HEALTHCHECKS_PING_KEY + remoteRef: + key: healthchecks + property: PING_KEY diff --git a/kubernetes/karakeep/helm/meilisearch/configmap.yaml b/kubernetes/karakeep/helm/meilisearch/configmap.yaml index ad8a60b03f..484ed6fb25 100644 --- a/kubernetes/karakeep/helm/meilisearch/configmap.yaml +++ b/kubernetes/karakeep/helm/meilisearch/configmap.yaml @@ -13,6 +13,8 @@ metadata: app.kubernetes.io/managed-by: Helm data: MEILI_ENV: "production" + MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: "true" + MEILI_MAX_INDEXING_MEMORY: "2Gb" MEILI_NO_ANALYTICS: "true" MEILI_UPGRADE_DB: "true" diff --git a/kubernetes/karakeep/helm/meilisearch/statefulset.yaml b/kubernetes/karakeep/helm/meilisearch/statefulset.yaml index 5ae10aba18..2f135295bb 100644 --- a/kubernetes/karakeep/helm/meilisearch/statefulset.yaml +++ b/kubernetes/karakeep/helm/meilisearch/statefulset.yaml @@ -28,7 +28,7 @@ spec: app.kubernetes.io/part-of: meilisearch app.kubernetes.io/managed-by: Helm annotations: - checksum/config: 4cb7c9de031634b65307b783c37ea7c3e2595a65f3f88d75bc6e509d673328fd + checksum/config: fc496f8e7457de8c064d49afeac3c06280869f29dcd1e2ddc37f4040576a20b8 spec: serviceAccountName: karakeep-meilisearch securityContext: diff --git a/kubernetes/karakeep/values.yaml b/kubernetes/karakeep/values.yaml index 1f9fbd60e1..c63d086309 100644 --- a/kubernetes/karakeep/values.yaml +++ b/kubernetes/karakeep/values.yaml @@ -108,6 +108,8 @@ meilisearch: enabled: true environment: MEILI_UPGRADE_DB: 'true' + MEILI_MAX_INDEXING_MEMORY: 2Gb + MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: 'true' auth: existingMasterKeySecret: karakeep-meilisearch persistence: diff --git a/kubernetes/openarchiver/meilisearch-statefulset.yaml b/kubernetes/openarchiver/meilisearch-statefulset.yaml index ec11fdac22..a3d722088e 100644 --- a/kubernetes/openarchiver/meilisearch-statefulset.yaml +++ b/kubernetes/openarchiver/meilisearch-statefulset.yaml @@ -33,6 +33,10 @@ spec: value: 'true' - name: MEILI_UPGRADE_DB value: 'true' + - name: MEILI_MAX_INDEXING_MEMORY + value: 2Gb + - name: MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE + value: 'true' - name: MEILI_MASTER_KEY valueFrom: secretKeyRef: diff --git a/kubernetes/openarchiver/namespace.yaml b/kubernetes/openarchiver/namespace.yaml index 655f890c5d..0bdb5e23d4 100644 --- a/kubernetes/openarchiver/namespace.yaml +++ b/kubernetes/openarchiver/namespace.yaml @@ -8,4 +8,4 @@ metadata: goldilocks.fairwinds.com/enabled: 'true' annotations: goldilocks.fairwinds.com/vpa-update-mode: InPlaceOrRecreate - goldilocks.fairwinds.com/vpa-resource-policy: '{"containerPolicies":[{"containerName":"*","controlledValues":"RequestsOnly"}]}' + goldilocks.fairwinds.com/vpa-resource-policy: '{"containerPolicies":[{"containerName":"*","controlledValues":"RequestsOnly"},{"containerName":"meilisearch","controlledValues":"RequestsOnly","maxAllowed":{"memory":"8Gi"}}]}' diff --git a/kubernetes/prometheus/config/rules.yml b/kubernetes/prometheus/config/rules.yml index 22869139c7..762d670824 100644 --- a/kubernetes/prometheus/config/rules.yml +++ b/kubernetes/prometheus/config/rules.yml @@ -8,6 +8,10 @@ groups: expr: hass_sensor_unit_m{entity="sensor.desk_ultrasonic_sensor"} > bool 0.9 - name: alert.rules rules: + - alert: Watchdog + expr: vector(1) + annotations: + message: Always firing; Alertmanager forwards it to Healthchecks as a heartbeat - alert: NodeDown expr: up{job="node resources", instance!="xtal:9100"} == 0 for: 5m diff --git a/kubernetes/prometheus/deploy.yaml b/kubernetes/prometheus/deploy.yaml index 26b459d528..0dd249f0e6 100644 --- a/kubernetes/prometheus/deploy.yaml +++ b/kubernetes/prometheus/deploy.yaml @@ -22,6 +22,7 @@ spec: app.kubernetes.io/name: prometheus app.kubernetes.io/instance: prometheus spec: + priorityClassName: monitoring serviceAccountName: prometheus containers: - name: prometheus diff --git a/kubernetes/prometheus/kustomization.yaml b/kubernetes/prometheus/kustomization.yaml index 593579c160..ce80167439 100644 --- a/kubernetes/prometheus/kustomization.yaml +++ b/kubernetes/prometheus/kustomization.yaml @@ -13,6 +13,7 @@ commonAnnotations: resources: - deploy.yaml - ingress.yaml +- priorityclass.yaml - pvc.yaml - rbac.yaml - service.yaml diff --git a/kubernetes/prometheus/priorityclass.yaml b/kubernetes/prometheus/priorityclass.yaml new file mode 100644 index 0000000000..c176c99a2a --- /dev/null +++ b/kubernetes/prometheus/priorityclass.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass + +metadata: + name: monitoring + +value: 1000000 +globalDefault: false +description: Prometheus and Alertmanager diff --git a/opentofu/healthchecks.tf b/opentofu/healthchecks.tf index b60edf6b91..3482e36848 100644 --- a/opentofu/healthchecks.tf +++ b/opentofu/healthchecks.tf @@ -159,6 +159,20 @@ resource "healthchecksio_check" "velero_daily" { channels = [data.healthchecksio_channel.selfhosted_email.id] } +# Prometheus/Alertmanager heartbeat - Alertmanager forwards the always-firing +# Watchdog alert every 5 minutes; silence means alerting is down +resource "healthchecksio_check" "prometheus_watchdog" { + provider = healthchecksio.selfhosted + + name = "prometheus-watchdog" + slug = "prometheus-watchdog" + desc = "Heartbeat from the Prometheus Watchdog alert via Alertmanager" + tags = ["monitoring", "kubernetes"] + timeout = 600 # 10 minutes + grace = 600 # 10 minutes + channels = [data.healthchecksio_channel.selfhosted_email.id] +} + # Kubernetes control-plane backup - runs daily resource "healthchecksio_check" "cluster_backup" { provider = healthchecksio.selfhosted