Keep Prometheus schedulable when memory is short

kubernetes/alertmanager/config.yaml

@@ -27,11 +27,21 @@ route:
 
     # The child route trees.
   routes:
+  - match:
+      alertname: Watchdog
+    receiver: healthchecks-watchdog
+    group_wait: 0s
+    group_interval: 1m
+    repeat_interval: 5m
   - match:
       realert: daily
     repeat_interval: 1d
 
 receivers:
+- name: healthchecks-watchdog
+  webhook_configs:
+  - url_file: /secrets/HEALTHCHECKS_WATCHDOG_URL
+    send_resolved: false
 - name: default
   slack_configs:
   - api_url_file: /secrets/SLACK_WEBHOOK_URL

kubernetes/alertmanager/deploy.yaml

@@ -22,6 +22,7 @@ spec:
         app.kubernetes.io/name: alertmanager
         app.kubernetes.io/instance: alertmanager
     spec:
+      priorityClassName: monitoring
       containers:
       - name: alertmanager
         image: prom/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a

kubernetes/alertmanager/externalsecret.yaml

@@ -9,6 +9,11 @@ spec:
   secretStoreRef:
     name: production
     kind: ClusterSecretStore
+  target:
+    template:
+      mergePolicy: Merge
+      data:
+        HEALTHCHECKS_WATCHDOG_URL: 'https://hc.k.oneill.net/ping/{{ .HEALTHCHECKS_PING_KEY }}/prometheus-watchdog'
   data:
   - secretKey: SMTP_PASSWORD
     remoteRef:
@@ -18,3 +23,7 @@ spec:
     remoteRef:
       key: alertmanager
       property: SLACK_WEBHOOK_URL
+  - secretKey: HEALTHCHECKS_PING_KEY
+    remoteRef:
+      key: healthchecks
+      property: PING_KEY

kubernetes/karakeep/helm/meilisearch/configmap.yaml

@@ -13,6 +13,8 @@ metadata:
     app.kubernetes.io/managed-by: Helm
 data:
   MEILI_ENV: "production"
+  MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: "true"
+  MEILI_MAX_INDEXING_MEMORY: "2Gb"
   MEILI_NO_ANALYTICS: "true"
   MEILI_UPGRADE_DB: "true"
 

kubernetes/karakeep/helm/meilisearch/statefulset.yaml

@@ -28,7 +28,7 @@ spec:
         app.kubernetes.io/part-of: meilisearch
         app.kubernetes.io/managed-by: Helm
       annotations:
-        checksum/config: 4cb7c9de031634b65307b783c37ea7c3e2595a65f3f88d75bc6e509d673328fd
+        checksum/config: fc496f8e7457de8c064d49afeac3c06280869f29dcd1e2ddc37f4040576a20b8
     spec:
       serviceAccountName: karakeep-meilisearch
       securityContext:

kubernetes/karakeep/values.yaml

@@ -108,6 +108,8 @@ meilisearch:
   enabled: true
   environment:
     MEILI_UPGRADE_DB: 'true'
+    MEILI_MAX_INDEXING_MEMORY: 2Gb
+    MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: 'true'
   auth:
     existingMasterKeySecret: karakeep-meilisearch
   persistence:

kubernetes/openarchiver/meilisearch-statefulset.yaml

@@ -33,6 +33,10 @@ spec:
           value: 'true'
         - name: MEILI_UPGRADE_DB
           value: 'true'
+        - name: MEILI_MAX_INDEXING_MEMORY
+          value: 2Gb
+        - name: MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE
+          value: 'true'
         - name: MEILI_MASTER_KEY
           valueFrom:
             secretKeyRef:

kubernetes/openarchiver/namespace.yaml

@@ -8,4 +8,4 @@ metadata:
     goldilocks.fairwinds.com/enabled: 'true'
   annotations:
     goldilocks.fairwinds.com/vpa-update-mode: InPlaceOrRecreate
-    goldilocks.fairwinds.com/vpa-resource-policy: '{"containerPolicies":[{"containerName":"*","controlledValues":"RequestsOnly"}]}'
+    goldilocks.fairwinds.com/vpa-resource-policy: '{"containerPolicies":[{"containerName":"*","controlledValues":"RequestsOnly"},{"containerName":"meilisearch","controlledValues":"RequestsOnly","maxAllowed":{"memory":"8Gi"}}]}'

kubernetes/prometheus/config/rules.yml

@@ -8,6 +8,10 @@ groups:
     expr: hass_sensor_unit_m{entity="sensor.desk_ultrasonic_sensor"} >  bool 0.9
 - name: alert.rules
   rules:
+  - alert: Watchdog
+    expr: vector(1)
+    annotations:
+      message: Always firing; Alertmanager forwards it to Healthchecks as a heartbeat
   - alert: NodeDown
     expr: up{job="node resources", instance!="xtal:9100"} == 0
     for: 5m

kubernetes/prometheus/deploy.yaml

@@ -22,6 +22,7 @@ spec:
         app.kubernetes.io/name: prometheus
         app.kubernetes.io/instance: prometheus
     spec:
+      priorityClassName: monitoring
       serviceAccountName: prometheus
       containers:
       - name: prometheus

kubernetes/prometheus/kustomization.yaml

@@ -13,6 +13,7 @@ commonAnnotations:
 resources:
 - deploy.yaml
 - ingress.yaml
+- priorityclass.yaml
 - pvc.yaml
 - rbac.yaml
 - service.yaml

kubernetes/prometheus/priorityclass.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+
+metadata:
+  name: monitoring
+
+value: 1000000
+globalDefault: false
+description: Prometheus and Alertmanager

opentofu/healthchecks.tf

@@ -159,6 +159,20 @@ resource "healthchecksio_check" "velero_daily" {
   channels = [data.healthchecksio_channel.selfhosted_email.id]
 }
 
+# Prometheus/Alertmanager heartbeat - Alertmanager forwards the always-firing
+# Watchdog alert every 5 minutes; silence means alerting is down
+resource "healthchecksio_check" "prometheus_watchdog" {
+  provider = healthchecksio.selfhosted
+
+  name     = "prometheus-watchdog"
+  slug     = "prometheus-watchdog"
+  desc     = "Heartbeat from the Prometheus Watchdog alert via Alertmanager"
+  tags     = ["monitoring", "kubernetes"]
+  timeout  = 600 # 10 minutes
+  grace    = 600 # 10 minutes
+  channels = [data.healthchecksio_channel.selfhosted_email.id]
+}
+
 # Kubernetes control-plane backup - runs daily
 resource "healthchecksio_check" "cluster_backup" {
   provider = healthchecksio.selfhosted