diff --git a/kubernetes/alertmanager/config.yaml b/kubernetes/alertmanager/config.yaml index fcb5ec58a4..c434f758ab 100644 --- a/kubernetes/alertmanager/config.yaml +++ b/kubernetes/alertmanager/config.yaml @@ -27,11 +27,21 @@ route: # The child route trees. routes: + - match: + alertname: Watchdog + receiver: healthchecks-watchdog + group_wait: 0s + group_interval: 1m + repeat_interval: 5m - match: realert: daily repeat_interval: 1d receivers: +- name: healthchecks-watchdog + webhook_configs: + - url_file: /secrets/HEALTHCHECKS_WATCHDOG_URL + send_resolved: false - name: default slack_configs: - api_url_file: /secrets/SLACK_WEBHOOK_URL diff --git a/kubernetes/alertmanager/deploy.yaml b/kubernetes/alertmanager/deploy.yaml index f3631c047a..938d9ce019 100644 --- a/kubernetes/alertmanager/deploy.yaml +++ b/kubernetes/alertmanager/deploy.yaml @@ -22,6 +22,7 @@ spec: app.kubernetes.io/name: alertmanager app.kubernetes.io/instance: alertmanager spec: + priorityClassName: monitoring containers: - name: alertmanager image: prom/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a diff --git a/kubernetes/alertmanager/externalsecret.yaml b/kubernetes/alertmanager/externalsecret.yaml index dadf20ee82..a3b3515e09 100644 --- a/kubernetes/alertmanager/externalsecret.yaml +++ b/kubernetes/alertmanager/externalsecret.yaml @@ -9,6 +9,11 @@ spec: secretStoreRef: name: production kind: ClusterSecretStore + target: + template: + mergePolicy: Merge + data: + HEALTHCHECKS_WATCHDOG_URL: 'https://hc.k.oneill.net/ping/{{ .HEALTHCHECKS_PING_KEY }}/prometheus-watchdog' data: - secretKey: SMTP_PASSWORD remoteRef: @@ -18,3 +23,7 @@ spec: remoteRef: key: alertmanager property: SLACK_WEBHOOK_URL + - secretKey: HEALTHCHECKS_PING_KEY + remoteRef: + key: healthchecks + property: PING_KEY diff --git a/kubernetes/karakeep/helm/meilisearch/configmap.yaml b/kubernetes/karakeep/helm/meilisearch/configmap.yaml index ad8a60b03f..484ed6fb25 100644 --- a/kubernetes/karakeep/helm/meilisearch/configmap.yaml +++ b/kubernetes/karakeep/helm/meilisearch/configmap.yaml @@ -13,6 +13,8 @@ metadata: app.kubernetes.io/managed-by: Helm data: MEILI_ENV: "production" + MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: "true" + MEILI_MAX_INDEXING_MEMORY: "2Gb" MEILI_NO_ANALYTICS: "true" MEILI_UPGRADE_DB: "true" diff --git a/kubernetes/karakeep/helm/meilisearch/statefulset.yaml b/kubernetes/karakeep/helm/meilisearch/statefulset.yaml index 5ae10aba18..2f135295bb 100644 --- a/kubernetes/karakeep/helm/meilisearch/statefulset.yaml +++ b/kubernetes/karakeep/helm/meilisearch/statefulset.yaml @@ -28,7 +28,7 @@ spec: app.kubernetes.io/part-of: meilisearch app.kubernetes.io/managed-by: Helm annotations: - checksum/config: 4cb7c9de031634b65307b783c37ea7c3e2595a65f3f88d75bc6e509d673328fd + checksum/config: fc496f8e7457de8c064d49afeac3c06280869f29dcd1e2ddc37f4040576a20b8 spec: serviceAccountName: karakeep-meilisearch securityContext: diff --git a/kubernetes/karakeep/values.yaml b/kubernetes/karakeep/values.yaml index 1f9fbd60e1..c63d086309 100644 --- a/kubernetes/karakeep/values.yaml +++ b/kubernetes/karakeep/values.yaml @@ -108,6 +108,8 @@ meilisearch: enabled: true environment: MEILI_UPGRADE_DB: 'true' + MEILI_MAX_INDEXING_MEMORY: 2Gb + MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: 'true' auth: existingMasterKeySecret: karakeep-meilisearch persistence: diff --git a/kubernetes/openarchiver/meilisearch-statefulset.yaml b/kubernetes/openarchiver/meilisearch-statefulset.yaml index ec11fdac22..a3d722088e 100644 --- a/kubernetes/openarchiver/meilisearch-statefulset.yaml +++ b/kubernetes/openarchiver/meilisearch-statefulset.yaml @@ -33,6 +33,10 @@ spec: value: 'true' - name: MEILI_UPGRADE_DB value: 'true' + - name: MEILI_MAX_INDEXING_MEMORY + value: 2Gb + - name: MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE + value: 'true' - name: MEILI_MASTER_KEY valueFrom: secretKeyRef: diff --git a/kubernetes/openarchiver/namespace.yaml b/kubernetes/openarchiver/namespace.yaml index 655f890c5d..0bdb5e23d4 100644 --- a/kubernetes/openarchiver/namespace.yaml +++ b/kubernetes/openarchiver/namespace.yaml @@ -8,4 +8,4 @@ metadata: goldilocks.fairwinds.com/enabled: 'true' annotations: goldilocks.fairwinds.com/vpa-update-mode: InPlaceOrRecreate - goldilocks.fairwinds.com/vpa-resource-policy: '{"containerPolicies":[{"containerName":"*","controlledValues":"RequestsOnly"}]}' + goldilocks.fairwinds.com/vpa-resource-policy: '{"containerPolicies":[{"containerName":"*","controlledValues":"RequestsOnly"},{"containerName":"meilisearch","controlledValues":"RequestsOnly","maxAllowed":{"memory":"8Gi"}}]}' diff --git a/kubernetes/prometheus/config/rules.yml b/kubernetes/prometheus/config/rules.yml index 22869139c7..762d670824 100644 --- a/kubernetes/prometheus/config/rules.yml +++ b/kubernetes/prometheus/config/rules.yml @@ -8,6 +8,10 @@ groups: expr: hass_sensor_unit_m{entity="sensor.desk_ultrasonic_sensor"} > bool 0.9 - name: alert.rules rules: + - alert: Watchdog + expr: vector(1) + annotations: + message: Always firing; Alertmanager forwards it to Healthchecks as a heartbeat - alert: NodeDown expr: up{job="node resources", instance!="xtal:9100"} == 0 for: 5m diff --git a/kubernetes/prometheus/deploy.yaml b/kubernetes/prometheus/deploy.yaml index 26b459d528..0dd249f0e6 100644 --- a/kubernetes/prometheus/deploy.yaml +++ b/kubernetes/prometheus/deploy.yaml @@ -22,6 +22,7 @@ spec: app.kubernetes.io/name: prometheus app.kubernetes.io/instance: prometheus spec: + priorityClassName: monitoring serviceAccountName: prometheus containers: - name: prometheus diff --git a/kubernetes/prometheus/kustomization.yaml b/kubernetes/prometheus/kustomization.yaml index 593579c160..ce80167439 100644 --- a/kubernetes/prometheus/kustomization.yaml +++ b/kubernetes/prometheus/kustomization.yaml @@ -13,6 +13,7 @@ commonAnnotations: resources: - deploy.yaml - ingress.yaml +- priorityclass.yaml - pvc.yaml - rbac.yaml - service.yaml diff --git a/kubernetes/prometheus/priorityclass.yaml b/kubernetes/prometheus/priorityclass.yaml new file mode 100644 index 0000000000..c176c99a2a --- /dev/null +++ b/kubernetes/prometheus/priorityclass.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass + +metadata: + name: monitoring + +value: 1000000 +globalDefault: false +description: Prometheus and Alertmanager diff --git a/opentofu/healthchecks.tf b/opentofu/healthchecks.tf index b60edf6b91..3482e36848 100644 --- a/opentofu/healthchecks.tf +++ b/opentofu/healthchecks.tf @@ -159,6 +159,20 @@ resource "healthchecksio_check" "velero_daily" { channels = [data.healthchecksio_channel.selfhosted_email.id] } +# Prometheus/Alertmanager heartbeat - Alertmanager forwards the always-firing +# Watchdog alert every 5 minutes; silence means alerting is down +resource "healthchecksio_check" "prometheus_watchdog" { + provider = healthchecksio.selfhosted + + name = "prometheus-watchdog" + slug = "prometheus-watchdog" + desc = "Heartbeat from the Prometheus Watchdog alert via Alertmanager" + tags = ["monitoring", "kubernetes"] + timeout = 600 # 10 minutes + grace = 600 # 10 minutes + channels = [data.healthchecksio_channel.selfhosted_email.id] +} + # Kubernetes control-plane backup - runs daily resource "healthchecksio_check" "cluster_backup" { provider = healthchecksio.selfhosted