docker-compose-observability.yaml

x-logging: &logging
  driver: json-file
  options:
    max-size: "20m"
    max-file: "10"

services:
  prometheus:
    image: prom/prometheus:v2.53.0
    container_name: datamap_prometheus
    restart: always
    logging: *logging
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=30d
      - --web.enable-lifecycle
    volumes:
      - ./infrastructure/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./infrastructure/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
      - prometheus_data:/prometheus
    # node-exporter is on the host network; this is how Prometheus reaches it.
    extra_hosts:
      - host.docker.internal:host-gateway
    # Not published: reached through Grafana, on the docker network.
    expose:
      - "9090"
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:9090/-/healthy"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - gatekeeper_gatekeeper-network

  grafana:
    image: grafana/grafana:11.1.0
    container_name: datamap_grafana
    restart: always
    logging: *logging
    env_file:
      - ${ENV_FILE_PATH}
    environment:
      - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/grafana/
      - GF_SERVER_SERVE_FROM_SUB_PATH=true
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_ANALYTICS_REPORTING_ENABLED=false
    volumes:
      - ./infrastructure/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./infrastructure/grafana/dashboards:/etc/grafana/dashboards:ro
      - grafana_data:/var/lib/grafana
    ports:
      - "127.0.0.1:3001:3000"
    depends_on:
      prometheus:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:3000/api/health"]
      interval: 10s
      timeout: 5s
      retries: 6
      start_period: 30s
    networks:
      - gatekeeper_gatekeeper-network

  loki:
    image: grafana/loki:3.7.8
    container_name: datamap_loki
    restart: always
    logging: *logging
    command: -config.file=/etc/loki/loki.yml
    volumes:
      - ./infrastructure/loki/loki.yml:/etc/loki/loki.yml:ro
      - loki_data:/loki
    expose:
      - "3100"
    # No healthcheck: the image is distroless, with no shell and no http client
    # to run one with. Prometheus scrapes it instead, and TargetDown alerts.
    networks:
      - gatekeeper_gatekeeper-network

  alloy:
    image: grafana/alloy:v1.19.2
    container_name: datamap_alloy
    restart: always
    logging: *logging
    command:
      - run
      - --server.http.listen-addr=0.0.0.0:12345
      - --storage.path=/var/lib/alloy/data
      - /etc/alloy/config.alloy
    volumes:
      - ./infrastructure/alloy/config.alloy:/etc/alloy/config.alloy:ro
      # Reading other containers' logs means talking to the daemon. See
      # docs/runbooks/observability.md for what that grants.
      - /var/run/docker.sock:/var/run/docker.sock:ro
      # The host nginx's access log; it is not a container.
      - /var/log/nginx:/var/log/nginx:ro
      - alloy_data:/var/lib/alloy/data
    depends_on:
      - loki
    # No healthcheck, for the same reason as Loki: the image carries a shell but
    # no http client to probe with, so the check could only ever fail. It ran
    # for a day reporting unhealthy while Prometheus scraped it successfully.
    # That scrape is the signal, and TargetDown is the alert.
    networks:
      - gatekeeper_gatekeeper-network

  # Host network, so the network panels show the host's interfaces rather than
  # the container's. It binds the docker0 address only: reachable from the
  # containers, never from outside the host. See docs/runbooks/observability.md.
  node-exporter:
    image: prom/node-exporter:v1.8.2
    container_name: datamap_node_exporter
    restart: always
    logging: *logging
    network_mode: host
    pid: host
    command:
      - --web.listen-address=${HOST_METRICS_ADDRESS:-172.17.0.1}:9100
      - --path.procfs=/host/proc
      - --path.sysfs=/host/sys
      - --path.rootfs=/rootfs
      # The database backup writes backup.prom here (docs/runbooks/database-backup.md).
      - --collector.textfile.directory=/textfile
      # An unresponsive NFS mount makes the filesystem collector hang, and a
      # hung collector costs every metric on the host, not just that mount.
      - --collector.filesystem.fs-types-exclude=^(autofs|nfs|nfs4|cifs|smb3|tmpfs|devtmpfs|overlay|squashfs|fuse.*)$$
      - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker/.+|rootfs/var/lib/docker/.+)($$|/)
      - --collector.netdev.device-exclude=^(lo|veth.*|br-.*|docker.*)$$
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
      - ${BACKUP_METRICS_DIR:-/home/datamap/metrics}:/textfile:ro

  # Sees every container on the host, including the neighbours'. Prometheus
  # keeps only datamap_* series (prometheus.yml), and the long housekeeping
  # interval is what keeps it cheap on a CPU-bound host.
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.55.1
    container_name: datamap_cadvisor
    restart: always
    logging: *logging
    privileged: true
    devices:
      - /dev/kmsg
    command:
      - --docker_only=true
      - --housekeeping_interval=30s
      - --store_container_labels=false
      - --disable_metrics=percpu,sched,tcp,udp,hugetlb,referenced_memory,cpu_topology,resctrl,advtcp,process,memory_numa
    volumes:
      - /:/rootfs:ro
      # Carries both docker.sock and containerd/containerd.sock: with Docker's
      # containerd image store, only containerd can say which layer is whose.
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    expose:
      - "8080"
    networks:
      - gatekeeper_gatekeeper-network

  # The application's own credentials, from the same env file the application
  # reads, so there is nothing separate to rotate.
  postgres-exporter:
    image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
    container_name: datamap_postgres_exporter
    restart: always
    logging: *logging
    env_file:
      - ${ENV_FILE_PATH}
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >-
        DATA_SOURCE_URI="datamap_gatekeeper_db:5432/$$POSTGRES_DB?sslmode=disable"
        DATA_SOURCE_USER="$$POSTGRES_USER" DATA_SOURCE_PASS="$$POSTGRES_PASSWORD"
        exec /bin/postgres_exporter
    expose:
      - "9187"
    networks:
      - gatekeeper_gatekeeper-network

  # nginx runs on the host, not in a container. Its stub_status is set up by
  # hand; docs/runbooks/observability.md has the steps.
  nginx-exporter:
    image: nginx/nginx-prometheus-exporter:1.3.0
    container_name: datamap_nginx_exporter
    restart: always
    logging: *logging
    command:
      - --nginx.scrape-uri=http://host.docker.internal:8081/stub_status
    extra_hosts:
      - host.docker.internal:host-gateway
    expose:
      - "9113"
    networks:
      - gatekeeper_gatekeeper-network

volumes:
  prometheus_data:
  grafana_data:
  loki_data:
  alloy_data:

networks:
  gatekeeper_gatekeeper-network:
    external: true
    name: gatekeeper_gatekeeper-network