docker-compose.yml

name: diffpype

services:
  db:
    build:
      context: .
      dockerfile: docker/db.Dockerfile
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
    ports:
      - "${POSTGRES_PORT:-5432}:5432"
    volumes:
      - diffpype_db_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
      interval: 5s
      timeout: 5s
      retries: 10
    networks:
      - diffpype_net

  redis:
    image: redis:7-alpine
    ports:
      - "${REDIS_PORT:-6379}:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 5s
      retries: 10
    networks:
      - diffpype_net

  jaeger:
    image: jaegertracing/all-in-one:1.57
    environment:
      COLLECTOR_OTLP_ENABLED: "true"
    ports:
      - "${JAEGER_UI_PORT:-16686}:16686"
    networks:
      - diffpype_net

  minio:
    image: minio/minio:RELEASE.2024-09-22T00-33-43Z
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID:-minioadmin}
      MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minioadmin}
    ports:
      # Container S3 API stays on 9000 (S3_ENDPOINT_URL=http://minio:9000 in-network);
      # host port 9000 is taken by portainer, so the host mapping uses 9002.
      - "${MINIO_API_PORT:-9002}:9000"
      - "${MINIO_CONSOLE_PORT:-9001}:9001"
    volumes:
      - minio_data:/data
    networks:
      - diffpype_net

  api:
    build:
      context: .
      dockerfile: docker/api.Dockerfile
    environment:
      DATABASE_URL: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
      REDIS_URL: redis://redis:6379/0
      TEST_DATABASE_URL: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/diffpype_test
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      ADMIN_USER: ${ADMIN_USER:-sysadmin}
      ADMIN_PASSWORD: ${ADMIN_PASSWORD:-changeme}
      ADMIN_SECRET_KEY: ${ADMIN_SECRET_KEY:-diffpype-dev-secret-key-change-in-production}
      CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:5173}
      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://jaeger:4317}
      OTEL_SERVICE_NAME: diffpype-api
      S3_ENDPOINT_URL: ${S3_ENDPOINT_URL:-http://minio:9000}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin}
      S3_BUCKET_NAME: ${S3_BUCKET_NAME:-diffpype-data}
      S3_REGION: ${S3_REGION:-us-east-1}
      STORAGE_BACKEND: ${STORAGE_BACKEND:-s3}
      LOCAL_STORAGE_ROOT: ${LOCAL_STORAGE_ROOT:-./data}
    ports:
      - "${API_PORT:-8000}:8000"
    volumes:
      - ./src:/app/src
      - ./migrations:/app/migrations
      - ${LOCAL_STORAGE_HOST_PATH:-./local_storage_data}:/data
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - diffpype_net

  worker_light:
    build:
      context: .
      dockerfile: docker/worker.Dockerfile
    environment:
      DATABASE_URL: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
      REDIS_URL: redis://redis:6379/0
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      CELERY_QUEUES: light,dead_letter
      CELERY_CONCURRENCY: ${CELERY_LIGHT_CONCURRENCY:-4}
      CELERY_MAX_MEMORY_PER_CHILD: ${CELERY_LIGHT_MAX_MEMORY_PER_CHILD:-200000}
      CELERY_TASK_MAX_RETRIES: ${CELERY_TASK_MAX_RETRIES:-3}
      CELERY_TASK_RETRY_DELAY: ${CELERY_TASK_RETRY_DELAY:-60}
      ENABLE_DB_BACKUP_CRON: ${ENABLE_DB_BACKUP_CRON:-false}
      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://jaeger:4317}
      OTEL_SERVICE_NAME: diffpype-worker-light
      S3_ENDPOINT_URL: ${S3_ENDPOINT_URL:-http://minio:9000}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin}
      S3_BUCKET_NAME: ${S3_BUCKET_NAME:-diffpype-data}
      S3_REGION: ${S3_REGION:-us-east-1}
      STORAGE_BACKEND: ${STORAGE_BACKEND:-s3}
      LOCAL_STORAGE_ROOT: ${LOCAL_STORAGE_ROOT:-./data}
    deploy:
      resources:
        limits:
          memory: ${WORKER_LIGHT_MEM_LIMIT:-512m}
    volumes:
      - ./src:/app/src
      - ${LOCAL_STORAGE_HOST_PATH:-./local_storage_data}:/data
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - diffpype_net

  worker_heavy:
    build:
      context: .
      dockerfile: docker/worker.Dockerfile
    environment:
      DATABASE_URL: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
      REDIS_URL: redis://redis:6379/0
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      CELERY_QUEUES: heavy_memory
      CELERY_CONCURRENCY: ${CELERY_HEAVY_CONCURRENCY:-1}
      CELERY_MAX_MEMORY_PER_CHILD: ${CELERY_HEAVY_MAX_MEMORY_PER_CHILD:-8000000}
      CELERY_TASK_MAX_RETRIES: ${CELERY_TASK_MAX_RETRIES:-3}
      CELERY_TASK_RETRY_DELAY: ${CELERY_TASK_RETRY_DELAY:-60}
      ENABLE_DB_BACKUP_CRON: ${ENABLE_DB_BACKUP_CRON:-false}
      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://jaeger:4317}
      OTEL_SERVICE_NAME: diffpype-worker-heavy
      S3_ENDPOINT_URL: ${S3_ENDPOINT_URL:-http://minio:9000}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin}
      S3_BUCKET_NAME: ${S3_BUCKET_NAME:-diffpype-data}
      S3_REGION: ${S3_REGION:-us-east-1}
      STORAGE_BACKEND: ${STORAGE_BACKEND:-s3}
      LOCAL_STORAGE_ROOT: ${LOCAL_STORAGE_ROOT:-./data}
    deploy:
      resources:
        limits:
          memory: ${WORKER_HEAVY_MEM_LIMIT:-8g}
    volumes:
      - ./src:/app/src
      - ${LOCAL_STORAGE_HOST_PATH:-./local_storage_data}:/data
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - diffpype_net

  beat:
    build:
      context: .
      dockerfile: docker/worker.Dockerfile
    # Dedicated Celery Beat process (exactly one, ever) using the database-backed
    # scheduler so schedules edited at runtime via SQLAdmin take effect without a
    # restart. Not a --beat flag on a worker: that would double-fire schedules if
    # the worker ever scaled past one replica. Requires the scheduler tables to
    # exist (run `alembic upgrade head` first); it consumes no task queues.
    command: celery -A src.worker.celery_app beat -S sqlalchemy_celery_beat.schedulers:DatabaseScheduler --loglevel=info
    environment:
      DATABASE_URL: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
      REDIS_URL: redis://redis:6379/0
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://jaeger:4317}
      OTEL_SERVICE_NAME: diffpype-beat
      STAGING_SYNC_INTERVAL_SECONDS: ${STAGING_SYNC_INTERVAL_SECONDS:-300}
      ENABLE_STAGING_SYNC_CRON: ${ENABLE_STAGING_SYNC_CRON:-true}
      JOB_STALENESS_TIMEOUT_SECONDS: ${JOB_STALENESS_TIMEOUT_SECONDS:-3600}
    volumes:
      - ./src:/app/src
    depends_on:
      db:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - diffpype_net

  flower:
    image: mher/flower:2.0.1
    environment:
      CELERY_BROKER_URL: redis://redis:6379/0
      FLOWER_PORT: ${FLOWER_PORT:-5555}
      FLOWER_BASIC_AUTH: ${FLOWER_BASIC_AUTH:-admin:changeme}
    ports:
      - "${FLOWER_PORT:-5555}:5555"
    depends_on:
      redis:
        condition: service_healthy
    networks:
      - diffpype_net

  portainer:
    image: portainer/portainer-ce:latest
    ports:
      - "${PORTAINER_PORT:-9000}:9000"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - portainer_data:/data
    networks:
      - diffpype_net

  ui:
    build:
      context: .
      dockerfile: docker/ui.Dockerfile
    environment:
      VITE_API_URL: ${VITE_API_URL:-http://localhost:8000}
    ports:
      - "${UI_PORT:-5173}:5173"
    volumes:
      - ./src/ui:/app
      - diffpype_ui_node_modules:/app/node_modules
    depends_on:
      - api
    networks:
      - diffpype_net

networks:
  diffpype_net:
    name: diffpype_net

volumes:
  diffpype_db_data:
  diffpype_ui_node_modules:
  portainer_data:
  minio_data: