From 8dbbd7835740dd103ed10c80106dd3a91e466a1b Mon Sep 17 00:00:00 2001 From: davecoulter Date: Tue, 28 Jul 2026 16:19:18 -0400 Subject: [PATCH 1/3] Add ingest/tile/epoch/mosaic domain services Populates the doc 26 domain schema end-to-end: real FITS ingest, spatial tiling, MJD-based epoch clustering, and mosaic job dispatch, so the future Aladin UI and JWST pipeline docs have real, tested services and data to build on instead of empty tables. --- .claude/skills/gen_tests.md | 4 +- .env.example | 20 +- .gitignore | 1 + CLAUDE.md | 7 +- docker-compose.yml | 12 + .../28_domain_graph_population.md | 145 ++++ docs/architecture/index.md | 1 + docs/conf.py | 5 + docs/index.rst | 40 + ...009_add_project_slug_and_ingest_batches.py | 109 +++ ...60727_0010_add_level3_mosaic_barycenter.py | 38 + pyproject.toml | 4 + scripts/smoke/populate_demo_project.sh | 82 ++ src/api/main.py | 10 + src/api/routes/epochs.py | 27 + src/api/routes/ingest.py | 28 + src/api/routes/mosaics.py | 35 + src/api/routes/projects.py | 21 + src/api/routes/tiles.py | 47 ++ src/api/schemas.py | 166 +++- src/api/tests/test_cli.py | 785 ++++++++++++++++++ src/api/tests/test_epochs.py | 99 +++ src/api/tests/test_ingest.py | 72 ++ src/api/tests/test_mosaics.py | 80 ++ src/api/tests/test_projects.py | 55 ++ src/api/tests/test_schemas.py | 26 +- src/api/tests/test_tiles.py | 105 +++ src/cli.py | 517 +++++++++++- src/core/config.py | 3 + src/db/models.py | 32 + src/db/seed.py | 49 +- src/db/spatial_types.py | 22 + src/db/tests/test_integration.py | 586 ++++++++++++- src/db/tests/test_spatial_types.py | 30 + src/services/epoch_service.py | 169 ++++ src/services/ingest_service.py | 194 +++++ src/services/mosaic_service.py | 129 +++ src/services/project_service.py | 23 + src/services/storage_service.py | 114 ++- src/services/tests/test_epoch_service.py | 67 ++ src/services/tests/test_ingest_service.py | 185 +++++ src/services/tests/test_mosaic_service.py | 136 +++ src/services/tests/test_project_service.py | 33 + src/services/tests/test_storage_service.py | 173 +++- src/services/tests/test_tile_service.py | 68 ++ src/services/tile_service.py | 220 +++++ src/worker/celery_app.py | 2 + src/worker/tasks.py | 126 ++- src/worker/tests/test_tasks.py | 254 +++++- uv.lock | 207 ++++- 50 files changed, 5319 insertions(+), 44 deletions(-) create mode 100644 docs/architecture/28_domain_graph_population.md create mode 100644 migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py create mode 100644 migrations/versions/20260727_0010_add_level3_mosaic_barycenter.py create mode 100755 scripts/smoke/populate_demo_project.sh create mode 100644 src/api/routes/epochs.py create mode 100644 src/api/routes/ingest.py create mode 100644 src/api/routes/mosaics.py create mode 100644 src/api/routes/projects.py create mode 100644 src/api/routes/tiles.py create mode 100644 src/api/tests/test_epochs.py create mode 100644 src/api/tests/test_ingest.py create mode 100644 src/api/tests/test_mosaics.py create mode 100644 src/api/tests/test_projects.py create mode 100644 src/api/tests/test_tiles.py create mode 100644 src/db/tests/test_spatial_types.py create mode 100644 src/services/epoch_service.py create mode 100644 src/services/ingest_service.py create mode 100644 src/services/mosaic_service.py create mode 100644 src/services/project_service.py create mode 100644 src/services/tests/test_epoch_service.py create mode 100644 src/services/tests/test_ingest_service.py create mode 100644 src/services/tests/test_mosaic_service.py create mode 100644 src/services/tests/test_project_service.py create mode 100644 src/services/tests/test_tile_service.py create mode 100644 src/services/tile_service.py diff --git a/.claude/skills/gen_tests.md b/.claude/skills/gen_tests.md index 924eb75..0f0f479 100644 --- a/.claude/skills/gen_tests.md +++ b/.claude/skills/gen_tests.md @@ -4,7 +4,9 @@ **Action:** Read `docs/architecture/[filename.md]`. Derive the complete test plan for this implementation. Then run it interactively, one step at a time — do not dump the full list and wait. Present each step, wait for the user to execute and report the result, then proceed. -**Presentation format (both phases):** Give each command its own small fenced code block — never combine multiple commands into one block. Most chat UIs put a copy button on each fenced block automatically, so one-command-per-block gives per-command copy ergonomics natively, no HTML/JS needed. Precede each command with a bold step label (e.g. `**06 · Pre-commit**`), follow it with an italicized `*Expect: ...*` line stating the observable success condition. Keep this entirely inline in chat — do not publish it as an Artifact. Test plans reference internal project details (branch names, env var names, infra topology) that don't need to leave the local conversation just for a nicer visual format; inline markdown is local, lightweight, and sufficient. +**Presentation format (both phases):** Give each command its own small fenced code block — never combine multiple commands into one block. Most chat UIs put a copy button on each fenced block automatically, so one-command-per-block gives per-command copy ergonomics natively, no HTML/JS needed. Precede each command with a bold step label (e.g. `**06 · Pre-commit**`), follow it with an italicized `*Expect: ...*` line stating the observable success condition. Keep this entirely inline in chat — do not publish it as an Artifact. Test plans reference internal project details (branch names, env var names, infra topology) that don't need to leave the local conversation just for a nicer visual format; inline markdown is local, lightweight, and sufficient. This format applies whenever steps are shown to the user — including when re-listing or recapping remaining/upcoming steps, not just when presenting the next step to execute. Never substitute a plain summary table for the full command+`Expect` format. + +**SQL in QA steps:** When a QA step involves a SQL query, present the raw SQL in its own fenced block — for pasting directly into DBeaver or another SQL client — rather than wrapping it in a `docker compose exec db psql -c "..."` invocation. Easier to read and copy on its own than escaped inside a shell command. --- diff --git a/.env.example b/.env.example index 65fb2f7..c005759 100644 --- a/.env.example +++ b/.env.example @@ -50,12 +50,30 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4317 # Production/staging deployment image tag (see doc 24) IMAGE_TAG=main -# S3-compatible object storage (see doc 27 — local MinIO mock) +# S3-compatible object storage (see doc 27 — local MinIO mock). In a real cloud +# deployment: leave S3_ENDPOINT_URL blank to use AWS's real regional endpoints, +# and leave AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY blank to let boto3's default +# credential chain resolve an IAM role instead of these MinIO dev-only static keys. S3_ENDPOINT_URL=http://minio:9000 AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin S3_BUCKET_NAME=diffpype-data +S3_REGION=us-east-1 # Host port mappings for the local MinIO container (container S3 API stays on 9000; # host 9000 is taken by portainer, so the MinIO host API port is 9002) MINIO_API_PORT=9002 MINIO_CONSOLE_PORT=9001 + +# Storage backend selection (see doc 28) — "s3" (default, MinIO/S3-compatible) or +# "local" (filesystem-backed, e.g. for a real fiducial FITS set outside MinIO). +# LOCAL_STORAGE_ROOT is the path the app reads/writes *inside* the container +# (fixed at /data via the docker-compose bind mount below) or, for a bare-metal +# run with no Docker at all, a plain relative path on the host. +# LOCAL_STORAGE_HOST_PATH is the *host* directory bind-mounted to /data in +# api/worker_light/worker_heavy — override in .env to point at a real dataset; +# defaults to a git-ignored directory in the repo so a fresh clone works with +# no setup. Put raw FITS files at /raw/ (flat, no +# subdirs — matches the raw/{base_filename} key layout). +STORAGE_BACKEND=s3 +LOCAL_STORAGE_ROOT=./data +LOCAL_STORAGE_HOST_PATH=./local_storage_data diff --git a/.gitignore b/.gitignore index bc446a8..e007ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ docs/_build/ # Local data files (Crucial for S3/fits development) data/ +local_storage_data/ *.fits **/.DS_Store diff --git a/CLAUDE.md b/CLAUDE.md index 5932681..b2dfddf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,7 @@ Claude may only suggest packages or write code that aligns with this strict base * **Command Authorization Rule:** Claude is explicitly authorized to autonomously execute standard, non-destructive development commands (e.g., `uv sync --all-groups`, `uv lock`, `pytest`, `docker compose build`, `docker compose up`, `docker compose down`) without pausing for human authorization. **Note:** Destructive commands that wipe data volumes (e.g., `docker compose down -v`) strictly require human authorization. * **Meta-Configuration Change Confirmation:** Before editing this file (`CLAUDE.md`), any file in `.claude/skills/`, `.claude/context/gemini_rules.md`, or `.claude/settings.local.json`, Claude must first propose the change in chat and wait for explicit user confirmation. These files govern how Claude and Gemini behave across every future session, so changes to them are never made silently as a side effect of other work — even when the change seems obviously correct. This does not apply when the user has directly dictated the exact change in the current message (no need to ask them to re-confirm their own instruction). * **API/CLI Parity:** Any feature, query, or mutation exposed via the FastAPI router must have a corresponding, fully functional command in the `diffpype-manage` CLI. Both boundaries must delegate to the exact same function in the Shared Service Layer. +* **QA/Debugging Fix Confirmation:** When a bug or unexpected failure surfaces during any phase of work — not just `genTests`, this applies universally, including ad-hoc debugging and live incident response — Claude must propose the fix in chat (what's broken, what the fix involves, which layer it belongs in: application code, container/infrastructure config, or both) and wait for explicit confirmation before writing or applying it. When a symptom spans layers, flag that explicitly rather than resolving it unilaterally in just one. * **Command Aliases:** If the user types `runPrompt on [filename.md]`, immediately read `.claude/skills/run_arch.md` and execute the instructions defined there. * **Skill Registry:** Claude has the following repeatable skills in `.claude/skills/`. When the user types the trigger command, immediately read the corresponding file and follow it exactly. @@ -44,6 +45,7 @@ Claude may only suggest packages or write code that aligns with this strict base * **Log Everything:** Any bug found and fixed at any phase — during `runPrompt` implementation, during `genTests` QA, or during ad-hoc debugging — must be logged as a dated entry in the `## Logs` section of the relevant architecture MD file immediately after the fix, not batched at the end. The entry must record what the bug was, what triggered its discovery, and what was changed. * **Pre-Commit Runs Locally, Not in Docker:** `pre-commit` requires a `.git` directory, which is never copied or mounted into the `api`/`worker` containers. Running it via `docker compose exec` fails with `FatalError: git failed`. Always run `uv run pre-commit run --all-files` on the host, exactly like Sphinx. * **Stuck Database States:** When modifying asynchronous tasks, always verify that database rows cannot be left in an orphaned `IN_PROCESS` state if the worker crashes. +* **Streaming & Bounded Resource Use:** Any task processing a batch of external files/objects (downloads, parses, uploads) must do so in a bounded, streaming fashion — one item (or a small fixed chunk) at a time — never materializing an entire batch before doing any work. Peak memory/disk must not scale unboundedly with batch size. Progress must be recoverable/visible (an incrementing counter, not a single all-or-nothing flip), and one item's failure must not take the whole task or its container down with it. * **Transactions:** Any global or framework-level error handler that intercepts a crash to update a database status MUST explicitly call `db.rollback()` before attempting to write the failure state, preventing `PendingRollbackError` crashes. * **Database Migrations:** If Claude modifies the SQLAlchemy models in `src/db/models.py`, it must automatically generate the Alembic migration script using `--autogenerate`. Afterward, it must explicitly pause and remind the user to run `alembic upgrade head` (or the equivalent Docker command) to apply the changes to the live database before proceeding. * **Git Workflow:** The repository uses GitHub Flow. Direct pushes to `main` are forbidden. The feature branch must be created **before** writing the architecture document — the arch doc file itself is a tracked change, and if it is committed on `main` first, it pollutes the PR diff. Claude must remind the user to `git checkout -b feature/[slug]` before Gemini begins drafting and before any implementation code is written. Commit changes on the feature branch and open a Pull Request against `main`. @@ -72,4 +74,7 @@ Claude may only suggest packages or write code that aligns with this strict base * **Action:** Updated the Core Tech Stack Constraints database line to drop the `healpix-alchemy` package name in favor of "native HEALPix range-type spatial columns." Discovered during `assessPrompt` on doc 27 that `healpix-alchemy==1.0.1` (unmaintained since 2021) has an unconstrained `mocpy` version dependency that now resolves to a modern `mocpy` release lacking the `IntervalSet` API its code requires — the package cannot be imported at all as currently pinned. Decided to vendor a small internal `src/db/spatial_types.py` module modeled on its `Tile`/`Point` types against a directly-pinned, working `mocpy` version instead of depending on the broken package. Tracked via GitHub issue #26. * **Action:** Extended the Model Evaluation Phase to require an effort-level recommendation (low/medium/high/xhigh/max) alongside the model recommendation, and synced the matching wording in `.claude/skills/run_arch.md`. Raised during `runPrompt` on doc 27 — the user wanted effort surfaced as its own axis, not folded into the model choice. * **Action:** Updated `.claude/skills/gen_tests.md`'s migration-verification step to stop treating an already-applied migration as a failure requiring a database reset. Host and containerized services share one Postgres instance, so a migration Claude already applied and verified live during `runPrompt` will correctly no-op when `genTests` re-runs `alembic upgrade head` — surfaced during `genTests` on doc 27 when this no-op read as a failure. The step now checks `alembic current` first and, when a live demonstration of the migration's code is actually wanted, uses a non-destructive `alembic downgrade -1` / `upgrade head` round-trip (only when the affected tables/columns are confirmed safe to round-trip) instead of a full database reset, which remains a human-authorized last resort only. -* **Action:** Added the CI Service Parity guardrail. Surfaced during `genTests` on doc 27: `minio` was added to `docker-compose.yml` for S3 storage, but `.github/workflows/ci.yml` was never updated to provision it, and this went unnoticed until a real (unmocked) integration test against it was proposed. Distinguished from `redis`, which correctly has no CI service today because nothing in the suite opens a real Redis connection (Celery dispatch is always mocked) — the guardrail's trigger is deliberately "a real integration test needs this service," not "a service exists in `docker-compose.yml`," so it doesn't over-fire on services like `flower`/`portainer`/`jaeger`/`ui` that never need CI wiring. \ No newline at end of file +* **Action:** Added the CI Service Parity guardrail. Surfaced during `genTests` on doc 27: `minio` was added to `docker-compose.yml` for S3 storage, but `.github/workflows/ci.yml` was never updated to provision it, and this went unnoticed until a real (unmocked) integration test against it was proposed. Distinguished from `redis`, which correctly has no CI service today because nothing in the suite opens a real Redis connection (Celery dispatch is always mocked) — the guardrail's trigger is deliberately "a real integration test needs this service," not "a service exists in `docker-compose.yml`," so it doesn't over-fire on services like `flower`/`portainer`/`jaeger`/`ui` that never need CI wiring. + +###### 2026-07-28 +* **Action:** Added the QA/Debugging Fix Confirmation guardrail and the Streaming & Bounded Resource Use checklist item. Surfaced during `genTests` on doc 28: `run_ingest_batch` downloaded an entire batch's files into a temp directory before doing any parsing, and a real ~20-file batch of real JWST FITS data OOM-killed the whole `worker_light` container (`WORKER_LIGHT_MEM_LIMIT`/`CELERY_LIGHT_MAX_MEMORY_PER_CHILD` were sized for the trivial Stage-0 dummy `sleep()` task and never revisited). Because an OOM kill (SIGKILL) isn't a catchable Python exception, the task's own crash-handling `except` block never ran, leaving its `IngestBatch` row permanently stuck `in_process` — a direct violation of the existing Stuck Database States guardrail. Claude proposed and applied a streaming per-file fix (download→parse→delete one file at a time, with real incremental progress) without pausing to confirm first, which prompted the QA/Debugging Fix Confirmation addition. Separately, `run_ingest_batch`/`run_mosaic_drizzle` were re-routed from `light` back to `heavy_memory` — real FITS/astropy ingest is memory-hungry even though it's not GPU-bound, and `light`'s multi-child concurrency (`CELERY_LIGHT_CONCURRENCY=4`, all children sharing one container memory ceiling) made it a poor fit regardless of any per-task memory tuning; `heavy` is redefined for now to mean "resource/memory intensive," not GPU-specific — `CeleryQueue.GPU` remains unused/reserved for whenever real GPU-bound work exists (most likely the JWST pipeline doc). \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 241aae9..f2d2e47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,11 +77,15 @@ services: AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin} S3_BUCKET_NAME: ${S3_BUCKET_NAME:-diffpype-data} + S3_REGION: ${S3_REGION:-us-east-1} + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_ROOT: ${LOCAL_STORAGE_ROOT:-./data} ports: - "${API_PORT:-8000}:8000" volumes: - ./src:/app/src - ./migrations:/app/migrations + - ${LOCAL_STORAGE_HOST_PATH:-./local_storage_data}:/data depends_on: db: condition: service_healthy @@ -110,12 +114,16 @@ services: AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin} S3_BUCKET_NAME: ${S3_BUCKET_NAME:-diffpype-data} + S3_REGION: ${S3_REGION:-us-east-1} + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_ROOT: ${LOCAL_STORAGE_ROOT:-./data} deploy: resources: limits: memory: ${WORKER_LIGHT_MEM_LIMIT:-512m} volumes: - ./src:/app/src + - ${LOCAL_STORAGE_HOST_PATH:-./local_storage_data}:/data depends_on: db: condition: service_healthy @@ -144,12 +152,16 @@ services: AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin} S3_BUCKET_NAME: ${S3_BUCKET_NAME:-diffpype-data} + S3_REGION: ${S3_REGION:-us-east-1} + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_ROOT: ${LOCAL_STORAGE_ROOT:-./data} deploy: resources: limits: memory: ${WORKER_HEAVY_MEM_LIMIT:-8g} volumes: - ./src:/app/src + - ${LOCAL_STORAGE_HOST_PATH:-./local_storage_data}:/data depends_on: db: condition: service_healthy diff --git a/docs/architecture/28_domain_graph_population.md b/docs/architecture/28_domain_graph_population.md new file mode 100644 index 0000000..bf83fbf --- /dev/null +++ b/docs/architecture/28_domain_graph_population.md @@ -0,0 +1,145 @@ +##### 28: Domain Graph Population & Service Boundaries +**Version:** 0.3 + +###### Preamble +This document establishes the headless domain graph population logic (Ingest, Tiles, Epochs, Mosaics). It enforces strict API/CLI parity, encapsulates Pandas entirely within service functions, and separates complex HEALPix/clustering computation from synchronous database persistence. It establishes the Storage Service factory pattern and populates the many-to-many associations that connect the domain graph. + +###### 1. Schema Additions (`Project.slug` & `IngestBatch`) +* **Directive:** Provide stable storage paths via immutable project slugs and track async ingestion. +* **Behavior:** + * **Project Slug:** Add a `slug` column (String, `unique=True`, `nullable=False`) to the `Project` model. When creating new projects, generate the slug via `python-slugify`; reject collisions with an HTTP 400. + * **IngestBatch Tracking:** Create an `IngestBatch` model (inheriting `TimestampMixin`) with `id`, `project_id` (ForeignKey), `s3_prefix` (String), `total_files` (Integer, default 0), `processed_files` (Integer, default 0), and `status` (Enum(`JobStatus`), default `JobStatus.PENDING`). + * **Migration Strategy (Hand-Authored):** Generate the migration using Alembic, but hand-edit it to handle the `Project.slug` backfill. Create the column as nullable, backfill existing rows by slugifying their names (auto-deduplicating collisions with a numeric suffix like `-2`, `-3`), and then alter the column to `nullable=False`. + +###### 2. Storage Service Factory & Key Layout +* **Directive:** Provide a common storage interface capable of swapping between S3 and local disk. +* **Behavior:** + * Create `src/services/storage_service.py` defining a `StorageBackend` Protocol (or ABC). + * Implement `S3StorageService` (using `boto3`) and `LocalStorageService` (using standard `shutil`/`os`). `LocalStorageService` must auto-create the `LOCAL_STORAGE_ROOT` directory if it is missing. + * Create a factory `get_storage_service(config) -> StorageBackend` that switches based on the `STORAGE_BACKEND` environment variable. + * **Key Layout Conventions:** Raw FITS files reside at `raw/{base_filename}` (flat, project-agnostic). Derived pipeline products reside at `projects/{project_slug}/working/{...}`. + +###### 3. Async Ingestion Service & Pipeline +* **Directive:** Scan a storage directory and execute idempotent bulk upserts. +* **Behavior:** + * Create `src/services/ingest_service.py` and Pydantic schema `IngestRequest(project_id, s3_prefix)`. + * **API/CLI Write:** `POST /api/v1/ingest` (CLI: `diffpype-manage ingest`) creates an `IngestBatch` row as `PENDING`, dispatches `run_ingest_batch`, and returns the ID. + * **API/CLI Read:** `GET /api/v1/ingest/{id}` (CLI: `diffpype-manage ingest-status --id `) returns the `IngestBatch` status and progress counts. + * **Worker:** The task fetches the manifest, updates `total_files`, parses headers into Pandas, bulk-upserts `Level2Image` and `Level2Calibration` for every row from the prefix, and updates `processed_files`. + +###### 4. Tile Generation & Association +* **Directive:** Isolate tessellation math and populate spatial associations. +* **Behavior:** + * Create `src/services/tile_service.py`. Schemas: `TileCreate` (name, ra, decl, delta_ra, delta_decl, footprint as `list[tuple[int, int]] | None`), `TileTessellationRequest`. + * **Compute (API/CLI):** `POST /api/v1/tiles/tessellate` (CLI: `diffpype-manage tessellate-tiles`) ports `GenerateSkyTiles` to return a preview `list[TileCreate]`. + * **Persistence (API/CLI):** `POST /api/v1/tiles` (CLI: `diffpype-manage create-tiles`) synchronously bulk-inserts `Tile` rows. + * **Association:** During persistence, the service queries for all `Level2Calibration` rows where `footprint && tile.footprint` (using the GiST index) and populates `tile_level2_calibration_association`. + +###### 5. Epoch Generation & Association +* **Directive:** Isolate temporal clustering and populate temporal associations. +* **Behavior:** + * Create `src/services/epoch_service.py`. Schemas: `EpochCreate`, `EpochClusterRequest`. + * **Compute (API/CLI):** `POST /api/v1/epochs/cluster` (CLI: `diffpype-manage cluster-epochs`) queries the DB for `Level2Calibration` records intersecting the `tile_id`/`band_id`, extracts MJDs, runs `CreateEpochsFromMJDs` (scipy/sklearn), and returns `list[EpochCreate]`. + * **Persistence (API/CLI):** `POST /api/v1/epochs` (CLI: `diffpype-manage create-epochs`) synchronously bulk-inserts `Epoch` rows. + * **Association:** During persistence, the service populates `epoch_level2_calibration_association` for all calibrations whose MJDs fall within the Epoch's `start_mjd` and `end_mjd`. + +###### 6. Mosaic Orchestration (Footprints & Barycenters) +* **Directive:** Establish the job-tracking entity for the drizzle pipeline and compute its spatial properties (Addresses #27). +* **Behavior:** + * Create `src/services/mosaic_service.py`. Schema: `MosaicCreate`. + * **API/CLI Write:** `POST /api/v1/mosaics` (CLI: `diffpype-manage create-mosaic`) synchronously creates a `Level3Mosaic` with `status=PENDING` and dispatches `run_mosaic_drizzle`. + * **API/CLI Read:** `GET /api/v1/mosaics/{id}` (CLI: `diffpype-manage mosaic-status --id `) polls the mosaic status. + * **Spatial Computation:** The service looks up the constituent `Level2Calibration` rows via the M2M tables. It uses `Get_Unioned_MOC` to merge their footprints into the mosaic's `footprint`. It computes the `ra` and `decl` barycenter from this union and stores it on the `Level3Mosaic`. + * **Worker:** The `run_mosaic_drizzle` task sleeps and transitions to `COMPLETE` (real drizzle deferred). + +###### 7. Dev Coordinator & Smoke Test +* **Directive:** Provide a developer tool to orchestrate the end-to-end population of the graph. +* **Behavior:** + * Add a `diffpype-manage populate-demo-project` CLI command. It sequentially invokes the Ingest, Tiles, Epochs, and Mosaics services. + * Tracked as Issue #31: Create a manual `scripts/smoke/` script to run against a live stack. + +###### 8. Environment Variables +* **Directive:** Ensure all configurations are synchronized. +* **Note:** `.env.example` and `.env` must remain identical. +| Name | Type | Default Value | Description | +| :--- | :--- | :--- | :--- | +| STORAGE_BACKEND | str | "s3" | The storage backend to use (must be "s3" or "local"). | +| LOCAL_STORAGE_ROOT | str | "./data" | The host directory for local storage, auto-created if missing. | + +###### 9. Dependencies & Packages +* **Packages:** Add `python-slugify`, `scipy`, and `scikit-learn` to `pyproject.toml` (Pandas/Mocpy/Boto3 are already installed). +* **Mocking:** Add `scipy` and `sklearn` to `autodoc_mock_imports` in `docs/conf.py`. + +###### 10. Testing Mandates +* **Associations:** Write explicit integration tests proving the M2M association logic correctly links Tiles (via spatial overlap) and Epochs (via MJD). +* **Mosaic Math:** Write unit tests for `Get_Unioned_MOC` and the barycenter extraction (Issue #27). +* **API Routes:** Every new FastAPI route must be tested for success (200), validation failure (422), and missing resources (404). +* **Dumb Pipes:** The synchronous DB write functions (`create_tiles`, `create_epochs`) must be explicitly unit/integration tested independently of their compute functions. + +###### 11. CLAUDE.md Compliance & Implementation Sequencing +* **Implementation Sequencing:** Implement §1 (Migrations) and §2 (Storage Factory) first. Then §3 (Ingest). Then §4/§5 (Tiles & Epochs). Finally, §6/§7 (Mosaics & Coordinator). +* **Documentation Registration:** Add the four new service modules to `docs/index.rst`. Add `28_domain_graph_population` to `docs/architecture/index.md`. + +###### Logs + +###### 2026-07-27 +* **Bug:** §4's tile/calibration association query cited "the Q3C/GiST index" for the `footprint && tile.footprint` overlap lookup. Q3C indexes `ra`/`decl` points (`q3c_ang2ipix`) and is unrelated to this query — only the GiST index on the `footprint` multirange column (`ix_tile_footprint_gist`, added in doc 27) backs `&&`. Left in, this risked an implementor reaching for Q3C functions on a query they don't apply to. +* **Trigger:** Found during `assessPrompt` review of v0.2. Fixed directly (deterministic, no design judgment involved) by editing "Q3C/GiST" → "GiST". Reverted when v0.3 came back from a full Gemini regeneration that didn't carry the direct edit forward; reapplied identically during the v0.3 `assessPrompt` pass. + +###### 2026-07-28 — `runPrompt` implementation +Full implementation of v0.3 on `feature/domain-graph-population`. All 11 sections built, tested (240 tests, 99.3% coverage), Sphinx `-W` build clean, `alembic check` shows zero drift, migrations 0009/0010 round-tripped downgrade→upgrade cleanly. Notable deviations and gaps found/filled during implementation, beyond straightforward execution of the doc: + +* **Project creation was never actually specified.** §1 says new projects "generate the slug via `python-slugify`; reject collisions with an HTTP 400," but no doc ever defined a Project-creation service, endpoint, or CLI command for this to live on. Added a minimal `src/services/project_service.py` (`create_project`, catching the DB `IntegrityError` on slug collision and re-raising `ValueError`), `POST /api/v1/projects`, and `diffpype-manage create-project` — the smallest addition that makes §1's own directive implementable, not a scope expansion. +* **`pandas` and `astropy` were never real dependencies.** §9 claimed "Pandas/Mocpy/Boto3 are already installed" — true for mocpy/boto3, false for pandas (not present anywhere in `uv.lock`, even transitively). Added `pandas==3.0.5` as a direct dependency. `astropy` (needed for FITS header/WCS parsing, tile tessellation, MJD clustering) is already resolved transitively via `mocpy`, matching the existing precedent of `numpy` being used directly without its own top-level pin — not re-declared as a direct dependency, but added to `docs/conf.py`'s `autodoc_mock_imports` alongside `pandas`/`scipy`/`sklearn`/`slugify` since all are now directly imported. +* **`MosaicCreate` was missing required model fields.** `Level3Mosaic.filename`/`target_plate_scale` are `NOT NULL`, but the doc's `MosaicCreate` only listed FK fields. Added `filename`/`target_plate_scale` to the schema and CLI/API surface — mechanical, no design judgment. +* **Issue #27 (`Level3Mosaic` ra/decl barycenter) required a schema change the doc never mentioned.** §6 says the computed barycenter is "stored on the Level3Mosaic," but no `ra`/`decl` columns exist on that model and §1 never added them. Added migration 0010: nullable `ra`/`decl` columns + a Q3C index (matching Tile/Level2Image), consistent with issue #27's own acceptance criteria and the earlier decision (this session) to fold #27 into doc 28. +* **Real correctness bug found and fixed before it shipped: epoch↔calibration association was unscoped by tile/band.** `Epoch` is modeled as "a temporal grouping... for a given Tile and Band," but the first draft of `_associate_epochs_with_calibrations_in_range` only filtered by `project_id` + MJD range — it would have linked calibrations from any tile or band whose MJD happened to overlap, silently corrupting the M2M associations a future mosaic/orchestrator step depends on. Caught while implementing `mosaic_service` (which consumes those associations) before any test was written against it. Fixed by joining through `tile_level2_calibration_association` and filtering `Level2Image.band_id`, mirroring `cluster_epochs`'s own scoping. Regression test added: `test_create_epochs_persists_and_associates_calibrations_in_mjd_range` now includes a same-MJD/wrong-tile calibration that must NOT be linked. +* **`DummyImage` decommission — deferred, tracked separately.** Removing the Stage-0 scaffolding (superseded by `IngestBatch`) was raised mid-implementation; scoped out of this doc per explicit decision (needs its own architecture doc under the human-in-the-loop rule, and `DiffpypeTask.on_failure`'s hardcoded `DummyImage` status-write needs a real generalized replacement, not a bare deletion). Filed as GitHub issue #33. +* **`minio-init` fiducial seed data was designed in conversation but never built in any doc.** §7's coordinator assumed a fiducial MinIO dataset that doesn't exist — no `minio-init` service, no seed FITS files anywhere in the repo. Per explicit decision, `populate-demo-project` takes real CLI arguments (`--s3-prefix`, `--ra`/`--decl`/`--radius-deg`, `--band-id`, `--instrument-id`, etc.) instead of a hardcoded fiducial path; its control flow (ingest → tiles → epochs → mosaic, polling both async steps to a terminal state) is fully implemented and tested. Filed as GitHub issue #34 — real astronomical FITS files are a data-acquisition task, not something fillable in code. +* **New Celery tasks (`run_ingest_batch`, `run_mosaic_drizzle`) intentionally not built on `DiffpypeTask`.** That base's `on_failure` hardcodes a `DummyImage` status write by `args[0]` — reusing it here would silently no-op, or worse, incorrectly mark an unrelated `DummyImage` row `FAILED` if one happened to share the same integer id as the batch/mosaic. Both new tasks handle their own crash-safety instead (explicit `try/except`, `db.rollback()` before writing failure state per CLAUDE.md's Transactions rule, mark the correct entity `FAILED`, re-raise), routed to the `heavy_memory` queue. +* **`StorageBackend` protocol gained a `list_prefix` method not in the original design.** §3 assumes ingest can "fetch the s3_prefix manifest," but no prior storage-service design (this session's or doc 27's) ever specified a manifest/listing method — only `upload_file`/`download_file` existed. Added `list_prefix(prefix) -> list[str]` to the protocol and both `S3StorageService` (paginated `list_objects_v2`) and `LocalStorageService` (`Path.rglob`), since ingest cannot function without it. + +**Verification commands run:** `uv run pytest --cov=src --cov-fail-under=90` (240 passed, 99.3%), `uv run sphinx-build -b html docs docs/_build/html -W` (clean), `uv run pre-commit run --all-files` (ruff, ruff-format, mypy all clean after fixing 7 real mypy findings — `Sequence` vs `list` param variance, an untyped `[]` literal, a `np.histogram` `range` tuple-vs-list mismatch, and two possibly-`None` attribute accesses in the coordinator's polling helpers), `alembic check` (zero drift against live dev DB), `alembic downgrade 0008` → `alembic upgrade head` round trip (clean). + +**Not yet done, reminders for the user:** +* Run `docker compose restart worker_light worker_heavy` — Celery task routing/signatures changed (`run_ingest_batch`, `run_mosaic_drizzle` added to `task_routes`). +* Run `docker compose build api worker_light worker_heavy` then `docker compose up -d` before relying on the containerized environment — new dependencies (`pandas`, `python-slugify`, `scipy`, `scikit-learn`) need to land in the built images. Migrations 0009/0010 are already applied to the shared dev DB from the host, so `docker compose exec api alembic upgrade head` will correctly no-op. +* `genTests` has not run yet — this Logs entry covers implementation-time verification only. + +###### 2026-07-28 — `genTests` bug: OOM-killed worker on a real ingest batch, orphaned `IngestBatch` row +* **Bug:** `run_ingest_batch` downloaded every file in the batch into a temp directory before parsing any of them. Against a real ~20-file batch of real JWST FITS data, this blew through `WORKER_LIGHT_MEM_LIMIT`/`CELERY_LIGHT_MAX_MEMORY_PER_CHILD` (sized for the trivial Stage-0 dummy `sleep()` task, never revisited) and the container was OOM-killed outright (`Killed` in the logs, no traceback). Because a SIGKILL isn't a catchable Python exception, the task's own `except`/rollback/mark-`FAILED` crash handling never ran — `IngestBatch(id=5)` was left permanently stuck at `in_process` with no worker ever going to touch it again, a direct violation of CLAUDE.md's Stuck Database States guardrail. Celery's `task_acks_late`/`task_reject_on_worker_lost` did not redeliver the task after the worker restarted (confirmed: `updated_at` never advanced). +* **Trigger:** Live QA against real fiducial FITS data (staged by the user at a bind-mounted host directory, `STORAGE_BACKEND=local`) — the first time `run_ingest_batch` had ever processed more than a zero-file prefix. +* **Fix:** `run_ingest_batch` now streams one file at a time — download → parse → `os.remove` — instead of materializing the whole batch on disk before any work happens, and updates `IngestBatch.processed_files` + logs an `ingest_file_processed` event after each file, giving real incremental progress instead of a single all-or-nothing jump at the end. `run_ingest_batch`/`run_mosaic_drizzle` were also re-routed from `light` back to `heavy_memory`: real FITS/astropy work is memory-hungry even though it isn't GPU-bound, and `light`'s multi-child concurrency (`CELERY_LIGHT_CONCURRENCY=4`, all children sharing one container memory ceiling) made it a poor fit regardless of per-task tuning — `heavy` is redefined for now to mean "resource/memory intensive," not GPU-specific. The orphaned `IngestBatch(id=5)` row was corrected manually (`UPDATE ... SET status = 'failed'`) since no automated path could recover it after the fact. New CLAUDE.md guardrails added as a result: QA/Debugging Fix Confirmation (propose before applying a fix, always) and Streaming & Bounded Resource Use (batch-processing tasks must stream, never materialize a whole batch up front). +* **Regression tests added:** `test_run_ingest_batch_parses_one_file_at_a_time`, `test_run_ingest_batch_increments_processed_files_progressively`, `test_run_ingest_batch_logs_progress_per_file`, `test_run_ingest_batch_removes_temp_file_after_each_file`, `test_run_ingest_batch_handles_zero_files_without_crashing` (`src/worker/tests/test_tasks.py`). + +###### 2026-07-28 — `genTests` bug: instrument/band matching rejected every real file +* **Bug:** Once the OOM fix above let a real ~20-file batch actually stream through, it still failed — `ValueError: Unknown instrument/band for files: [...all 20...]`. Two distinct causes: (1) `_resolve_reference_ids` matched `INSTRUME`/`FILTER` header values against seeded `Instrument`/`Band` names with an exact, case-sensitive string match — real JWST headers write `INSTRUME='NIRCAM'` (uppercase), while the seed data used mixed-case `"NIRCam"`, so every NIRCam file failed the lookup. (2) Independently, the real files' `FILTER='F115W'` wasn't seeded at all — the reference `Band` table only ever had two placeholder rows (`F150W`, `F277W`) from Stage 0, not a real filter list. +* **Trigger:** Live QA re-dispatch of the real ingest batch immediately after the OOM fix (batch_id=6) — confirmed via a read-only header inspection (`astropy.io.fits.getheader`) before proposing any fix, per the newly-added QA/Debugging Fix Confirmation guardrail. +* **Fix:** `_resolve_reference_ids` now matches case-insensitively (`.str.upper()` on both the DB names and the header values). `src/db/seed.py`'s `_SEED_BANDS` replaced the 2-filter placeholder with the complete standard NIRCam (wide/medium/narrow) + MIRI imaging filter set, so the next not-yet-seen real filter doesn't reproduce the same gap. +* **Regression test added:** `test_resolve_reference_ids_matches_case_insensitively` (`src/services/tests/test_ingest_service.py`). + +###### 2026-07-28 — `genTests` bug: `mosaic-status` CLI table unreadable, dumping raw MOC + bare FK ids +* **Bug:** `cmd_mosaic_status` printed every `Level3Mosaic` column through the generic `_print_entity_table` helper, including the raw `footprint` `mocpy.MOC` object (tabulate falls back to its Python repr, which is enormous and not human-readable) and five bare foreign-key ids (`instrument_id`, `band_id`, `tile_id`, `epoch_id`, `project_id`, `job_configuration_id`) that aren't meaningful without a join. The command's actual purpose — "did my job finish" — was buried in noise. +* **Trigger:** Live QA Step 9a against a real mosaic row. +* **Fix:** `_print_entity_table` gained an optional `fields` parameter to restrict/order printed columns, and `cmd_mosaic_status` now passes `["id", "filename", "status", "ra", "decl", "created_at", "updated_at"]` — status-focused, using the already-computed barycenter (`ra`/`decl`, from issue #27) instead of the raw footprint. Separately, `_print_entity_table` also gained a generic `_format_moc_for_display` fallback (renders any `MOC` value as `MOC()`) as defense-in-depth for any other current or future table call that surfaces a MOC column and hasn't been curated — `mosaic-status` itself no longer hits this path at all since footprint isn't in its field list. +* **Regression tests added:** `test_format_moc_for_display_reduces_moc_to_compact_sq_deg_string`, `test_format_moc_for_display_passes_through_non_moc_values`, `test_print_entity_table_restricts_and_orders_columns_when_fields_given`, plus updated `test_cmd_mosaic_status_prints_table_for_found_mosaic` to assert `footprint`/FK ids are absent from output (`src/api/tests/test_cli.py`). + +###### 2026-07-28 — `genTests` follow-up: `GET /api/v1/mosaics/{id}` also carried the same footprint fat +* **Bug:** After the CLI table fix above, live QA Step 9b hit the API boundary directly and found `MosaicStatus` (the `GET /api/v1/mosaics/{id}` response model) still returns the full `footprint` range-list on every status poll — same complaint as the CLI, just on the other boundary: a status check doesn't need spatial payload, only a plotting/spatial consumer would. +* **Trigger:** Live QA Step 9b (API/CLI parity check for `mosaic-status`). +* **Fix:** Removed `footprint` (and its `_coerce_footprint` validator binding) from `MosaicStatus` in `src/api/schemas.py`. `_coerce_moc_footprint` itself is unchanged and still used by `TileCreate`/`TileRead`, which legitimately need footprint. `GET /api/v1/mosaics/{id}` needed no route change — FastAPI's `response_model` already drops any ORM attribute not declared on the schema. `MosaicStatus` now returns `id, filename, target_plate_scale, ra, decl, status, project_id, tile_id, epoch_id, band_id, instrument_id` — bare FK ids stay (legitimate for API consumers, unlike the CLI table) but no MOC-derived payload, matching the CLI status view on both boundaries. +* **Regression test added:** `test_get_mosaic_status_returns_mosaic` updated to assert `"footprint" not in body` (`src/api/tests/test_mosaics.py`). + +###### 2026-07-28 — `genTests` bug: Redis-down at dispatch time permanently orphans the job row +* **Bug:** `mosaic_service.create_mosaic` and `ingest_service.create_ingest_batch` both commit their `Level3Mosaic`/`IngestBatch` row as `PENDING` *before* calling `.delay()` to dispatch the Celery task, with no `try/except` around that call. When the broker is unreachable at that exact moment, `.delay()` raises and propagates uncaught — the row is already committed, so it's left `PENDING` forever with no task ever entered into the queue. Unlike the worker-crash case (Step 10), there's no eventual redelivery here (nothing was ever queued to redeliver), so this is strictly worse: a permanently orphaned row, a direct violation of the Stuck Database States guardrail. +* **Trigger:** Live QA Step 12 (`docker compose stop redis` immediately before `create-mosaic`) — confirmed via the actual traceback (`kombu.exceptions.OperationalError` raised from `mosaic_service.py:113`, after `db.commit()` at line 110) and a direct DB check showing the row committed `PENDING` with no further activity. +* **Fix:** both functions now wrap their `.delay()` call in `try/except Exception`; on failure they mark the just-created row `FAILED`, commit that, and re-raise the original exception so the caller still sees a clear error — mirrors the crash-handling pattern `run_mosaic_drizzle`/`run_ingest_batch` already use internally, just applied to the dispatch-time gap instead of the mid-task gap. +* **Regression tests added:** `test_create_mosaic_marks_failed_and_reraises_when_dispatch_fails` (`src/services/tests/test_mosaic_service.py`), `test_create_ingest_batch_marks_failed_and_reraises_when_dispatch_fails` (`src/services/tests/test_ingest_service.py`). +* **Related, deferred to doc 29:** the worker-crash case (Step 10) surfaced a separate, harder gap — a task that's already `IN_PROCESS` when its worker is killed has no reliable/prompt recovery path either (Celery's Redis transport defaults to a 3600s visibility timeout before redelivery, effectively inert for QA-timescale observation). That needs a periodic reconciliation/watchdog mechanism, not a dispatch-time try/except — tracked in memory as part of doc 29's scope, not fixed here. + +###### 2026-07-28 — `genTests` bug: `populate-demo-project` picked an arbitrary tile, not one with real data +* **Bug:** `cmd_populate_demo_project` tessellated the target region into a tile grid, then did `tile = tiles[0]` — the first tile in generation order, with no relationship to which tiles actually overlap real ingested data. Live QA Step 13 against a real 100-file batch produced 22 tiles, only 4 of which had any associated `Level2Calibration` rows (confirmed via `tile_level2_calibration_association` counts); `tiles[0]` was one of the 18 empty ones, so `cluster_epochs` correctly found nothing and the run aborted with "No epochs generated (no calibrations in range)." A regular tessellation grid over an arbitrary region routinely produces empty edge tiles — this isn't specific to this dataset. +* **Trigger:** Live QA Step 13 (`populate-demo-project` full coordinator run against real fiducial data). +* **Fix:** added `tile_service.tile_with_most_calibrations(db, tile_ids) -> int | None`, which queries `tile_level2_calibration_association` and returns whichever tile actually has the most real overlap (or `None` if every tile has zero). `cmd_populate_demo_project` now uses that instead of positional indexing, and aborts with a clearer message ("No tile in the tessellated region overlaps any ingested calibration") when `None`, rather than silently proceeding on an empty tile and failing two steps later with a message that doesn't point at the real cause. General principle applied per explicit user direction: never dereference a collection by position without a reason to pick that particular element — always select by an explicit criterion (`np.any()`-style), even in one-off coordinator/dev-tooling code. +* **Regression tests added:** `test_tile_with_most_calibrations_returns_the_best_tile_id`, `test_tile_with_most_calibrations_returns_none_when_no_tile_has_any` (`src/services/tests/test_tile_service.py`); `test_cmd_populate_demo_project_aborts_when_no_tile_overlaps_data` (`src/api/tests/test_cli.py`). +* **Scope note:** the broader question of whether `populate-demo-project` should ship with a small curated fiducial dataset (so it doesn't require a user to hand-supply real files + hand-derive `--ra`/`--decl`/`--band-id`/`--instrument-id` via SQL) is GitHub issue #34, already open — explicitly left as-is per user decision, since this command is a manually-invoked smoke path, never run in CI, and doesn't warrant building fixture infrastructure for a single test. \ No newline at end of file diff --git a/docs/architecture/index.md b/docs/architecture/index.md index 8f65cab..238e63c 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -34,4 +34,5 @@ work for that stage. 25_documentation_diagrams 26_domain_models 27_schema_storage_spatial_types +28_domain_graph_population ``` diff --git a/docs/conf.py b/docs/conf.py index d4407e9..2810be7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,6 +48,7 @@ # Mock heavy external deps so RTD does not need the full scientific Python stack. autodoc_mock_imports = [ + "astropy", "bcrypt", "boto3", "celery", @@ -56,10 +57,14 @@ "mocpy", "numpy", "opentelemetry", + "pandas", "psycopg2", "pydantic", "pydantic_settings", "redis", + "scipy", + "sklearn", + "slugify", "sqladmin", "sqlalchemy", "src.api.admin", diff --git a/docs/index.rst b/docs/index.rst index d1a1b70..9acc12a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,26 @@ Services :members: :undoc-members: +.. automodule:: src.services.project_service + :members: + :undoc-members: + +.. automodule:: src.services.ingest_service + :members: + :undoc-members: + +.. automodule:: src.services.tile_service + :members: + :undoc-members: + +.. automodule:: src.services.epoch_service + :members: + :undoc-members: + +.. automodule:: src.services.mosaic_service + :members: + :undoc-members: + API (FastAPI) ------------- @@ -63,6 +83,26 @@ API (FastAPI) :members: :undoc-members: +.. automodule:: src.api.routes.projects + :members: + :undoc-members: + +.. automodule:: src.api.routes.ingest + :members: + :undoc-members: + +.. automodule:: src.api.routes.tiles + :members: + :undoc-members: + +.. automodule:: src.api.routes.epochs + :members: + :undoc-members: + +.. automodule:: src.api.routes.mosaics + :members: + :undoc-members: + Worker (Celery) --------------- diff --git a/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py b/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py new file mode 100644 index 0000000..1251565 --- /dev/null +++ b/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py @@ -0,0 +1,109 @@ +"""add project slug (with backfill) and ingest_batches + +Revision ID: 0009 +Revises: 0008 +Create Date: 2026-07-27 + +Adds an immutable, unique `slug` column to `projects` (generated via +python-slugify at creation time, rejecting collisions with an HTTP 400 — +see src/services/project_service.py) and creates the `ingest_batches` +table that tracks the progress of an async ingest run (doc 28). + +Hand-authored, not autogenerated: the `slug` column needs a data backfill +step for any pre-existing Project rows, which autogenerate cannot produce. +The column is added nullable first, backfilled by slugifying each row's +`name` and auto-deduplicating collisions with a numeric suffix (-2, -3, +...), and only then altered to NOT NULL. Auto-dedupe (rather than the +interactive-rejection behavior used for new project creation) is used here +because there is no user to reject a collision against mid-migration. +""" + +import sqlalchemy as sa +from alembic import op +from slugify import slugify +from sqlalchemy.dialects import postgresql + +revision = "0009" +down_revision = "0008" +branch_labels = None +depends_on = None + + +def _job_status_enum() -> postgresql.ENUM: + """Reference the job_status enum type created in migration 0001 (never re-create it).""" + return postgresql.ENUM( + "pending", + "in_process", + "complete", + "failed", + name="job_status", + create_type=False, + ) + + +def _unique_slug(base_slug: str, taken: set[str]) -> str: + """Return base_slug, or base_slug-2/base_slug-3/... the first value not already in `taken`.""" + if base_slug not in taken: + return base_slug + suffix = 2 + candidate = f"{base_slug}-{suffix}" + while candidate in taken: + suffix += 1 + candidate = f"{base_slug}-{suffix}" + return candidate + + +def upgrade() -> None: + # --- projects.slug: add nullable, backfill, then enforce NOT NULL + unique --- + op.add_column("projects", sa.Column("slug", sa.String(), nullable=True)) + + bind = op.get_bind() + projects = sa.table( + "projects", + sa.column("id", sa.Integer), + sa.column("name", sa.String), + sa.column("slug", sa.String), + ) + existing_rows = bind.execute(sa.select(projects.c.id, projects.c.name)).fetchall() + + taken_slugs: set[str] = set() + for row in existing_rows: + slug = _unique_slug(slugify(row.name), taken_slugs) + taken_slugs.add(slug) + bind.execute( + projects.update().where(projects.c.id == row.id).values(slug=slug) + ) + + op.alter_column("projects", "slug", nullable=False) + op.create_unique_constraint("uq_project_slug", "projects", ["slug"]) + + # --- ingest_batches: async ingest progress tracking --- + op.create_table( + "ingest_batches", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("project_id", sa.Integer(), nullable=False), + sa.Column("s3_prefix", sa.String(), nullable=False), + sa.Column("total_files", sa.Integer(), nullable=False), + sa.Column("processed_files", sa.Integer(), nullable=False), + sa.Column("status", _job_status_enum(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"]), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("ingest_batches") + op.drop_constraint("uq_project_slug", "projects", type_="unique") + op.drop_column("projects", "slug") diff --git a/migrations/versions/20260727_0010_add_level3_mosaic_barycenter.py b/migrations/versions/20260727_0010_add_level3_mosaic_barycenter.py new file mode 100644 index 0000000..310e9a8 --- /dev/null +++ b/migrations/versions/20260727_0010_add_level3_mosaic_barycenter.py @@ -0,0 +1,38 @@ +"""add ra/decl barycenter columns and Q3C index to level3_mosaics + +Revision ID: 0010 +Revises: 0009 +Create Date: 2026-07-27 + +Addresses GitHub issue #27: Level3Mosaic gains its own ra/decl columns, +computed as the barycenter of its footprint at write time (see +src/services/mosaic_service.py), Q3C-indexed like Tile/Level2Image. Nullable +because a mosaic with zero constituent calibrations has no footprint to +derive a barycenter from yet. The table is empty in every environment (doc 28 +is the first thing that ever writes a Level3Mosaic row), so this is a plain +additive migration with no backfill. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0010" +down_revision = "0009" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("level3_mosaics", sa.Column("ra", sa.Float(), nullable=True)) + op.add_column("level3_mosaics", sa.Column("decl", sa.Float(), nullable=True)) + op.create_index( + "ix_level3_mosaic_q3c", + "level3_mosaics", + [sa.text("q3c_ang2ipix(ra, decl)")], + ) + + +def downgrade() -> None: + op.drop_index("ix_level3_mosaic_q3c", table_name="level3_mosaics") + op.drop_column("level3_mosaics", "decl") + op.drop_column("level3_mosaics", "ra") diff --git a/pyproject.toml b/pyproject.toml index b07fa55..fe56ef7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,10 @@ dependencies = [ # opentelemetry-instrumentation 0.48b0 imports pkg_resources, which setuptools # removed in 81+. Pin below that so the instrumentors can import. "setuptools<81", + "python-slugify==8.0.4", + "scipy==1.18.0", + "scikit-learn==1.9.0", + "pandas==3.0.5", ] [project.scripts] diff --git a/scripts/smoke/populate_demo_project.sh b/scripts/smoke/populate_demo_project.sh new file mode 100755 index 0000000..0ef8de7 --- /dev/null +++ b/scripts/smoke/populate_demo_project.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Smoke test (GitHub issue #31): runs the doc-28 dev coordinator +# (`diffpype-manage populate-demo-project`) against a live `docker compose up` +# stack and asserts the domain graph actually got populated. +# +# Deliberately not wired into CI: it needs real Celery workers consuming the +# queue, and CI currently starts only db/minio directly, not the full stack. +# Run manually, locally, against your own docker compose stack. +# +# Built on top of the coordinator rather than reimplementing its sequence — +# the coordinator already polls each Celery task (ingest, mosaic) to a +# terminal state internally, so this script does not need its own poll loop. +# +# PREREQUISITE (GitHub issue #34, not yet resolved): this needs a real +# --S3_PREFIX already populated with FITS files in MinIO. There is currently +# no minio-init fiducial seed dataset, so this script cannot run end-to-end +# until #34 lands — the assertions below document what "success" means once +# it does. +set -euo pipefail + +: "${PROJECT_NAME:=SmokeTestProject}" +: "${USER_ID:?USER_ID must be set to an existing User id}" +: "${S3_PREFIX:?S3_PREFIX must be set to a populated storage prefix (see issue #34)}" +: "${RA:?RA must be set (deg)}" +: "${DECL:?DECL must be set (deg)}" +: "${RADIUS_DEG:?RADIUS_DEG must be set (deg)}" +: "${BAND_ID:?BAND_ID must be set to an existing Band id}" +: "${INSTRUMENT_ID:?INSTRUMENT_ID must be set to an existing Instrument id}" + +echo "Running populate-demo-project against the live docker compose stack..." +docker compose exec -T api diffpype-manage populate-demo-project \ + --project-name "$PROJECT_NAME" \ + --user-id "$USER_ID" \ + --s3-prefix "$S3_PREFIX" \ + --ra "$RA" \ + --decl "$DECL" \ + --radius-deg "$RADIUS_DEG" \ + --band-id "$BAND_ID" \ + --instrument-id "$INSTRUMENT_ID" + +echo "" +echo "Asserting the domain graph was actually populated..." +docker compose exec -T db psql -U "${POSTGRES_USER:-diffpype}" -d "${POSTGRES_DB:-diffpype}" -v ON_ERROR_STOP=1 < JSONR app.include_router(jobs_router, prefix="/api/v1") app.include_router(meta_router, prefix="/api/v1") +app.include_router(projects_router, prefix="/api/v1") +app.include_router(ingest_router, prefix="/api/v1") +app.include_router(tiles_router, prefix="/api/v1") +app.include_router(epochs_router, prefix="/api/v1") +app.include_router(mosaics_router, prefix="/api/v1") # Instrument the app, the SQLAlchemy engine, and Celery. Called last so the router # and middleware stack are fully assembled before OTel wraps the ASGI app. diff --git a/src/api/routes/epochs.py b/src/api/routes/epochs.py new file mode 100644 index 0000000..19b29ba --- /dev/null +++ b/src/api/routes/epochs.py @@ -0,0 +1,27 @@ +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from src.api.schemas import EpochBulkCreateRequest, EpochClusterRequest, EpochCreate, EpochRead +from src.db.session import get_db +from src.services import epoch_service + +router = APIRouter(prefix="/epochs", tags=["epochs"]) + + +@router.post("/cluster", response_model=list[EpochCreate]) +def cluster_epochs( + body: EpochClusterRequest, db: Session = Depends(get_db) +) -> list[EpochCreate]: + epochs = epoch_service.cluster_epochs( + db, body.project_id, body.tile_id, body.band_id, body.peak_distance_thresh + ) + return [EpochCreate(**e) for e in epochs] + + +@router.post("", response_model=list[EpochRead]) +def create_epochs( + body: EpochBulkCreateRequest, db: Session = Depends(get_db) +) -> list[EpochRead]: + epoch_dicts = [e.model_dump() for e in body.epochs] + created = epoch_service.create_epochs(db, body.project_id, epoch_dicts) + return [EpochRead.model_validate(e) for e in created] diff --git a/src/api/routes/ingest.py b/src/api/routes/ingest.py new file mode 100644 index 0000000..d66550b --- /dev/null +++ b/src/api/routes/ingest.py @@ -0,0 +1,28 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from src.api.schemas import IngestBatchStatus, IngestDispatchResponse, IngestRequest +from src.db.session import get_db +from src.services import ingest_service + +router = APIRouter(prefix="/ingest", tags=["ingest"]) + + +@router.post("", response_model=IngestDispatchResponse) +def create_ingest_batch( + body: IngestRequest, db: Session = Depends(get_db) +) -> IngestDispatchResponse: + job_id, batch_id = ingest_service.create_ingest_batch( + db, body.project_id, body.s3_prefix + ) + return IngestDispatchResponse(job_id=job_id, batch_id=batch_id) + + +@router.get("/{batch_id}", response_model=IngestBatchStatus) +def get_ingest_batch_status( + batch_id: int, db: Session = Depends(get_db) +) -> IngestBatchStatus: + batch = ingest_service.get_ingest_batch(db, batch_id) + if batch is None: + raise HTTPException(status_code=404, detail="IngestBatch not found") + return IngestBatchStatus.model_validate(batch) diff --git a/src/api/routes/mosaics.py b/src/api/routes/mosaics.py new file mode 100644 index 0000000..3e04784 --- /dev/null +++ b/src/api/routes/mosaics.py @@ -0,0 +1,35 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from src.api.schemas import MosaicCreate, MosaicDispatchResponse, MosaicStatus +from src.db.session import get_db +from src.services import mosaic_service + +router = APIRouter(prefix="/mosaics", tags=["mosaics"]) + + +@router.post("", response_model=MosaicDispatchResponse) +def create_mosaic( + body: MosaicCreate, db: Session = Depends(get_db) +) -> MosaicDispatchResponse: + job_id, mosaic_id = mosaic_service.create_mosaic( + db, + body.project_id, + body.tile_id, + body.epoch_id, + body.band_id, + body.instrument_id, + body.filename, + body.target_plate_scale, + ) + return MosaicDispatchResponse(job_id=job_id, mosaic_id=mosaic_id) + + +@router.get("/{mosaic_id}", response_model=MosaicStatus) +def get_mosaic_status( + mosaic_id: int, db: Session = Depends(get_db) +) -> MosaicStatus: + mosaic = mosaic_service.get_mosaic(db, mosaic_id) + if mosaic is None: + raise HTTPException(status_code=404, detail="Level3Mosaic not found") + return MosaicStatus.model_validate(mosaic) diff --git a/src/api/routes/projects.py b/src/api/routes/projects.py new file mode 100644 index 0000000..bd637c6 --- /dev/null +++ b/src/api/routes/projects.py @@ -0,0 +1,21 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from src.api.schemas import ProjectCreate, ProjectRead +from src.db.session import get_db +from src.services import project_service + +router = APIRouter(prefix="/projects", tags=["projects"]) + + +@router.post("", response_model=ProjectRead) +def create_project( + body: ProjectCreate, db: Session = Depends(get_db) +) -> ProjectRead: + try: + project = project_service.create_project( + db, body.name, body.description, body.user_id + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return ProjectRead.model_validate(project) diff --git a/src/api/routes/tiles.py b/src/api/routes/tiles.py new file mode 100644 index 0000000..753b4b6 --- /dev/null +++ b/src/api/routes/tiles.py @@ -0,0 +1,47 @@ +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from src.api.schemas import TileBulkCreateRequest, TileCreate, TileRead, TileTessellationRequest +from src.db.session import get_db +from src.db.spatial_types import moc_to_ranges, ranges_to_moc +from src.services import tile_service + +router = APIRouter(prefix="/tiles", tags=["tiles"]) + + +@router.post("/tessellate", response_model=list[TileCreate]) +def tessellate_tiles(body: TileTessellationRequest) -> list[TileCreate]: + moc_to_tile = ranges_to_moc(body.moc_to_tile) + tiles = tile_service.generate_tile_tessellation( + body.tile_side_length_arc_min, moc_to_tile, body.overlap_in_arc_min + ) + return [ + TileCreate( + name=t["name"], + ra=t["ra"], + decl=t["decl"], + delta_ra=t["delta_ra"], + delta_decl=t["delta_decl"], + footprint=moc_to_ranges(t["footprint"]), + ) + for t in tiles + ] + + +@router.post("", response_model=list[TileRead]) +def create_tiles( + body: TileBulkCreateRequest, db: Session = Depends(get_db) +) -> list[TileRead]: + tile_dicts = [ + { + "name": t.name, + "ra": t.ra, + "decl": t.decl, + "delta_ra": t.delta_ra, + "delta_decl": t.delta_decl, + "footprint": ranges_to_moc(t.footprint), + } + for t in body.tiles + ] + created = tile_service.create_tiles(db, body.project_id, tile_dicts) + return [TileRead.model_validate(t) for t in created] diff --git a/src/api/schemas.py b/src/api/schemas.py index 4ebd1c8..ec387a7 100644 --- a/src/api/schemas.py +++ b/src/api/schemas.py @@ -1,8 +1,9 @@ from datetime import datetime -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from src.db.enums import JobStatus +from src.db.spatial_types import moc_to_ranges class JobDispatchResponse(BaseModel): @@ -45,3 +46,166 @@ class PaginationParams(BaseModel): limit: int = Field(default=100, ge=1, le=1000) offset: int = Field(default=0, ge=0) + + +class ProjectCreate(BaseModel): + """Request body for creating a Project; its slug is derived from name server-side.""" + + name: str + description: str | None = None + user_id: int + + +class ProjectRead(BaseModel): + id: int + name: str + slug: str + description: str | None + user_id: int + + model_config = {"from_attributes": True} + + +class IngestRequest(BaseModel): + """Request body dispatching an async ingest run over a storage prefix.""" + + project_id: int + s3_prefix: str + + +class IngestDispatchResponse(BaseModel): + job_id: str + batch_id: int + + +class IngestBatchStatus(BaseModel): + id: int + project_id: int + s3_prefix: str + total_files: int + processed_files: int + status: JobStatus + + model_config = {"from_attributes": True} + + +def _coerce_moc_footprint(value): + """Convert a mocpy.MOC ORM attribute into its wire-safe depth-29 range list.""" + from mocpy import MOC + + if isinstance(value, MOC): + return moc_to_ranges(value) + return value + + +class TileCreate(BaseModel): + name: str + ra: float + decl: float + delta_ra: float # degrees, not arcmin — see Tile.delta_ra + delta_decl: float # degrees, not arcmin — see Tile.delta_decl + footprint: list[tuple[int, int]] | None = None + + _coerce_footprint = field_validator("footprint", mode="before")( + _coerce_moc_footprint + ) + + +class EpochCreate(BaseModel): + start_date: datetime + end_date: datetime + start_mjd: float + end_mjd: float + tile_id: int + band_id: int + + +class EpochClusterRequest(BaseModel): + """Request body for a no-write MJD-clustering preview over a tile+band.""" + + project_id: int + tile_id: int + band_id: int + peak_distance_thresh: float + + +class EpochBulkCreateRequest(BaseModel): + project_id: int + epochs: list[EpochCreate] + + +class MosaicCreate(BaseModel): + """Request body creating a Level3Mosaic job-metadata row and dispatching its drizzle task.""" + + project_id: int + tile_id: int + epoch_id: int + band_id: int + instrument_id: int + filename: str + target_plate_scale: float + + +class MosaicDispatchResponse(BaseModel): + job_id: str + mosaic_id: int + + +class MosaicStatus(BaseModel): + id: int + filename: str + target_plate_scale: float + ra: float | None + decl: float | None + status: JobStatus + project_id: int + tile_id: int + epoch_id: int + band_id: int + instrument_id: int + + model_config = {"from_attributes": True} + + +class EpochRead(BaseModel): + id: int + start_date: datetime + end_date: datetime + start_mjd: float | None + end_mjd: float | None + project_id: int + tile_id: int + band_id: int + + model_config = {"from_attributes": True} + + +class TileTessellationRequest(BaseModel): + """Request body for a no-DB tile-tessellation preview over a target region.""" + + project_id: int + tile_side_length_arc_min: float + moc_to_tile: list[tuple[int, int]] + overlap_in_arc_min: float = 0.0 + + +class TileBulkCreateRequest(BaseModel): + project_id: int + tiles: list[TileCreate] + + +class TileRead(BaseModel): + id: int + name: str + ra: float + decl: float + delta_ra: float + delta_decl: float + footprint: list[tuple[int, int]] | None + project_id: int + + model_config = {"from_attributes": True} + + _coerce_footprint = field_validator("footprint", mode="before")( + _coerce_moc_footprint + ) diff --git a/src/api/tests/test_cli.py b/src/api/tests/test_cli.py index 6d85c3d..b33d599 100644 --- a/src/api/tests/test_cli.py +++ b/src/api/tests/test_cli.py @@ -4,16 +4,31 @@ import pytest from src.cli import ( + _colorize_status, + _colors_enabled, _elapsed_label, _entity_to_dict, + _format_moc_for_display, + _poll_until_terminal, _print_entity_table, build_parser, + cmd_cluster_epochs, + cmd_create_epochs, + cmd_create_mosaic, + cmd_create_project, + cmd_create_tiles, cmd_get_dummy, + cmd_ingest, + cmd_ingest_status, + cmd_mosaic_status, + cmd_populate_demo_project, cmd_reset_db, cmd_run_dummy, cmd_seed_db, + cmd_tessellate_tiles, main, ) +from src.db.enums import JobStatus def test_entity_to_dict_serializes_sqlalchemy_object(): @@ -50,6 +65,47 @@ def test_entity_to_dict_serializes_pydantic_model(): } +def test_format_moc_for_display_reduces_moc_to_compact_sq_deg_string(): + import astropy.units as u + from mocpy import MOC + + moc = MOC.from_cone(lon=10 * u.deg, lat=0 * u.deg, radius=1 * u.deg, max_depth=10) + + result = _format_moc_for_display(moc) + + assert result.startswith("MOC(") + assert "sq.deg)" in result + + +def test_format_moc_for_display_passes_through_non_moc_values(): + assert _format_moc_for_display(42) == 42 + assert _format_moc_for_display(None) is None + assert _format_moc_for_display("pending") == "pending" + + +def test_print_entity_table_restricts_and_orders_columns_when_fields_given(capsys): + cols = [] + for name in ("id", "filename", "footprint", "status"): + col = MagicMock() + col.name = name + cols.append(col) + entity = MagicMock() + entity.__table__ = MagicMock() + entity.__table__.columns = cols + entity.id = 1 + entity.filename = "x.fits" + entity.footprint = "raw-moc-object" + entity.status = "pending" + del entity.model_dump + + _print_entity_table([entity], fields=["id", "status"]) + + out = capsys.readouterr().out + assert "filename" not in out + assert "footprint" not in out + assert "id" in out and "status" in out + + def test_print_entity_table_outputs_column_headers_and_values(mocker, capsys): col = MagicMock() col.name = "id" @@ -66,6 +122,82 @@ def test_print_entity_table_outputs_column_headers_and_values(mocker, capsys): assert "42" in out +def test_colors_enabled_false_when_not_a_tty(mocker): + mocker.patch("sys.stdout.isatty", return_value=False) + assert _colors_enabled() is False + + +def test_colors_enabled_false_when_no_color_set(mocker): + mocker.patch("sys.stdout.isatty", return_value=True) + mocker.patch.dict("os.environ", {"NO_COLOR": "1"}) + assert _colors_enabled() is False + + +def test_colors_enabled_true_in_a_plain_tty(mocker): + mocker.patch("sys.stdout.isatty", return_value=True) + mocker.patch.dict("os.environ", {}, clear=True) + assert _colors_enabled() is True + + +@pytest.mark.parametrize( + "status,color_code", + [ + ("pending", "\033[33m"), + ("in_process", "\033[36m"), + ("complete", "\033[32m"), + ("failed", "\033[31m"), + ], +) +def test_colorize_status_wraps_known_statuses(status, color_code): + assert _colorize_status(status) == f"{color_code}{status}\033[0m" + + +def test_colorize_status_passes_through_unknown_value(): + assert _colorize_status("some_new_status") == "some_new_status" + + +def _make_status_entity(status): + """A MagicMock shaped like a real ORM row, carrying a genuine JobStatus enum member.""" + col_id, col_status = MagicMock(), MagicMock() + col_id.name, col_status.name = "id", "status" + entity = MagicMock() + entity.__table__ = MagicMock() + entity.__table__.columns = [col_id, col_status] + entity.id = 1 + entity.status = status + del entity.model_dump + return entity + + +def test_print_entity_table_normalizes_real_enum_member_to_its_value(capsys): + """Regression: tabulate previously printed a real enum member as "JobStatus.COMPLETE".""" + _print_entity_table([_make_status_entity(JobStatus.COMPLETE)]) + + out = capsys.readouterr().out + assert "complete" in out + assert "JobStatus" not in out + + +def test_print_entity_table_colorizes_status_when_colors_enabled(mocker, capsys): + mocker.patch("src.cli._colors_enabled", return_value=True) + + _print_entity_table([_make_status_entity(JobStatus.COMPLETE)]) + + out = capsys.readouterr().out + assert "\033[32m" in out + assert "\033[0m" in out + + +def test_print_entity_table_leaves_status_plain_when_colors_disabled(mocker, capsys): + mocker.patch("src.cli._colors_enabled", return_value=False) + + _print_entity_table([_make_status_entity(JobStatus.COMPLETE)]) + + out = capsys.readouterr().out + assert "\033[" not in out + assert "complete" in out + + def test_parser_recognises_get_dummy_command(): args = build_parser().parse_args(["get-dummy", "--id", "5"]) assert args.command == "get-dummy" @@ -300,3 +432,656 @@ def test_cmd_run_dummy_logs_job_id_to_stdout(mocker, capsys): out = capsys.readouterr().out assert "abc-123" in out assert "7" in out + + +def test_parser_recognises_create_project_command(): + args = build_parser().parse_args( + ["create-project", "--name", "My Survey", "--user-id", "1"] + ) + assert args.command == "create-project" + assert args.name == "My Survey" + assert args.description is None + assert args.user_id == 1 + + +def test_main_routes_create_project_to_cmd_create_project(mocker): + mock_cmd = mocker.patch("src.cli.cmd_create_project") + main(["create-project", "--name", "My Survey", "--user-id", "1"]) + mock_cmd.assert_called_once() + + +def test_cmd_create_project_prints_id_and_slug(mocker, capsys): + from src.db.models import Project + + fake_project = Project(id=3, name="My Survey", slug="my-survey", user_id=1) + mocker.patch( + "src.services.project_service.create_project", return_value=fake_project + ) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_create_project( + argparse.Namespace( + command="create-project", + name="My Survey", + description=None, + user_id=1, + ) + ) + + out = capsys.readouterr().out + assert "id=3" in out + assert "slug=my-survey" in out + + +def test_cmd_create_project_prints_error_on_slug_collision(mocker, capsys): + mocker.patch( + "src.services.project_service.create_project", + side_effect=ValueError("A project with slug 'my-survey' already exists"), + ) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_create_project( + argparse.Namespace( + command="create-project", + name="My Survey", + description=None, + user_id=1, + ) + ) + + out = capsys.readouterr().out + assert "Error" in out + assert "already exists" in out + + +def test_cmd_create_project_closes_session(mocker): + mocker.patch( + "src.services.project_service.create_project", + return_value=MagicMock(id=1, slug="x"), + ) + mock_session = MagicMock() + mocker.patch("src.db.session.SessionLocal", return_value=mock_session) + + cmd_create_project( + argparse.Namespace( + command="create-project", + name="My Survey", + description=None, + user_id=1, + ) + ) + + mock_session.close.assert_called_once() + + +def test_parser_recognises_ingest_command(): + args = build_parser().parse_args( + ["ingest", "--project-id", "1", "--s3-prefix", "raw/"] + ) + assert args.command == "ingest" + assert args.project_id == 1 + assert args.s3_prefix == "raw/" + + +def test_main_routes_ingest_to_cmd_ingest(mocker): + mock_cmd = mocker.patch("src.cli.cmd_ingest") + main(["ingest", "--project-id", "1", "--s3-prefix", "raw/"]) + mock_cmd.assert_called_once() + + +def test_cmd_ingest_calls_service_and_prints_ids(mocker, capsys): + mocker.patch( + "src.services.ingest_service.create_ingest_batch", + return_value=("task-abc", 9), + ) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_ingest(argparse.Namespace(command="ingest", project_id=1, s3_prefix="raw/")) + + out = capsys.readouterr().out + assert "task-abc" in out + assert "9" in out + + +def test_parser_recognises_ingest_status_command(): + args = build_parser().parse_args(["ingest-status", "--id", "9"]) + assert args.command == "ingest-status" + assert args.id == 9 + + +def test_main_routes_ingest_status_to_cmd_ingest_status(mocker): + mock_cmd = mocker.patch("src.cli.cmd_ingest_status") + main(["ingest-status", "--id", "9"]) + mock_cmd.assert_called_once() + + +def test_cmd_ingest_status_prints_table_for_found_batch(mocker, capsys): + from src.db.models import IngestBatch + + fake_batch = IngestBatch(id=9, project_id=1, s3_prefix="raw/", status="complete") + mocker.patch( + "src.services.ingest_service.get_ingest_batch", return_value=fake_batch + ) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_ingest_status(argparse.Namespace(command="ingest-status", id=9)) + + out = capsys.readouterr().out + assert "9" in out + assert "complete" in out + + +def test_cmd_ingest_status_prints_error_for_missing_batch(mocker, capsys): + mocker.patch("src.services.ingest_service.get_ingest_batch", return_value=None) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_ingest_status(argparse.Namespace(command="ingest-status", id=404)) + + out = capsys.readouterr().out + assert "404" in out + assert "Error" in out + + +TESSELLATION_ARGS = [ + "--tile-side-arcmin", + "6.0", + "--ra", + "10.0", + "--decl", + "20.0", + "--radius-deg", + "0.1", +] + + +def test_parser_recognises_tessellate_tiles_command(): + args = build_parser().parse_args(["tessellate-tiles", *TESSELLATION_ARGS]) + assert args.command == "tessellate-tiles" + assert args.tile_side_arcmin == 6.0 + assert args.overlap_arcmin == 0.0 + + +def test_main_routes_tessellate_tiles_to_cmd_tessellate_tiles(mocker): + mock_cmd = mocker.patch("src.cli.cmd_tessellate_tiles") + main(["tessellate-tiles", *TESSELLATION_ARGS]) + mock_cmd.assert_called_once() + + +def test_cmd_tessellate_tiles_prints_generated_tiles(mocker, capsys): + mocker.patch( + "src.services.tile_service.generate_tile_tessellation", + return_value=[ + {"name": "Tile_1", "ra": 10.0, "decl": 20.0}, + {"name": "Tile_2", "ra": 10.1, "decl": 20.1}, + ], + ) + + cmd_tessellate_tiles( + argparse.Namespace( + command="tessellate-tiles", + tile_side_arcmin=6.0, + ra=10.0, + decl=20.0, + radius_deg=0.1, + overlap_arcmin=0.0, + ) + ) + + out = capsys.readouterr().out + assert "Generated 2 tile(s)" in out + assert "Tile_1" in out + assert "Tile_2" in out + + +def test_parser_recognises_create_tiles_command(): + args = build_parser().parse_args( + ["create-tiles", "--project-id", "1", *TESSELLATION_ARGS] + ) + assert args.command == "create-tiles" + assert args.project_id == 1 + + +def test_main_routes_create_tiles_to_cmd_create_tiles(mocker): + mock_cmd = mocker.patch("src.cli.cmd_create_tiles") + main(["create-tiles", "--project-id", "1", *TESSELLATION_ARGS]) + mock_cmd.assert_called_once() + + +def test_cmd_create_tiles_persists_and_prints_count(mocker, capsys): + mocker.patch( + "src.services.tile_service.generate_tile_tessellation", + return_value=[{"name": "Tile_1"}, {"name": "Tile_2"}], + ) + mock_create = mocker.patch( + "src.services.tile_service.create_tiles", + return_value=[MagicMock(), MagicMock()], + ) + mock_session = MagicMock() + mocker.patch("src.db.session.SessionLocal", return_value=mock_session) + + cmd_create_tiles( + argparse.Namespace( + command="create-tiles", + project_id=3, + tile_side_arcmin=6.0, + ra=10.0, + decl=20.0, + radius_deg=0.1, + overlap_arcmin=0.0, + ) + ) + + mock_create.assert_called_once() + assert mock_create.call_args[0][1] == 3 + mock_session.close.assert_called_once() + out = capsys.readouterr().out + assert "Created 2 tile(s) for project_id=3" in out + + +EPOCH_ARGS = [ + "--project-id", + "1", + "--tile-id", + "2", + "--band-id", + "3", + "--peak-distance-thresh", + "5.0", +] + + +def test_parser_recognises_cluster_epochs_command(): + args = build_parser().parse_args(["cluster-epochs", *EPOCH_ARGS]) + assert args.command == "cluster-epochs" + assert args.tile_id == 2 + assert args.peak_distance_thresh == 5.0 + + +def test_main_routes_cluster_epochs_to_cmd_cluster_epochs(mocker): + mock_cmd = mocker.patch("src.cli.cmd_cluster_epochs") + main(["cluster-epochs", *EPOCH_ARGS]) + mock_cmd.assert_called_once() + + +def test_cmd_cluster_epochs_prints_generated_epochs(mocker, capsys): + mocker.patch( + "src.services.epoch_service.cluster_epochs", + return_value=[{"start_mjd": 60300.0, "end_mjd": 60301.0}], + ) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_cluster_epochs( + argparse.Namespace( + command="cluster-epochs", + project_id=1, + tile_id=2, + band_id=3, + peak_distance_thresh=5.0, + ) + ) + + out = capsys.readouterr().out + assert "Generated 1 epoch(s)" in out + assert "60300.00" in out + + +def test_parser_recognises_create_epochs_command(): + args = build_parser().parse_args(["create-epochs", *EPOCH_ARGS]) + assert args.command == "create-epochs" + + +def test_main_routes_create_epochs_to_cmd_create_epochs(mocker): + mock_cmd = mocker.patch("src.cli.cmd_create_epochs") + main(["create-epochs", *EPOCH_ARGS]) + mock_cmd.assert_called_once() + + +def test_cmd_create_epochs_clusters_then_persists_and_prints_count(mocker, capsys): + mocker.patch( + "src.services.epoch_service.cluster_epochs", + return_value=[{"start_mjd": 60300.0, "end_mjd": 60301.0}], + ) + mock_create = mocker.patch( + "src.services.epoch_service.create_epochs", return_value=[MagicMock()] + ) + mock_session = MagicMock() + mocker.patch("src.db.session.SessionLocal", return_value=mock_session) + + cmd_create_epochs( + argparse.Namespace( + command="create-epochs", + project_id=4, + tile_id=2, + band_id=3, + peak_distance_thresh=5.0, + ) + ) + + mock_create.assert_called_once() + assert mock_create.call_args[0][1] == 4 + mock_session.close.assert_called_once() + out = capsys.readouterr().out + assert "Created 1 epoch(s) for project_id=4" in out + + +MOSAIC_ARGS = [ + "--project-id", + "1", + "--tile-id", + "2", + "--epoch-id", + "3", + "--band-id", + "4", + "--instrument-id", + "5", + "--filename", + "mosaic_1.fits", + "--target-plate-scale", + "0.03", +] + + +def test_parser_recognises_create_mosaic_command(): + args = build_parser().parse_args(["create-mosaic", *MOSAIC_ARGS]) + assert args.command == "create-mosaic" + assert args.filename == "mosaic_1.fits" + assert args.target_plate_scale == 0.03 + + +def test_main_routes_create_mosaic_to_cmd_create_mosaic(mocker): + mock_cmd = mocker.patch("src.cli.cmd_create_mosaic") + main(["create-mosaic", *MOSAIC_ARGS]) + mock_cmd.assert_called_once() + + +def test_cmd_create_mosaic_calls_service_and_prints_ids(mocker, capsys): + mocker.patch( + "src.services.mosaic_service.create_mosaic", + return_value=("task-xyz", 11), + ) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_create_mosaic( + argparse.Namespace( + command="create-mosaic", + project_id=1, + tile_id=2, + epoch_id=3, + band_id=4, + instrument_id=5, + filename="mosaic_1.fits", + target_plate_scale=0.03, + ) + ) + + out = capsys.readouterr().out + assert "task-xyz" in out + assert "11" in out + + +def test_parser_recognises_mosaic_status_command(): + args = build_parser().parse_args(["mosaic-status", "--id", "11"]) + assert args.command == "mosaic-status" + assert args.id == 11 + + +def test_main_routes_mosaic_status_to_cmd_mosaic_status(mocker): + mock_cmd = mocker.patch("src.cli.cmd_mosaic_status") + main(["mosaic-status", "--id", "11"]) + mock_cmd.assert_called_once() + + +def test_cmd_mosaic_status_prints_table_for_found_mosaic(mocker, capsys): + from src.db.models import Level3Mosaic + + fake_mosaic = Level3Mosaic( + id=11, + filename="mosaic_1.fits", + target_plate_scale=0.03, + status="pending", + ra=10.5, + decl=41.2, + ) + mocker.patch("src.services.mosaic_service.get_mosaic", return_value=fake_mosaic) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_mosaic_status(argparse.Namespace(command="mosaic-status", id=11)) + + out = capsys.readouterr().out + assert "11" in out + assert "mosaic_1.fits" in out + assert "10.5" in out + # Status view is curated for readability -- no bare FK ids, no raw footprint. + assert "footprint" not in out + assert "instrument_id" not in out + assert "tile_id" not in out + + +def test_cmd_mosaic_status_prints_error_for_missing_mosaic(mocker, capsys): + mocker.patch("src.services.mosaic_service.get_mosaic", return_value=None) + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + + cmd_mosaic_status(argparse.Namespace(command="mosaic-status", id=404)) + + out = capsys.readouterr().out + assert "404" in out + assert "Error" in out + + +# --------------------------------------------------------------------------- +# populate-demo-project coordinator +# --------------------------------------------------------------------------- + + +def test_poll_until_terminal_returns_immediately_on_complete(): + assert ( + _poll_until_terminal(lambda: JobStatus.COMPLETE, "X", timeout_s=1.0) + == JobStatus.COMPLETE + ) + + +def test_poll_until_terminal_returns_failed_status(): + assert ( + _poll_until_terminal(lambda: JobStatus.FAILED, "X", timeout_s=1.0) + == JobStatus.FAILED + ) + + +def test_poll_until_terminal_raises_timeout_error_when_never_terminal(): + with pytest.raises(TimeoutError, match="did not reach a terminal state"): + _poll_until_terminal( + lambda: JobStatus.IN_PROCESS, "X", timeout_s=0.05, interval_s=0.01 + ) + + +POPULATE_DEMO_ARGS = dict( + command="populate-demo-project", + project_name="Demo Project", + user_id=1, + s3_prefix="raw/", + ra=10.0, + decl=20.0, + radius_deg=0.1, + tile_side_arcmin=6.0, + overlap_arcmin=0.0, + band_id=2, + instrument_id=3, + peak_distance_thresh=5.0, + target_plate_scale=0.03, + ingest_timeout=5.0, + mosaic_timeout=5.0, +) + + +def test_parser_recognises_populate_demo_project_command(): + args = build_parser().parse_args( + [ + "populate-demo-project", + "--project-name", + "Demo Project", + "--user-id", + "1", + "--s3-prefix", + "raw/", + "--ra", + "10.0", + "--decl", + "20.0", + "--radius-deg", + "0.1", + "--band-id", + "2", + "--instrument-id", + "3", + ] + ) + assert args.command == "populate-demo-project" + assert args.tile_side_arcmin == 6.0 + assert args.target_plate_scale == 0.03 + + +def test_main_routes_populate_demo_project_to_cmd(mocker): + mock_cmd = mocker.patch("src.cli.cmd_populate_demo_project") + main( + [ + "populate-demo-project", + "--project-name", + "Demo Project", + "--user-id", + "1", + "--s3-prefix", + "raw/", + "--ra", + "10.0", + "--decl", + "20.0", + "--radius-deg", + "0.1", + "--band-id", + "2", + "--instrument-id", + "3", + ] + ) + mock_cmd.assert_called_once() + + +def _mock_populate_services( + mocker, + ingest_status=JobStatus.COMPLETE, + mosaic_status=JobStatus.COMPLETE, + tiles=None, + epochs=None, + best_tile_id=5, +): + mocker.patch("src.db.session.SessionLocal", return_value=MagicMock()) + project = MagicMock(id=1, slug="demo-project") + mocker.patch("src.services.project_service.create_project", return_value=project) + mocker.patch( + "src.services.ingest_service.create_ingest_batch", + return_value=("ingest-task", 10), + ) + mocker.patch( + "src.services.ingest_service.get_ingest_batch", + return_value=MagicMock(status=ingest_status), + ) + mocker.patch( + "src.services.tile_service.generate_tile_tessellation", + return_value=[{"name": "Tile_1"}], + ) + mocker.patch( + "src.services.tile_service.create_tiles", + return_value=tiles if tiles is not None else [MagicMock(id=5)], + ) + mocker.patch( + "src.services.tile_service.tile_with_most_calibrations", + return_value=best_tile_id, + ) + mocker.patch( + "src.services.epoch_service.cluster_epochs", return_value=[{"start_mjd": 1}] + ) + mocker.patch( + "src.services.epoch_service.create_epochs", + return_value=epochs if epochs is not None else [MagicMock(id=6)], + ) + mocker.patch( + "src.services.mosaic_service.create_mosaic", + return_value=("mosaic-task", 20), + ) + mocker.patch( + "src.services.mosaic_service.get_mosaic", + return_value=MagicMock(status=mosaic_status), + ) + return project + + +def test_cmd_populate_demo_project_happy_path(mocker, capsys): + _mock_populate_services(mocker) + + cmd_populate_demo_project(argparse.Namespace(**POPULATE_DEMO_ARGS)) + + out = capsys.readouterr().out + assert "Created project" in out + assert "Ingest complete." in out + assert "Created 1 tile(s)" in out + assert "Created 1 epoch(s)" in out + assert "Mosaic finished with status=complete" in out + assert "Demo project population complete. project_id=1" in out + + +def test_cmd_populate_demo_project_aborts_on_ingest_failure(mocker, capsys): + _mock_populate_services(mocker, ingest_status=JobStatus.FAILED) + mock_create_tiles = mocker.patch("src.services.tile_service.create_tiles") + + cmd_populate_demo_project(argparse.Namespace(**POPULATE_DEMO_ARGS)) + + out = capsys.readouterr().out + assert "Ingest batch failed. Aborting." in out + mock_create_tiles.assert_not_called() + + +def test_cmd_populate_demo_project_aborts_when_no_tiles_generated(mocker, capsys): + _mock_populate_services(mocker, tiles=[]) + mock_cluster = mocker.patch("src.services.epoch_service.cluster_epochs") + + cmd_populate_demo_project(argparse.Namespace(**POPULATE_DEMO_ARGS)) + + out = capsys.readouterr().out + assert "No tiles generated" in out + mock_cluster.assert_not_called() + + +def test_cmd_populate_demo_project_aborts_when_no_tile_overlaps_data(mocker, capsys): + """A tessellated grid can produce tiles with zero real spatial overlap (edge + cells) even when tiles themselves were generated -- must not silently proceed + with an arbitrary empty tile.""" + _mock_populate_services(mocker, best_tile_id=None) + mock_cluster = mocker.patch("src.services.epoch_service.cluster_epochs") + + cmd_populate_demo_project(argparse.Namespace(**POPULATE_DEMO_ARGS)) + + out = capsys.readouterr().out + assert "No tile in the tessellated region overlaps any ingested calibration" in out + mock_cluster.assert_not_called() + + +def test_cmd_populate_demo_project_aborts_when_no_epochs_generated(mocker, capsys): + _mock_populate_services(mocker, epochs=[]) + mock_create_mosaic = mocker.patch("src.services.mosaic_service.create_mosaic") + + cmd_populate_demo_project(argparse.Namespace(**POPULATE_DEMO_ARGS)) + + out = capsys.readouterr().out + assert "No epochs generated" in out + mock_create_mosaic.assert_not_called() + + +def test_cmd_populate_demo_project_closes_session(mocker): + _mock_populate_services(mocker) + mock_session = MagicMock() + mocker.patch("src.db.session.SessionLocal", return_value=mock_session) + + cmd_populate_demo_project(argparse.Namespace(**POPULATE_DEMO_ARGS)) + + mock_session.close.assert_called_once() diff --git a/src/api/tests/test_epochs.py b/src/api/tests/test_epochs.py new file mode 100644 index 0000000..da1b9b9 --- /dev/null +++ b/src/api/tests/test_epochs.py @@ -0,0 +1,99 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from src.api.main import app +from src.db.models import Epoch +from src.db.session import get_db + + +@pytest.fixture +def mock_db(): + db = MagicMock() + app.dependency_overrides[get_db] = lambda: db + yield db + app.dependency_overrides.pop(get_db, None) + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_cluster_epochs_returns_preview_without_writing(client, mock_db, mocker): + start = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2024, 1, 2, tzinfo=timezone.utc) + mocker.patch( + "src.services.epoch_service.cluster_epochs", + return_value=[ + { + "start_date": start, + "end_date": end, + "start_mjd": 60300.0, + "end_mjd": 60301.0, + "tile_id": 1, + "band_id": 2, + } + ], + ) + + response = client.post( + "/api/v1/epochs/cluster", + json={ + "project_id": 1, + "tile_id": 1, + "band_id": 2, + "peak_distance_thresh": 5.0, + }, + ) + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + assert body[0]["start_mjd"] == 60300.0 + assert body[0]["tile_id"] == 1 + + +def test_create_epochs_returns_persisted_epochs(client, mock_db, mocker): + start = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2024, 1, 2, tzinfo=timezone.utc) + mocker.patch( + "src.services.epoch_service.create_epochs", + return_value=[ + Epoch( + id=1, + start_date=start, + end_date=end, + start_mjd=60300.0, + end_mjd=60301.0, + project_id=1, + tile_id=1, + band_id=2, + ) + ], + ) + + response = client.post( + "/api/v1/epochs", + json={ + "project_id": 1, + "epochs": [ + { + "start_date": start.isoformat(), + "end_date": end.isoformat(), + "start_mjd": 60300.0, + "end_mjd": 60301.0, + "tile_id": 1, + "band_id": 2, + } + ], + }, + ) + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + assert body[0]["id"] == 1 + assert body[0]["start_mjd"] == 60300.0 diff --git a/src/api/tests/test_ingest.py b/src/api/tests/test_ingest.py new file mode 100644 index 0000000..def109a --- /dev/null +++ b/src/api/tests/test_ingest.py @@ -0,0 +1,72 @@ +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from src.api.main import app +from src.db.enums import JobStatus +from src.db.models import IngestBatch +from src.db.session import get_db + + +@pytest.fixture +def mock_db(): + db = MagicMock() + app.dependency_overrides[get_db] = lambda: db + yield db + app.dependency_overrides.pop(get_db, None) + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_create_ingest_batch_returns_job_and_batch_id(client, mock_db, mocker): + mocker.patch( + "src.services.ingest_service.create_ingest_batch", + return_value=("fake-task-id", 7), + ) + + response = client.post( + "/api/v1/ingest", json={"project_id": 1, "s3_prefix": "raw/"} + ) + + assert response.status_code == 200 + assert response.json() == {"job_id": "fake-task-id", "batch_id": 7} + + +def test_get_ingest_batch_status_returns_batch(client, mock_db, mocker): + mocker.patch( + "src.services.ingest_service.get_ingest_batch", + return_value=IngestBatch( + id=7, + project_id=1, + s3_prefix="raw/", + total_files=10, + processed_files=10, + status=JobStatus.COMPLETE, + ), + ) + + response = client.get("/api/v1/ingest/7") + + assert response.status_code == 200 + assert response.json() == { + "id": 7, + "project_id": 1, + "s3_prefix": "raw/", + "total_files": 10, + "processed_files": 10, + "status": "complete", + } + + +def test_get_ingest_batch_status_404_when_missing(client, mock_db, mocker): + mocker.patch( + "src.services.ingest_service.get_ingest_batch", return_value=None + ) + + response = client.get("/api/v1/ingest/999") + + assert response.status_code == 404 diff --git a/src/api/tests/test_mosaics.py b/src/api/tests/test_mosaics.py new file mode 100644 index 0000000..4ff269c --- /dev/null +++ b/src/api/tests/test_mosaics.py @@ -0,0 +1,80 @@ +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from src.api.main import app +from src.db.enums import JobStatus +from src.db.models import Level3Mosaic +from src.db.session import get_db + +VALID_PAYLOAD = { + "project_id": 1, + "tile_id": 2, + "epoch_id": 3, + "band_id": 4, + "instrument_id": 5, + "filename": "mosaic_1.fits", + "target_plate_scale": 0.03, +} + + +@pytest.fixture +def mock_db(): + db = MagicMock() + app.dependency_overrides[get_db] = lambda: db + yield db + app.dependency_overrides.pop(get_db, None) + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_create_mosaic_returns_job_and_mosaic_id(client, mock_db, mocker): + mocker.patch( + "src.services.mosaic_service.create_mosaic", + return_value=("fake-task-id", 8), + ) + + response = client.post("/api/v1/mosaics", json=VALID_PAYLOAD) + + assert response.status_code == 200 + assert response.json() == {"job_id": "fake-task-id", "mosaic_id": 8} + + +def test_get_mosaic_status_returns_mosaic(client, mock_db, mocker): + mocker.patch( + "src.services.mosaic_service.get_mosaic", + return_value=Level3Mosaic( + id=8, + filename="mosaic_1.fits", + target_plate_scale=0.03, + footprint=None, + ra=None, + decl=None, + status=JobStatus.PENDING, + project_id=1, + tile_id=2, + epoch_id=3, + band_id=4, + instrument_id=5, + ), + ) + + response = client.get("/api/v1/mosaics/8") + + assert response.status_code == 200 + body = response.json() + assert body["id"] == 8 + assert body["status"] == "pending" + assert "footprint" not in body + + +def test_get_mosaic_status_404_when_missing(client, mock_db, mocker): + mocker.patch("src.services.mosaic_service.get_mosaic", return_value=None) + + response = client.get("/api/v1/mosaics/999") + + assert response.status_code == 404 diff --git a/src/api/tests/test_projects.py b/src/api/tests/test_projects.py new file mode 100644 index 0000000..3c5c521 --- /dev/null +++ b/src/api/tests/test_projects.py @@ -0,0 +1,55 @@ +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +from src.api.main import app +from src.db.models import Project +from src.db.session import get_db + +VALID_PAYLOAD = {"name": "My Survey", "description": "desc", "user_id": 1} + + +@pytest.fixture +def mock_db(): + db = MagicMock() + app.dependency_overrides[get_db] = lambda: db + yield db + app.dependency_overrides.pop(get_db, None) + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_create_project_returns_created_project(client, mock_db, mocker): + mocker.patch( + "src.services.project_service.create_project", + return_value=Project( + id=1, name="My Survey", slug="my-survey", description="desc", user_id=1 + ), + ) + + response = client.post("/api/v1/projects", json=VALID_PAYLOAD) + + assert response.status_code == 200 + assert response.json() == { + "id": 1, + "name": "My Survey", + "slug": "my-survey", + "description": "desc", + "user_id": 1, + } + + +def test_create_project_returns_400_on_slug_collision(client, mock_db, mocker): + mocker.patch( + "src.services.project_service.create_project", + side_effect=ValueError("A project with slug 'my-survey' already exists"), + ) + + response = client.post("/api/v1/projects", json=VALID_PAYLOAD) + + assert response.status_code == 400 + assert "already exists" in response.json()["detail"] diff --git a/src/api/tests/test_schemas.py b/src/api/tests/test_schemas.py index 48c3a9b..4c61538 100644 --- a/src/api/tests/test_schemas.py +++ b/src/api/tests/test_schemas.py @@ -1,9 +1,12 @@ """Unit tests for API schema models.""" +import astropy.units as u import pytest +from mocpy import MOC from pydantic import ValidationError -from src.api.schemas import PaginationParams +from src.api.schemas import PaginationParams, TileRead +from src.db.spatial_types import moc_to_ranges def test_pagination_params_defaults(): @@ -35,3 +38,24 @@ def test_pagination_params_negative_offset_rejected(): """offset=-1 must raise ValidationError (ge=0).""" with pytest.raises(ValidationError): PaginationParams(offset=-1) + + +def test_tile_read_coerces_a_real_moc_footprint_to_a_range_list(): + """model_validate against an ORM-shaped object with a real mocpy.MOC footprint attribute.""" + moc = MOC.from_cone( + lon=10 * u.deg, lat=20 * u.deg, radius=0.1 * u.deg, max_depth=10 + ) + + class _FakeTile: + id = 1 + name = "Tile_1" + ra = 10.0 + decl = 20.0 + delta_ra = 0.2 + delta_decl = 0.2 + footprint = moc + project_id = 1 + + result = TileRead.model_validate(_FakeTile(), from_attributes=True) + + assert result.footprint == moc_to_ranges(moc) diff --git a/src/api/tests/test_tiles.py b/src/api/tests/test_tiles.py new file mode 100644 index 0000000..3008607 --- /dev/null +++ b/src/api/tests/test_tiles.py @@ -0,0 +1,105 @@ +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient +from mocpy import MOC + +from src.api.main import app +from src.db.models import Tile +from src.db.session import get_db + + +@pytest.fixture +def mock_db(): + db = MagicMock() + app.dependency_overrides[get_db] = lambda: db + yield db + app.dependency_overrides.pop(get_db, None) + + +@pytest.fixture +def client(): + return TestClient(app) + + +def test_tessellate_tiles_returns_preview_without_writing(client, mocker): + fake_moc = MOC.new_empty(10) + mocker.patch( + "src.services.tile_service.generate_tile_tessellation", + return_value=[ + { + "name": "Tile_1", + "ra": 10.0, + "decl": 20.0, + "delta_ra": 0.1, + "delta_decl": 0.1, + "footprint": fake_moc, + } + ], + ) + + response = client.post( + "/api/v1/tiles/tessellate", + json={ + "project_id": 1, + "tile_side_length_arc_min": 6.0, + "moc_to_tile": [[0, 10]], + "overlap_in_arc_min": 0.0, + }, + ) + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + assert body[0]["name"] == "Tile_1" + assert body[0]["footprint"] == [] # new_empty MOC has no ranges + + +def test_create_tiles_returns_persisted_tiles(client, mock_db, mocker): + mocker.patch( + "src.services.tile_service.create_tiles", + return_value=[ + Tile( + id=1, + name="Tile_1", + ra=10.0, + decl=20.0, + delta_ra=0.1, + delta_decl=0.1, + footprint=None, + project_id=1, + ) + ], + ) + + response = client.post( + "/api/v1/tiles", + json={ + "project_id": 1, + "tiles": [ + { + "name": "Tile_1", + "ra": 10.0, + "decl": 20.0, + "delta_ra": 0.1, + "delta_decl": 0.1, + "footprint": None, + } + ], + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body == [ + { + "id": 1, + "name": "Tile_1", + "ra": 10.0, + "decl": 20.0, + "delta_ra": 0.1, + "delta_decl": 0.1, + "footprint": None, + "project_id": 1, + } + ] diff --git a/src/cli.py b/src/cli.py index bc02230..c717878 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1,6 +1,30 @@ import argparse +import enum +import os import sys +# All four codes are the same string length ("\033[NNm", 2-digit N) so every +# colorized status cell carries identical invisible overhead — tabulate's width +# calculation ends up uniformly wider than necessary (harmless), never uneven. +_STATUS_ANSI = { + "pending": "\033[33m", # yellow + "in_process": "\033[36m", # cyan + "complete": "\033[32m", # green + "failed": "\033[31m", # red +} +_ANSI_RESET = "\033[0m" + + +def _colors_enabled() -> bool: + """Colorize only in an interactive terminal, honoring the NO_COLOR convention (no-color.org).""" + return sys.stdout.isatty() and not os.environ.get("NO_COLOR") + + +def _colorize_status(value: str) -> str: + """Wrap a known JobStatus string in its ANSI color; unrecognized values pass through unchanged.""" + code = _STATUS_ANSI.get(value) + return f"{code}{value}{_ANSI_RESET}" if code else value + def _entity_to_dict(entity) -> dict: """Serialize a SQLAlchemy ORM object or Pydantic model to a plain dictionary.""" @@ -9,11 +33,49 @@ def _entity_to_dict(entity) -> dict: return {col.name: getattr(entity, col.name) for col in entity.__table__.columns} -def _print_entity_table(entities: list) -> None: - """Print a list of domain entities or Pydantic models as an ASCII grid table.""" +def _format_moc_for_display(value): + """Reduce a mocpy.MOC to a compact sq.deg string; pass through any other value unchanged. + + Safety net for any table column that happens to hold a raw MOC (e.g. a + future status command that surfaces footprint) — tabulate otherwise dumps + the object's internal repr, which is unreadable and can be enormous. + """ + from mocpy import MOC + + if isinstance(value, MOC): + return f"MOC({value.sky_fraction * 41253.0:.4f} sq.deg)" + return value + + +def _print_entity_table(entities: list, fields: list[str] | None = None) -> None: + """Print a list of domain entities or Pydantic models as an ASCII grid table. + + `fields`, if given, restricts and orders the printed columns — use it to + keep a status-check table salient (e.g. skip bare foreign-key ids that + aren't meaningful without a join) rather than dumping every column. + + The `status` column is normalized to its plain value (tabulate otherwise + prints a real enum member as e.g. "JobStatus.COMPLETE", since `str()`/ + `format()` on a `str`+`Enum` mixin doesn't reduce to the bare value) and + ANSI-colored when stdout is an interactive terminal (and NO_COLOR is + unset) — plain text otherwise, so piping/redirecting output never leaks + escape codes into a file or another program. Any raw MOC value is reduced + to a compact sq.deg string rather than printed as-is. + """ from tabulate import tabulate rows = [_entity_to_dict(e) for e in entities] + if fields is not None: + rows = [{key: row[key] for key in fields} for row in rows] + colors_on = _colors_enabled() + for row in rows: + status = row.get("status") + if isinstance(status, enum.Enum): + status = status.value + if status is not None: + row["status"] = _colorize_status(status) if colors_on else status + for key, value in row.items(): + row[key] = _format_moc_for_display(value) print(tabulate(rows, headers="keys", tablefmt="grid")) @@ -98,6 +160,298 @@ def cmd_run_dummy(args: argparse.Namespace) -> None: print(f"Dispatched dummy job. job_id={job_id}, image_id={image_id}") +def cmd_create_project(args: argparse.Namespace) -> None: + """Create a Project through the service layer and print its identifiers.""" + from src.db.session import SessionLocal + from src.services import project_service + + db = SessionLocal() + try: + try: + project = project_service.create_project( + db, args.name, args.description, args.user_id + ) + except ValueError as exc: + print(f"Error: {exc}") + return + finally: + db.close() + + print(f"Created project. id={project.id}, slug={project.slug}") + + +def cmd_ingest(args: argparse.Namespace) -> None: + """Dispatch an ingest batch through the service layer and print its identifiers.""" + from src.db.session import SessionLocal + from src.services import ingest_service + + db = SessionLocal() + try: + job_id, batch_id = ingest_service.create_ingest_batch( + db, args.project_id, args.s3_prefix + ) + finally: + db.close() + + print(f"Dispatched ingest batch. job_id={job_id}, batch_id={batch_id}") + + +def cmd_ingest_status(args: argparse.Namespace) -> None: + """Fetch an IngestBatch by ID from the database and print it as an ASCII table.""" + from src.db.session import SessionLocal + from src.services import ingest_service + + db = SessionLocal() + try: + batch = ingest_service.get_ingest_batch(db, args.id) + finally: + db.close() + + if batch is None: + print(f"Error: No IngestBatch found with id={args.id}.") + return + + _print_entity_table([batch]) + + +def _cone_moc(ra: float, decl: float, radius_deg: float): + """Build a MOC covering a cone region, the CLI's way to specify a tile-tessellation target.""" + import astropy.units as u + from mocpy import MOC + + return MOC.from_cone( + lon=ra * u.deg, lat=decl * u.deg, radius=radius_deg * u.deg, max_depth=10 + ) + + +def cmd_tessellate_tiles(args: argparse.Namespace) -> None: + """Preview a tile tessellation over a cone region and print it, without writing to the DB.""" + from src.services import tile_service + + moc_to_tile = _cone_moc(args.ra, args.decl, args.radius_deg) + tiles = tile_service.generate_tile_tessellation( + args.tile_side_arcmin, moc_to_tile, args.overlap_arcmin + ) + print(f"Generated {len(tiles)} tile(s):") + for t in tiles: + print(f" {t['name']}: ra={t['ra']:.4f}, decl={t['decl']:.4f}") + + +def cmd_create_tiles(args: argparse.Namespace) -> None: + """Generate a tile tessellation over a cone region and persist it through the service layer.""" + from src.db.session import SessionLocal + from src.services import tile_service + + moc_to_tile = _cone_moc(args.ra, args.decl, args.radius_deg) + tiles = tile_service.generate_tile_tessellation( + args.tile_side_arcmin, moc_to_tile, args.overlap_arcmin + ) + + db = SessionLocal() + try: + created = tile_service.create_tiles(db, args.project_id, tiles) + finally: + db.close() + + print(f"Created {len(created)} tile(s) for project_id={args.project_id}.") + + +def cmd_cluster_epochs(args: argparse.Namespace) -> None: + """Preview an MJD-based epoch clustering for a tile+band and print it, without writing.""" + from src.db.session import SessionLocal + from src.services import epoch_service + + db = SessionLocal() + try: + epochs = epoch_service.cluster_epochs( + db, args.project_id, args.tile_id, args.band_id, args.peak_distance_thresh + ) + finally: + db.close() + + print(f"Generated {len(epochs)} epoch(s):") + for e in epochs: + print(f" start_mjd={e['start_mjd']:.2f}, end_mjd={e['end_mjd']:.2f}") + + +def cmd_create_epochs(args: argparse.Namespace) -> None: + """Cluster a tile+band's MJDs into epochs and persist them through the service layer.""" + from src.db.session import SessionLocal + from src.services import epoch_service + + db = SessionLocal() + try: + epochs = epoch_service.cluster_epochs( + db, args.project_id, args.tile_id, args.band_id, args.peak_distance_thresh + ) + created = epoch_service.create_epochs(db, args.project_id, epochs) + finally: + db.close() + + print(f"Created {len(created)} epoch(s) for project_id={args.project_id}.") + + +def cmd_create_mosaic(args: argparse.Namespace) -> None: + """Create a Level3Mosaic through the service layer and print its identifiers.""" + from src.db.session import SessionLocal + from src.services import mosaic_service + + db = SessionLocal() + try: + job_id, mosaic_id = mosaic_service.create_mosaic( + db, + args.project_id, + args.tile_id, + args.epoch_id, + args.band_id, + args.instrument_id, + args.filename, + args.target_plate_scale, + ) + finally: + db.close() + + print(f"Dispatched mosaic. job_id={job_id}, mosaic_id={mosaic_id}") + + +def cmd_mosaic_status(args: argparse.Namespace) -> None: + """Fetch a Level3Mosaic by ID from the database and print it as an ASCII table.""" + from src.db.session import SessionLocal + from src.services import mosaic_service + + db = SessionLocal() + try: + mosaic = mosaic_service.get_mosaic(db, args.id) + finally: + db.close() + + if mosaic is None: + print(f"Error: No Level3Mosaic found with id={args.id}.") + return + + _print_entity_table( + [mosaic], + fields=["id", "filename", "status", "ra", "decl", "created_at", "updated_at"], + ) + + +def _poll_until_terminal( + fetch_status, label: str, timeout_s: float, interval_s: float = 1.0 +): + """Poll `fetch_status()` (returning a JobStatus) until COMPLETE/FAILED or timeout.""" + import time as _time + + from src.db.enums import JobStatus + + deadline = _time.monotonic() + timeout_s + while _time.monotonic() < deadline: + status = fetch_status() + if status in (JobStatus.COMPLETE, JobStatus.FAILED): + return status + _time.sleep(interval_s) + raise TimeoutError(f"{label} did not reach a terminal state within {timeout_s}s") + + +def cmd_populate_demo_project(args: argparse.Namespace) -> None: + """Sequentially invoke Ingest, Tiles, Epochs, and Mosaics to populate one demo project end-to-end. + + Dev/demo tooling only (no API route, matching cmd_reset_db/cmd_seed_db + precedent). Requires a real --s3-prefix already populated in storage — + see GitHub issue #34 for the still-outstanding minio-init fiducial seed. + """ + from src.db.enums import JobStatus + from src.db.session import SessionLocal + from src.services import ( + epoch_service, + ingest_service, + mosaic_service, + project_service, + tile_service, + ) + + db = SessionLocal() + try: + project = project_service.create_project( + db, args.project_name, None, args.user_id + ) + print(f"Created project. id={project.id}, slug={project.slug}") + + _job_id, batch_id = ingest_service.create_ingest_batch( + db, project.id, args.s3_prefix + ) + print( + f"Dispatched ingest batch. batch_id={batch_id} - waiting for completion..." + ) + + def _ingest_status(): + batch = ingest_service.get_ingest_batch(db, batch_id) + assert batch is not None, f"IngestBatch {batch_id} disappeared mid-poll" + return batch.status + + status = _poll_until_terminal( + _ingest_status, "IngestBatch", args.ingest_timeout + ) + if status == JobStatus.FAILED: + print("Ingest batch failed. Aborting.") + return + print("Ingest complete.") + + moc_to_tile = _cone_moc(args.ra, args.decl, args.radius_deg) + tessellation = tile_service.generate_tile_tessellation( + args.tile_side_arcmin, moc_to_tile, args.overlap_arcmin + ) + tiles = tile_service.create_tiles(db, project.id, tessellation) + print(f"Created {len(tiles)} tile(s).") + if not tiles: + print("No tiles generated for the given region. Aborting.") + return + best_tile_id = tile_service.tile_with_most_calibrations( + db, [t.id for t in tiles] + ) + if best_tile_id is None: + print( + "No tile in the tessellated region overlaps any ingested " + "calibration. Aborting." + ) + return + tile = next(t for t in tiles if t.id == best_tile_id) + + epoch_dicts = epoch_service.cluster_epochs( + db, project.id, tile.id, args.band_id, args.peak_distance_thresh + ) + epochs = epoch_service.create_epochs(db, project.id, epoch_dicts) + print(f"Created {len(epochs)} epoch(s).") + if not epochs: + print("No epochs generated (no calibrations in range). Aborting.") + return + epoch = epochs[0] + + _job_id, mosaic_id = mosaic_service.create_mosaic( + db, + project.id, + tile.id, + epoch.id, + args.band_id, + args.instrument_id, + filename=f"{project.slug}_mosaic.fits", + target_plate_scale=args.target_plate_scale, + ) + print(f"Dispatched mosaic. mosaic_id={mosaic_id} - waiting for completion...") + + def _mosaic_status(): + mosaic = mosaic_service.get_mosaic(db, mosaic_id) + assert mosaic is not None, f"Level3Mosaic {mosaic_id} disappeared mid-poll" + return mosaic.status + + status = _poll_until_terminal( + _mosaic_status, "Level3Mosaic", args.mosaic_timeout + ) + print(f"Mosaic finished with status={status.value}.") + print(f"Demo project population complete. project_id={project.id}") + finally: + db.close() + + def build_parser() -> argparse.ArgumentParser: """Construct the argparse parser with all diffpype-manage subcommands.""" parser = argparse.ArgumentParser( @@ -138,6 +492,145 @@ def build_parser() -> argparse.ArgumentParser: help="Sleep duration in seconds (1-10, default 5).", ) + create_project = subparsers.add_parser( + "create-project", help="Create a Project with a name-derived, unique slug." + ) + create_project.add_argument("--name", required=True, help="Project name.") + create_project.add_argument( + "--description", default=None, help="Optional project description." + ) + create_project.add_argument( + "--user-id", type=int, required=True, help="Owning User's integer ID." + ) + + ingest = subparsers.add_parser( + "ingest", help="Dispatch an async ingest run over a storage prefix." + ) + ingest.add_argument( + "--project-id", type=int, required=True, help="Owning Project's integer ID." + ) + ingest.add_argument( + "--s3-prefix", required=True, help="Storage prefix to scan for FITS files." + ) + + ingest_status = subparsers.add_parser( + "ingest-status", help="Fetch and display the status of an IngestBatch by ID." + ) + ingest_status.add_argument( + "--id", type=int, required=True, metavar="ID", help="IngestBatch integer ID." + ) + + def _add_tessellation_args(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument( + "--tile-side-arcmin", + type=float, + required=True, + help="Tile side length (arcmin).", + ) + subparser.add_argument( + "--ra", type=float, required=True, help="Target cone center RA (deg)." + ) + subparser.add_argument( + "--decl", type=float, required=True, help="Target cone center Dec (deg)." + ) + subparser.add_argument( + "--radius-deg", type=float, required=True, help="Target cone radius (deg)." + ) + subparser.add_argument( + "--overlap-arcmin", type=float, default=0.0, help="Tile overlap (arcmin)." + ) + + tessellate_tiles = subparsers.add_parser( + "tessellate-tiles", + help="Preview a tile tessellation over a cone region, without writing to the DB.", + ) + _add_tessellation_args(tessellate_tiles) + + create_tiles = subparsers.add_parser( + "create-tiles", + help="Generate and persist a tile tessellation over a cone region.", + ) + create_tiles.add_argument( + "--project-id", type=int, required=True, help="Owning Project's integer ID." + ) + _add_tessellation_args(create_tiles) + + def _add_epoch_cluster_args(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument( + "--project-id", type=int, required=True, help="Owning Project's integer ID." + ) + subparser.add_argument( + "--tile-id", + type=int, + required=True, + help="Tile integer ID to cluster over.", + ) + subparser.add_argument( + "--band-id", + type=int, + required=True, + help="Band integer ID to cluster over.", + ) + subparser.add_argument( + "--peak-distance-thresh", + type=float, + required=True, + help="Minimum MJD spacing between histogram peaks to count as a new epoch.", + ) + + cluster_epochs = subparsers.add_parser( + "cluster-epochs", + help="Preview an MJD-based epoch clustering for a tile+band, without writing to the DB.", + ) + _add_epoch_cluster_args(cluster_epochs) + + create_epochs = subparsers.add_parser( + "create-epochs", help="Cluster a tile+band's MJDs into epochs and persist them." + ) + _add_epoch_cluster_args(create_epochs) + + create_mosaic = subparsers.add_parser( + "create-mosaic", + help="Create a Level3Mosaic job-metadata row and dispatch its drizzle task.", + ) + create_mosaic.add_argument("--project-id", type=int, required=True) + create_mosaic.add_argument("--tile-id", type=int, required=True) + create_mosaic.add_argument("--epoch-id", type=int, required=True) + create_mosaic.add_argument("--band-id", type=int, required=True) + create_mosaic.add_argument("--instrument-id", type=int, required=True) + create_mosaic.add_argument("--filename", required=True) + create_mosaic.add_argument("--target-plate-scale", type=float, required=True) + + mosaic_status = subparsers.add_parser( + "mosaic-status", help="Fetch and display the status of a Level3Mosaic by ID." + ) + mosaic_status.add_argument( + "--id", type=int, required=True, metavar="ID", help="Level3Mosaic integer ID." + ) + + populate_demo = subparsers.add_parser( + "populate-demo-project", + help="Sequentially ingest, tile, cluster, and mosaic one demo project end-to-end.", + ) + populate_demo.add_argument("--project-name", required=True) + populate_demo.add_argument("--user-id", type=int, required=True) + populate_demo.add_argument( + "--s3-prefix", + required=True, + help="Storage prefix already populated with FITS files.", + ) + populate_demo.add_argument("--ra", type=float, required=True) + populate_demo.add_argument("--decl", type=float, required=True) + populate_demo.add_argument("--radius-deg", type=float, required=True) + populate_demo.add_argument("--tile-side-arcmin", type=float, default=6.0) + populate_demo.add_argument("--overlap-arcmin", type=float, default=0.0) + populate_demo.add_argument("--band-id", type=int, required=True) + populate_demo.add_argument("--instrument-id", type=int, required=True) + populate_demo.add_argument("--peak-distance-thresh", type=float, default=5.0) + populate_demo.add_argument("--target-plate-scale", type=float, default=0.03) + populate_demo.add_argument("--ingest-timeout", type=float, default=120.0) + populate_demo.add_argument("--mosaic-timeout", type=float, default=60.0) + return parser @@ -154,6 +647,26 @@ def main(argv: list[str] | None = None) -> None: cmd_reset_db(args) elif args.command == "run-dummy": cmd_run_dummy(args) + elif args.command == "create-project": + cmd_create_project(args) + elif args.command == "ingest": + cmd_ingest(args) + elif args.command == "ingest-status": + cmd_ingest_status(args) + elif args.command == "tessellate-tiles": + cmd_tessellate_tiles(args) + elif args.command == "create-tiles": + cmd_create_tiles(args) + elif args.command == "cluster-epochs": + cmd_cluster_epochs(args) + elif args.command == "create-epochs": + cmd_create_epochs(args) + elif args.command == "create-mosaic": + cmd_create_mosaic(args) + elif args.command == "mosaic-status": + cmd_mosaic_status(args) + elif args.command == "populate-demo-project": + cmd_populate_demo_project(args) if __name__ == "__main__": diff --git a/src/core/config.py b/src/core/config.py index e8948de..2a16b28 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -26,6 +26,9 @@ class Settings(BaseSettings): aws_access_key_id: str = "minioadmin" aws_secret_access_key: str = "minioadmin" s3_bucket_name: str = "diffpype-data" + s3_region: str = "us-east-1" + storage_backend: str = "s3" + local_storage_root: str = "./data" settings = Settings() # type: ignore[call-arg] # database_url/redis_url are populated from the environment at runtime; mypy can't see that. diff --git a/src/db/models.py b/src/db/models.py index acc5707..5d99ee8 100644 --- a/src/db/models.py +++ b/src/db/models.py @@ -66,12 +66,39 @@ class Project(TimestampMixin, Base): id: Mapped[int] = mapped_column(sa.Integer, primary_key=True) name: Mapped[str] = mapped_column(sa.String, nullable=False) + slug: Mapped[str] = mapped_column(sa.String, unique=True, nullable=False) description: Mapped[str | None] = mapped_column(sa.String, nullable=True) user_id: Mapped[int] = mapped_column(sa.ForeignKey("users.id"), nullable=False) user: Mapped["User"] = relationship(back_populates="projects") +class IngestBatch(TimestampMixin, Base): + """Tracks the progress of one asynchronous ingest run over a storage prefix.""" + + __tablename__ = "ingest_batches" + + id: Mapped[int] = mapped_column(sa.Integer, primary_key=True) + project_id: Mapped[int] = mapped_column( + sa.ForeignKey("projects.id"), nullable=False + ) + s3_prefix: Mapped[str] = mapped_column(sa.String, nullable=False) + total_files: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0) + processed_files: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0) + status: Mapped[JobStatus] = mapped_column( + sa.Enum( + JobStatus, + name="job_status", + create_type=False, + values_callable=lambda x: [e.value for e in x], + ), + nullable=False, + default=JobStatus.PENDING, + ) + + project: Mapped["Project"] = relationship() + + class StepDefinition(TimestampMixin, Base): """Maps a pipeline action to its Celery task name and execution queue.""" @@ -210,6 +237,8 @@ class Tile(TimestampMixin, Base): name: Mapped[str] = mapped_column(sa.String, nullable=False) ra: Mapped[float] = mapped_column(sa.Float, nullable=False) decl: Mapped[float] = mapped_column(sa.Float, nullable=False) + # Degrees, not arcmin — tile_service.generate_tile_tessellation converts its + # tile_side_length_arc_min input to degrees (/ 60.0) before this gets set. delta_ra: Mapped[float] = mapped_column(sa.Float, nullable=False) delta_decl: Mapped[float] = mapped_column(sa.Float, nullable=False) footprint: Mapped["MOC | None"] = mapped_column(MOCType, nullable=True) @@ -351,12 +380,15 @@ class Level3Mosaic(TimestampMixin, Base): "footprint", postgresql_using="gist", ), + sa.Index("ix_level3_mosaic_q3c", sa.text("q3c_ang2ipix(ra, decl)")), ) id: Mapped[int] = mapped_column(sa.Integer, primary_key=True) filename: Mapped[str] = mapped_column(sa.String, nullable=False) target_plate_scale: Mapped[float] = mapped_column(sa.Float, nullable=False) footprint: Mapped["MOC | None"] = mapped_column(MOCType, nullable=True) + ra: Mapped[float | None] = mapped_column(sa.Float, nullable=True) + decl: Mapped[float | None] = mapped_column(sa.Float, nullable=True) instrument_id: Mapped[int] = mapped_column( sa.ForeignKey("instruments.id"), nullable=False ) diff --git a/src/db/seed.py b/src/db/seed.py index e8bf378..f9238f4 100644 --- a/src/db/seed.py +++ b/src/db/seed.py @@ -7,9 +7,54 @@ from src.db.session import SessionLocal # Baseline JWST reference data so a fresh sandbox is immediately usable. Central -# wavelengths are the filter pivot wavelengths in microns. +# wavelengths are the filter pivot wavelengths in microns. The complete standard +# NIRCam (wide/medium/narrow) + MIRI imaging filter sets — not a curated subset — +# since a partial list means every not-yet-seen real filter is a fresh ingest +# failure (see the F115W gap found live during doc-28 genTests). _SEED_INSTRUMENTS = ["NIRCam", "MIRI"] -_SEED_BANDS = [("F150W", 1.501), ("F277W", 2.776)] +_SEED_BANDS = [ + # NIRCam wide filters (short + long wavelength channel) + ("F070W", 0.704), + ("F090W", 0.902), + ("F115W", 1.154), + ("F150W", 1.501), + ("F150W2", 1.659), + ("F200W", 1.990), + ("F277W", 2.776), + ("F356W", 3.563), + ("F444W", 4.421), + # NIRCam medium filters + ("F140M", 1.404), + ("F162M", 1.626), + ("F182M", 1.845), + ("F210M", 2.095), + ("F250M", 2.503), + ("F300M", 2.996), + ("F335M", 3.365), + ("F360M", 3.621), + ("F410M", 4.092), + ("F430M", 4.280), + ("F460M", 4.624), + ("F480M", 4.834), + # NIRCam narrow filters + ("F164N", 1.644), + ("F187N", 1.874), + ("F212N", 2.121), + ("F323N", 3.237), + ("F405N", 4.055), + ("F466N", 4.654), + ("F470N", 4.707), + # MIRI imaging filters + ("F560W", 5.6), + ("F770W", 7.7), + ("F1000W", 10.0), + ("F1130W", 11.3), + ("F1280W", 12.8), + ("F1500W", 15.0), + ("F1800W", 18.0), + ("F2100W", 21.0), + ("F2550W", 25.5), +] def _seed_reference_data(db: Session) -> None: diff --git a/src/db/spatial_types.py b/src/db/spatial_types.py index 6d691b9..4830ed8 100644 --- a/src/db/spatial_types.py +++ b/src/db/spatial_types.py @@ -89,3 +89,25 @@ def compare_values(self, x, y) -> bool: if not isinstance(x, MOC) or not isinstance(y, MOC): return x is y return np.array_equal(x.to_depth29_ranges, y.to_depth29_ranges) + + +def moc_to_ranges(moc: MOC | None) -> list[tuple[int, int]] | None: + """Convert a MOC to a plain list of depth-29 ``[lo, hi)`` integer range pairs. + + The wire-safe representation for a footprint at every Pydantic/API/CLI + boundary: exactly the ranges ``MOCType`` binds/reads at the DB boundary, so + there is no separate encode/decode logic and no fidelity loss. + """ + if moc is None: + return None + return [(int(lo), int(hi)) for lo, hi in moc.to_depth29_ranges] + + +def ranges_to_moc(ranges: list[tuple[int, int]] | None) -> MOC | None: + """Convert a plain list of depth-29 ``[lo, hi)`` integer range pairs back to a MOC.""" + if ranges is None: + return None + if not ranges: + return MOC.new_empty(MAX_DEPTH) + array = np.array(ranges, dtype=np.uint64) + return MOC.from_depth29_ranges(MAX_DEPTH, array) diff --git a/src/db/tests/test_integration.py b/src/db/tests/test_integration.py index b084fdd..3f8717b 100644 --- a/src/db/tests/test_integration.py +++ b/src/db/tests/test_integration.py @@ -8,10 +8,13 @@ """ from datetime import datetime, timezone +from unittest.mock import MagicMock import astropy.units as u +import pandas as pd import pytest from mocpy import MOC +from slugify import slugify from sqlalchemy import literal, select, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import sessionmaker @@ -31,6 +34,7 @@ StepDefinition, Tile, User, + epoch_level2_calibration_association, tile_level2_calibration_association, ) @@ -351,7 +355,7 @@ def _make_calibration(db, image, project, plate_scale=0.031): def _make_project(db, user, name="DomainTestProject"): - project = Project(name=name, user_id=user.id) + project = Project(name=name, slug=slugify(name), user_id=user.id) db.add(project) db.flush() return project @@ -388,6 +392,16 @@ def test_band_name_is_unique(db): db.flush() +def test_project_slug_is_unique(db, user): + """A duplicate Project slug is rejected by the database.""" + db.add(Project(name="Same Name", slug="same-name", user_id=user.id)) + db.flush() + with pytest.raises(IntegrityError): + with db.begin_nested(): + db.add(Project(name="Same Name Again", slug="same-name", user_id=user.id)) + db.flush() + + def test_seed_reference_data_is_idempotent(mocker, test_engine): """Calling seed_step_definitions() twice must not raise or duplicate reference rows.""" from src.db.seed import seed_step_definitions @@ -412,9 +426,7 @@ def test_seed_reference_data_is_idempotent(mocker, test_engine): def test_tile_and_epoch_roundtrip_with_project_fk(db, user): """Tile and Epoch persist and their foreign keys resolve back to the parent objects.""" - project = Project(name="DomainTestProject", user_id=user.id) - db.add(project) - db.flush() + project = _make_project(db, user) _instrument, band = _make_ref(db) tile = _make_tile(db, project) epoch = _make_epoch(db, project, tile, band) @@ -611,9 +623,7 @@ def test_footprint_overlap_query_returns_only_spatial_matches(db, user): def test_calibration_associates_with_many_tiles_and_epochs(db, user): """A single Level2Calibration can belong to multiple Tiles and Epochs via the junction tables.""" - project = Project(name="DomainTestProject", user_id=user.id) - db.add(project) - db.flush() + project = _make_project(db, user) instrument, band = _make_ref(db) tile_a = _make_tile(db, project, name="Tile-A") tile_b = _make_tile(db, project, name="Tile-B") @@ -640,9 +650,7 @@ def test_calibration_associates_with_many_tiles_and_epochs(db, user): def test_duplicate_tile_association_is_rejected(db, user): """The association composite primary key rejects a duplicate tile/calibration pairing.""" - project = Project(name="DomainTestProject", user_id=user.id) - db.add(project) - db.flush() + project = _make_project(db, user) instrument, band = _make_ref(db) tile = _make_tile(db, project) cal = _make_calibration(db, _make_image(db, instrument, band), project) @@ -665,9 +673,7 @@ def test_duplicate_tile_association_is_rejected(db, user): def test_level3_mosaic_roundtrip_and_identity_uniqueness(db, user): """Level3Mosaic round-trips, and a second mosaic with the same identity tuple is rejected.""" - project = Project(name="DomainTestProject", user_id=user.id) - db.add(project) - db.flush() + project = _make_project(db, user) instrument, band = _make_ref(db) tile = _make_tile(db, project) epoch = _make_epoch(db, project, tile, band) @@ -705,3 +711,557 @@ def test_level3_mosaic_roundtrip_and_identity_uniqueness(db, user): ) ) db.flush() + + +def test_create_tiles_persists_and_associates_overlapping_calibrations(test_engine): + """create_tiles bulk-inserts Tile rows and links only spatially-overlapping calibrations. + + Runs on its own session for the same reason as the ingest bulk-upsert test: + create_tiles calls db.commit() internally. + """ + from src.services.tile_service import create_tiles + + TestSession = sessionmaker(bind=test_engine) + db = TestSession() + try: + user = User( + username="tileserviceowner", + email="tileserviceowner@diffpype.local", + is_active=True, + hashed_password="dummy_hash_for_testing", + ) + db.add(user) + db.flush() + project = Project( + name="TileServiceProject", slug="tile-service-project", user_id=user.id + ) + db.add(project) + instrument = Instrument(name="NIRCam-tilesvc") + band = Band(name="F150W-tilesvc", central_lambda=1.501) + db.add_all([instrument, band]) + db.flush() + + def _make_cal(base_filename, ra, decl): + image = Level2Image( + base_filename=base_filename, + ra=ra, + decl=decl, + exp_time=1.0, + target_name="TILESVC-TARGET", + obs_start=_utc(2024, 1, 1), + instrument_id=instrument.id, + band_id=band.id, + ) + db.add(image) + db.flush() + cal = Level2Calibration( + level2_image_id=image.id, + project_id=project.id, + current_file_ext=".fits", + plate_scale=0.03, + footprint=MOC.from_cone( + lon=ra * u.deg, lat=decl * u.deg, radius=0.05 * u.deg, max_depth=12 + ), + ) + db.add(cal) + db.flush() + return cal + + near_cal = _make_cal("tile_svc_near.fits", 10.0, 20.0) + _make_cal("tile_svc_far.fits", 200.0, -40.0) # must NOT be associated + db.commit() + + tile_footprint = MOC.from_cone( + lon=10 * u.deg, lat=20 * u.deg, radius=0.1 * u.deg, max_depth=10 + ) + created = create_tiles( + db, + project.id, + [ + { + "name": "AssocTile", + "ra": 10.0, + "decl": 20.0, + "delta_ra": 0.2, + "delta_decl": 0.2, + "footprint": tile_footprint, + } + ], + ) + + assert len(created) == 1 + tile = created[0] + assert tile.id is not None + assoc_ids = { + row.level2_calibration_id + for row in db.execute( + text( + "SELECT level2_calibration_id FROM tile_level2_calibration_association " + "WHERE tile_id = :tid" + ), + {"tid": tile.id}, + ).all() + } + assert assoc_ids == {near_cal.id} + finally: + db.execute( + text( + "DELETE FROM tile_level2_calibration_association " + "WHERE tile_id IN (SELECT id FROM tiles WHERE project_id IN " + "(SELECT id FROM projects WHERE slug = 'tile-service-project'))" + ) + ) + db.query(Tile).filter( + Tile.project_id.in_( + db.query(Project.id).filter_by(slug="tile-service-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Calibration).filter( + Level2Calibration.project_id.in_( + db.query(Project.id).filter_by(slug="tile-service-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Image).filter( + Level2Image.base_filename.in_(["tile_svc_near.fits", "tile_svc_far.fits"]) + ).delete(synchronize_session=False) + db.query(Project).filter_by(slug="tile-service-project").delete() + db.query(Band).filter_by(name="F150W-tilesvc").delete() + db.query(Instrument).filter_by(name="NIRCam-tilesvc").delete() + db.query(User).filter_by(username="tileserviceowner").delete() + db.commit() + db.close() + + +def test_create_epochs_persists_and_associates_calibrations_in_mjd_range(test_engine): + """create_epochs bulk-inserts Epoch rows and links only calibrations whose MJD is in range. + + Runs on its own session for the same reason as the other service-layer + integration tests: create_epochs calls db.commit() internally. + """ + from src.services.epoch_service import create_epochs + + TestSession = sessionmaker(bind=test_engine) + db = TestSession() + try: + user = User( + username="epochserviceowner", + email="epochserviceowner@diffpype.local", + is_active=True, + hashed_password="dummy_hash_for_testing", + ) + db.add(user) + db.flush() + project = Project( + name="EpochServiceProject", slug="epoch-service-project", user_id=user.id + ) + db.add(project) + instrument = Instrument(name="NIRCam-epochsvc") + band = Band(name="F150W-epochsvc", central_lambda=1.501) + db.add_all([instrument, band]) + db.flush() + tile = Tile( + name="EpochSvcTile", + ra=10.0, + decl=20.0, + delta_ra=0.2, + delta_decl=0.2, + coord_sys=2000, + project_id=project.id, + ) + other_tile = Tile( + name="EpochSvcOtherTile", + ra=200.0, + decl=-40.0, + delta_ra=0.2, + delta_decl=0.2, + coord_sys=2000, + project_id=project.id, + ) + db.add_all([tile, other_tile]) + db.flush() + + def _make_cal(base_filename, mjd_avg, associated_tile): + image = Level2Image( + base_filename=base_filename, + ra=10.0, + decl=20.0, + exp_time=1.0, + mjd_avg=mjd_avg, + target_name="EPOCHSVC-TARGET", + obs_start=_utc(2024, 1, 1), + instrument_id=instrument.id, + band_id=band.id, + ) + db.add(image) + db.flush() + cal = Level2Calibration( + level2_image_id=image.id, + project_id=project.id, + current_file_ext=".fits", + plate_scale=0.03, + ) + db.add(cal) + db.flush() + db.execute( + tile_level2_calibration_association.insert().values( + tile_id=associated_tile.id, level2_calibration_id=cal.id + ) + ) + return cal + + in_range_cal = _make_cal("epoch_svc_in_range.fits", 60300.5, tile) + _make_cal( + "epoch_svc_out_of_range.fits", 61000.0, tile + ) # right tile, wrong MJD -> must NOT be associated + _make_cal( + "epoch_svc_wrong_tile.fits", 60300.6, other_tile + ) # right MJD, wrong tile -> must NOT be associated + db.commit() + + created = create_epochs( + db, + project.id, + [ + { + "start_date": _utc(2024, 1, 1), + "end_date": _utc(2024, 1, 2), + "start_mjd": 60300.0, + "end_mjd": 60301.0, + "tile_id": tile.id, + "band_id": band.id, + } + ], + ) + + assert len(created) == 1 + epoch = created[0] + assert epoch.id is not None + assoc_ids = { + row.level2_calibration_id + for row in db.execute( + text( + "SELECT level2_calibration_id FROM epoch_level2_calibration_association " + "WHERE epoch_id = :eid" + ), + {"eid": epoch.id}, + ).all() + } + assert assoc_ids == {in_range_cal.id} + finally: + db.execute( + text( + "DELETE FROM epoch_level2_calibration_association " + "WHERE epoch_id IN (SELECT id FROM epochs WHERE project_id IN " + "(SELECT id FROM projects WHERE slug = 'epoch-service-project'))" + ) + ) + db.execute( + text( + "DELETE FROM tile_level2_calibration_association " + "WHERE tile_id IN (SELECT id FROM tiles WHERE project_id IN " + "(SELECT id FROM projects WHERE slug = 'epoch-service-project'))" + ) + ) + db.query(Epoch).filter( + Epoch.project_id.in_( + db.query(Project.id).filter_by(slug="epoch-service-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Calibration).filter( + Level2Calibration.project_id.in_( + db.query(Project.id).filter_by(slug="epoch-service-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Image).filter( + Level2Image.base_filename.in_( + [ + "epoch_svc_in_range.fits", + "epoch_svc_out_of_range.fits", + "epoch_svc_wrong_tile.fits", + ] + ) + ).delete(synchronize_session=False) + db.query(Tile).filter( + Tile.name.in_(["EpochSvcTile", "EpochSvcOtherTile"]) + ).delete(synchronize_session=False) + db.query(Project).filter_by(slug="epoch-service-project").delete() + db.query(Band).filter_by(name="F150W-epochsvc").delete() + db.query(Instrument).filter_by(name="NIRCam-epochsvc").delete() + db.query(User).filter_by(username="epochserviceowner").delete() + db.commit() + db.close() + + +def test_create_mosaic_computes_footprint_and_barycenter_from_constituent_calibrations( + test_engine, mocker +): + """create_mosaic unions footprints of calibrations linked via BOTH tile and epoch M2M tables. + + Runs on its own session for the same reason as the other service-layer + integration tests: create_mosaic calls db.commit() internally. Mocks the + Celery dispatch: worker_heavy is a real running container consuming this + queue, so an un-mocked .delay() risks a race where it flips the mosaic's + status before this test's assertions read it — the DB footprint/barycenter + computation is what this test verifies, not the dispatch itself (already + covered by unit tests). + """ + from src.services.mosaic_service import create_mosaic + + mocker.patch( + "src.worker.tasks.run_mosaic_drizzle.delay", + return_value=MagicMock(id="fake-mosaic-task-id"), + ) + + TestSession = sessionmaker(bind=test_engine) + db = TestSession() + try: + user = User( + username="mosaicserviceowner", + email="mosaicserviceowner@diffpype.local", + is_active=True, + hashed_password="dummy_hash_for_testing", + ) + db.add(user) + db.flush() + project = Project( + name="MosaicServiceProject", slug="mosaic-service-project", user_id=user.id + ) + db.add(project) + instrument = Instrument(name="NIRCam-mosaicsvc") + band = Band(name="F150W-mosaicsvc", central_lambda=1.501) + other_band = Band(name="F277W-mosaicsvc", central_lambda=2.776) + db.add_all([instrument, band, other_band]) + db.flush() + tile = Tile( + name="MosaicSvcTile", + ra=10.0, + decl=20.0, + delta_ra=0.5, + delta_decl=0.5, + coord_sys=2000, + project_id=project.id, + ) + db.add(tile) + db.flush() + epoch = Epoch( + start_date=_utc(2024, 1, 1), + end_date=_utc(2024, 1, 2), + start_mjd=60300.0, + end_mjd=60301.0, + project_id=project.id, + tile_id=tile.id, + band_id=band.id, + ) + db.add(epoch) + db.flush() + + def _make_associated_cal(base_filename, ra, band_row): + image = Level2Image( + base_filename=base_filename, + ra=ra, + decl=20.0, + exp_time=1.0, + mjd_avg=60300.5, + target_name="MOSAICSVC-TARGET", + obs_start=_utc(2024, 1, 1), + instrument_id=instrument.id, + band_id=band_row.id, + ) + db.add(image) + db.flush() + cal = Level2Calibration( + level2_image_id=image.id, + project_id=project.id, + current_file_ext=".fits", + plate_scale=0.03, + footprint=MOC.from_cone( + lon=ra * u.deg, lat=20.0 * u.deg, radius=0.05 * u.deg, max_depth=12 + ), + ) + db.add(cal) + db.flush() + db.execute( + tile_level2_calibration_association.insert().values( + tile_id=tile.id, level2_calibration_id=cal.id + ) + ) + db.execute( + epoch_level2_calibration_association.insert().values( + epoch_id=epoch.id, level2_calibration_id=cal.id + ) + ) + return cal + + _make_associated_cal("mosaic_svc_a.fits", 10.0, band) + _make_associated_cal("mosaic_svc_b.fits", 10.05, band) + # Right tile+epoch, wrong band -> must NOT contribute to the union. + _make_associated_cal("mosaic_svc_wrong_band.fits", 50.0, other_band) + db.commit() + + job_id, mosaic_id = create_mosaic( + db, + project.id, + tile.id, + epoch.id, + band.id, + instrument.id, + filename="mosaic_svc.fits", + target_plate_scale=0.03, + ) + + assert job_id + mosaic = db.get(Level3Mosaic, mosaic_id) + assert mosaic.footprint is not None + assert mosaic.ra == pytest.approx(10.025, abs=0.1) + assert mosaic.decl == pytest.approx(20.0, abs=0.1) + assert mosaic.status == JobStatus.PENDING + finally: + db.execute( + text( + "DELETE FROM epoch_level2_calibration_association " + "WHERE epoch_id IN (SELECT id FROM epochs WHERE project_id IN " + "(SELECT id FROM projects WHERE slug = 'mosaic-service-project'))" + ) + ) + db.execute( + text( + "DELETE FROM tile_level2_calibration_association " + "WHERE tile_id IN (SELECT id FROM tiles WHERE project_id IN " + "(SELECT id FROM projects WHERE slug = 'mosaic-service-project'))" + ) + ) + db.query(Level3Mosaic).filter( + Level3Mosaic.project_id.in_( + db.query(Project.id).filter_by(slug="mosaic-service-project") + ) + ).delete(synchronize_session=False) + db.query(Epoch).filter( + Epoch.project_id.in_( + db.query(Project.id).filter_by(slug="mosaic-service-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Calibration).filter( + Level2Calibration.project_id.in_( + db.query(Project.id).filter_by(slug="mosaic-service-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Image).filter( + Level2Image.base_filename.in_( + [ + "mosaic_svc_a.fits", + "mosaic_svc_b.fits", + "mosaic_svc_wrong_band.fits", + ] + ) + ).delete(synchronize_session=False) + db.query(Tile).filter_by(name="MosaicSvcTile").delete() + db.query(Project).filter_by(slug="mosaic-service-project").delete() + db.query(Band).filter( + Band.name.in_(["F150W-mosaicsvc", "F277W-mosaicsvc"]) + ).delete(synchronize_session=False) + db.query(Instrument).filter_by(name="NIRCam-mosaicsvc").delete() + db.query(User).filter_by(username="mosaicserviceowner").delete() + db.commit() + db.close() + + +def test_bulk_upsert_images_and_calibrations_is_idempotent(test_engine): + """A real Core bulk upsert: first call inserts, an identical second call is a safe no-op. + + Runs on its own session (not the transactional `db` fixture) because + bulk_upsert_images_and_calibrations calls db.commit() internally, which + would otherwise end the fixture's outer transaction early. Committed rows + are explicitly deleted at the end per the integration-test isolation rule. + """ + from src.services.ingest_service import bulk_upsert_images_and_calibrations + + TestSession = sessionmaker(bind=test_engine) + db = TestSession() + try: + user = User( + username="bulkupsertowner", + email="bulkupsertowner@diffpype.local", + is_active=True, + hashed_password="dummy_hash_for_testing", + ) + db.add(user) + db.flush() + project = Project( + name="BulkUpsertProject", slug="bulk-upsert-project", user_id=user.id + ) + db.add(project) + instrument = Instrument(name="NIRCam-bulk") + band = Band(name="F150W-bulk", central_lambda=1.501) + db.add_all([instrument, band]) + db.flush() + db.commit() + + moc = MOC.from_cone( + lon=10 * u.deg, lat=20 * u.deg, radius=0.01 * u.deg, max_depth=10 + ) + df = pd.DataFrame( + [ + { + "base_filename": "bulk_upsert_test_001.fits", + "current_file_ext": ".fits", + "ra": 10.0, + "decl": 20.0, + "exp_time": 100.0, + "mjd_avg": 60300.0, + "target_name": "BULKTEST", + "obs_start": _utc(2024, 1, 1), + "instrument_name": "NIRCam-bulk", + "band_name": "F150W-bulk", + "plate_scale": 0.03, + "footprint": moc, + } + ] + ) + + count_first = bulk_upsert_images_and_calibrations(db, project.id, df) + assert count_first == 1 + + image = ( + db.query(Level2Image) + .filter_by(base_filename="bulk_upsert_test_001.fits") + .one() + ) + cal = ( + db.query(Level2Calibration) + .filter_by(level2_image_id=image.id, project_id=project.id) + .one() + ) + assert cal.plate_scale == 0.03 + assert cal.status == JobStatus.COMPLETE + + count_second = bulk_upsert_images_and_calibrations(db, project.id, df) + assert ( + count_second == 1 + ) # still "processed" 1 df row, but persisted no duplicate + assert ( + db.query(Level2Image) + .filter_by(base_filename="bulk_upsert_test_001.fits") + .count() + == 1 + ) + assert ( + db.query(Level2Calibration) + .filter_by(level2_image_id=image.id, project_id=project.id) + .count() + == 1 + ) + finally: + db.query(Level2Calibration).filter( + Level2Calibration.project_id.in_( + db.query(Project.id).filter_by(slug="bulk-upsert-project") + ) + ).delete(synchronize_session=False) + db.query(Level2Image).filter_by( + base_filename="bulk_upsert_test_001.fits" + ).delete() + db.query(Project).filter_by(slug="bulk-upsert-project").delete() + db.query(Band).filter_by(name="F150W-bulk").delete() + db.query(Instrument).filter_by(name="NIRCam-bulk").delete() + db.query(User).filter_by(username="bulkupsertowner").delete() + db.commit() + db.close() diff --git a/src/db/tests/test_spatial_types.py b/src/db/tests/test_spatial_types.py new file mode 100644 index 0000000..9dc4d5d --- /dev/null +++ b/src/db/tests/test_spatial_types.py @@ -0,0 +1,30 @@ +import astropy.units as u +from mocpy import MOC + +from src.db.spatial_types import moc_to_ranges, ranges_to_moc + + +def test_moc_to_ranges_returns_none_for_none(): + assert moc_to_ranges(None) is None + + +def test_ranges_to_moc_returns_none_for_none(): + assert ranges_to_moc(None) is None + + +def test_ranges_to_moc_returns_empty_moc_for_empty_list(): + moc = ranges_to_moc([]) + assert moc.empty() + + +def test_moc_to_ranges_round_trips_through_ranges_to_moc(): + original = MOC.from_cone( + lon=10 * u.deg, lat=20 * u.deg, radius=0.5 * u.deg, max_depth=10 + ) + + ranges = moc_to_ranges(original) + restored = ranges_to_moc(ranges) + + assert isinstance(ranges, list) + assert all(isinstance(r, tuple) and len(r) == 2 for r in ranges) + assert (restored.to_depth29_ranges == original.to_depth29_ranges).all() diff --git a/src/services/epoch_service.py b/src/services/epoch_service.py new file mode 100644 index 0000000..bb15cf7 --- /dev/null +++ b/src/services/epoch_service.py @@ -0,0 +1,169 @@ +"""Business logic for temporal (MJD) clustering, persistence, and association. + +Isolates the clustering math (``cluster_epochs``) from persistence +(``create_epochs``, the one synchronous write path), matching tile_service's +compute/persist split. +""" + +from collections.abc import Sequence + +import numpy as np +import pandas as pd +import sqlalchemy as sa +from astropy.time import Time +from scipy.signal import find_peaks +from sklearn.cluster import KMeans +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from src.db.models import ( + Epoch, + Level2Calibration, + Level2Image, + epoch_level2_calibration_association, + tile_level2_calibration_association, +) + + +def _fetch_calibration_mjds( + db: Session, project_id: int, tile_id: int, band_id: int +) -> pd.DataFrame: + """Query the MJDs of Level2Calibration rows tied to a tile (spatially) and a band.""" + stmt = ( + sa.select(Level2Calibration.id, Level2Image.mjd_avg) + .join(Level2Image, Level2Calibration.level2_image_id == Level2Image.id) + .join( + tile_level2_calibration_association, + tile_level2_calibration_association.c.level2_calibration_id + == Level2Calibration.id, + ) + .where( + Level2Calibration.project_id == project_id, + tile_level2_calibration_association.c.tile_id == tile_id, + Level2Image.band_id == band_id, + Level2Image.mjd_avg.isnot(None), + ) + ) + rows = db.execute(stmt).all() + return pd.DataFrame(rows, columns=["level2_calibration_id", "mjd_avg"]) + + +def _cluster_mjds(mjd_list: list[float], peak_distance_thresh: float) -> np.ndarray: + """Cluster MJDs into [start, end] epoch intervals via peak-finding + KMeans. + + Pure computation, ports the prototype's ``CreateEpochsFromMJDs``. Guards + against zero detected peaks (which the prototype did not: an all-flat MJD + histogram would otherwise call ``KMeans(n_clusters=0)`` and crash) by + falling back to a single cluster. + """ + mjd = np.array(mjd_list) + mjd_reshaped = mjd.reshape(-1, 1) + + nbins = (int(np.ceil(mjd.max())) - int(np.floor(mjd.min()))) + 2 + hist, _ = np.histogram(mjd, bins=nbins, range=(mjd.min() - 1, mjd.max() + 1)) + peaks, _ = find_peaks(hist, height=None, distance=peak_distance_thresh) + num_clusters = max(len(peaks), 1) + + kmeans = KMeans(n_clusters=num_clusters, random_state=0) + kmeans.fit(mjd_reshaped) + + cluster_centers = np.sort(kmeans.cluster_centers_.flatten()) + return np.asarray( + [ + [ + int(np.floor(center - peak_distance_thresh)), + int(np.ceil(center + peak_distance_thresh)), + ] + for center in cluster_centers + ] + ) + + +def cluster_epochs( + db: Session, + project_id: int, + tile_id: int, + band_id: int, + peak_distance_thresh: float, +) -> list[dict]: + """Cluster a tile+band's calibration MJDs into epoch dicts. Queries MJDs, writes nothing. + + Returns plain dicts (start_date, end_date, start_mjd, end_mjd, tile_id, + band_id) — the API/CLI boundary wraps each into an ``EpochCreate``. + """ + df = _fetch_calibration_mjds(db, project_id, tile_id, band_id) + if df.empty: + return [] + + intervals = _cluster_mjds(df["mjd_avg"].tolist(), peak_distance_thresh) + + return [ + { + "start_mjd": float(lo), + "end_mjd": float(hi), + "start_date": Time(float(lo), format="mjd").datetime, + "end_date": Time(float(hi), format="mjd").datetime, + "tile_id": tile_id, + "band_id": band_id, + } + for lo, hi in intervals + ] + + +def create_epochs(db: Session, project_id: int, epochs: list[dict]) -> list[Epoch]: + """Bulk-insert Epoch rows and associate each with calibrations whose MJD falls in its range.""" + if not epochs: + return [] + + rows = [{**epoch, "project_id": project_id} for epoch in epochs] + created = db.execute(sa.insert(Epoch).returning(Epoch), rows).scalars().all() + db.flush() + + _associate_epochs_with_calibrations_in_range(db, project_id, created) + db.commit() + return list(created) + + +def _associate_epochs_with_calibrations_in_range( + db: Session, project_id: int, epochs: Sequence[Epoch] +) -> None: + """Populate epoch_level2_calibration_association for each epoch's tile+band+MJD range. + + Scoped to the epoch's own tile (via the same spatial association tile_service + already populated) and band, not just project+MJD — an Epoch is defined as a + temporal grouping for one specific Tile and Band (see the Epoch model's + docstring), so an unscoped MJD-only match would wrongly link calibrations + from an unrelated tile or band whose MJD happens to overlap. + """ + association_rows: list[dict] = [] + for epoch in epochs: + stmt = ( + sa.select(Level2Calibration.id) + .join(Level2Image, Level2Calibration.level2_image_id == Level2Image.id) + .join( + tile_level2_calibration_association, + tile_level2_calibration_association.c.level2_calibration_id + == Level2Calibration.id, + ) + .where( + Level2Calibration.project_id == project_id, + tile_level2_calibration_association.c.tile_id == epoch.tile_id, + Level2Image.band_id == epoch.band_id, + Level2Image.mjd_avg.isnot(None), + Level2Image.mjd_avg >= epoch.start_mjd, + Level2Image.mjd_avg <= epoch.end_mjd, + ) + ) + association_rows.extend( + {"epoch_id": epoch.id, "level2_calibration_id": cal_id} + for (cal_id,) in db.execute(stmt).all() + ) + + if association_rows: + db.execute( + pg_insert(epoch_level2_calibration_association) + .values(association_rows) + .on_conflict_do_nothing( + index_elements=["epoch_id", "level2_calibration_id"] + ) + ) diff --git a/src/services/ingest_service.py b/src/services/ingest_service.py new file mode 100644 index 0000000..1f93b64 --- /dev/null +++ b/src/services/ingest_service.py @@ -0,0 +1,194 @@ +"""Business logic for scanning a storage prefix and ingesting FITS headers. + +Pandas stays entirely inside this module (``parse_fits_headers`` and its +callers here) — it never crosses the API/CLI/Celery boundary, matching the +project's object-model convention: Pydantic at every boundary, pandas only +as an internal computation detail. +""" + +import os + +import numpy as np +import pandas as pd +import sqlalchemy as sa +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.time import Time +from astropy.wcs import WCS +from astropy.wcs.utils import proj_plane_pixel_scales +from mocpy import MOC +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from src.core.logger import get_logger +from src.db.enums import JobStatus +from src.db.models import Band, IngestBatch, Instrument, Level2Calibration, Level2Image + +# HEALPix order used for per-image footprints computed from a WCS polygon — matches +# the prototype's ported convention for detector-scale (not all-sky-tile-scale) +# precision. spatial_types.MOCType normalizes any input depth to depth 29 on +# persistence, so this only affects computation precision, not storage. +_FOOTPRINT_MAX_DEPTH = 21 + + +def parse_fits_headers(local_paths: list[str]) -> pd.DataFrame: + """Extract Level2Image/Level2Calibration fields from local FITS files into a DataFrame. + + One row per file. ``instrument_name``/``band_name`` are the raw header strings + (INSTRUME/FILTER); resolving them to Instrument/Band ids is a separate, + DB-touching step so this function stays a pure, file-in/DataFrame-out unit. + """ + rows = [] + for path in local_paths: + h_sci = fits.getheader(path, "SCI") + h_pri = fits.getheader(path, "PRIMARY") + wcs = WCS(h_sci) + + footprint_deg = wcs.calc_footprint() + sky_poly = SkyCoord(footprint_deg, unit="deg", frame="icrs") + moc = MOC.from_polygon_skycoord( + sky_poly, complement=False, max_depth=_FOOTPRINT_MAX_DEPTH + ) + plate_scale = float(np.mean(proj_plane_pixel_scales(wcs)) * 3600.0) + + mjd_avg = float(h_sci["MJD-AVG"]) + base_filename = os.path.basename(path) + + rows.append( + { + "base_filename": base_filename, + "current_file_ext": os.path.splitext(base_filename)[-1], + "ra": float(h_sci["CRVAL1"]), + "decl": float(h_sci["CRVAL2"]), + "exp_time": float(h_sci["XPOSURE"]), + "mjd_avg": mjd_avg, + "target_name": h_pri["TARGPROP"], + "obs_start": Time(mjd_avg, format="mjd").datetime, + "instrument_name": h_pri["INSTRUME"], + "band_name": h_pri["FILTER"], + "plate_scale": plate_scale, + "footprint": moc, + } + ) + return pd.DataFrame(rows) + + +def _resolve_reference_ids(db: Session, df: pd.DataFrame) -> pd.DataFrame: + """Replace instrument_name/band_name columns with instrument_id/band_id via DB lookup. + + Matched case-insensitively: real FITS headers conventionally write + INSTRUME/FILTER in uppercase (e.g. "NIRCAM"), which doesn't necessarily + match this project's reference data naming convention (e.g. "NIRCam"). + """ + instrument_ids = {i.name.upper(): i.id for i in db.query(Instrument).all()} + band_ids = {b.name.upper(): b.id for b in db.query(Band).all()} + + df = df.copy() + df["instrument_id"] = df["instrument_name"].str.upper().map(instrument_ids) + df["band_id"] = df["band_name"].str.upper().map(band_ids) + + missing = df[df["instrument_id"].isna() | df["band_id"].isna()] + if not missing.empty: + raise ValueError( + "Unknown instrument/band for files: " + f"{missing['base_filename'].tolist()}" + ) + + df["instrument_id"] = df["instrument_id"].astype(int) + df["band_id"] = df["band_id"].astype(int) + return df.drop(columns=["instrument_name", "band_name"]) + + +def bulk_upsert_images_and_calibrations( + db: Session, project_id: int, df: pd.DataFrame +) -> int: + """Bulk upsert Level2Image + project-scoped Level2Calibration rows from a parsed DataFrame. + + Idempotent: re-ingesting an already-ingested base_filename/project pair is a + safe no-op via ON CONFLICT DO NOTHING on both tables' unique constraints. + Returns the number of rows processed from ``df``. + """ + if df.empty: + return 0 + + df = _resolve_reference_ids(db, df) + + image_columns = [ + "base_filename", + "ra", + "decl", + "exp_time", + "mjd_avg", + "target_name", + "obs_start", + "instrument_id", + "band_id", + ] + image_rows = df[image_columns].to_dict("records") + db.execute( + pg_insert(Level2Image) + .values(image_rows) + .on_conflict_do_nothing(index_elements=["base_filename"]) + ) + db.flush() + + base_filenames = df["base_filename"].tolist() + image_ids = { + row.base_filename: row.id + for row in db.execute( + sa.select(Level2Image.id, Level2Image.base_filename).where( + Level2Image.base_filename.in_(base_filenames) + ) + ).all() + } + + calibration_rows = [ + { + "level2_image_id": image_ids[record["base_filename"]], + "project_id": project_id, + "footprint": record["footprint"], + "current_file_ext": record["current_file_ext"], + "plate_scale": record["plate_scale"], + "status": JobStatus.COMPLETE, + } + for record in df.to_dict("records") + ] + db.execute( + pg_insert(Level2Calibration) + .values(calibration_rows) + .on_conflict_do_nothing( + index_elements=["level2_image_id", "project_id"] + ) + ) + db.commit() + return len(df) + + +def create_ingest_batch(db: Session, project_id: int, s3_prefix: str) -> tuple[str, int]: + """Persist a PENDING IngestBatch, dispatch the ingest Celery task, return (job_id, batch_id).""" + from src.worker.tasks import run_ingest_batch # lazy: avoids a circular import + + batch = IngestBatch( + project_id=project_id, s3_prefix=s3_prefix, status=JobStatus.PENDING + ) + db.add(batch) + db.commit() + db.refresh(batch) + + try: + async_result = run_ingest_batch.delay(batch.id) + except Exception: + get_logger().error("ingest_batch_dispatch_failed", batch_id=batch.id, exc_info=True) + batch.status = JobStatus.FAILED + db.commit() + raise + + get_logger().info( + "ingest_batch_dispatched", batch_id=batch.id, job_id=async_result.id + ) + return async_result.id, batch.id + + +def get_ingest_batch(db: Session, batch_id: int) -> IngestBatch | None: + """Return the IngestBatch with the given primary key, or None if not found.""" + return db.get(IngestBatch, batch_id) diff --git a/src/services/mosaic_service.py b/src/services/mosaic_service.py new file mode 100644 index 0000000..bdd4476 --- /dev/null +++ b/src/services/mosaic_service.py @@ -0,0 +1,129 @@ +"""Business logic for mosaic job-metadata creation: footprint union + barycenter. + +A Level3Mosaic row is real job metadata, not a placeholder: once created with +its real (tile, epoch, band, instrument, project) combination, it *is* the +complete job specification a future drizzle pipeline queries via the M2M +association tables to find its constituent Level2Calibrations. The pixel-level +drizzle itself is deferred to a future JWST-pipeline doc. +""" + +import sqlalchemy as sa +from mocpy import MOC +from sqlalchemy.orm import Session + +from src.core.logger import get_logger +from src.db.enums import JobStatus +from src.db.models import ( + Level2Calibration, + Level2Image, + Level3Mosaic, + epoch_level2_calibration_association, + tile_level2_calibration_association, +) + + +def _unioned_footprint_and_barycenter(mocs: list[MOC]) -> tuple[MOC, float, float]: + """Union a list of MOCs and return (union, ra_barycenter_deg, decl_barycenter_deg). + + Ports the prototype's ``Get_Unioned_MOC``, plus barycenter extraction — + addresses GitHub issue #27 in the same write path that already has to + compute the union. + """ + union = mocs[0] + for moc in mocs[1:]: + union = union.union(moc) + barycenter = union.barycenter() + return union, float(barycenter.ra.degree), float(barycenter.dec.degree) + + +def _fetch_constituent_footprints( + db: Session, project_id: int, tile_id: int, epoch_id: int, band_id: int +) -> list[MOC]: + """Return footprints of Level2Calibrations linked to both this tile and this epoch, for this band. + + The M2M association tables (populated by tile_service/epoch_service) are + the source of truth for "which calibrations make up this mosaic" — this + mirrors exactly the query a future orchestrator would run to construct the + real drizzle job's input file list. + """ + stmt = ( + sa.select(Level2Calibration.footprint) + .join(Level2Image, Level2Calibration.level2_image_id == Level2Image.id) + .join( + tile_level2_calibration_association, + tile_level2_calibration_association.c.level2_calibration_id + == Level2Calibration.id, + ) + .join( + epoch_level2_calibration_association, + epoch_level2_calibration_association.c.level2_calibration_id + == Level2Calibration.id, + ) + .where( + Level2Calibration.project_id == project_id, + tile_level2_calibration_association.c.tile_id == tile_id, + epoch_level2_calibration_association.c.epoch_id == epoch_id, + Level2Image.band_id == band_id, + Level2Calibration.footprint.isnot(None), + ) + ) + return [row[0] for row in db.execute(stmt).all()] + + +def create_mosaic( + db: Session, + project_id: int, + tile_id: int, + epoch_id: int, + band_id: int, + instrument_id: int, + filename: str, + target_plate_scale: float, +) -> tuple[str, int]: + """Create a Level3Mosaic with its computed footprint/barycenter, dispatch the drizzle task. + + Returns (job_id, mosaic_id). + """ + from src.worker.tasks import run_mosaic_drizzle # lazy: avoids a circular import + + footprints = _fetch_constituent_footprints( + db, project_id, tile_id, epoch_id, band_id + ) + footprint = ra = decl = None + if footprints: + footprint, ra, decl = _unioned_footprint_and_barycenter(footprints) + + mosaic = Level3Mosaic( + filename=filename, + target_plate_scale=target_plate_scale, + footprint=footprint, + ra=ra, + decl=decl, + instrument_id=instrument_id, + band_id=band_id, + epoch_id=epoch_id, + tile_id=tile_id, + project_id=project_id, + status=JobStatus.PENDING, + ) + db.add(mosaic) + db.commit() + db.refresh(mosaic) + + try: + async_result = run_mosaic_drizzle.delay(mosaic.id) + except Exception: + get_logger().error("mosaic_dispatch_failed", mosaic_id=mosaic.id, exc_info=True) + mosaic.status = JobStatus.FAILED + db.commit() + raise + + get_logger().info( + "mosaic_dispatched", mosaic_id=mosaic.id, job_id=async_result.id + ) + return async_result.id, mosaic.id + + +def get_mosaic(db: Session, mosaic_id: int) -> Level3Mosaic | None: + """Return the Level3Mosaic with the given primary key, or None if not found.""" + return db.get(Level3Mosaic, mosaic_id) diff --git a/src/services/project_service.py b/src/services/project_service.py new file mode 100644 index 0000000..a88eb76 --- /dev/null +++ b/src/services/project_service.py @@ -0,0 +1,23 @@ +"""Business logic for creating Projects, shared by the API and CLI boundaries.""" + +from slugify import slugify +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from src.db.models import Project + + +def create_project( + db: Session, name: str, description: str | None, user_id: int +) -> Project: + """Create a Project with a name-derived slug, raising ValueError if the slug already exists.""" + slug = slugify(name) + project = Project(name=name, slug=slug, description=description, user_id=user_id) + db.add(project) + try: + db.commit() + except IntegrityError: + db.rollback() + raise ValueError(f"A project with slug '{slug}' already exists") from None + db.refresh(project) + return project diff --git a/src/services/storage_service.py b/src/services/storage_service.py index dc2917f..7ead455 100644 --- a/src/services/storage_service.py +++ b/src/services/storage_service.py @@ -1,28 +1,114 @@ -"""S3-compatible object storage access for FITS payloads, shared by API and CLI.""" +"""Object storage access for FITS payloads, shared by API and CLI. + +Two interchangeable backends implement the same `StorageBackend` protocol: +`S3StorageService` (S3-compatible, e.g. MinIO) and `LocalStorageService` +(host filesystem, for local dev without Docker/MinIO). `get_storage_service` +selects between them via the `STORAGE_BACKEND` setting. +""" + +import shutil +from pathlib import Path +from typing import Protocol, runtime_checkable import boto3 from src.core.config import Settings, settings +@runtime_checkable +class StorageBackend(Protocol): + """Common interface for uploading and downloading files by storage key.""" + + def upload_file(self, local_path: str, key: str) -> None: + """Upload a local file to storage under the given key.""" + ... + + def download_file(self, key: str, local_path: str) -> None: + """Download the object at the given key from storage to a local path.""" + ... + + def list_prefix(self, prefix: str) -> list[str]: + """Return all object keys under the given prefix.""" + ... + + class S3StorageService: """Thin boto3 wrapper for uploading and downloading files to an S3-compatible bucket.""" def __init__(self, config: Settings = settings) -> None: - """Build an S3 client and bind the target bucket from application settings.""" + """Build an S3 client and bind the target bucket from application settings. + + Credentials and endpoint are only passed to boto3 when actually + configured (non-empty) — leaving them unset lets boto3's default + credential chain resolve real IAM role credentials (EC2/ECS/IRSA) and + AWS's real regional endpoints in a cloud deployment, instead of + forcing MinIO's dev-only static-credential shape everywhere. Region is + configurable rather than hardcoded, since real deployments aren't + always us-east-1. + """ self._bucket = config.s3_bucket_name - self._client = boto3.client( - "s3", - endpoint_url=config.s3_endpoint_url, - aws_access_key_id=config.aws_access_key_id, - aws_secret_access_key=config.aws_secret_access_key, - region_name="us-east-1", + client_kwargs: dict = {"region_name": config.s3_region} + if config.s3_endpoint_url: + client_kwargs["endpoint_url"] = config.s3_endpoint_url + if config.aws_access_key_id and config.aws_secret_access_key: + client_kwargs["aws_access_key_id"] = config.aws_access_key_id + client_kwargs["aws_secret_access_key"] = config.aws_secret_access_key + self._client = boto3.client("s3", **client_kwargs) + + def upload_file(self, local_path: str, key: str) -> None: + """Upload a local file to the configured bucket under the given key.""" + self._client.upload_file(local_path, self._bucket, key) + + def download_file(self, key: str, local_path: str) -> None: + """Download the object at the given key from the configured bucket to a local path.""" + self._client.download_file(self._bucket, key, local_path) + + def list_prefix(self, prefix: str) -> list[str]: + """Return all object keys under the given prefix in the configured bucket.""" + paginator = self._client.get_paginator("list_objects_v2") + keys: list[str] = [] + for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix): + keys.extend(obj["Key"] for obj in page.get("Contents", [])) + return keys + + +class LocalStorageService: + """Filesystem-backed storage service rooted at a configured local directory. + + Auto-creates its root directory if missing (a different risk profile than + the S3 bucket, which is deliberately never auto-created by this codebase). + """ + + def __init__(self, config: Settings = settings) -> None: + """Bind the storage root from application settings and create it if missing.""" + self._root = Path(config.local_storage_root) + self._root.mkdir(parents=True, exist_ok=True) + + def upload_file(self, local_path: str, key: str) -> None: + """Copy a local file into the storage root under the given key.""" + destination = self._root / key + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(local_path, destination) + + def download_file(self, key: str, local_path: str) -> None: + """Copy the file at the given key from the storage root to a local path.""" + source = self._root / key + if not source.is_file(): + raise FileNotFoundError(f"No object at key '{key}' under {self._root}") + shutil.copyfile(source, local_path) + + def list_prefix(self, prefix: str) -> list[str]: + """Return all relative keys under the given prefix in the storage root.""" + base = self._root / prefix + if not base.is_dir(): + return [] + return sorted( + str(p.relative_to(self._root)) for p in base.rglob("*") if p.is_file() ) - def upload_file(self, local_path: str, s3_key: str) -> None: - """Upload a local file to the configured bucket under the given S3 key.""" - self._client.upload_file(local_path, self._bucket, s3_key) - def download_file(self, s3_key: str, local_path: str) -> None: - """Download the object at the given S3 key from the configured bucket to a local path.""" - self._client.download_file(self._bucket, s3_key, local_path) +def get_storage_service(config: Settings = settings) -> StorageBackend: + """Return the configured storage backend: `S3StorageService` or `LocalStorageService`.""" + if config.storage_backend == "local": + return LocalStorageService(config) + return S3StorageService(config) diff --git a/src/services/tests/test_epoch_service.py b/src/services/tests/test_epoch_service.py new file mode 100644 index 0000000..28aba20 --- /dev/null +++ b/src/services/tests/test_epoch_service.py @@ -0,0 +1,67 @@ +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from src.services.epoch_service import _cluster_mjds, cluster_epochs, create_epochs + + +def test_cluster_mjds_groups_two_tight_clusters_into_two_epochs(): + """Real, in-memory clustering math: two tight MJD groups separated by a gap.""" + cluster_a = [60300.0, 60300.1, 60300.2, 60300.3] + cluster_b = [60350.0, 60350.1, 60350.2, 60350.3] + + intervals = _cluster_mjds(cluster_a + cluster_b, peak_distance_thresh=5.0) + + assert len(intervals) == 2 + (lo1, hi1), (lo2, hi2) = sorted(intervals.tolist()) + assert lo1 <= min(cluster_a) and hi1 >= max(cluster_a) + assert lo2 <= min(cluster_b) and hi2 >= max(cluster_b) + + +def test_cluster_mjds_falls_back_to_one_cluster_when_no_peaks_detected(): + """A single tight group with a huge distance threshold must not crash KMeans(n_clusters=0).""" + intervals = _cluster_mjds([60300.0, 60300.1, 60300.2], peak_distance_thresh=1000.0) + + assert len(intervals) == 1 + + +def test_cluster_epochs_returns_empty_list_when_no_calibrations(mocker): + mock_db = MagicMock() + mock_db.execute.return_value.all.return_value = [] + + result = cluster_epochs( + mock_db, project_id=1, tile_id=1, band_id=1, peak_distance_thresh=5.0 + ) + + assert result == [] + + +def test_cluster_epochs_builds_epoch_dicts_from_clustered_mjds(mocker): + mock_db = MagicMock() + mock_db.execute.return_value.all.return_value = [ + (1, 60300.0), + (2, 60300.1), + ] + mocker.patch( + "src.services.epoch_service._cluster_mjds", + return_value=np.array([[60299, 60302]]), + ) + + result = cluster_epochs( + mock_db, project_id=1, tile_id=7, band_id=2, peak_distance_thresh=5.0 + ) + + assert len(result) == 1 + epoch = result[0] + assert epoch["start_mjd"] == 60299.0 + assert epoch["end_mjd"] == 60302.0 + assert epoch["tile_id"] == 7 + assert epoch["band_id"] == 2 + assert "start_date" in epoch and "end_date" in epoch + + +def test_create_epochs_returns_empty_list_for_no_epochs(): + mock_db = MagicMock() + assert create_epochs(mock_db, project_id=1, epochs=[]) == [] + mock_db.execute.assert_not_called() diff --git a/src/services/tests/test_ingest_service.py b/src/services/tests/test_ingest_service.py new file mode 100644 index 0000000..ef51124 --- /dev/null +++ b/src/services/tests/test_ingest_service.py @@ -0,0 +1,185 @@ +from unittest.mock import MagicMock, patch + +import numpy as np +import pandas as pd +import pytest + +from src.db.enums import JobStatus +from src.db.models import IngestBatch +from src.services.ingest_service import ( + _resolve_reference_ids, + bulk_upsert_images_and_calibrations, + create_ingest_batch, + get_ingest_batch, + parse_fits_headers, +) + + +def _fake_wcs(): + wcs = MagicMock() + wcs.calc_footprint.return_value = np.array( + [[10.0, 20.0], [10.1, 20.0], [10.1, 20.1], [10.0, 20.1]] + ) + return wcs + + +def test_parse_fits_headers_extracts_expected_fields(mocker): + header_sci = { + "CRVAL1": 10.05, + "CRVAL2": 20.05, + "XPOSURE": 150.0, + "MJD-AVG": 60310.5, + } + header_pri = {"TARGPROP": "TEST-TARGET", "INSTRUME": "NIRCam", "FILTER": "F150W"} + + def fake_getheader(path, ext): + return header_sci if ext == "SCI" else header_pri + + mocker.patch( + "src.services.ingest_service.fits.getheader", side_effect=fake_getheader + ) + mocker.patch("src.services.ingest_service.WCS", return_value=_fake_wcs()) + mocker.patch( + "src.services.ingest_service.proj_plane_pixel_scales", + return_value=np.array([2.5e-5, 2.5e-5]), + ) + fake_moc = MagicMock() + mocker.patch( + "src.services.ingest_service.MOC.from_polygon_skycoord", return_value=fake_moc + ) + + df = parse_fits_headers(["/tmp/jw00001_sci.fits"]) + + assert len(df) == 1 + row = df.iloc[0] + assert row["base_filename"] == "jw00001_sci.fits" + assert row["current_file_ext"] == ".fits" + assert row["ra"] == 10.05 + assert row["decl"] == 20.05 + assert row["exp_time"] == 150.0 + assert row["mjd_avg"] == 60310.5 + assert row["target_name"] == "TEST-TARGET" + assert row["instrument_name"] == "NIRCam" + assert row["band_name"] == "F150W" + assert row["plate_scale"] == pytest.approx(2.5e-5 * 3600.0) + assert row["footprint"] is fake_moc + + +def test_resolve_reference_ids_maps_names_to_ids(): + mock_db = MagicMock() + # MagicMock(name=...) sets the mock's repr name, not a `.name` attribute; set explicitly. + instrument = MagicMock() + instrument.name = "NIRCam" + instrument.id = 1 + band = MagicMock() + band.name = "F150W" + band.id = 2 + mock_db.query.return_value.all.side_effect = [[instrument], [band]] + + df = pd.DataFrame( + [{"base_filename": "a.fits", "instrument_name": "NIRCam", "band_name": "F150W"}] + ) + + result = _resolve_reference_ids(mock_db, df) + + assert result.iloc[0]["instrument_id"] == 1 + assert result.iloc[0]["band_id"] == 2 + assert "instrument_name" not in result.columns + assert "band_name" not in result.columns + + +def test_resolve_reference_ids_matches_case_insensitively(): + """Regression: real FITS headers write INSTRUME/FILTER in uppercase (e.g. "NIRCAM"), + which doesn't match this project's mixed-case reference data ("NIRCam") on an + exact-string lookup — a real ~20-file batch of real JWST data failed on this.""" + mock_db = MagicMock() + instrument = MagicMock() + instrument.name = "NIRCam" + instrument.id = 1 + band = MagicMock() + band.name = "F115W" + band.id = 2 + mock_db.query.return_value.all.side_effect = [[instrument], [band]] + + df = pd.DataFrame( + [{"base_filename": "a.fits", "instrument_name": "NIRCAM", "band_name": "F115W"}] + ) + + result = _resolve_reference_ids(mock_db, df) + + assert result.iloc[0]["instrument_id"] == 1 + assert result.iloc[0]["band_id"] == 2 + + +def test_resolve_reference_ids_raises_on_unknown_instrument(): + mock_db = MagicMock() + mock_db.query.return_value.all.side_effect = [[], []] + + df = pd.DataFrame( + [{"base_filename": "a.fits", "instrument_name": "Unknown", "band_name": "F150W"}] + ) + + with pytest.raises(ValueError, match="Unknown instrument/band"): + _resolve_reference_ids(mock_db, df) + + +def test_bulk_upsert_returns_zero_for_empty_dataframe(): + mock_db = MagicMock() + assert bulk_upsert_images_and_calibrations(mock_db, 1, pd.DataFrame()) == 0 + mock_db.execute.assert_not_called() + + +def test_create_ingest_batch_dispatches_and_returns_ids(mocker): + mock_db = MagicMock() + mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 11) + fake_result = MagicMock(id="ingest-task-id") + mock_delay = mocker.patch( + "src.worker.tasks.run_ingest_batch.delay", return_value=fake_result + ) + + job_id, batch_id = create_ingest_batch(mock_db, project_id=1, s3_prefix="raw/") + + assert job_id == "ingest-task-id" + assert batch_id == 11 + mock_db.add.assert_called_once() + added = mock_db.add.call_args[0][0] + assert isinstance(added, IngestBatch) + assert added.project_id == 1 + assert added.s3_prefix == "raw/" + mock_delay.assert_called_once_with(11) + + +def test_create_ingest_batch_marks_failed_and_reraises_when_dispatch_fails(mocker): + """If .delay() fails (e.g. broker unreachable), the already-committed row must not + be left permanently PENDING with no task ever queued to redeliver it.""" + mock_db = MagicMock() + mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 21) + mocker.patch( + "src.worker.tasks.run_ingest_batch.delay", + side_effect=ConnectionError("broker unreachable"), + ) + + with pytest.raises(ConnectionError): + create_ingest_batch(mock_db, project_id=1, s3_prefix="raw/") + + added = mock_db.add.call_args[0][0] + assert added.status == JobStatus.FAILED + assert mock_db.commit.call_count == 2 + + +def test_get_ingest_batch_returns_batch_for_known_id(): + fake_batch = MagicMock(spec=IngestBatch) + mock_db = MagicMock() + mock_db.get.return_value = fake_batch + + result = get_ingest_batch(mock_db, 5) + + mock_db.get.assert_called_once_with(IngestBatch, 5) + assert result is fake_batch + + +def test_get_ingest_batch_returns_none_for_unknown_id(): + mock_db = MagicMock() + mock_db.get.return_value = None + + assert get_ingest_batch(mock_db, 999) is None diff --git a/src/services/tests/test_mosaic_service.py b/src/services/tests/test_mosaic_service.py new file mode 100644 index 0000000..23dff80 --- /dev/null +++ b/src/services/tests/test_mosaic_service.py @@ -0,0 +1,136 @@ +from unittest.mock import MagicMock + +import astropy.units as u +import pytest +from mocpy import MOC + +from src.db.enums import JobStatus +from src.db.models import Level3Mosaic +from src.services.mosaic_service import ( + _unioned_footprint_and_barycenter, + create_mosaic, + get_mosaic, +) + + +def test_unioned_footprint_and_barycenter_unions_and_extracts_center(): + """Real, in-memory MOC math: two adjacent cones union into one footprint with a center between them.""" + moc_a = MOC.from_cone(lon=10 * u.deg, lat=0 * u.deg, radius=0.2 * u.deg, max_depth=12) + moc_b = MOC.from_cone(lon=10.3 * u.deg, lat=0 * u.deg, radius=0.2 * u.deg, max_depth=12) + + union, ra, decl = _unioned_footprint_and_barycenter([moc_a, moc_b]) + + assert union.sky_fraction >= moc_a.sky_fraction + assert union.sky_fraction >= moc_b.sky_fraction + assert 9.5 <= ra <= 10.8 + assert -0.5 <= decl <= 0.5 + + +def test_unioned_footprint_and_barycenter_single_moc_is_a_no_op_union(): + moc = MOC.from_cone(lon=50 * u.deg, lat=20 * u.deg, radius=0.1 * u.deg, max_depth=10) + + union, ra, decl = _unioned_footprint_and_barycenter([moc]) + + assert (union.to_depth29_ranges == moc.to_depth29_ranges).all() + + +def test_create_mosaic_dispatches_and_returns_ids(mocker): + mock_db = MagicMock() + mock_db.execute.return_value.all.return_value = [] # no constituent calibrations + mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 42) + fake_result = MagicMock(id="mosaic-task-id") + mock_delay = mocker.patch( + "src.worker.tasks.run_mosaic_drizzle.delay", return_value=fake_result + ) + + job_id, mosaic_id = create_mosaic( + mock_db, + project_id=1, + tile_id=2, + epoch_id=3, + band_id=4, + instrument_id=5, + filename="mosaic_1.fits", + target_plate_scale=0.03, + ) + + assert job_id == "mosaic-task-id" + assert mosaic_id == 42 + added = mock_db.add.call_args[0][0] + assert isinstance(added, Level3Mosaic) + assert added.footprint is None + assert added.ra is None + assert added.decl is None + mock_delay.assert_called_once_with(42) + + +def test_create_mosaic_computes_footprint_and_barycenter_when_calibrations_exist(mocker): + moc_a = MOC.from_cone(lon=10 * u.deg, lat=20 * u.deg, radius=0.05 * u.deg, max_depth=12) + mock_db = MagicMock() + mock_db.execute.return_value.all.return_value = [(moc_a,)] + mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 7) + mocker.patch( + "src.worker.tasks.run_mosaic_drizzle.delay", return_value=MagicMock(id="x") + ) + + create_mosaic( + mock_db, + project_id=1, + tile_id=2, + epoch_id=3, + band_id=4, + instrument_id=5, + filename="mosaic_2.fits", + target_plate_scale=0.03, + ) + + added = mock_db.add.call_args[0][0] + assert added.footprint is not None + assert added.ra == pytest.approx(10.0, abs=0.1) + assert added.decl == pytest.approx(20.0, abs=0.1) + + +def test_create_mosaic_marks_failed_and_reraises_when_dispatch_fails(mocker): + """If .delay() fails (e.g. broker unreachable), the already-committed row must not + be left permanently PENDING with no task ever queued to redeliver it.""" + mock_db = MagicMock() + mock_db.execute.return_value.all.return_value = [] + mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 13) + mocker.patch( + "src.worker.tasks.run_mosaic_drizzle.delay", + side_effect=ConnectionError("broker unreachable"), + ) + + with pytest.raises(ConnectionError): + create_mosaic( + mock_db, + project_id=1, + tile_id=2, + epoch_id=3, + band_id=4, + instrument_id=5, + filename="mosaic_3.fits", + target_plate_scale=0.03, + ) + + added = mock_db.add.call_args[0][0] + assert added.status == JobStatus.FAILED + assert mock_db.commit.call_count == 2 + + +def test_get_mosaic_returns_mosaic_for_known_id(): + fake_mosaic = MagicMock(spec=Level3Mosaic) + mock_db = MagicMock() + mock_db.get.return_value = fake_mosaic + + result = get_mosaic(mock_db, 9) + + mock_db.get.assert_called_once_with(Level3Mosaic, 9) + assert result is fake_mosaic + + +def test_get_mosaic_returns_none_for_unknown_id(): + mock_db = MagicMock() + mock_db.get.return_value = None + + assert get_mosaic(mock_db, 999) is None diff --git a/src/services/tests/test_project_service.py b/src/services/tests/test_project_service.py new file mode 100644 index 0000000..630e480 --- /dev/null +++ b/src/services/tests/test_project_service.py @@ -0,0 +1,33 @@ +from unittest.mock import MagicMock + +import pytest +from sqlalchemy.exc import IntegrityError + +from src.db.models import Project +from src.services.project_service import create_project + + +def test_create_project_generates_slug_from_name(mocker): + mock_db = MagicMock() + mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 1) + + project = create_project(mock_db, "My First Survey!", "desc", user_id=1) + + assert project.slug == "my-first-survey" + mock_db.add.assert_called_once() + added = mock_db.add.call_args[0][0] + assert isinstance(added, Project) + assert added.name == "My First Survey!" + assert added.description == "desc" + assert added.user_id == 1 + mock_db.commit.assert_called_once() + + +def test_create_project_raises_value_error_on_slug_collision(): + mock_db = MagicMock() + mock_db.commit.side_effect = IntegrityError("stmt", "params", Exception("orig")) + + with pytest.raises(ValueError, match="already exists"): + create_project(mock_db, "Duplicate Name", None, user_id=1) + + mock_db.rollback.assert_called_once() diff --git a/src/services/tests/test_storage_service.py b/src/services/tests/test_storage_service.py index e273092..349b87b 100644 --- a/src/services/tests/test_storage_service.py +++ b/src/services/tests/test_storage_service.py @@ -1,18 +1,25 @@ from types import SimpleNamespace from unittest.mock import MagicMock -from src.services.storage_service import S3StorageService +import pytest + +from src.services.storage_service import ( + LocalStorageService, + S3StorageService, + get_storage_service, +) FAKE_CONFIG = SimpleNamespace( s3_endpoint_url="http://minio:9000", aws_access_key_id="testkey", aws_secret_access_key="testsecret", s3_bucket_name="test-bucket", + s3_region="us-east-1", ) def test_init_builds_client_from_settings(mocker): - """The service constructs a boto3 S3 client using endpoint and credentials from settings.""" + """The service constructs a boto3 S3 client using endpoint, region, and credentials from settings.""" mock_client_factory = mocker.patch( "src.services.storage_service.boto3.client", return_value=MagicMock() ) @@ -21,10 +28,67 @@ def test_init_builds_client_from_settings(mocker): mock_client_factory.assert_called_once_with( "s3", + region_name="us-east-1", endpoint_url="http://minio:9000", aws_access_key_id="testkey", aws_secret_access_key="testsecret", + ) + + +def test_init_uses_configured_region(mocker): + mock_client_factory = mocker.patch( + "src.services.storage_service.boto3.client", return_value=MagicMock() + ) + config = SimpleNamespace( + s3_endpoint_url="http://minio:9000", + aws_access_key_id="testkey", + aws_secret_access_key="testsecret", + s3_bucket_name="test-bucket", + s3_region="eu-west-1", + ) + + S3StorageService(config=config) + + assert mock_client_factory.call_args.kwargs["region_name"] == "eu-west-1" + + +def test_init_omits_credentials_when_unset_so_boto3_falls_back_to_iam(mocker): + """A cloud deployment leaves these blank to use its IAM role instead of static creds.""" + mock_client_factory = mocker.patch( + "src.services.storage_service.boto3.client", return_value=MagicMock() + ) + config = SimpleNamespace( + s3_endpoint_url="", + aws_access_key_id="", + aws_secret_access_key="", + s3_bucket_name="test-bucket", + s3_region="us-east-1", + ) + + S3StorageService(config=config) + + mock_client_factory.assert_called_once_with("s3", region_name="us-east-1") + + +def test_init_omits_only_endpoint_when_credentials_are_still_set(mocker): + mock_client_factory = mocker.patch( + "src.services.storage_service.boto3.client", return_value=MagicMock() + ) + config = SimpleNamespace( + s3_endpoint_url="", + aws_access_key_id="testkey", + aws_secret_access_key="testsecret", + s3_bucket_name="test-bucket", + s3_region="us-east-1", + ) + + S3StorageService(config=config) + + mock_client_factory.assert_called_once_with( + "s3", region_name="us-east-1", + aws_access_key_id="testkey", + aws_secret_access_key="testsecret", ) @@ -52,3 +116,108 @@ def test_download_file_delegates_to_client_with_bucket(mocker): fake_client.download_file.assert_called_once_with( "test-bucket", "prefix/remote.fits", "/tmp/local.fits" ) + + +def test_local_storage_service_creates_root_if_missing(tmp_path): + """Instantiating LocalStorageService auto-creates a missing storage root.""" + root = tmp_path / "does" / "not" / "exist" + config = SimpleNamespace(local_storage_root=str(root)) + + LocalStorageService(config=config) + + assert root.is_dir() + + +def test_local_storage_service_upload_then_download_round_trips_content(tmp_path): + """A file uploaded under a nested key and downloaded back has identical bytes.""" + config = SimpleNamespace(local_storage_root=str(tmp_path / "root")) + svc = LocalStorageService(config=config) + src = tmp_path / "upload.txt" + src.write_text("diffpype-local-roundtrip") + + svc.upload_file(str(src), "raw/upload.txt") + dst = tmp_path / "download.txt" + svc.download_file("raw/upload.txt", str(dst)) + + assert dst.read_text() == "diffpype-local-roundtrip" + + +def test_s3_storage_service_list_prefix_paginates_all_pages(mocker): + """list_prefix aggregates keys across multiple paginator pages.""" + fake_paginator = MagicMock() + fake_paginator.paginate.return_value = [ + {"Contents": [{"Key": "raw/a.fits"}, {"Key": "raw/b.fits"}]}, + {"Contents": [{"Key": "raw/c.fits"}]}, + ] + fake_client = MagicMock() + fake_client.get_paginator.return_value = fake_paginator + mocker.patch("src.services.storage_service.boto3.client", return_value=fake_client) + + svc = S3StorageService(config=FAKE_CONFIG) + keys = svc.list_prefix("raw/") + + assert keys == ["raw/a.fits", "raw/b.fits", "raw/c.fits"] + fake_client.get_paginator.assert_called_once_with("list_objects_v2") + + +def test_s3_storage_service_list_prefix_empty_when_no_contents(mocker): + fake_paginator = MagicMock() + fake_paginator.paginate.return_value = [{}] + fake_client = MagicMock() + fake_client.get_paginator.return_value = fake_paginator + mocker.patch("src.services.storage_service.boto3.client", return_value=fake_client) + + svc = S3StorageService(config=FAKE_CONFIG) + + assert svc.list_prefix("empty/") == [] + + +def test_local_storage_service_list_prefix_returns_files_under_prefix(tmp_path): + config = SimpleNamespace(local_storage_root=str(tmp_path / "root")) + svc = LocalStorageService(config=config) + svc.upload_file(str(_write(tmp_path, "a.fits")), "raw/a.fits") + svc.upload_file(str(_write(tmp_path, "b.fits")), "raw/b.fits") + svc.upload_file(str(_write(tmp_path, "c.fits")), "other/c.fits") + + assert svc.list_prefix("raw") == ["raw/a.fits", "raw/b.fits"] + + +def test_local_storage_service_list_prefix_empty_when_prefix_missing(tmp_path): + config = SimpleNamespace(local_storage_root=str(tmp_path / "root")) + svc = LocalStorageService(config=config) + + assert svc.list_prefix("nope") == [] + + +def _write(tmp_path, name: str): + path = tmp_path / name + path.write_text("x") + return path + + +def test_local_storage_service_download_of_missing_key_raises(tmp_path): + """Downloading a key that was never uploaded raises FileNotFoundError, not a silent no-op.""" + config = SimpleNamespace(local_storage_root=str(tmp_path / "root")) + svc = LocalStorageService(config=config) + + with pytest.raises(FileNotFoundError): + svc.download_file("raw/missing.txt", str(tmp_path / "out.txt")) + + +def test_get_storage_service_returns_s3_by_default(mocker): + mocker.patch("src.services.storage_service.boto3.client", return_value=MagicMock()) + config = SimpleNamespace(storage_backend="s3", **FAKE_CONFIG.__dict__) + + svc = get_storage_service(config=config) + + assert isinstance(svc, S3StorageService) + + +def test_get_storage_service_returns_local_when_configured(tmp_path): + config = SimpleNamespace( + storage_backend="local", local_storage_root=str(tmp_path / "root") + ) + + svc = get_storage_service(config=config) + + assert isinstance(svc, LocalStorageService) diff --git a/src/services/tests/test_tile_service.py b/src/services/tests/test_tile_service.py new file mode 100644 index 0000000..0539a28 --- /dev/null +++ b/src/services/tests/test_tile_service.py @@ -0,0 +1,68 @@ +from unittest.mock import MagicMock + +import astropy.units as u +from mocpy import MOC + +from src.services.tile_service import ( + create_tiles, + generate_tile_tessellation, + tile_with_most_calibrations, +) + + +def test_generate_tile_tessellation_covers_a_small_cone_region(): + """Real, in-memory tessellation math: no mocking, matching the doc's testing directive.""" + moc_to_tile = MOC.from_cone( + lon=180 * u.deg, lat=0 * u.deg, radius=0.1 * u.deg, max_depth=10 + ) + + tiles = generate_tile_tessellation( + tile_side_length_arc_min=6.0, moc_to_tile=moc_to_tile, overlap_in_arc_min=0.0 + ) + + assert len(tiles) >= 1 + for tile in tiles: + assert tile["name"].startswith("Tile_") + assert isinstance(tile["footprint"], MOC) + assert 178 <= tile["ra"] <= 182 + assert -2 <= tile["decl"] <= 2 + assert tile["delta_ra"] > 0 + assert tile["delta_decl"] > 0 + + +def test_generate_tile_tessellation_only_keeps_tiles_that_intersect_the_region(): + """A larger tessellation grid than the target region still returns only overlapping tiles.""" + moc_to_tile = MOC.from_cone( + lon=90 * u.deg, lat=45 * u.deg, radius=0.05 * u.deg, max_depth=12 + ) + + tiles = generate_tile_tessellation( + tile_side_length_arc_min=3.0, moc_to_tile=moc_to_tile + ) + + for tile in tiles: + assert moc_to_tile.intersection(tile["footprint"]).sky_fraction > 0 + + +def test_create_tiles_returns_empty_list_for_no_tiles(): + mock_db = MagicMock() + assert create_tiles(mock_db, project_id=1, tiles=[]) == [] + mock_db.execute.assert_not_called() + + +def test_tile_with_most_calibrations_returns_the_best_tile_id(): + mock_db = MagicMock() + mock_db.execute.return_value.first.return_value = MagicMock(tile_id=39) + + result = tile_with_most_calibrations(mock_db, [28, 34, 35, 39, 40]) + + assert result == 39 + + +def test_tile_with_most_calibrations_returns_none_when_no_tile_has_any(): + mock_db = MagicMock() + mock_db.execute.return_value.first.return_value = None + + result = tile_with_most_calibrations(mock_db, [28, 29, 30]) + + assert result is None diff --git a/src/services/tile_service.py b/src/services/tile_service.py new file mode 100644 index 0000000..d34286c --- /dev/null +++ b/src/services/tile_service.py @@ -0,0 +1,220 @@ +"""Business logic for tile tessellation, persistence, and spatial association. + +Isolates the MOC tessellation math (``generate_tile_tessellation``, a pure +function with no DB access) from persistence (``create_tiles``, the one +synchronous write path) so the tessellation can be previewed without writing, +regardless of whether the tile list came from the tessellation math below or +a future hand-drawn UI widget. +""" + +import math +from collections.abc import Sequence + +import astropy.units as u +import numpy as np +import sqlalchemy as sa +from astropy.coordinates import SkyCoord +from mocpy import MOC +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +from src.db.models import Level2Calibration, Tile, tile_level2_calibration_association +from src.db.spatial_types import MOCType + +# HEALPix order for generated tile footprints, matching the prototype's ported +# convention for tile-scale (not detector-scale) precision. MOCType normalizes +# any input depth to depth 29 on persistence, so this only affects computation. +_TILE_FOOTPRINT_MAX_DEPTH = 21 + + +def generate_tile_tessellation( + tile_side_length_arc_min: float, + moc_to_tile: MOC, + overlap_in_arc_min: float = 0.0, +) -> list[dict]: + """Generate a regular tile grid covering ``moc_to_tile``. Pure computation, no DB access. + + Ports the prototype's ``GenerateSkyTiles``. Returns a list of plain dicts + (name, ra, decl, delta_ra, delta_decl, footprint) — the API/CLI boundary + wraps each into a ``TileCreate`` (encoding ``footprint`` as a depth-29 + range list) before this ever leaves the service layer. Inputs are arcmin + (matching the CLI/API parameter names); ``delta_ra``/``delta_decl`` in the + returned dicts (and the persisted ``Tile`` columns) are degrees, not + arcmin — converted once here and never converted back. + """ + orig_deg_height = tile_side_length_arc_min / 60.0 + orig_deg_width = tile_side_length_arc_min / 60.0 + deg_height = (tile_side_length_arc_min - overlap_in_arc_min) / 60.0 + deg_width = (tile_side_length_arc_min - overlap_in_arc_min) / 60.0 + + # Avoids generating duplicate tiles at the 0/360 and +/-90 wrap boundaries. + threshold = 0.1 / 3600.0 + northern_limit = 90.0 - threshold - deg_height + southern_limit = -90.0 + eastern_limit = 360.0 - threshold - deg_width + western_limit = 0.0 + + dec_range = northern_limit - southern_limit + ra_range = eastern_limit - western_limit + + frac_dec_tile, num_dec_tiles = math.modf(dec_range / deg_height) + total_dec_tiles = int(num_dec_tiles) + 1 + if num_dec_tiles == 0: + num_dec_tiles = 1 + dec_differential = (deg_height - (frac_dec_tile * deg_height)) / num_dec_tiles + dec_delta = deg_height - dec_differential + starting_dec = southern_limit + deg_height / 2.0 + + decs = [starting_dec + i * dec_delta for i in range(total_dec_tiles)] + + ras_over_decs = [] + for d in decs: + adjusted_tile_width = deg_width / np.abs(np.cos(np.radians(d))) + frac_ra_tile, num_ra_tiles = math.modf(ra_range / adjusted_tile_width) + total_ra_tiles = int(num_ra_tiles) + 1 + if num_ra_tiles == 0: + num_ra_tiles = 1 + ra_differential = ( + adjusted_tile_width - (frac_ra_tile * adjusted_tile_width) + ) / num_ra_tiles + ra_delta = adjusted_tile_width - ra_differential + starting_ra = adjusted_tile_width / 2.0 + ras_over_decs.append( + [starting_ra + i * ra_delta for i in range(total_ra_tiles)] + ) + + ra_values: list[float] = [] + dec_values: list[float] = [] + for d, ras in zip(decs, ras_over_decs): + for r in ras: + ra_values.append(float(r)) + dec_values.append(float(d)) + + all_sky_coords = SkyCoord( + np.asarray(ra_values), + np.asarray(dec_values), + unit=(u.degree, u.degree), + frame="icrs", + ) + + covered_area = moc_to_tile.sky_fraction * 41252.96 + bary = moc_to_tile.barycenter() + fov_degrees = 2 * np.sqrt(covered_area) + nearby_coords = all_sky_coords[all_sky_coords.separation(bary).degree <= fov_degrees] + + x_step = orig_deg_width / 2.0 + y_step = orig_deg_height / 2.0 + + tiles = [] + tile_num = 1 + for coord in nearby_coords: + x_step_adjusted = x_step / np.cos(np.radians(coord.dec.degree)) + corner_ras = np.asarray( + [ + coord.ra.degree + x_step_adjusted, + coord.ra.degree + x_step_adjusted, + coord.ra.degree - x_step_adjusted, + coord.ra.degree - x_step_adjusted, + ] + ) + corner_decs = np.asarray( + [ + coord.dec.degree + y_step, + coord.dec.degree - y_step, + coord.dec.degree - y_step, + coord.dec.degree + y_step, + ] + ) + corner_coords = SkyCoord(corner_ras, corner_decs, unit="deg", frame="icrs") + tile_moc = MOC.from_polygon_skycoord( + corner_coords, complement=False, max_depth=_TILE_FOOTPRINT_MAX_DEPTH + ) + + if moc_to_tile.intersection(tile_moc).sky_fraction > 0: + tiles.append( + { + "name": f"Tile_{tile_num}", + "ra": float(coord.ra.degree), + "decl": float(coord.dec.degree), + "delta_ra": float(deg_width), + "delta_decl": float(deg_height), + "footprint": tile_moc, + } + ) + tile_num += 1 + + return tiles + + +def create_tiles(db: Session, project_id: int, tiles: list[dict]) -> list[Tile]: + """Bulk-insert Tile rows and associate each with its overlapping Level2Calibrations. + + ``tiles`` are plain dicts already carrying real ``mocpy.MOC`` footprints + (name, ra, decl, delta_ra, delta_decl, footprint) — the API/CLI boundary is + responsible for decoding each ``TileCreate.footprint`` range list into a MOC + via ``ranges_to_moc`` before calling this. + """ + if not tiles: + return [] + + rows = [{**tile, "project_id": project_id} for tile in tiles] + created = ( + db.execute(sa.insert(Tile).returning(Tile), rows).scalars().all() + ) + db.flush() + + _associate_tiles_with_overlapping_calibrations(db, project_id, created) + db.commit() + return list(created) + + +def _associate_tiles_with_overlapping_calibrations( + db: Session, project_id: int, tiles: Sequence[Tile] +) -> None: + """Populate tile_level2_calibration_association for each tile's spatial overlap.""" + association_rows: list[dict] = [] + for tile in tiles: + if tile.footprint is None: + continue + overlap_stmt = sa.select(Level2Calibration.id).where( + Level2Calibration.project_id == project_id, + Level2Calibration.footprint.isnot(None), + Level2Calibration.footprint.op("&&")( + sa.literal(tile.footprint, type_=MOCType) + ), + ) + association_rows.extend( + {"tile_id": tile.id, "level2_calibration_id": cal_id} + for (cal_id,) in db.execute(overlap_stmt).all() + ) + + if association_rows: + db.execute( + pg_insert(tile_level2_calibration_association) + .values(association_rows) + .on_conflict_do_nothing( + index_elements=["tile_id", "level2_calibration_id"] + ) + ) + + +def tile_with_most_calibrations(db: Session, tile_ids: Sequence[int]) -> int | None: + """Return whichever of ``tile_ids`` has the most associated Level2Calibrations, or None if all have zero. + + A regular tessellation grid over an arbitrary region routinely produces + tiles with no real spatial overlap at all (edge cells, sparse coverage) — + callers that need one "representative" tile (e.g. a demo/smoke run) must + pick by actual data content, never by position in the list. + """ + stmt = ( + sa.select( + tile_level2_calibration_association.c.tile_id, + sa.func.count().label("n"), + ) + .where(tile_level2_calibration_association.c.tile_id.in_(tile_ids)) + .group_by(tile_level2_calibration_association.c.tile_id) + .order_by(sa.desc("n")) + .limit(1) + ) + row = db.execute(stmt).first() + return row.tile_id if row else None diff --git a/src/worker/celery_app.py b/src/worker/celery_app.py index 93e9a45..7cd018f 100644 --- a/src/worker/celery_app.py +++ b/src/worker/celery_app.py @@ -28,6 +28,8 @@ task_routes={ "src.worker.tasks.sleep_and_update_status": {"queue": CeleryQueue.LIGHT}, "src.worker.tasks.dlq_dump": {"queue": "dead_letter"}, + "src.worker.tasks.run_ingest_batch": {"queue": CeleryQueue.HEAVY_MEMORY}, + "src.worker.tasks.run_mosaic_drizzle": {"queue": CeleryQueue.HEAVY_MEMORY}, }, ) diff --git a/src/worker/tasks.py b/src/worker/tasks.py index 452d7cd..5576b94 100644 --- a/src/worker/tasks.py +++ b/src/worker/tasks.py @@ -1,12 +1,17 @@ +import os import subprocess +import tempfile import time +import pandas as pd from sqlalchemy import func from src.core.logger import get_logger from src.db.enums import JobStatus -from src.db.models import DummyImage, JobConfiguration +from src.db.models import DummyImage, IngestBatch, JobConfiguration, Level3Mosaic from src.db.session import SessionLocal +from src.services import ingest_service +from src.services.storage_service import get_storage_service from src.worker.base_task import DiffpypeTask from src.worker.celery_app import celery_app from src.worker.utils import build_cli_command @@ -43,6 +48,125 @@ def sleep_and_update_status(image_id: int, sleep_duration: int = 5) -> None: log.info("task_completed", image_id=image_id) +@celery_app.task(name="src.worker.tasks.run_ingest_batch") +def run_ingest_batch(batch_id: int) -> None: + """Scan an IngestBatch's storage prefix and bulk-upsert Level2Image/Level2Calibration rows. + + Not built on DiffpypeTask: that base's on_failure hardcodes a DummyImage + status write, which would silently no-op (or worse, mutate an unrelated + DummyImage row sharing the same integer id) for this batch_id. Handles its + own crash-safety instead, matching the same guarantee (no orphaned + IN_PROCESS row) scoped to the entity this task actually tracks. + """ + log = get_logger() + log.info("ingest_batch_started", batch_id=batch_id) + + db = SessionLocal() + try: + batch = db.get(IngestBatch, batch_id) + assert batch is not None, f"IngestBatch {batch_id} not found" + batch.status = JobStatus.IN_PROCESS + db.commit() + + storage = get_storage_service() + keys = storage.list_prefix(batch.s3_prefix) + batch.total_files = len(keys) + batch.processed_files = 0 + db.commit() + + # One file on disk at a time, not all of them at once: downloading every + # file into the temp dir up front (the previous shape) meant peak memory/ + # disk footprint scaled with total batch size, which OOM-killed the whole + # container on a real ~20-file batch of real FITS data (WORKER_LIGHT_MEM_LIMIT + # is 512m). Streaming download->parse->delete per file keeps peak footprint + # to ~one file, and doubles as real per-file progress (processed_files, + # ingest_file_processed) instead of a single all-or-nothing jump at the end. + header_frames: list[pd.DataFrame] = [] + with tempfile.TemporaryDirectory() as tmp_dir: + for index, key in enumerate(keys, start=1): + local_path = os.path.join(tmp_dir, os.path.basename(key)) + storage.download_file(key, local_path) + header_frames.append(ingest_service.parse_fits_headers([local_path])) + os.remove(local_path) + + batch.processed_files = index + db.commit() + log.info( + "ingest_file_processed", + batch_id=batch_id, + key=key, + index=index, + total=len(keys), + ) + + header_df = ( + pd.concat(header_frames, ignore_index=True) + if header_frames + else pd.DataFrame() + ) + ingest_service.bulk_upsert_images_and_calibrations( + db, batch.project_id, header_df + ) + + batch.status = JobStatus.COMPLETE + db.commit() + log.info( + "ingest_batch_completed", batch_id=batch_id, total_files=batch.total_files + ) + except Exception as exc: + log.error( + "ingest_batch_failed", batch_id=batch_id, error=str(exc), exc_info=True + ) + db.rollback() + batch = db.get(IngestBatch, batch_id) + if batch is not None: + batch.status = JobStatus.FAILED + db.commit() + raise + finally: + db.close() + + +@celery_app.task(name="src.worker.tasks.run_mosaic_drizzle") +def run_mosaic_drizzle(mosaic_id: int) -> None: + """Placeholder drizzle execution: sleeps and transitions to COMPLETE. + + The Level3Mosaic row itself (footprint, barycenter, and its M2M-derived + constituent calibrations) is already the complete job specification — + only the actual pixel-level drizzle is deferred, to a future + JWST-pipeline-specific doc. Not built on DiffpypeTask for the same reason + as run_ingest_batch: that base's on_failure hardcodes a DummyImage status + write. + """ + log = get_logger() + log.info("mosaic_drizzle_started", mosaic_id=mosaic_id) + + db = SessionLocal() + try: + mosaic = db.get(Level3Mosaic, mosaic_id) + assert mosaic is not None, f"Level3Mosaic {mosaic_id} not found" + mosaic.status = JobStatus.IN_PROCESS + db.commit() + + time.sleep(5) + + mosaic.status = JobStatus.COMPLETE + db.commit() + log.info("mosaic_drizzle_completed", mosaic_id=mosaic_id) + except Exception as exc: + log.error( + "mosaic_drizzle_failed", mosaic_id=mosaic_id, error=str(exc), exc_info=True + ) + db.rollback() + mosaic = db.get(Level3Mosaic, mosaic_id) + if mosaic is not None: + mosaic.status = JobStatus.FAILED + db.commit() + raise + finally: + db.close() + + @celery_app.task(name="src.worker.tasks.dlq_dump") def dlq_dump(failed_task_name: str, task_kwargs: dict, error_msg: str) -> None: """Log a permanently failed task payload to the dead letter queue.""" diff --git a/src/worker/tests/test_tasks.py b/src/worker/tests/test_tasks.py index 3dd3c04..8f2c30d 100644 --- a/src/worker/tests/test_tasks.py +++ b/src/worker/tests/test_tasks.py @@ -1,11 +1,18 @@ import subprocess from unittest.mock import MagicMock +import pandas as pd import pytest from src.db.enums import JobStatus -from src.db.models import DummyImage, JobConfiguration -from src.worker.tasks import dlq_dump, execute_cli_tool, sleep_and_update_status +from src.db.models import DummyImage, IngestBatch, JobConfiguration, Level3Mosaic +from src.worker.tasks import ( + dlq_dump, + execute_cli_tool, + run_ingest_batch, + run_mosaic_drizzle, + sleep_and_update_status, +) def test_dlq_dump_logs_failed_task_payload(mocker): @@ -141,6 +148,249 @@ def test_execute_cli_tool_closes_session_on_subprocess_error(mocker): mock_session.close.assert_called_once() +# --------------------------------------------------------------------------- +# run_ingest_batch +# --------------------------------------------------------------------------- + + +def _make_ingest_session(mocker, fake_batch=None): + batch = fake_batch or MagicMock( + spec=IngestBatch, id=1, project_id=1, s3_prefix="raw/" + ) + mock_session = MagicMock() + mock_session.get.return_value = batch + mocker.patch("src.worker.tasks.SessionLocal", return_value=mock_session) + # download_file is mocked in these tests (writes nothing real to disk), so + # os.remove has nothing real to delete unless a test overrides this itself. + mocker.patch("src.worker.tasks.os.remove") + return mock_session, batch + + +def test_run_ingest_batch_transitions_pending_to_complete(mocker): + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = ["raw/a.fits", "raw/b.fits"] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mocker.patch( + "src.worker.tasks.ingest_service.parse_fits_headers", + side_effect=lambda paths: pd.DataFrame([{"base_filename": paths[0]}]), + ) + mock_upsert = mocker.patch( + "src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations" + ) + + run_ingest_batch(1) + + assert batch.status == JobStatus.COMPLETE + assert batch.total_files == 2 + assert batch.processed_files == 2 + mock_upsert.assert_called_once() + upserted_df = mock_upsert.call_args[0][2] + assert len(upserted_df) == 2 # both files' parsed rows concatenated + assert mock_session.close.call_count == 1 + + +def test_run_ingest_batch_downloads_each_listed_key(mocker): + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = ["raw/a.fits"] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mocker.patch( + "src.worker.tasks.ingest_service.parse_fits_headers", + return_value=pd.DataFrame(), + ) + mocker.patch("src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations") + + run_ingest_batch(1) + + fake_storage.download_file.assert_called_once() + assert fake_storage.download_file.call_args[0][0] == "raw/a.fits" + + +def test_run_ingest_batch_parses_one_file_at_a_time(mocker): + """parse_fits_headers is called per-file (a single-element list each time), not + once for the whole batch — this is what keeps peak memory/disk to ~one file.""" + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = ["raw/a.fits", "raw/b.fits", "raw/c.fits"] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mock_parse = mocker.patch( + "src.worker.tasks.ingest_service.parse_fits_headers", + return_value=pd.DataFrame(), + ) + mocker.patch("src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations") + + run_ingest_batch(1) + + assert mock_parse.call_count == 3 + for call in mock_parse.call_args_list: + assert len(call.args[0]) == 1 # always a single-element path list + + +def test_run_ingest_batch_increments_processed_files_progressively(mocker): + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = ["raw/a.fits", "raw/b.fits"] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mocker.patch( + "src.worker.tasks.ingest_service.parse_fits_headers", + return_value=pd.DataFrame(), + ) + mocker.patch("src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations") + + observed_progress = [] + mock_session.commit.side_effect = lambda: observed_progress.append( + batch.processed_files + ) + + run_ingest_batch(1) + + # processed_files must have visibly been 1, then 2, before the task finished — + # not just jump straight from 0 to 2 at the very end. + assert 1 in observed_progress + assert 2 in observed_progress + + +def test_run_ingest_batch_logs_progress_per_file(mocker): + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = ["raw/a.fits", "raw/b.fits"] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mocker.patch( + "src.worker.tasks.ingest_service.parse_fits_headers", + return_value=pd.DataFrame(), + ) + mocker.patch("src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations") + mock_logger = MagicMock() + mocker.patch("src.worker.tasks.get_logger", return_value=mock_logger) + + run_ingest_batch(1) + + progress_calls = [ + c + for c in mock_logger.info.call_args_list + if c.args[0] == "ingest_file_processed" + ] + assert len(progress_calls) == 2 + assert progress_calls[0].kwargs["index"] == 1 + assert progress_calls[1].kwargs["index"] == 2 + assert progress_calls[1].kwargs["total"] == 2 + + +def test_run_ingest_batch_removes_temp_file_after_each_file(mocker): + """Confirms the streaming shape: each file is deleted right after parsing, + not left on disk until the whole batch finishes.""" + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = ["raw/a.fits"] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mocker.patch( + "src.worker.tasks.ingest_service.parse_fits_headers", + return_value=pd.DataFrame(), + ) + mocker.patch("src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations") + mock_remove = mocker.patch("src.worker.tasks.os.remove") + + run_ingest_batch(1) + + mock_remove.assert_called_once() + assert mock_remove.call_args[0][0].endswith("a.fits") + + +def test_run_ingest_batch_handles_zero_files_without_crashing(mocker): + mock_session, batch = _make_ingest_session(mocker) + fake_storage = MagicMock() + fake_storage.list_prefix.return_value = [] + mocker.patch("src.worker.tasks.get_storage_service", return_value=fake_storage) + mock_parse = mocker.patch("src.worker.tasks.ingest_service.parse_fits_headers") + mock_upsert = mocker.patch( + "src.worker.tasks.ingest_service.bulk_upsert_images_and_calibrations" + ) + + run_ingest_batch(1) + + mock_parse.assert_not_called() + mock_upsert.assert_called_once() + empty_df = mock_upsert.call_args[0][2] + assert empty_df.empty + assert batch.status == JobStatus.COMPLETE + assert batch.processed_files == 0 + + +def test_run_ingest_batch_marks_failed_and_reraises_on_error(mocker): + mock_session, batch = _make_ingest_session(mocker) + mocker.patch( + "src.worker.tasks.get_storage_service", + side_effect=RuntimeError("storage unreachable"), + ) + + with pytest.raises(RuntimeError, match="storage unreachable"): + run_ingest_batch(1) + + assert batch.status == JobStatus.FAILED + mock_session.rollback.assert_called_once() + mock_session.close.assert_called_once() + + +def test_run_ingest_batch_missing_batch_raises_assertion(mocker): + mock_session = MagicMock() + mock_session.get.return_value = None + mocker.patch("src.worker.tasks.SessionLocal", return_value=mock_session) + + with pytest.raises(AssertionError, match="IngestBatch 99 not found"): + run_ingest_batch(99) + + mock_session.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# run_mosaic_drizzle +# --------------------------------------------------------------------------- + + +def _make_mosaic_session(mocker, fake_mosaic=None): + mosaic = fake_mosaic or MagicMock(spec=Level3Mosaic, id=1) + mock_session = MagicMock() + mock_session.get.return_value = mosaic + mocker.patch("src.worker.tasks.SessionLocal", return_value=mock_session) + return mock_session, mosaic + + +def test_run_mosaic_drizzle_transitions_pending_to_complete(mocker): + mock_session, mosaic = _make_mosaic_session(mocker) + mocker.patch("src.worker.tasks.time.sleep") + + run_mosaic_drizzle(1) + + assert mosaic.status == JobStatus.COMPLETE + assert mock_session.close.call_count == 1 + + +def test_run_mosaic_drizzle_marks_failed_and_reraises_on_error(mocker): + mock_session, mosaic = _make_mosaic_session(mocker) + mocker.patch( + "src.worker.tasks.time.sleep", side_effect=RuntimeError("drizzle boom") + ) + + with pytest.raises(RuntimeError, match="drizzle boom"): + run_mosaic_drizzle(1) + + assert mosaic.status == JobStatus.FAILED + mock_session.rollback.assert_called_once() + mock_session.close.assert_called_once() + + +def test_run_mosaic_drizzle_missing_mosaic_raises_assertion(mocker): + mock_session = MagicMock() + mock_session.get.return_value = None + mocker.patch("src.worker.tasks.SessionLocal", return_value=mock_session) + + with pytest.raises(AssertionError, match="Level3Mosaic 99 not found"): + run_mosaic_drizzle(99) + + mock_session.close.assert_called_once() + + def test_execute_cli_tool_handles_none_job_kwargs(mocker): mock_session, _ = _make_cli_session(mocker, None) mock_session.get.return_value = MagicMock(spec=JobConfiguration, job_kwargs=None) diff --git a/uv.lock b/uv.lock index 7906408..6db8fd1 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,18 @@ version = 1 revision = 3 requires-python = ">=3.12" resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version >= '3.13' and python_full_version < '3.15'", - "python_full_version < '3.13'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -477,11 +486,15 @@ dependencies = [ { name = "opentelemetry-instrumentation-fastapi" }, { name = "opentelemetry-instrumentation-sqlalchemy" }, { name = "opentelemetry-sdk" }, + { name = "pandas" }, { name = "prometheus-client" }, { name = "psycopg2-binary" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "python-slugify" }, { name = "redis" }, + { name = "scikit-learn" }, + { name = "scipy" }, { name = "setuptools" }, { name = "sqladmin" }, { name = "sqlalchemy" }, @@ -518,11 +531,15 @@ requires-dist = [ { name = "opentelemetry-instrumentation-fastapi", specifier = "==0.48b0" }, { name = "opentelemetry-instrumentation-sqlalchemy", specifier = "==0.48b0" }, { name = "opentelemetry-sdk", specifier = "==1.27.0" }, + { name = "pandas", specifier = "==3.0.5" }, { name = "prometheus-client", specifier = "==0.21.0" }, { name = "psycopg2-binary", specifier = "==2.9.9" }, { name = "pydantic", specifier = "==2.9.2" }, { name = "pydantic-settings", specifier = "==2.5.2" }, + { name = "python-slugify", specifier = "==8.0.4" }, { name = "redis", specifier = "==5.0.8" }, + { name = "scikit-learn", specifier = "==1.9.0" }, + { name = "scipy", specifier = "==1.18.0" }, { name = "setuptools", specifier = "<81" }, { name = "sqladmin", specifier = "==0.18.0" }, { name = "sqlalchemy", specifier = "==2.0.35" }, @@ -846,6 +863,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + [[package]] name = "kombu" version = "5.6.2" @@ -1130,6 +1156,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/b4/b036f8fdb667587bb37df29dc6644681dd78b7a2a6321a34684b79412b28/myst_parser-4.0.0-py3-none-any.whl", hash = "sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d", size = 84563, upload-time = "2024-08-05T14:02:43.767Z" }, ] +[[package]] +name = "narwhals" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/58946e5aab18393e793bd4add6985b95d0e01c3a2d832f38f54468b10dcd/narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d", size = 661143, upload-time = "2026-07-13T10:49:19.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -1366,6 +1401,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -1636,6 +1717,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, ] +[[package]] +name = "python-slugify" +version = "8.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "text-unidecode" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/c7/5e1547c44e31da50a460df93af11a535ace568ef89d7a811069ead340c4a/python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856", size = 10921, upload-time = "2024-02-08T18:32:45.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1739,6 +1832,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e", size = 83175, upload-time = "2024-11-20T21:06:03.961Z" }, ] +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/20/75f915ff375d6249e6550ac740fdbbd66159a068fd3af1400ff62036b07a/scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2bd41b0d201bc81575531b96b713d3eb5e5f50fb0b82101ff0f92294fdc236ac", size = 8741122, upload-time = "2026-06-02T11:53:24.08Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d5/2b5148f2279196775e1db2aeb85d14b70ac80e7e32b3b28e7ebeafb0901d/scikit_learn-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5be45aa4a42a68a533913a6ed736cf309de2226411c79ef8d609a5456f1939b1", size = 8261512, upload-time = "2026-06-02T11:53:27.183Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ee/5adbc77656b71f9456a2f5a7a9fdb4bcf9207a6b962889f1c2f9323afa4e/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f", size = 8837603, upload-time = "2026-06-02T11:53:30.328Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, + { url = "https://files.pythonhosted.org/packages/83/a4/c8e67227c680e2259c8864ae72ff48b06e16a6f51253a22167aa02a8aa4e/scikit_learn-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:4306775fad04cc4b472a1b15af1ae9cede1540fbfcc17fbce3767cd8dc7ae283", size = 8211173, upload-time = "2026-06-02T11:53:36.602Z" }, + { url = "https://files.pythonhosted.org/packages/cf/fd/3c0863792e98e67e9184aa4029288a175935eb65443afcd30d4f143450cf/scikit_learn-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:26e22435f63bcdcf396b574273f29f13dd531f5ea035801f5be10ba1540a4e60", size = 7867451, upload-time = "2026-06-02T11:53:39.075Z" }, + { url = "https://files.pythonhosted.org/packages/3c/01/cf3310626b6d48d3e9be69a1223f9180360b5e6edb045f50fade723ce494/scikit_learn-1.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:80746d63bd4b6eaca54d36fe5feaf4d28bb38dc6f9470f81c7cad7c40155f119", size = 8705188, upload-time = "2026-06-02T11:53:41.964Z" }, + { url = "https://files.pythonhosted.org/packages/3e/04/5acd7ae280c5f93b6ac5ef6cdec14eef4c8d1cd91d85b3292989c94d96b1/scikit_learn-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5b934c45c252844a91d69fda3a34cff5e7307e1db10d77cb10a3980312c74713", size = 8228299, upload-time = "2026-06-02T11:53:44.817Z" }, + { url = "https://files.pythonhosted.org/packages/0c/39/ffe829a5b8ecb40a518724a997794657fdc354ada5e8fe8e64d998c0bac9/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38c3dcb9a1ffb85505ec53d54c7b4aea0cff70050425a7760c2af661ac85df05", size = 8789690, upload-time = "2026-06-02T11:53:47.461Z" }, + { url = "https://files.pythonhosted.org/packages/1f/88/8dab5de10c638c083772a6be83a3d8106ced492f74a928c8693638e5bb50/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da76d09304a4706db7cc1e3ebaa3b6b98a67365cc11d2996c4f1e58ba47df714", size = 9087723, upload-time = "2026-06-02T11:53:50.702Z" }, + { url = "https://files.pythonhosted.org/packages/20/3f/7917ca72464038f6240ec70c29f94862d08a34a74291ae4d4ec5eb8186a0/scikit_learn-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:5808d98f15c6bf6d9d96d2348c1997392a5888ce7097e664105f930c4bca1277", size = 8184330, upload-time = "2026-06-02T11:53:53.396Z" }, + { url = "https://files.pythonhosted.org/packages/78/c7/15739eb2f61fda3c54639e9942414e5a19ad8a8d1f5a3266afad7cb7df80/scikit_learn-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:d77f54c017633791bc0225a43e2f8d03745fdcfe4880268fcc4df15f505dec2e", size = 7840653, upload-time = "2026-06-02T11:53:56.035Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7d/c9a35cf59b20a86fec24d306f1547b78dec194b08d367ce2a3e4854169d9/scikit_learn-1.9.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9656acd4e93f74e0b66c8a36c88830a99252dfa900044d36bc2212ae89a47162", size = 8713289, upload-time = "2026-06-02T11:53:58.788Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a7/552a7821597c632b907f7bfe8f36f9f572777af8ef8a48353041cf8e091a/scikit_learn-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:24360002ae845e7866522b0a5bbf690802e7bc388cac8663502e78aa98598aa2", size = 8245141, upload-time = "2026-06-02T11:54:01.694Z" }, + { url = "https://files.pythonhosted.org/packages/7d/79/f4a0c4fe9711154cddabf913471153af79056382ddc612cfe5ee0ff4b72e/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5162ad10a418c8a282dde04c9aa06965de3e9a65f33c1440c0ae69bb1a09d913", size = 8847671, upload-time = "2026-06-02T11:54:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/f0/af/4d72d9e475ac83719160c662619e4bf7b95c19507cd582e7d0167a3c3dae/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fea2cc5677ab49d6f5bade978c866da44957b712d92e9635e8b4f723013c3cb", size = 9118104, upload-time = "2026-06-02T11:54:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/a2/d5/6a58eea2cb9abbb9b3f2bb8b2cfb3243d1152d69f442d256c7af71304769/scikit_learn-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:64fa347efc1c839c487433e40c5144d38c336e8a2b59c81aa8660373945c2673", size = 8290674, upload-time = "2026-06-02T11:54:10.087Z" }, + { url = "https://files.pythonhosted.org/packages/65/5b/d4c879cf358f1187141cf90ced473f087183489090244f50c124a2ee478b/scikit_learn-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:1b944b6db288f6b926e3650026ddafb988929de95d11fc2cc5fa117773c9ba42", size = 7978807, upload-time = "2026-06-02T11:54:12.769Z" }, + { url = "https://files.pythonhosted.org/packages/8a/43/bfae3121ec67ae09150d453c442c7c1cc166e9aefe056e6ab3b7728a5cfc/scikit_learn-1.9.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4ccacf04ca5f4b492158a5f28afe0ace43f81b2571e4b9a66d34848b46128949", size = 9031941, upload-time = "2026-06-02T11:54:15.436Z" }, + { url = "https://files.pythonhosted.org/packages/75/b0/20a4546eb17f3b25d3c66df15810411c14ed5065bcfab50b53c96fb627b2/scikit_learn-1.9.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ee1a8db2c18c08e34c7412d4b10be1cac214cd4ea7dc9715a6a327eb49a37c96", size = 8613528, upload-time = "2026-06-02T11:54:18.842Z" }, + { url = "https://files.pythonhosted.org/packages/18/3c/e440e039bb82cd19004edaaad00acbde0fb9b461083c3ecf37941c557312/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:147e9329ef0e39f75d4cffa02b2aa48d827832684926cd5210d9a2cb5c57246b", size = 8855050, upload-time = "2026-06-02T11:54:21.699Z" }, + { url = "https://files.pythonhosted.org/packages/43/26/b341b8dab5998da6270a3a42c2152c578501354d36f944b5856757035ef8/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bad8f8b9950321b54c965fdcbac6c6c55e79e16646b49977bcf3668d3870a1a", size = 9097190, upload-time = "2026-06-02T11:54:24.454Z" }, + { url = "https://files.pythonhosted.org/packages/fb/de/b650b4d69b84468cfa2e28a3ff7b8103743029e6446ce1a97fe060ef688c/scikit_learn-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:78fc56eafd4edb9575d2d8950d1dd152061abb573341a1cb7e099fc40f6c6666", size = 8963204, upload-time = "2026-06-02T11:54:27.428Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f3/ff83d76d7418112e5a61326443cdda87be3545dd8d6599c95b2481a4419e/scikit_learn-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:051075bda8b7aab87b1906ab3d4740a1e1224a19d7b3781a576736edc94e76aa", size = 8222661, upload-time = "2026-06-02T11:54:30.192Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, + { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, + { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, + { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, + { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9c0136c2de7ae0779b7b366447766cec6d9f0702c56bb8ffeb04c8fd3af4/scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446", size = 31036107, upload-time = "2026-06-19T15:00:14.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520", size = 28663303, upload-time = "2026-06-19T15:00:16.819Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0f/10ffa0b697a572f4e0d48b92a88895d366422f019f723e7e14a84c050dac/scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197", size = 20404960, upload-time = "2026-06-19T15:00:19.635Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d2/e896cea21ba8edd6c81d4c55b1ffcc717e79698dcbebf9641b4cfb4c6622/scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b", size = 23034074, upload-time = "2026-06-19T15:00:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/e83ea34279a52c03374477c74006256ec78df65fc877baa4617d6de1d202/scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468", size = 33942038, upload-time = "2026-06-19T15:00:24.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f", size = 35266390, upload-time = "2026-06-19T15:00:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/3a/49/2c5cbb907b56695fc67517811d1db234dfd83381a84814ec220aded2794d/scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f", size = 35551324, upload-time = "2026-06-19T15:00:31.014Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/eda39f7a2d306ff0ffc574afd13c0bbb6d10a603d9a413998ee269487a80/scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4", size = 37404785, upload-time = "2026-06-19T15:00:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d2/ae881ee28d014f38e0ccbfd974a06a919ba9af34f1f74bf42b5301891d63/scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7", size = 36554943, upload-time = "2026-06-19T15:00:36.903Z" }, + { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + [[package]] name = "setuptools" version = "80.10.2" @@ -1977,6 +2160,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] +[[package]] +name = "text-unidecode" +version = "1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, +] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From b8014f48d2bcd8d44b8a5e347f25ae6cc55f6eb2 Mon Sep 17 00:00:00 2001 From: davecoulter Date: Tue, 28 Jul 2026 16:26:23 -0400 Subject: [PATCH 2/3] Fixing ruff format issues. --- ...009_add_project_slug_and_ingest_batches.py | 4 +--- src/api/routes/epochs.py | 7 ++++++- src/api/routes/mosaics.py | 4 +--- src/api/routes/projects.py | 4 +--- src/api/routes/tiles.py | 7 ++++++- src/api/tests/test_ingest.py | 4 +--- src/services/ingest_service.py | 15 +++++++------- src/services/mosaic_service.py | 4 +--- src/services/tests/test_epoch_service.py | 1 - src/services/tests/test_ingest_service.py | 10 ++++++++-- src/services/tests/test_mosaic_service.py | 20 ++++++++++++++----- src/services/tile_service.py | 12 +++++------ 12 files changed, 53 insertions(+), 39 deletions(-) diff --git a/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py b/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py index 1251565..e2d2778 100644 --- a/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py +++ b/migrations/versions/20260727_0009_add_project_slug_and_ingest_batches.py @@ -70,9 +70,7 @@ def upgrade() -> None: for row in existing_rows: slug = _unique_slug(slugify(row.name), taken_slugs) taken_slugs.add(slug) - bind.execute( - projects.update().where(projects.c.id == row.id).values(slug=slug) - ) + bind.execute(projects.update().where(projects.c.id == row.id).values(slug=slug)) op.alter_column("projects", "slug", nullable=False) op.create_unique_constraint("uq_project_slug", "projects", ["slug"]) diff --git a/src/api/routes/epochs.py b/src/api/routes/epochs.py index 19b29ba..ca11156 100644 --- a/src/api/routes/epochs.py +++ b/src/api/routes/epochs.py @@ -1,7 +1,12 @@ from fastapi import APIRouter, Depends from sqlalchemy.orm import Session -from src.api.schemas import EpochBulkCreateRequest, EpochClusterRequest, EpochCreate, EpochRead +from src.api.schemas import ( + EpochBulkCreateRequest, + EpochClusterRequest, + EpochCreate, + EpochRead, +) from src.db.session import get_db from src.services import epoch_service diff --git a/src/api/routes/mosaics.py b/src/api/routes/mosaics.py index 3e04784..23e6645 100644 --- a/src/api/routes/mosaics.py +++ b/src/api/routes/mosaics.py @@ -26,9 +26,7 @@ def create_mosaic( @router.get("/{mosaic_id}", response_model=MosaicStatus) -def get_mosaic_status( - mosaic_id: int, db: Session = Depends(get_db) -) -> MosaicStatus: +def get_mosaic_status(mosaic_id: int, db: Session = Depends(get_db)) -> MosaicStatus: mosaic = mosaic_service.get_mosaic(db, mosaic_id) if mosaic is None: raise HTTPException(status_code=404, detail="Level3Mosaic not found") diff --git a/src/api/routes/projects.py b/src/api/routes/projects.py index bd637c6..66458b3 100644 --- a/src/api/routes/projects.py +++ b/src/api/routes/projects.py @@ -9,9 +9,7 @@ @router.post("", response_model=ProjectRead) -def create_project( - body: ProjectCreate, db: Session = Depends(get_db) -) -> ProjectRead: +def create_project(body: ProjectCreate, db: Session = Depends(get_db)) -> ProjectRead: try: project = project_service.create_project( db, body.name, body.description, body.user_id diff --git a/src/api/routes/tiles.py b/src/api/routes/tiles.py index 753b4b6..45be634 100644 --- a/src/api/routes/tiles.py +++ b/src/api/routes/tiles.py @@ -1,7 +1,12 @@ from fastapi import APIRouter, Depends from sqlalchemy.orm import Session -from src.api.schemas import TileBulkCreateRequest, TileCreate, TileRead, TileTessellationRequest +from src.api.schemas import ( + TileBulkCreateRequest, + TileCreate, + TileRead, + TileTessellationRequest, +) from src.db.session import get_db from src.db.spatial_types import moc_to_ranges, ranges_to_moc from src.services import tile_service diff --git a/src/api/tests/test_ingest.py b/src/api/tests/test_ingest.py index def109a..d005932 100644 --- a/src/api/tests/test_ingest.py +++ b/src/api/tests/test_ingest.py @@ -63,9 +63,7 @@ def test_get_ingest_batch_status_returns_batch(client, mock_db, mocker): def test_get_ingest_batch_status_404_when_missing(client, mock_db, mocker): - mocker.patch( - "src.services.ingest_service.get_ingest_batch", return_value=None - ) + mocker.patch("src.services.ingest_service.get_ingest_batch", return_value=None) response = client.get("/api/v1/ingest/999") diff --git a/src/services/ingest_service.py b/src/services/ingest_service.py index 1f93b64..e8217ce 100644 --- a/src/services/ingest_service.py +++ b/src/services/ingest_service.py @@ -90,8 +90,7 @@ def _resolve_reference_ids(db: Session, df: pd.DataFrame) -> pd.DataFrame: missing = df[df["instrument_id"].isna() | df["band_id"].isna()] if not missing.empty: raise ValueError( - "Unknown instrument/band for files: " - f"{missing['base_filename'].tolist()}" + "Unknown instrument/band for files: " f"{missing['base_filename'].tolist()}" ) df["instrument_id"] = df["instrument_id"].astype(int) @@ -156,15 +155,15 @@ def bulk_upsert_images_and_calibrations( db.execute( pg_insert(Level2Calibration) .values(calibration_rows) - .on_conflict_do_nothing( - index_elements=["level2_image_id", "project_id"] - ) + .on_conflict_do_nothing(index_elements=["level2_image_id", "project_id"]) ) db.commit() return len(df) -def create_ingest_batch(db: Session, project_id: int, s3_prefix: str) -> tuple[str, int]: +def create_ingest_batch( + db: Session, project_id: int, s3_prefix: str +) -> tuple[str, int]: """Persist a PENDING IngestBatch, dispatch the ingest Celery task, return (job_id, batch_id).""" from src.worker.tasks import run_ingest_batch # lazy: avoids a circular import @@ -178,7 +177,9 @@ def create_ingest_batch(db: Session, project_id: int, s3_prefix: str) -> tuple[s try: async_result = run_ingest_batch.delay(batch.id) except Exception: - get_logger().error("ingest_batch_dispatch_failed", batch_id=batch.id, exc_info=True) + get_logger().error( + "ingest_batch_dispatch_failed", batch_id=batch.id, exc_info=True + ) batch.status = JobStatus.FAILED db.commit() raise diff --git a/src/services/mosaic_service.py b/src/services/mosaic_service.py index bdd4476..230695c 100644 --- a/src/services/mosaic_service.py +++ b/src/services/mosaic_service.py @@ -118,9 +118,7 @@ def create_mosaic( db.commit() raise - get_logger().info( - "mosaic_dispatched", mosaic_id=mosaic.id, job_id=async_result.id - ) + get_logger().info("mosaic_dispatched", mosaic_id=mosaic.id, job_id=async_result.id) return async_result.id, mosaic.id diff --git a/src/services/tests/test_epoch_service.py b/src/services/tests/test_epoch_service.py index 28aba20..96b0356 100644 --- a/src/services/tests/test_epoch_service.py +++ b/src/services/tests/test_epoch_service.py @@ -1,7 +1,6 @@ from unittest.mock import MagicMock import numpy as np -import pytest from src.services.epoch_service import _cluster_mjds, cluster_epochs, create_epochs diff --git a/src/services/tests/test_ingest_service.py b/src/services/tests/test_ingest_service.py index ef51124..8be9472 100644 --- a/src/services/tests/test_ingest_service.py +++ b/src/services/tests/test_ingest_service.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import numpy as np import pandas as pd @@ -116,7 +116,13 @@ def test_resolve_reference_ids_raises_on_unknown_instrument(): mock_db.query.return_value.all.side_effect = [[], []] df = pd.DataFrame( - [{"base_filename": "a.fits", "instrument_name": "Unknown", "band_name": "F150W"}] + [ + { + "base_filename": "a.fits", + "instrument_name": "Unknown", + "band_name": "F150W", + } + ] ) with pytest.raises(ValueError, match="Unknown instrument/band"): diff --git a/src/services/tests/test_mosaic_service.py b/src/services/tests/test_mosaic_service.py index 23dff80..7bc6bba 100644 --- a/src/services/tests/test_mosaic_service.py +++ b/src/services/tests/test_mosaic_service.py @@ -15,8 +15,12 @@ def test_unioned_footprint_and_barycenter_unions_and_extracts_center(): """Real, in-memory MOC math: two adjacent cones union into one footprint with a center between them.""" - moc_a = MOC.from_cone(lon=10 * u.deg, lat=0 * u.deg, radius=0.2 * u.deg, max_depth=12) - moc_b = MOC.from_cone(lon=10.3 * u.deg, lat=0 * u.deg, radius=0.2 * u.deg, max_depth=12) + moc_a = MOC.from_cone( + lon=10 * u.deg, lat=0 * u.deg, radius=0.2 * u.deg, max_depth=12 + ) + moc_b = MOC.from_cone( + lon=10.3 * u.deg, lat=0 * u.deg, radius=0.2 * u.deg, max_depth=12 + ) union, ra, decl = _unioned_footprint_and_barycenter([moc_a, moc_b]) @@ -27,7 +31,9 @@ def test_unioned_footprint_and_barycenter_unions_and_extracts_center(): def test_unioned_footprint_and_barycenter_single_moc_is_a_no_op_union(): - moc = MOC.from_cone(lon=50 * u.deg, lat=20 * u.deg, radius=0.1 * u.deg, max_depth=10) + moc = MOC.from_cone( + lon=50 * u.deg, lat=20 * u.deg, radius=0.1 * u.deg, max_depth=10 + ) union, ra, decl = _unioned_footprint_and_barycenter([moc]) @@ -64,8 +70,12 @@ def test_create_mosaic_dispatches_and_returns_ids(mocker): mock_delay.assert_called_once_with(42) -def test_create_mosaic_computes_footprint_and_barycenter_when_calibrations_exist(mocker): - moc_a = MOC.from_cone(lon=10 * u.deg, lat=20 * u.deg, radius=0.05 * u.deg, max_depth=12) +def test_create_mosaic_computes_footprint_and_barycenter_when_calibrations_exist( + mocker, +): + moc_a = MOC.from_cone( + lon=10 * u.deg, lat=20 * u.deg, radius=0.05 * u.deg, max_depth=12 + ) mock_db = MagicMock() mock_db.execute.return_value.all.return_value = [(moc_a,)] mock_db.refresh.side_effect = lambda obj: setattr(obj, "id", 7) diff --git a/src/services/tile_service.py b/src/services/tile_service.py index d34286c..e2a3db8 100644 --- a/src/services/tile_service.py +++ b/src/services/tile_service.py @@ -100,7 +100,9 @@ def generate_tile_tessellation( covered_area = moc_to_tile.sky_fraction * 41252.96 bary = moc_to_tile.barycenter() fov_degrees = 2 * np.sqrt(covered_area) - nearby_coords = all_sky_coords[all_sky_coords.separation(bary).degree <= fov_degrees] + nearby_coords = all_sky_coords[ + all_sky_coords.separation(bary).degree <= fov_degrees + ] x_step = orig_deg_width / 2.0 y_step = orig_deg_height / 2.0 @@ -158,9 +160,7 @@ def create_tiles(db: Session, project_id: int, tiles: list[dict]) -> list[Tile]: return [] rows = [{**tile, "project_id": project_id} for tile in tiles] - created = ( - db.execute(sa.insert(Tile).returning(Tile), rows).scalars().all() - ) + created = db.execute(sa.insert(Tile).returning(Tile), rows).scalars().all() db.flush() _associate_tiles_with_overlapping_calibrations(db, project_id, created) @@ -192,9 +192,7 @@ def _associate_tiles_with_overlapping_calibrations( db.execute( pg_insert(tile_level2_calibration_association) .values(association_rows) - .on_conflict_do_nothing( - index_elements=["tile_id", "level2_calibration_id"] - ) + .on_conflict_do_nothing(index_elements=["tile_id", "level2_calibration_id"]) ) From a9bc784ff4892e43fa83d32ab9377ce30fe724d8 Mon Sep 17 00:00:00 2001 From: davecoulter Date: Tue, 28 Jul 2026 16:36:45 -0400 Subject: [PATCH 3/3] Updating the pre-commit checks to include checking for uncommitted files. --- .claude/skills/gen_tests.md | 1 + .claude/skills/run_arch.md | 3 ++- CLAUDE.md | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.claude/skills/gen_tests.md b/.claude/skills/gen_tests.md index 0f0f479..f389c81 100644 --- a/.claude/skills/gen_tests.md +++ b/.claude/skills/gen_tests.md @@ -35,6 +35,7 @@ Derive the exact commands from context: never do this automatically as part of a QA step. - `docker compose exec api uv run pytest --cov=src --cov-fail-under=90 -q` - Sphinx locally: `DATABASE_URL="postgresql+psycopg2://dummy:dummy@localhost/dummy" REDIS_URL="redis://localhost:6379/0" uv run sphinx-build -b html docs docs/_build/html -W` +- Pre-commit locally: `uv run pre-commit run --all-files`. Before running this, check `git status --short` for `??` (untracked) entries. If any exist, stop and ask the user to commit them first — never stage or commit on their behalf. Pre-commit's `--all-files` only checks git-tracked files (confirmed empirically during doc 28's `genTests`), so running it against an untracked file gives a false "Passed" that doesn't reflect what will actually be pushed. Re-check any time a `genTests`-driven bug fix creates a new file — it needs to be committed again before the next pre-commit run means anything. Wait for the user to confirm all CLI steps pass before proceeding to Phase 2. If any CLI step fails: diagnose, fix, and ask the user to re-run that step before continuing. diff --git a/.claude/skills/run_arch.md b/.claude/skills/run_arch.md index add7c84..d7a54dd 100644 --- a/.claude/skills/run_arch.md +++ b/.claude/skills/run_arch.md @@ -22,4 +22,5 @@ Once authorized: 3. `docker compose up -d api worker_light worker_heavy` — recreates containers from fresh images; also clears stale worker memory 4. `docker compose exec api alembic upgrade head` — skip if no SQLAlchemy model changes 5. `docker compose exec api uv run pytest --cov=src --cov-fail-under=90 -q` — full test suite including integration tests against the real DB -6. **Sphinx — run locally, not in Docker** (docs/ is not mounted in the container): `DATABASE_URL="postgresql+psycopg2://dummy:dummy@localhost/dummy" REDIS_URL="redis://localhost:6379/0" uv run sphinx-build -b html docs docs/_build/html -W`" \ No newline at end of file +6. **Sphinx — run locally, not in Docker** (docs/ is not mounted in the container): `DATABASE_URL="postgresql+psycopg2://dummy:dummy@localhost/dummy" REDIS_URL="redis://localhost:6379/0" uv run sphinx-build -b html docs docs/_build/html -W` +7. Commit your changes (`git add` + `git commit`) before starting `genTests` — pre-commit's `--all-files` only checks git-tracked files, so QA validation is unreliable until new files are actually committed." \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index b2dfddf..ac640ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,7 @@ Claude may only suggest packages or write code that aligns with this strict base * **Docker State:** If a Celery worker's routing, queue topology, or task signature is changed, Claude must remind the user to run `docker compose restart worker_light worker_heavy` to clear stale memory. * **Bug Regression Tests:** If Claude fixes a bug, it must proactively write at least one specific unit test designed to catch that exact class of error in the future. * **Log Everything:** Any bug found and fixed at any phase — during `runPrompt` implementation, during `genTests` QA, or during ad-hoc debugging — must be logged as a dated entry in the `## Logs` section of the relevant architecture MD file immediately after the fix, not batched at the end. The entry must record what the bug was, what triggered its discovery, and what was changed. -* **Pre-Commit Runs Locally, Not in Docker:** `pre-commit` requires a `.git` directory, which is never copied or mounted into the `api`/`worker` containers. Running it via `docker compose exec` fails with `FatalError: git failed`. Always run `uv run pre-commit run --all-files` on the host, exactly like Sphinx. +* **Pre-Commit Runs Locally, Not in Docker:** `pre-commit` requires a `.git` directory, which is never copied or mounted into the `api`/`worker` containers. Running it via `docker compose exec` fails with `FatalError: git failed`. Always run `uv run pre-commit run --all-files` on the host, exactly like Sphinx. **`--all-files` silently skips untracked files** (confirmed empirically during doc 28's `genTests`: every file that later failed CI had been sitting uncommitted the whole session, and every local pre-commit run reported a false "Passed" for them) — this check is only meaningful once new files are actually committed, not just present on disk. * **Stuck Database States:** When modifying asynchronous tasks, always verify that database rows cannot be left in an orphaned `IN_PROCESS` state if the worker crashes. * **Streaming & Bounded Resource Use:** Any task processing a batch of external files/objects (downloads, parses, uploads) must do so in a bounded, streaming fashion — one item (or a small fixed chunk) at a time — never materializing an entire batch before doing any work. Peak memory/disk must not scale unboundedly with batch size. Progress must be recoverable/visible (an incrementing counter, not a single all-or-nothing flip), and one item's failure must not take the whole task or its container down with it. * **Transactions:** Any global or framework-level error handler that intercepts a crash to update a database status MUST explicitly call `db.rollback()` before attempting to write the failure state, preventing `PendingRollbackError` crashes. @@ -77,4 +77,5 @@ Claude may only suggest packages or write code that aligns with this strict base * **Action:** Added the CI Service Parity guardrail. Surfaced during `genTests` on doc 27: `minio` was added to `docker-compose.yml` for S3 storage, but `.github/workflows/ci.yml` was never updated to provision it, and this went unnoticed until a real (unmocked) integration test against it was proposed. Distinguished from `redis`, which correctly has no CI service today because nothing in the suite opens a real Redis connection (Celery dispatch is always mocked) — the guardrail's trigger is deliberately "a real integration test needs this service," not "a service exists in `docker-compose.yml`," so it doesn't over-fire on services like `flower`/`portainer`/`jaeger`/`ui` that never need CI wiring. ###### 2026-07-28 +* **Action:** Amended the Pre-Commit Runs Locally gotcha, and added a commit checkpoint to `.claude/skills/run_arch.md`'s Post-Implementation Verification Checklist plus an explicit, gated pre-commit step to `.claude/skills/gen_tests.md`'s Phase 1. Surfaced after doc 28's PR: CI failed on `ruff`/`ruff-format` even though every local `pre-commit run --all-files` during the entire `genTests` session had reported clean. Root cause, confirmed empirically (a fresh untracked file with a deliberate lint violation got a false "Passed"): `--all-files` only checks git-tracked files, and the whole doc-28 implementation had been sitting uncommitted the entire session — pre-commit silently never checked any of the ~24 new files until the user finally committed and pushed, at which point CI caught them for the first time. Fix is workflow, not automation: the user commits (Claude does not auto-stage or auto-commit, by explicit preference), but `genTests` now checks `git status --short` for untracked entries before trusting a pre-commit run, and `runPrompt`'s checklist now ends with an explicit reminder to commit before `genTests` begins. * **Action:** Added the QA/Debugging Fix Confirmation guardrail and the Streaming & Bounded Resource Use checklist item. Surfaced during `genTests` on doc 28: `run_ingest_batch` downloaded an entire batch's files into a temp directory before doing any parsing, and a real ~20-file batch of real JWST FITS data OOM-killed the whole `worker_light` container (`WORKER_LIGHT_MEM_LIMIT`/`CELERY_LIGHT_MAX_MEMORY_PER_CHILD` were sized for the trivial Stage-0 dummy `sleep()` task and never revisited). Because an OOM kill (SIGKILL) isn't a catchable Python exception, the task's own crash-handling `except` block never ran, leaving its `IngestBatch` row permanently stuck `in_process` — a direct violation of the existing Stuck Database States guardrail. Claude proposed and applied a streaming per-file fix (download→parse→delete one file at a time, with real incremental progress) without pausing to confirm first, which prompted the QA/Debugging Fix Confirmation addition. Separately, `run_ingest_batch`/`run_mosaic_drizzle` were re-routed from `light` back to `heavy_memory` — real FITS/astropy ingest is memory-hungry even though it's not GPU-bound, and `light`'s multi-child concurrency (`CELERY_LIGHT_CONCURRENCY=4`, all children sharing one container memory ceiling) made it a poor fit regardless of any per-task memory tuning; `heavy` is redefined for now to mean "resource/memory intensive," not GPU-specific — `CeleryQueue.GPU` remains unused/reserved for whenever real GPU-bound work exists (most likely the JWST pipeline doc). \ No newline at end of file