diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85db8f3..573be00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - +# CI gates for raglogs. Enforces what the project actually cares about: +# ruff lint, unit tests with a src/core coverage floor, and integration tests +# against a real Postgres+pgvector service — across the supported Python range. name: CI on: @@ -11,54 +11,87 @@ on: paths-ignore: - "**/*.md" - "CODEOWNERS" - - "Makefile" pull_request: branches: [ "main" ] paths-ignore: - "**/*.md" - "CODEOWNERS" - - "Makefile" permissions: contents: read jobs: - build: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.12"] + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: raglogs + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DB_URL: postgresql+psycopg://postgres:postgres@localhost:5432/raglogs + steps: + - uses: actions/checkout@v7 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Lint (ruff) + run: make lint + - name: Unit tests (with src/core coverage floor) + # Floor starts at the current level (~87%); ratchet upward over time, + # never set an aspirational number that turns main red. + run: pytest tests/unit/ --cov=src/core --cov-report=term-missing --cov-fail-under=85 + - name: Integration tests (real Postgres + pgvector) + run: make test-int + openapi: runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Export OpenAPI + run: PYTHONPATH=. python scripts/export_openapi.py + - name: Upload OpenAPI artifact + uses: actions/upload-artifact@v7 + with: + name: openapi.json + path: clients/openapi.json + docker: + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - name: Set up Python 3.10 - uses: actions/setup-python@v7 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest - - name: Export OpenAPI - run: | - PYTHONPATH=. python scripts/export_openapi.py - - name: Upload OpenAPI artifact - uses: actions/upload-artifact@v7 - with: - name: openapi.json - path: clients/openapi.json + - uses: actions/checkout@v7 + - name: Docker build smoke test + run: docker build -t raglogs:ci . publish-openapi-release: if: startsWith(github.ref, 'refs/tags/') - needs: build + needs: openapi runs-on: ubuntu-latest permissions: contents: write diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b7c0426..3396ef7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ This document covers how to get set up, what the codebase expects, and how to su git clone https://github.com/leo-aa88/raglogs cd raglogs python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt && pip install -e . +pip install -e ".[dev]" cp .env.example .env # Edit .env — set DB_URL at minimum docker compose up postgres -d diff --git a/Dockerfile b/Dockerfile index 7812a50..1dfc95d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,10 +7,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpq-dev \ && rm -rf /var/lib/apt/lists/* -COPY pyproject.toml . -RUN pip install --no-cache-dir -e ".[dev]" - +# Copy the whole project before installing: the hatchling build backend reads +# README.md (readme=) and src/ (packages=["src"]) to generate metadata, so an +# editable install can't run against pyproject.toml alone. COPY . . +RUN pip install --no-cache-dir -e ".[dev]" EXPOSE 8000 diff --git a/Makefile b/Makefile index 4351bf2..be94aee 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,6 @@ help: # ── Setup ───────────────────────────────────────────────────────────────────── -install-requirements: - $(PIP) install -r requirements.txt - install-dev: $(PIP) install -e ".[dev]" diff --git a/README.md b/README.md index 3b7ae06..5570d82 100644 --- a/README.md +++ b/README.md @@ -1305,8 +1305,8 @@ injects the header, or keep auth off on loopback). `/health` stays public. ## Development ```bash -# Install everything -pip install -r requirements.txt && pip install -e . +# Install everything (runtime + dev tooling) +pip install -e ".[dev]" # Unit tests (no DB needed) make test-unit diff --git a/pyproject.toml b/pyproject.toml index 269fecd..956e1a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dev = [ "pytest-cov>=5.0.0", "anyio>=4.0.0", "moto[logs]>=5.0.0", + "ruff>=0.6.0", ] [project.scripts] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6059671..0000000 --- a/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -typer>=0.12.0 -fastapi>=0.111.0 -uvicorn[standard]>=0.30.0 -sqlalchemy[asyncio]>=2.0.0 -alembic>=1.13.0 -psycopg[binary,pool]>=3.1.0 -pgvector>=0.3.0 -pydantic>=2.7.0 -pydantic-settings>=2.3.0 -rich>=13.7.0 -orjson>=3.10.0 -httpx>=0.27.0 -tenacity>=8.3.0 -boto3>=1.34.0 -numpy>=1.26.0 -scikit-learn>=1.5.0 -python-dateutil>=2.9.0 -structlog>=24.1.0 -python-dotenv>=1.0.0 -jinja2>=3.1.0 -argon2-cffi>=23.1.0 -PyJWT>=2.8.0 -cryptography>=42.0.0 -prometheus_client>=0.21.0 -opentelemetry-api>=1.27.0 -opentelemetry-sdk>=1.27.0 -opentelemetry-exporter-otlp-proto-http>=1.27.0 diff --git a/tests/integration/test_cloudwatch_ingest.py b/tests/integration/test_cloudwatch_ingest.py index ccf87b7..3301447 100644 --- a/tests/integration/test_cloudwatch_ingest.py +++ b/tests/integration/test_cloudwatch_ingest.py @@ -29,6 +29,9 @@ def db_session(): with engine.connect() as conn: conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) conn.commit() + # Clean slate so tests stay isolated when they share one database + # (e.g. a single Postgres service across the whole CI run). + Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with get_db() as db: @@ -50,19 +53,22 @@ def test_cloudwatch_ingest_then_explain(db_session): client.create_log_stream(logGroupName="/aws/lambda/my-service", logStreamName="stream-1") now_ms = int(datetime.now(tz=timezone.utc).timestamp() * 1000) + # Distinct messages per event: identical content would be collapsed by + # the ingest content-dedup (see test_ingest_dedup), leaving one row and + # defeating this test's point — that all five persist with a timestamp. client.put_log_events( logGroupName="/aws/lambda/my-service", logStreamName="stream-1", logEvents=[ { - "timestamp": now_ms, + "timestamp": now_ms + i * 1000, "message": ( '{"level": "error", "message": ' - '"Stripe webhook signature verification failed", ' + f'"Stripe webhook signature verification failed (attempt {i})", ' '"service": "billing-worker"}' ), } - for _ in range(5) + for i in range(5) ], ) diff --git a/tests/integration/test_full_flow.py b/tests/integration/test_full_flow.py index 7c33011..f7c39fe 100644 --- a/tests/integration/test_full_flow.py +++ b/tests/integration/test_full_flow.py @@ -29,6 +29,9 @@ def db_session(): with engine.connect() as conn: conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) conn.commit() + # Clean slate so tests stay isolated when they share one database + # (e.g. a single Postgres service across the whole CI run). + Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with get_db() as db: diff --git a/tests/integration/test_ingest_dedup.py b/tests/integration/test_ingest_dedup.py index 74d6c38..946259a 100644 --- a/tests/integration/test_ingest_dedup.py +++ b/tests/integration/test_ingest_dedup.py @@ -27,6 +27,9 @@ def db_session(): with engine.connect() as conn: conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) conn.commit() + # Clean slate so tests stay isolated when they share one database + # (e.g. a single Postgres service across the whole CI run). + Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with get_db() as db: diff --git a/tests/integration/test_trigger_search.py b/tests/integration/test_trigger_search.py index b626d4a..3b09de5 100644 --- a/tests/integration/test_trigger_search.py +++ b/tests/integration/test_trigger_search.py @@ -62,6 +62,9 @@ def db_session(): with engine.connect() as conn: conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) conn.commit() + # Clean slate so tests stay isolated when they share one database + # (e.g. a single Postgres service across the whole CI run). + Base.metadata.drop_all(engine) Base.metadata.create_all(engine) with get_db() as db: