edi json, access token, edi pipeline, edi envelope, cdc fix, outbound path testing

.agents/AGENTS.md

@@ -4,3 +4,6 @@
 - Use proper separation of concerns (e.g. TanStack layout routes instead of polluting `__root.tsx`).
 - Implement robust error handling, proper typing, and scalable folder structures from the very first commit.
 - Never use anti-patterns to save time. If a proper implementation takes more steps, take the time to do it right.
+
+# Package Manager
+- ALWAYS use `pnpm` for frontend/Node.js package management instead of `npm`. Do not use `npm install`.

Makefile

@@ -45,19 +45,19 @@ dev:
 
 dev-as2:
 	@echo "Starting AS2 Server with hot-reload for local development..."
-	ENVIRONMENT=development uv run uvicorn as2_server.main:app --reload --port 8000
+	ENVIRONMENT=development uv run uvicorn as2_server.main:app --reload --host 0.0.0.0 --port 8000
 
 dev-api:
 	@echo "Starting API Gateway with hot-reload for local development..."
-	ENVIRONMENT=development uv run uvicorn api.main:app --reload --port 8001
+	ENVIRONMENT=development uv run uvicorn api.main:app --reload --host 0.0.0.0 --port 8001
 
 dev-web:
 	@echo "Starting React Frontend with Vite..."
 	cd frontend/web && pnpm dev
 
 dev-worker:
-	@echo "Starting Provision Worker for local development..."
-	ENVIRONMENT=development PYTHONPATH=services/worker/src:libs/database/src:libs/config/src:libs/pipeline/src uv run python services/worker/src/worker/provision/main.py
+	@echo "Starting Unified Worker (Data + Provision) for local development..."
+	ENVIRONMENT=development PYTHONPATH=services/worker/src:libs/database/src:libs/config/src:libs/pipeline/src:libs/domain/src:libs/transformer/src uv run python services/worker/src/worker/main.py
 
 db-init:
 	@echo "Waiting for databases to be ready..."
@@ -69,16 +69,23 @@ db-init:
 
 db-reset:
 	@echo "Wiping application databases (leaving Zitadel intact)..."
-	docker compose stop postgres_global postgres_shard_1
-	docker compose rm -f -v postgres_global postgres_shard_1
-	-docker volume rm $$(docker volume ls -q | grep -E "postgres_global_data|postgres_shard_[0-9]+_data") 2>/dev/null
-	@echo "Restarting application databases..."
-	docker compose up -d postgres_global postgres_shard_1
+	docker compose stop postgres_global postgres_shard_1 debezium_shard_1
+	docker compose rm -f -v postgres_global postgres_shard_1 debezium_shard_1
+	-docker volume rm $$(docker volume ls -q | grep -E "postgres_global_data|postgres_shard_[0-9]+_data|debezium_data") 2>/dev/null
+	@echo "Restarting application databases and Debezium..."
+	docker compose up -d postgres_global postgres_shard_1 debezium_shard_1
 	@echo "Waiting for databases to initialize..."
 	sleep 5
 	@echo "Re-running migrations and seeding..."
 	$(MAKE) db-init
 
+sqs-purge:
+	@echo "Purging all LocalStack SQS queues..."
+	uv run python scripts/purge_sqs.py
+
+db-sqs-reset: db-reset sqs-purge
+	@echo "Database and SQS queues have been completely reset."
+
 seed: db-init
 
 # --- Docker Infrastructure (Postgres, LocalStack, OTel) ---

TECHNICAL_DEBT.md

@@ -1,10 +1,39 @@
-# Technical Debt
+# Technical Debt Register
 
-A living record of known gaps, shortcuts, and missing enterprise capabilities.
+This document tracks identified technical debt, proposed refactoring, and estimated effort to resolve.
 
----
+## 1. Modernization of Bots Core (Stateless AST Migration)
 
-## AS2 Protocol
+**Description:**
+The open-source `bots_core` library heavily relies on a legacy, stateful `Node` class pattern to build, traverse, and count EDI segments (e.g., `SE01`, `GE01` counting via `node.getcount()`). Currently, we are bypassing this internal engine completely in our modern microservices architecture by injecting raw stateless Python JSON dictionaries (AST format) into the `bots_core.facade`.
+
+Because `bots_core` doesn't natively parse our stateless AST for trailing counts, we temporarily built an `ASTUtils.count_segments()` workaround in the `transformer` domain layer.
+
+**Proposed Resolution:**
+To achieve the long-term vision of completely merging and modernizing `bots_core` as a first-class citizen of our modern Python stack:
+1. Refactor the inner workings of `bots_core` (specifically `message.py`, `outmessage.py`, and `node.py`) to natively accept, traverse, and serialize stateless dictionary ASTs without forcing instantiation of legacy `Node` objects.
+2. Move AST utility functions (like `count_segments`) natively into `bots_core/domain/ast_utils.py`.
+3. Eliminate the legacy mapping engine scripts (`inn.get()`, `out.put()`) internally inside `bots_core` if they are fully deprecated by the API gateway.
+
+**Estimated Effort:** Medium-High
+**Estimated Time:** 1 to 2 Sprint Weeks (40 - 80 hours)
+**Impact:** Will result in a completely modernized, lightning-fast, fully stateless fork of the `bots` EDI engine perfectly aligned with our cloud-native event-driven architecture.
+
+## 2. Outbox Sweeper (CDC Fallback Relay)
+
+**Description:**
+The system currently relies exclusively on Debezium (CDC) reading the PostgreSQL Write-Ahead Log (WAL) to route `Outbox` events to SQS. If Debezium crashes, loses offsets, or experiences network partitioning, `PENDING` outbox events will be permanently trapped in the database, breaking the asynchronous event pipeline.
+
+**Proposed Resolution:**
+Implement an Outbox Sweeper background worker that acts as a robust enterprise fallback and garbage collector:
+1. **Fallback Poller:** A cron/scheduled task that periodically queries `SELECT * FROM outbox WHERE status = 'PENDING'` for events older than a configured threshold (e.g., 60 seconds) and manually relays them to SQS.
+2. **Garbage Collector:** A cleanup task that runs `DELETE FROM outbox WHERE status = 'COMPLETED'` for events older than 7 days to prevent unbounded database growth.
+
+**Estimated Effort:** Low
+**Estimated Time:** 1 to 2 days
+**Impact:** Essential for enterprise-grade high availability. Guarantees no messages are ever lost due to CDC infrastructure failures and keeps the database optimized over time.
+
+## 3. AS2 Protocol
 
 ### Async MDN — Inbound Callback Not Implemented
 **Priority:** High
@@ -25,11 +54,17 @@ A living record of known gaps, shortcuts, and missing enterprise capabilities.
   store the sent `Message-ID` and `MIC` so they can be matched when the callback arrives.
 - Database: Add an `outbound_mdn_pending` table `(message_id, mic, trace_id, expires_at)`.
 
----
-
-## Testing
+## 4. Testing
 
 ### No Frontend Test Runner (Vitest)
+
 **Priority:** Medium
 `make test` skips frontend tests with a placeholder comment.
 React component tests and TanStack Query mutation tests are not covered.
+
+## 5. Database Schema
+
+### Missing `edi_headers` Table
+
+**Priority:** Medium
+**Description:** We currently lack an `edi_headers` table to store extracted EDI header metadata (e.g. ST/GS segments). This table needs to be created and linked via foreign key to the `outbound_route` table so that EDI messages can be properly tracked and correlated with their configured outbound routes.

docker-compose.yml

@@ -1,17 +1,7 @@
-version: "3.9"
-
-  # Include the observability and identity stacks
+version: '3.9'
 include:
-  # - libs/observability/docker/docker-compose.yml
-  - libs/identity/docker/docker-compose.yml
-
+- libs/identity/docker/docker-compose.yml
 services:
-
-  # ---------------------------------------------------------------------------
-  # Postgres Databases (Hybrid Multi-Tenancy Architecture)
-  # ---------------------------------------------------------------------------
-
-  # 1. Global Control Plane DB
   postgres_global:
     image: postgres:15-alpine
     container_name: edi_postgres_global
@@ -20,127 +10,110 @@ services:
       POSTGRES_PASSWORD: edi_password
       POSTGRES_DB: edi_global
     ports:
-      - "5432:5432"
+    - 5432:5432
     volumes:
-      - postgres_global_data:/var/lib/postgresql/data
+    - postgres_global_data:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U edi"]
+      test:
+      - CMD-SHELL
+      - pg_isready -U edi
       interval: 5s
       timeout: 5s
       retries: 5
-
-  # 2. Standard Tier (Pooled Shard) DB
   postgres_shard_1:
     image: postgres:15-alpine
     container_name: edi_postgres_shard_1
     environment:
       POSTGRES_USER: edi
       POSTGRES_PASSWORD: edi_password
       POSTGRES_DB: edi_shard_1
-    command: ["postgres", "-c", "wal_level=logical"]
+    command:
+    - postgres
+    - -c
+    - wal_level=logical
     ports:
-      - "5433:5432"
+    - 5433:5432
     volumes:
-      - postgres_shard_1_data:/var/lib/postgresql/data
+    - postgres_shard_1_data:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U edi"]
+      test:
+      - CMD-SHELL
+      - pg_isready -U edi
       interval: 5s
       timeout: 5s
       retries: 5
-
-  # ---------------------------------------------------------------------------
-  # LocalStack (AWS Cloud Emulator for S3, SQS, SNS, KMS, etc.)
-  # ---------------------------------------------------------------------------
   localstack:
     image: localstack/localstack:3.8.0
     container_name: edi_localstack
     ports:
-      - "4566:4566"            # LocalStack Gateway
-      - "4510-4559:4510-4559"  # external services port range
+    - 4566:4566
+    - 4510-4559:4510-4559
     environment:
-      - DEBUG=${DEBUG:-0}
-      - DOCKER_HOST=unix:///var/run/docker.sock
+    - DEBUG=${DEBUG:-0}
+    - DOCKER_HOST=unix:///var/run/docker.sock
     volumes:
-      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
-      - "/var/run/docker.sock:/var/run/docker.sock"
-      # Auto-init scripts are executed when LocalStack boots
-      - ./docker/localstack/init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh
+    - ${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack
+    - /var/run/docker.sock:/var/run/docker.sock
+    - ./docker/localstack/init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
+      test:
+      - CMD
+      - curl
+      - -f
+      - http://localhost:4566/_localstack/health
       interval: 5s
       timeout: 5s
       retries: 5
-
-  # ---------------------------------------------------------------------------
-  # HashiCorp Vault (Secrets & PKI)
-  # ---------------------------------------------------------------------------
   vault:
     image: hashicorp/vault:1.15
     container_name: edi_vault
     ports:
-      - "127.0.0.1:8200:8200"
+    - 127.0.0.1:8200:8200
     environment:
-      VAULT_DEV_ROOT_TOKEN_ID: "${VAULT_DEV_ROOT_TOKEN_ID}"
-      VAULT_DEV_LISTEN_ADDRESS: "0.0.0.0:8200"
+      VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_DEV_ROOT_TOKEN_ID:-root}
+      VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
     cap_add:
-      - IPC_LOCK
+    - IPC_LOCK
     healthcheck:
-      test: ["CMD", "vault", "status", "-address=http://127.0.0.1:8200"]
+      test:
+      - CMD
+      - vault
+      - status
+      - -address=http://127.0.0.1:8200
       interval: 5s
       timeout: 5s
       retries: 5
-
-#  # ---------------------------------------------------------------------------
-#  # EDI AS2 Server (FastAPI)
-#  # ---------------------------------------------------------------------------
-#  as2-server:
-#    build:
-#      context: .
-#      dockerfile: soopaedi/services/as2_server/Dockerfile
-#    container_name: edi_as2_server
-#    ports:
-#      - "8000:8000"
-#    environment:
-#      - APP_ENV=development
-#      - LOG_LEVEL=DEBUG
-#      - DB_URL=postgresql+asyncpg://edi:edi_password@postgres:5432/edi
-#      - S3_BUCKET=edi-as2-payloads
-#      - S3_ENDPOINT_URL=http://localstack:4566
-#      - S3_REGION=us-east-1
-#      - S3_ACCESS_KEY_ID=test
-#      - S3_SECRET_ACCESS_KEY=test
-#      - OTEL_SERVICE_NAME=edi-as2-server
-#      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
-#    depends_on:
-#      postgres:
-#        condition: service_healthy
-#      localstack:
-#        condition: service_healthy
-
-  # ---------------------------------------------------------------------------
-  # Debezium CDC Server
-  # ---------------------------------------------------------------------------
-  debezium:
+  cdcproxy:
+    image: nginx:alpine
+    container_name: edi_cdcproxy
+    ports:
+    - 8002:80
+    command: "/bin/sh -c \"echo '\nserver {\n    listen 80;\n    location / {\n  \
+      \      proxy_pass http://host.docker.internal:8001;\n        proxy_set_header\
+      \ Connection \\\"\\\";\n        proxy_set_header Upgrade \\\"\\\";\n    }\n\
+      }' > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'\""
+    extra_hosts:
+    - host.docker.internal:host-gateway
+  debezium_shard_1:
     image: quay.io/debezium/server:2.7.3.Final
-    container_name: edi_debezium
+    container_name: edi_debezium_shard_1
     environment:
-      - DEBEZIUM_SOURCE_DATABASE_HOSTNAME=postgres_shard_1
-      - DEBEZIUM_SOURCE_DATABASE_PORT=5432
-      - DEBEZIUM_SOURCE_DATABASE_USER=edi
-      - DEBEZIUM_SOURCE_DATABASE_PASSWORD=edi_password
-      - DEBEZIUM_SOURCE_DATABASE_DBNAME=edi_shard_1
-      - DEBEZIUM_SINK_URL=http://host.docker.internal:8001/internal/cdc/relay
+    - DEBEZIUM_SOURCE_DATABASE_HOSTNAME=postgres_shard_1
+    - DEBEZIUM_SOURCE_DATABASE_PORT=5432
+    - DEBEZIUM_SOURCE_DATABASE_USER=edi
+    - DEBEZIUM_SOURCE_DATABASE_PASSWORD=edi_password
+    - DEBEZIUM_SOURCE_DATABASE_DBNAME=edi_shard_1
+    - DEBEZIUM_SINK_URL=http://cdcproxy/internal/cdc/relay
     extra_hosts:
-      - "host.docker.internal:host-gateway"
+    - host.docker.internal:host-gateway
     volumes:
-      - ./docker/debezium:/debezium/conf
-      - debezium_data:/debezium/data
+    - ./docker/debezium/application.properties:/debezium/conf/application.properties
+    - debezium_data:/debezium/data
     depends_on:
       postgres_shard_1:
         condition: service_healthy
-
 volumes:
-  postgres_global_data:
-  postgres_shard_1_data:
-  postgres_enterprise_1_data:
-  debezium_data:
+  postgres_global_data: null
+  postgres_shard_1_data: null
+  postgres_enterprise_1_data: null
+  debezium_data: null

docker/debezium/application-global.properties

@@ -0,0 +1,32 @@
+# ── Source: Postgres WAL ──────────────────────────────────────────────────────
+debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector
+debezium.source.topic.prefix=soopa_global
+debezium.source.database.hostname=${DEBEZIUM_SOURCE_DATABASE_HOSTNAME:postgres_global}
+debezium.source.database.port=${DEBEZIUM_SOURCE_DATABASE_PORT:5432}
+debezium.source.database.user=${DEBEZIUM_SOURCE_DATABASE_USER:edi}
+debezium.source.database.password=${DEBEZIUM_SOURCE_DATABASE_PASSWORD:edi_password}
+debezium.source.database.dbname=${DEBEZIUM_SOURCE_DATABASE_DBNAME:edi_global}
+debezium.source.plugin.name=pgoutput
+debezium.source.publication.name=edi_global_cdc
+debezium.source.slot.name=edi_global_slot
+
+debezium.source.table.include.list=.*\\.outbox
+debezium.source.snapshot.mode=initial
+
+# ── Sink: HTTP → FastAPI CdcRelay ───────────────────────────────────
+debezium.sink.type=http
+debezium.sink.http.url=${DEBEZIUM_SINK_URL:http://host.docker.internal:8001/internal/cdc/relay}
+debezium.sink.http.timeout.ms=10000
+debezium.sink.http.retry.count=10
+debezium.sink.http.retry.delay.ms=500
+debezium.format.value=json
+debezium.format.value.schemas.enable=false
+debezium.format.key=json
+debezium.transforms=unwrap
+debezium.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState
+debezium.transforms.unwrap.drop.tombstones=true
+debezium.transforms.unwrap.add.fields=table,schema,op
+
+debezium.source.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore
+debezium.source.offset.storage.file.filename=/debezium/data/offsets-global.dat
+debezium.source.offset.flush.interval.ms=5000

docker/localstack/init-aws.sh

@@ -14,4 +14,18 @@ DLQ_ARN=$(awslocal sqs get-queue-attributes --queue-url http://localhost:4566/00
 # Create main queue with redrive policy
 awslocal sqs create-queue --queue-name EdiTransformerQueue --attributes "{\"RedrivePolicy\":\"{\\\"deadLetterTargetArn\\\":\\\"$DLQ_ARN\\\",\\\"maxReceiveCount\\\":\\\"3\\\"}\"}"
 
+# Create Data Plane CDC Queues and DLQs
+awslocal sqs create-queue --queue-name TranslateQueue-DLQ
+TRANSLATE_DLQ_ARN=$(awslocal sqs get-queue-attributes --queue-url http://localhost:4566/000000000000/TranslateQueue-DLQ --attribute-names QueueArn --query 'Attributes.QueueArn' --output text)
+awslocal sqs create-queue --queue-name TranslateQueue --attributes "{\"RedrivePolicy\":\"{\\\"deadLetterTargetArn\\\":\\\"$TRANSLATE_DLQ_ARN\\\",\\\"maxReceiveCount\\\":\\\"3\\\"}\"}"
+
+awslocal sqs create-queue --queue-name DeliverQueue-DLQ
+DELIVER_DLQ_ARN=$(awslocal sqs get-queue-attributes --queue-url http://localhost:4566/000000000000/DeliverQueue-DLQ --attribute-names QueueArn --query 'Attributes.QueueArn' --output text)
+awslocal sqs create-queue --queue-name DeliverQueue --attributes "{\"RedrivePolicy\":\"{\\\"deadLetterTargetArn\\\":\\\"$DELIVER_DLQ_ARN\\\",\\\"maxReceiveCount\\\":\\\"3\\\"}\"}"
+
+# Create Control Plane CDC Queues and DLQs
+awslocal sqs create-queue --queue-name ProvisioningQueue-DLQ
+PROVISIONING_DLQ_ARN=$(awslocal sqs get-queue-attributes --queue-url http://localhost:4566/000000000000/ProvisioningQueue-DLQ --attribute-names QueueArn --query 'Attributes.QueueArn' --output text)
+awslocal sqs create-queue --queue-name ProvisioningQueue --attributes "{\"RedrivePolicy\":\"{\\\"deadLetterTargetArn\\\":\\\"$PROVISIONING_DLQ_ARN\\\",\\\"maxReceiveCount\\\":\\\"3\\\"}\"}"
+
 echo "LocalStack Initialization Complete."