feat: validate lexicons

compose.override.yaml

@@ -0,0 +1,51 @@
+# Development overrides for docker-compose
+# This file is automatically loaded by docker-compose and overrides compose.yaml
+# For production, use: docker-compose -f compose.yaml up
+#
+# In development, run lexhub locally with: npm run dev
+# Only nexus and postgres run in Docker for development
+
+services:
+  lexhub:
+    # Disable lexhub service in development
+    # Run it locally instead with: npm run dev
+    scale: 0
+
+  nexus:
+    restart: no
+    network_mode: host
+    ports: []
+    volumes: []
+    environment:
+      # Reduced parallelism for lighter resource usage
+      - NEXUS_FIREHOSE_PARALLELISM=5
+      - NEXUS_RESYNC_PARALLELISM=2
+      - NEXUS_OUTBOX_PARALLELISM=2
+      # Debug logging for development
+      - NEXUS_LOG_LEVEL=debug
+      - NEXUS_WEBHOOK_URL=http://host.docker.internal:10000/api/ingest
+
+  postgres:
+    restart: no
+    command:
+      - "postgres"
+      - "-c"
+      - "shared_preload_libraries=pg_stat_statements"
+      - "-c"
+      - "pg_stat_statements.track=all"
+      # Reduced resources for development
+      - "-c"
+      - "max_connections=100"
+      - "-c"
+      - "shared_buffers=128MB"
+      - "-c"
+      - "effective_cache_size=512MB"
+      - "-c"
+      - "work_mem=8MB"
+      - "-c"
+      - "maintenance_work_mem=64MB"
+      # SQL query logging for debugging
+      - "-c"
+      - "log_statement=all"
+      - "-c"
+      - "log_duration=on"

drizzle/0000_init.sql

@@ -1,26 +1,31 @@
-CREATE TABLE "lexicons" (
-	"id" varchar(317) NOT NULL,
+-- Create valid_lexicons table
+CREATE TABLE "valid_lexicons" (
+	"nsid" varchar(317) NOT NULL,
 	"cid" varchar(100) NOT NULL,
+	"repo_did" varchar(256) NOT NULL,
+	"repo_rev" varchar(256) NOT NULL,
 	"data" jsonb NOT NULL,
-	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
-	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
-	CONSTRAINT "lexicons_id_cid_pk" PRIMARY KEY("id","cid")
+	"ingested_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "valid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did")
 );
 --> statement-breakpoint
-CREATE INDEX "lexicons_id_idx" ON "lexicons" USING btree ("id");--> statement-breakpoint
-CREATE INDEX "lexicons_created_at_idx" ON "lexicons" USING btree ("created_at");--> statement-breakpoint
-CREATE INDEX "lexicons_data_gin_idx" ON "lexicons" USING gin ("data");--> statement-breakpoint
-
-
-CREATE OR REPLACE FUNCTION update_updated_at_column()
-RETURNS TRIGGER AS $$
-BEGIN
-    NEW.updated_at = NOW();
-    RETURN NEW;
-END;
-$$ language 'plpgsql';
-
-CREATE TRIGGER update_lexicons_updated_at
-	BEFORE UPDATE ON "lexicons"
-	FOR EACH ROW
-	EXECUTE FUNCTION update_updated_at_column();
+-- Create invalid_lexicons table
+CREATE TABLE "invalid_lexicons" (
+	"nsid" varchar(317) NOT NULL,
+	"cid" varchar(100) NOT NULL,
+	"repo_did" varchar(256) NOT NULL,
+	"repo_rev" varchar(256) NOT NULL,
+	"raw_data" jsonb NOT NULL,
+	"validation_errors" jsonb NOT NULL,
+	"ingested_at" timestamp with time zone DEFAULT now() NOT NULL,
+	CONSTRAINT "invalid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did")
+);
+--> statement-breakpoint
+-- Create indexes for valid_lexicons
+CREATE INDEX "valid_lexicons_nsid_idx" ON "valid_lexicons" USING btree ("nsid");--> statement-breakpoint
+CREATE INDEX "valid_lexicons_repo_did_idx" ON "valid_lexicons" USING btree ("repo_did");--> statement-breakpoint
+CREATE INDEX "valid_lexicons_data_gin_idx" ON "valid_lexicons" USING gin ("data");--> statement-breakpoint
+-- Create indexes for invalid_lexicons
+CREATE INDEX "invalid_lexicons_nsid_idx" ON "invalid_lexicons" USING btree ("nsid");--> statement-breakpoint
+CREATE INDEX "invalid_lexicons_repo_did_idx" ON "invalid_lexicons" USING btree ("repo_did");--> statement-breakpoint
+CREATE INDEX "invalid_lexicons_raw_data_gin_idx" ON "invalid_lexicons" USING gin ("raw_data");

drizzle/meta/_journal.json

@@ -5,7 +5,7 @@
     {
       "idx": 0,
       "version": "7",
-      "when": 1763597119878,
+      "when": 1733093280000,
       "tag": "0000_init",
       "breakpoints": true
     }