From 27960dd89a379f7ed15ef7a65b3260fc96da9083 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Tue, 25 Nov 2025 15:07:43 -0800 Subject: [PATCH 01/15] feat: add @atproto/lexicon-resolver package --- package-lock.json | 560 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 561 insertions(+) diff --git a/package-lock.json b/package-lock.json index 0df87a0..a856c0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "lexhub", "version": "0.1.0", "dependencies": { + "@atproto/lexicon-resolver": "^0.2.4", "drizzle-kit": "^0.31.7", "drizzle-orm": "^0.44.7", "next": "16.0.3", @@ -23,6 +24,189 @@ "typescript": "^5" } }, + "node_modules/@atproto-labs/fetch": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@atproto-labs/fetch/-/fetch-0.2.3.tgz", + "integrity": "sha512-NZtbJOCbxKUFRFKMpamT38PUQMY0hX0p7TG5AEYOPhZKZEP7dHZ1K2s1aB8MdVH0qxmqX7nQleNrrvLf09Zfdw==", + "license": "MIT", + "dependencies": { + "@atproto-labs/pipe": "0.1.1" + } + }, + "node_modules/@atproto-labs/fetch-node": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@atproto-labs/fetch-node/-/fetch-node-0.2.0.tgz", + "integrity": "sha512-Krq09nH/aeoiU2s9xdHA0FjTEFWG9B5FFenipv1iRixCcPc7V3DhTNDawxG9gI8Ny0k4dBVS9WTRN/IDzBx86Q==", + "license": "MIT", + "dependencies": { + "@atproto-labs/fetch": "0.2.3", + "@atproto-labs/pipe": "0.1.1", + "ipaddr.js": "^2.1.0", + "undici": "^6.14.1" + }, + "engines": { + "node": ">=18.7.0" + } + }, + "node_modules/@atproto-labs/pipe": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@atproto-labs/pipe/-/pipe-0.1.1.tgz", + "integrity": "sha512-hdNw2oUs2B6BN1lp+32pF7cp8EMKuIN5Qok2Vvv/aOpG/3tNSJ9YkvfI0k6Zd188LeDDYRUpYpxcoFIcGH/FNg==", + "license": "MIT" + }, + "node_modules/@atproto/common": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@atproto/common/-/common-0.5.1.tgz", + "integrity": "sha512-0S57sjzw4r9OLc5srJFi6uAz/aTKYl6btz3x36tSnGriL716m6h0x2IVtgd+FhUfIQfisevrqcqw8SfaGk8VTw==", + "license": "MIT", + "dependencies": { + "@atproto/common-web": "^0.4.5", + "@atproto/lex-cbor": "0.0.1", + "@atproto/lex-data": "0.0.1", + "iso-datestring-validator": "^2.2.2", + "multiformats": "^9.9.0", + "pino": "^8.21.0" + }, + "engines": { + "node": ">=18.7.0" + } + }, + "node_modules/@atproto/common-web": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/@atproto/common-web/-/common-web-0.4.5.tgz", + "integrity": "sha512-Tx0xUafLm3vRvOQpbBl5eb9V8xlC7TaRXs6dAulHRkDG3Kb+P9qn3pkDteq+aeMshbVXbVa1rm3Ok4vFyuoyYA==", + "license": "MIT", + "dependencies": { + "@atproto/lex-data": "0.0.1", + "@atproto/lex-json": "0.0.1", + "zod": "^3.23.8" + } + }, + "node_modules/@atproto/crypto": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/@atproto/crypto/-/crypto-0.4.4.tgz", + "integrity": "sha512-Yq9+crJ7WQl7sxStVpHgie5Z51R05etaK9DLWYG/7bR5T4bhdcIgF6IfklLShtZwLYdVVj+K15s0BqW9a8PSDA==", + "license": "MIT", + "dependencies": { + "@noble/curves": "^1.7.0", + "@noble/hashes": "^1.6.1", + "uint8arrays": "3.0.0" + }, + "engines": { + "node": ">=18.7.0" + } + }, + "node_modules/@atproto/identity": { + "version": "0.4.10", + "resolved": "https://registry.npmjs.org/@atproto/identity/-/identity-0.4.10.tgz", + "integrity": "sha512-nQbzDLXOhM8p/wo0cTh5DfMSOSHzj6jizpodX37LJ4S1TZzumSxAjHEZa5Rev3JaoD5uSWMVE0MmKEGWkPPvfQ==", + "license": "MIT", + "dependencies": { + "@atproto/common-web": "^0.4.4", + "@atproto/crypto": "^0.4.4" + }, + "engines": { + "node": ">=18.7.0" + } + }, + "node_modules/@atproto/lex-cbor": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@atproto/lex-cbor/-/lex-cbor-0.0.1.tgz", + "integrity": "sha512-GCgowcC041tYmsoIxalIECJq4ZRHgREk6lFa4BzNRUZarMqwz57YF/7eUlo2Q6hoaMUL7Bjr6FvXwcZFaKrhvA==", + "license": "MIT", + "dependencies": { + "@atproto/lex-data": "0.0.1", + "multiformats": "^9.9.0", + "tslib": "^2.8.1" + } + }, + "node_modules/@atproto/lex-data": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@atproto/lex-data/-/lex-data-0.0.1.tgz", + "integrity": "sha512-DrS/8cQcQs3s5t9ELAFNtyDZ8/PdiCx47ALtFEP2GnX2uCBHZRkqWG7xmu6ehjc787nsFzZBvlnz3T/gov5fGA==", + "license": "MIT", + "dependencies": { + "@atproto/syntax": "0.4.1", + "multiformats": "^9.9.0", + "tslib": "^2.8.1", + "uint8arrays": "3.0.0", + "unicode-segmenter": "^0.14.0" + } + }, + "node_modules/@atproto/lex-json": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@atproto/lex-json/-/lex-json-0.0.1.tgz", + "integrity": "sha512-ivcF7+pDRuD/P97IEKQ/9TruunXj0w58Khvwk3M6psaI5eZT6LRsRZ4cWcKaXiFX4SHnjy+x43g0f7pPtIsERg==", + "license": "MIT", + "dependencies": { + "@atproto/lex-data": "0.0.1", + "tslib": "^2.8.1" + } + }, + "node_modules/@atproto/lexicon": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/@atproto/lexicon/-/lexicon-0.5.2.tgz", + "integrity": "sha512-lRmJgMA8f5j7VB5Iu5cp188ald5FuI4FlmZ7nn6EBrk1dgOstWVrI5Ft6K3z2vjyLZRG6nzknlsw+tDP63p7bQ==", + "license": "MIT", + "dependencies": { + "@atproto/common-web": "^0.4.4", + "@atproto/syntax": "^0.4.1", + "iso-datestring-validator": "^2.2.2", + "multiformats": "^9.9.0", + "zod": "^3.23.8" + } + }, + "node_modules/@atproto/lexicon-resolver": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@atproto/lexicon-resolver/-/lexicon-resolver-0.2.4.tgz", + "integrity": "sha512-tPa7ylTLtvT5d5hHNPBPLtVaJdONZQB9VroyjD95aLJMeM8OHGhMnGABro25V58V5ipFv2bsm1Jldb6XsXP44g==", + "license": "MIT", + "dependencies": { + "@atproto-labs/fetch-node": "^0.2.0", + "@atproto/identity": "^0.4.10", + "@atproto/lexicon": "^0.5.2", + "@atproto/repo": "^0.8.11", + "@atproto/syntax": "^0.4.1", + "@atproto/xrpc": "^0.7.6", + "multiformats": "^9.9.0" + } + }, + "node_modules/@atproto/repo": { + "version": "0.8.11", + "resolved": "https://registry.npmjs.org/@atproto/repo/-/repo-0.8.11.tgz", + "integrity": "sha512-b/WCu5ITws4ILHoXiZz0XXB5U9C08fUVzkBQDwpnme62GXv8gUaAPL/ttG61OusW09ARwMMQm4vxoP0hTFg+zA==", + "license": "MIT", + "dependencies": { + "@atproto/common": "^0.5.0", + "@atproto/common-web": "^0.4.4", + "@atproto/crypto": "^0.4.4", + "@atproto/lexicon": "^0.5.2", + "@ipld/dag-cbor": "^7.0.0", + "multiformats": "^9.9.0", + "uint8arrays": "3.0.0", + "varint": "^6.0.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.7.0" + } + }, + "node_modules/@atproto/syntax": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@atproto/syntax/-/syntax-0.4.1.tgz", + "integrity": "sha512-CJdImtLAiFO+0z3BWTtxwk6aY5w4t8orHTMVJgkf++QRJWTxPbIFko/0hrkADB7n2EruDxDSeAgfUGehpH6ngw==", + "license": "MIT" + }, + "node_modules/@atproto/xrpc": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/@atproto/xrpc/-/xrpc-0.7.6.tgz", + "integrity": "sha512-RvCf4j0JnKYWuz3QzsYCntJi3VuiAAybQsMIUw2wLWcHhchO9F7UaBZINLL2z0qc/cYWPv5NSwcVydMseoCZLA==", + "license": "MIT", + "dependencies": { + "@atproto/lexicon": "^0.5.2", + "zod": "^3.23.8" + } + }, "node_modules/@babel/helper-string-parser": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", @@ -1366,6 +1550,16 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ipld/dag-cbor": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-7.0.3.tgz", + "integrity": "sha512-1VVh2huHsuohdXC1bGJNE8WR72slZ9XE2T3wbBBq31dm7ZBatmKLLxrB+XAqafxfRFjv08RZmj/W/ZqaM13AuA==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "cborg": "^1.6.0", + "multiformats": "^9.5.4" + } + }, "node_modules/@next/env": { "version": "16.0.3", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.0.3.tgz", @@ -1500,6 +1694,33 @@ "node": ">= 10" } }, + "node_modules/@noble/curves": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.7.tgz", + "integrity": "sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@swc/helpers": { "version": "0.5.15", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", @@ -1539,6 +1760,27 @@ "@types/react": "^19.2.0" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/babel-plugin-react-compiler": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/babel-plugin-react-compiler/-/babel-plugin-react-compiler-1.0.0.tgz", @@ -1549,6 +1791,50 @@ "@babel/types": "^7.26.0" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -1575,6 +1861,15 @@ ], "license": "CC-BY-4.0" }, + "node_modules/cborg": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.10.2.tgz", + "integrity": "sha512-b3tFPA9pUr2zCUiCfRd2+wok2/LBSNUMKOuRRok+WlvvAgEt/PlbgPTsZUcwCOs53IJvLgTp0eotwtosE6njug==", + "license": "Apache-2.0", + "bin": { + "cborg": "cli.js" + } + }, "node_modules/client-only": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", @@ -1808,6 +2103,33 @@ "esbuild": ">=0.12 <1" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/get-tsconfig": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", @@ -1820,12 +2142,53 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/iso-datestring-validator": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/iso-datestring-validator/-/iso-datestring-validator-2.2.2.tgz", + "integrity": "sha512-yLEMkBbLZTlVQqOnQ4FiMujR6T4DEcCb1xizmvXS+OxuhwcbtynoosRzdMA69zZCShCNAbi+gJ71FxZBBXx1SA==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -1896,12 +2259,59 @@ } } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "license": "ISC" }, + "node_modules/pino": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.21.0.tgz", + "integrity": "sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^1.2.0", + "pino-std-serializers": "^6.0.0", + "process-warning": "^3.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^3.7.0", + "thread-stream": "^2.6.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.2.0.tgz", + "integrity": "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q==", + "license": "MIT", + "dependencies": { + "readable-stream": "^4.0.0", + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", + "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==", + "license": "MIT" + }, "node_modules/postcss": { "version": "8.4.31", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", @@ -1943,6 +2353,27 @@ "url": "https://github.com/sponsors/porsager" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-warning": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", + "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==", + "license": "MIT" + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, "node_modules/react": { "version": "19.2.0", "resolved": "https://registry.npmjs.org/react/-/react-19.2.0.tgz", @@ -1964,6 +2395,31 @@ "react": "^19.2.0" } }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -1973,6 +2429,35 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", @@ -2037,6 +2522,15 @@ "@img/sharp-win32-x64": "0.34.5" } }, + "node_modules/sonic-boom": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz", + "integrity": "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -2065,6 +2559,24 @@ "source-map": "^0.6.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/styled-jsx": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", @@ -2088,6 +2600,15 @@ } } }, + "node_modules/thread-stream": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.7.0.tgz", + "integrity": "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -2108,12 +2629,51 @@ "node": ">=14.17" } }, + "node_modules/uint8arrays": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-3.0.0.tgz", + "integrity": "sha512-HRCx0q6O9Bfbp+HHSfQQKD7wU70+lydKVt4EghkdOvlK/NlrF90z+eXV34mUd48rNvVJXwkrMSPpCATkct8fJA==", + "license": "MIT", + "dependencies": { + "multiformats": "^9.4.2" + } + }, + "node_modules/undici": { + "version": "6.22.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.22.0.tgz", + "integrity": "sha512-hU/10obOIu62MGYjdskASR3CUAiYaFTtC9Pa6vHyf//mAipSvSQg6od2CnJswq7fvzNS3zJhxoRkgNVaHurWKw==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true, "license": "MIT" + }, + "node_modules/unicode-segmenter": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/unicode-segmenter/-/unicode-segmenter-0.14.0.tgz", + "integrity": "sha512-AH4lhPCJANUnSLEKnM4byboctePJzltF4xj8b+NbNiYeAkAXGh7px2K/4NANFp7dnr6+zB3e6HLu8Jj8SKyvYg==", + "license": "MIT" + }, + "node_modules/varint": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", + "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==", + "license": "MIT" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 6635bea..65ed516 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "db:studio": "drizzle-kit studio" }, "dependencies": { + "@atproto/lexicon-resolver": "^0.2.4", "drizzle-kit": "^0.31.7", "drizzle-orm": "^0.44.7", "next": "16.0.3", From 475008f9141e080a32d168c00f1c82ef55f81b00 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Tue, 25 Nov 2025 15:17:42 -0800 Subject: [PATCH 02/15] feat: add @atproto/syntax package --- package-lock.json | 1 + package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/package-lock.json b/package-lock.json index a856c0d..7f04fa6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@atproto/lexicon-resolver": "^0.2.4", + "@atproto/syntax": "^0.4.1", "drizzle-kit": "^0.31.7", "drizzle-orm": "^0.44.7", "next": "16.0.3", diff --git a/package.json b/package.json index 65ed516..d5047fc 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@atproto/lexicon-resolver": "^0.2.4", + "@atproto/syntax": "^0.4.1", "drizzle-kit": "^0.31.7", "drizzle-orm": "^0.44.7", "next": "16.0.3", From be7409b9b3de08f8af5a9df181241dd81d6d4322 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Tue, 25 Nov 2025 15:35:05 -0800 Subject: [PATCH 03/15] feat: add @atproto/lexicon package --- package-lock.json | 1 + package.json | 1 + 2 files changed, 2 insertions(+) diff --git a/package-lock.json b/package-lock.json index 7f04fa6..3b1b0b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "lexhub", "version": "0.1.0", "dependencies": { + "@atproto/lexicon": "^0.5.2", "@atproto/lexicon-resolver": "^0.2.4", "@atproto/syntax": "^0.4.1", "drizzle-kit": "^0.31.7", diff --git a/package.json b/package.json index d5047fc..8c51864 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "db:studio": "drizzle-kit studio" }, "dependencies": { + "@atproto/lexicon": "^0.5.2", "@atproto/lexicon-resolver": "^0.2.4", "@atproto/syntax": "^0.4.1", "drizzle-kit": "^0.31.7", From 37dc82bea7716ac4db5a94de2ec599ca6ed159ce Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Tue, 25 Nov 2025 16:13:19 -0800 Subject: [PATCH 04/15] feat(util): add LexiconSchemaRecord type and type guard for lexicon NSID resolution --- src/util/lexicon.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/util/lexicon.ts diff --git a/src/util/lexicon.ts b/src/util/lexicon.ts new file mode 100644 index 0000000..c7388d2 --- /dev/null +++ b/src/util/lexicon.ts @@ -0,0 +1,16 @@ +import { + LexiconSchemaRecord as OriginalLexiconSchemaRecord, + LEXICON_SCHEMA_NSID, +} from "@atproto/lexicon-resolver"; + +export type LexiconSchemaRecord = OriginalLexiconSchemaRecord & { + id: string; +}; + +/** + * Type guard to check if a value is a LexiconSchemaRecord. + * Extend the type and check for 'id' property to help with resolving the NSID. + */ +export function isLexiconSchemaRecord(v: any): v is LexiconSchemaRecord { + return v?.["$type"] === LEXICON_SCHEMA_NSID && Object.hasOwn(v, "id"); +} From a377825859e51a493172bc0d275a857744752f98 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Tue, 25 Nov 2025 16:13:34 -0800 Subject: [PATCH 05/15] feat(api): validate lexicon schema record and enforce DID authority in ingest route - Use LexiconSchemaRecord type and type guard for incoming records - Resolve and check NSID DID authority matches record DID - Return error if record is invalid or authority does not match - Refactor to use new @atproto/lexicon and @atproto/lexicon-resolver utilities --- src/app/api/ingest/route.ts | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/app/api/ingest/route.ts b/src/app/api/ingest/route.ts index 41127bd..f2607f3 100644 --- a/src/app/api/ingest/route.ts +++ b/src/app/api/ingest/route.ts @@ -2,6 +2,8 @@ import { NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; import { lexicons } from "@/db/schema"; +import { isLexiconSchemaRecord, LexiconSchemaRecord } from "@/util/lexicon"; +import { resolveLexiconDidAuthority } from "@atproto/lexicon-resolver"; interface UserEvent { id: number; @@ -20,10 +22,7 @@ interface RecordEvent { action: "create" | "update" | "delete"; cid: string; live: boolean; - record: { - id: string; - [key: string]: any; - }; + record: LexiconSchemaRecord; }; } @@ -77,11 +76,20 @@ export async function POST(request: NextRequest) { return ackEvent("Event type not desired"); } - const event = body.record; - const { cid, record: lexiconRecord } = event; + const commit = body.record; + const { cid, record: lexiconRecord } = commit; - // NOTE(caidanw): most lexicon records use 'id', but I've encountered ones using 'nsid' - const nsid = lexiconRecord.id ?? lexiconRecord.nsid; + if (!isLexiconSchemaRecord(lexiconRecord)) { + return ackEvent("Not a valid lexicon schema record"); + } + + const nsid = lexiconRecord.id; + const did = await resolveLexiconDidAuthority(nsid); + + // Ensure the DID authority from the NSID matches the record's DID to prevent spoofing + if (did === undefined || did !== commit.did) { + return ackEvent("NSID DID authority does not match record DID"); + } // Insert or update the lexicon record in the database await db @@ -102,7 +110,7 @@ export async function POST(request: NextRequest) { id: body.id, nsid: nsid, cid: cid, - action: event.action, + action: commit.action, }); return ackEvent("Event ingested successfully"); From 6ff2ae891e8ec550f2a93dab1f29fe7f4e1b888f Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Tue, 25 Nov 2025 16:37:07 -0800 Subject: [PATCH 06/15] feat: add zod package --- package-lock.json | 45 +++++++++++++++++++++++++++++++++++++++++---- package.json | 3 ++- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3b1b0b3..84f7aa5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "next": "16.0.3", "postgres": "^3.4.7", "react": "19.2.0", - "react-dom": "19.2.0" + "react-dom": "19.2.0", + "zod": "^4.1.13" }, "devDependencies": { "@types/node": "^20", @@ -84,6 +85,15 @@ "zod": "^3.23.8" } }, + "node_modules/@atproto/common-web/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/@atproto/crypto": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/@atproto/crypto/-/crypto-0.4.4.tgz", @@ -173,6 +183,15 @@ "multiformats": "^9.9.0" } }, + "node_modules/@atproto/lexicon/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/@atproto/repo": { "version": "0.8.11", "resolved": "https://registry.npmjs.org/@atproto/repo/-/repo-0.8.11.tgz", @@ -193,6 +212,15 @@ "node": ">=18.7.0" } }, + "node_modules/@atproto/repo/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/@atproto/syntax": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@atproto/syntax/-/syntax-0.4.1.tgz", @@ -209,6 +237,15 @@ "zod": "^3.23.8" } }, + "node_modules/@atproto/xrpc/node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/@babel/helper-string-parser": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", @@ -2669,9 +2706,9 @@ "license": "MIT" }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.13.tgz", + "integrity": "sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 8c51864..1b01f52 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "next": "16.0.3", "postgres": "^3.4.7", "react": "19.2.0", - "react-dom": "19.2.0" + "react-dom": "19.2.0", + "zod": "^4.1.13" }, "devDependencies": { "@types/node": "^20", From 7a07c86ecf16061f6098e9af50362874408698da Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Wed, 26 Nov 2025 16:58:34 -0800 Subject: [PATCH 07/15] refactor(util): redefine LexiconSchemaRecord to require 'id' field for NSID resolution Remove extension from @atproto/lexicon-resolver type and explicitly define LexiconSchemaRecord with required 'id' property. Update type guard and documentation to clarify purpose for DID authority resolution. --- src/util/lexicon.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/util/lexicon.ts b/src/util/lexicon.ts index c7388d2..64a47bb 100644 --- a/src/util/lexicon.ts +++ b/src/util/lexicon.ts @@ -1,15 +1,16 @@ -import { - LexiconSchemaRecord as OriginalLexiconSchemaRecord, - LEXICON_SCHEMA_NSID, -} from "@atproto/lexicon-resolver"; +import { LEXICON_SCHEMA_NSID } from "@atproto/lexicon-resolver"; -export type LexiconSchemaRecord = OriginalLexiconSchemaRecord & { +/** + * Extend the type from `@atproto/lexicon-resolver` to require the 'id' field. + * This extension is necessary because we need the NSID to resolve the DID authority. + */ +export type LexiconSchemaRecord = { + $type: typeof LEXICON_SCHEMA_NSID; id: string; }; /** * Type guard to check if a value is a LexiconSchemaRecord. - * Extend the type and check for 'id' property to help with resolving the NSID. */ export function isLexiconSchemaRecord(v: any): v is LexiconSchemaRecord { return v?.["$type"] === LEXICON_SCHEMA_NSID && Object.hasOwn(v, "id"); From 898286341b07a147515a43d2e6f3f0a1a2fc0681 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Wed, 26 Nov 2025 16:58:38 -0800 Subject: [PATCH 08/15] feat(api): parse and validate lexicon records using parseLexiconDoc and zod in ingest route Parse incoming lexicon records with parseLexiconDoc, validate with zod, and improve error handling for invalid records. Store parsed LexiconDoc in the database instead of raw input. --- src/app/api/ingest/route.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/app/api/ingest/route.ts b/src/app/api/ingest/route.ts index f2607f3..17b30aa 100644 --- a/src/app/api/ingest/route.ts +++ b/src/app/api/ingest/route.ts @@ -3,7 +3,9 @@ import { NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; import { lexicons } from "@/db/schema"; import { isLexiconSchemaRecord, LexiconSchemaRecord } from "@/util/lexicon"; +import { LexiconDoc, parseLexiconDoc } from "@atproto/lexicon"; import { resolveLexiconDidAuthority } from "@atproto/lexicon-resolver"; +import z from "zod"; interface UserEvent { id: number; @@ -91,24 +93,40 @@ export async function POST(request: NextRequest) { return ackEvent("NSID DID authority does not match record DID"); } + let lexiconDoc: LexiconDoc; + try { + lexiconDoc = parseLexiconDoc(lexiconRecord); + } catch (error) { + if (error instanceof z.ZodError) { + console.warn( + "Lexicon record failed validation:\n", + JSON.stringify(error.issues, null, 2), + ); + // TODO(caidanw): store the commit and lexicon record for further analysis, mark as invalid + } else { + console.error("Unknown error while parsing lexicon record:", error); + } + return ackEvent("Lexicon record failed to parse"); + } + // Insert or update the lexicon record in the database await db .insert(lexicons) .values({ - id: nsid, + id: lexiconDoc.id, cid: cid, - data: lexiconRecord, + data: lexiconDoc, }) .onConflictDoUpdate({ target: [lexicons.id, lexicons.cid], set: { - data: lexiconRecord, + data: lexiconDoc, }, }); console.log("Record event ingested:", { id: body.id, - nsid: nsid, + nsid: lexiconDoc.id, cid: cid, action: commit.action, }); From dc5c7dd03337e38437082544da8ec17a8e3cb90c Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 14:49:04 -0800 Subject: [PATCH 09/15] refactor(db): separate valid and invalid lexicons into distinct tables - Replace single lexicons table with valid_lexicons and invalid_lexicons tables - Add repo_did to primary key [nsid, cid, repo_did] to track lexicon migrations - Store validation errors in invalid_lexicons for developer debugging - Use different column names (data vs raw_data) for semantic clarity - Add indexes on nsid and repo_did for both tables - Include repo_rev field to track repository state at ingestion time --- ...001_update_schema_valid_invalid_tables.sql | 37 ++++++ drizzle/meta/0001_snapshot.json | 118 ++++++++++++++++++ drizzle/meta/_journal.json | 7 ++ src/db/schema.ts | 117 +++++++++++++---- 4 files changed, 257 insertions(+), 22 deletions(-) create mode 100644 drizzle/0001_update_schema_valid_invalid_tables.sql create mode 100644 drizzle/meta/0001_snapshot.json diff --git a/drizzle/0001_update_schema_valid_invalid_tables.sql b/drizzle/0001_update_schema_valid_invalid_tables.sql new file mode 100644 index 0000000..131cd6f --- /dev/null +++ b/drizzle/0001_update_schema_valid_invalid_tables.sql @@ -0,0 +1,37 @@ +-- Custom SQL migration file, put your code below! -- + +-- Drop old lexicons table +DROP TABLE IF EXISTS "lexicons"; + +-- Create valid_lexicons table +CREATE TABLE "valid_lexicons" ( + "nsid" varchar(317) NOT NULL, + "cid" varchar(100) NOT NULL, + "repo_did" varchar(256) NOT NULL, + "repo_rev" varchar(256) NOT NULL, + "data" jsonb NOT NULL, + "ingested_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "valid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did") +); + +-- Create invalid_lexicons table +CREATE TABLE "invalid_lexicons" ( + "nsid" varchar(317) NOT NULL, + "cid" varchar(100) NOT NULL, + "repo_did" varchar(256) NOT NULL, + "repo_rev" varchar(256) NOT NULL, + "raw_data" jsonb NOT NULL, + "validation_errors" jsonb NOT NULL, + "ingested_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "invalid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did") +); + +-- Create indexes for valid_lexicons +CREATE INDEX "valid_lexicons_nsid_idx" ON "valid_lexicons" USING btree ("nsid"); +CREATE INDEX "valid_lexicons_repo_did_idx" ON "valid_lexicons" USING btree ("repo_did"); +CREATE INDEX "valid_lexicons_data_gin_idx" ON "valid_lexicons" USING gin ("data"); + +-- Create indexes for invalid_lexicons +CREATE INDEX "invalid_lexicons_nsid_idx" ON "invalid_lexicons" USING btree ("nsid"); +CREATE INDEX "invalid_lexicons_repo_did_idx" ON "invalid_lexicons" USING btree ("repo_did"); +CREATE INDEX "invalid_lexicons_raw_data_gin_idx" ON "invalid_lexicons" USING gin ("raw_data"); diff --git a/drizzle/meta/0001_snapshot.json b/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..8b81c6e --- /dev/null +++ b/drizzle/meta/0001_snapshot.json @@ -0,0 +1,118 @@ +{ + "id": "9d4c7501-b55c-430c-9334-1c092fe00410", + "prevId": "1f9ba7ab-95a0-4e5b-b8d5-89a953fef8fb", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.lexicons": { + "name": "lexicons", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(317)", + "primaryKey": false, + "notNull": true + }, + "cid": { + "name": "cid", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "lexicons_id_idx": { + "name": "lexicons_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "lexicons_created_at_idx": { + "name": "lexicons_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "btree", + "concurrently": false + }, + "lexicons_data_gin_idx": { + "name": "lexicons_data_gin_idx", + "columns": [ + { + "expression": "data", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "with": {}, + "method": "gin", + "concurrently": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "lexicons_id_cid_pk": { + "name": "lexicons_id_cid_pk", + "columns": [ + "id", + "cid" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "views": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 6163bae..6d05a99 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1763597119878, "tag": "0000_init", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1764629271549, + "tag": "0001_update_schema_valid_invalid_tables", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index f2673b3..1343d2a 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -8,19 +8,20 @@ import { } from "drizzle-orm/pg-core"; /** - * Lexicons table stores ATProto Lexicon files. + * Valid lexicons table stores ATProto Lexicon schemas that have passed both + * DNS validation and schema validation. * - * Each row represents a specific version of a lexicon, identified by the combination - * of NSID (id) and CID (content hash). This allows tracking lexicons over their lifetime. + * Primary key is [nsid, cid, repo_did] to handle migrations where the same + * lexicon content (same CID) is published from a different DID. */ -export const lexicons = pgTable( - "lexicons", +export const validLexicons = pgTable( + "valid_lexicons", { /** * NSID - Namespaced Identifier (e.g., com.atproto.sync.subscribeRepos) * Max length: 317 characters per NSID spec */ - id: varchar({ length: 317 }).notNull(), + nsid: varchar({ length: 317 }).notNull(), /** * CID - Content Identifier hash (e.g., bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a) @@ -30,40 +31,112 @@ export const lexicons = pgTable( cid: varchar({ length: 100 }).notNull(), /** - * Full lexicon record in JSON format. + * Repository DID that published this lexicon version. + * DNS validation ensures this DID matches the NSID's DNS TXT record. + * Max length: 256 chars for DID identifiers + */ + repoDid: varchar("repo_did", { length: 256 }).notNull(), + + /** + * Repository revision at time of ingestion + */ + repoRev: varchar("repo_rev", { length: 256 }).notNull(), + + /** + * Validated, parsed LexiconDoc in JSON format. * Contains: id, $type, lexicon, defs, description */ data: jsonb().notNull(), /** - * Timestamp when this lexicon record was first ingested + * Timestamp when this lexicon version was ingested */ - createdAt: timestamp("created_at", { withTimezone: true }) + ingestedAt: timestamp("ingested_at", { withTimezone: true }) .notNull() .defaultNow(), + }, + (table) => [ + // Composite primary key includes repo_did to track migrations + primaryKey({ columns: [table.nsid, table.cid, table.repoDid] }), + + // Index on nsid for quick lookup of all versions of a lexicon + index("valid_lexicons_nsid_idx").on(table.nsid), + + // Index on repo_did to find all lexicons from a specific repository + index("valid_lexicons_repo_did_idx").on(table.repoDid), + // JSONB GIN index for efficient querying within lexicon content + index("valid_lexicons_data_gin_idx").using("gin", table.data), + ], +); + +/** + * Invalid lexicons table stores lexicon records that passed DNS validation + * but failed schema validation. Used for debugging and helping developers + * identify why their lexicons are broken. + * + * Primary key is [nsid, cid, repo_did] matching valid_lexicons structure. + */ +export const invalidLexicons = pgTable( + "invalid_lexicons", + { /** - * Timestamp when this lexicon record was last updated + * NSID - Namespaced Identifier */ - updatedAt: timestamp("updated_at", { withTimezone: true }) + nsid: varchar({ length: 317 }).notNull(), + + /** + * CID - Content Identifier hash + */ + cid: varchar({ length: 100 }).notNull(), + + /** + * Repository DID that published this invalid lexicon. + * DNS was validated, but schema validation failed. + */ + repoDid: varchar("repo_did", { length: 256 }).notNull(), + + /** + * Repository revision at time of ingestion + */ + repoRev: varchar("repo_rev", { length: 256 }).notNull(), + + /** + * Raw, unvalidated data that failed schema validation. + * May not conform to LexiconDoc structure. + */ + rawData: jsonb("raw_data").notNull(), + + /** + * Validation errors from Zod parser. + * Contains ZodError issues array with details about what failed. + */ + validationErrors: jsonb("validation_errors").notNull(), + + /** + * Timestamp when this invalid lexicon was ingested + */ + ingestedAt: timestamp("ingested_at", { withTimezone: true }) .notNull() .defaultNow(), }, (table) => [ - // Composite primary key: each NSID+CID combination is unique - primaryKey({ columns: [table.id, table.cid] }), + // Composite primary key matches valid_lexicons + primaryKey({ columns: [table.nsid, table.cid, table.repoDid] }), - // Index on id alone for quick lookup of all versions of a lexicon - index("lexicons_id_idx").on(table.id), + // Index on nsid for debugging specific lexicons + index("invalid_lexicons_nsid_idx").on(table.nsid), - // Index on createdAt for chronological queries - index("lexicons_created_at_idx").on(table.createdAt), + // Index on repo_did to find all invalid lexicons from a repository + index("invalid_lexicons_repo_did_idx").on(table.repoDid), - // JSONB GIN index for efficient querying of the entire data field - // Useful for full-text searches and containment queries within the lexicon data - index("lexicons_data_gin_idx").using("gin", table.data), + // JSONB GIN index for searching validation errors + index("invalid_lexicons_raw_data_gin_idx").using("gin", table.rawData), ], ); -export type Lexicon = typeof lexicons.$inferSelect; -export type NewLexicon = typeof lexicons.$inferInsert; +export type ValidLexicon = typeof validLexicons.$inferSelect; +export type NewValidLexicon = typeof validLexicons.$inferInsert; + +export type InvalidLexicon = typeof invalidLexicons.$inferSelect; +export type NewInvalidLexicon = typeof invalidLexicons.$inferInsert; From ee77d08af4f4f423ddfe2944d6cd1d97210f2163 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 14:49:19 -0800 Subject: [PATCH 10/15] feat(api): store valid and invalid lexicons in separate tables - Store valid lexicons in valid_lexicons table after successful parsing - Store invalid lexicons in invalid_lexicons table with validation errors - DNS validation acts as gate before any storage (prevents DDOS) - Include repo_did and repo_rev in all stored records - Improve logging to distinguish valid vs invalid ingestion events - Remove onConflictDoUpdate logic (primary key now includes repo_did) --- src/app/api/ingest/route.ts | 75 +++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/src/app/api/ingest/route.ts b/src/app/api/ingest/route.ts index 17b30aa..487725d 100644 --- a/src/app/api/ingest/route.ts +++ b/src/app/api/ingest/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; -import { lexicons } from "@/db/schema"; +import { invalidLexicons, validLexicons } from "@/db/schema"; import { isLexiconSchemaRecord, LexiconSchemaRecord } from "@/util/lexicon"; import { LexiconDoc, parseLexiconDoc } from "@atproto/lexicon"; import { resolveLexiconDidAuthority } from "@atproto/lexicon-resolver"; @@ -88,50 +88,61 @@ export async function POST(request: NextRequest) { const nsid = lexiconRecord.id; const did = await resolveLexiconDidAuthority(nsid); - // Ensure the DID authority from the NSID matches the record's DID to prevent spoofing + // DNS validation gate: Reject if DNS doesn't resolve or doesn't match the repo DID + // This helps prevents spoofing and DDOS attacks by only storing lexicons with valid DNS authority if (did === undefined || did !== commit.did) { return ackEvent("NSID DID authority does not match record DID"); } + // Attempt to parse and validate the lexicon schema let lexiconDoc: LexiconDoc; try { lexiconDoc = parseLexiconDoc(lexiconRecord); - } catch (error) { - if (error instanceof z.ZodError) { - console.warn( - "Lexicon record failed validation:\n", - JSON.stringify(error.issues, null, 2), - ); - // TODO(caidanw): store the commit and lexicon record for further analysis, mark as invalid - } else { - console.error("Unknown error while parsing lexicon record:", error); - } - return ackEvent("Lexicon record failed to parse"); - } - // Insert or update the lexicon record in the database - await db - .insert(lexicons) - .values({ - id: lexiconDoc.id, + // Valid lexicon: store in valid_lexicons table + await db.insert(validLexicons).values({ + nsid: nsid, cid: cid, + repoDid: commit.did, + repoRev: commit.rev, data: lexiconDoc, - }) - .onConflictDoUpdate({ - target: [lexicons.id, lexicons.cid], - set: { - data: lexiconDoc, - }, }); - console.log("Record event ingested:", { - id: body.id, - nsid: lexiconDoc.id, - cid: cid, - action: commit.action, - }); + console.log("Valid lexicon ingested:", { + eventId: body.id, + nsid: nsid, + cid: cid, + repoDid: commit.did, + action: commit.action, + }); - return ackEvent("Event ingested successfully"); + return ackEvent("Valid lexicon ingested successfully"); + } catch (error) { + if (error instanceof z.ZodError) { + // Invalid lexicon: store in invalid_lexicons table for debugging + await db.insert(invalidLexicons).values({ + nsid: nsid, + cid: cid, + repoDid: commit.did, + repoRev: commit.rev, + rawData: lexiconRecord, + validationErrors: error.issues, + }); + + console.warn("Invalid lexicon ingested:", { + eventId: body.id, + nsid: nsid, + cid: cid, + repoDid: commit.did, + errorCount: error.issues.length, + }); + + return ackEvent("Invalid lexicon stored for debugging"); + } else { + console.error("Unknown error while parsing lexicon record:", error); + return ackEvent("Lexicon record failed to parse"); + } + } } catch (error) { console.error("Error processing ingest request:", error); From 28a66b08daa8dd4656c6e833dbda9b8101c7bd57 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 16:55:34 -0800 Subject: [PATCH 11/15] refactor(db): squash migrations into single init migration - Combine initial lexicons table and updated schema into one migration - Remove intermediate migration file (0001) - Clean schema shows final valid_lexicons and invalid_lexicons tables --- drizzle/0000_init.sql | 49 ++++---- ...001_update_schema_valid_invalid_tables.sql | 37 ------ drizzle/meta/0001_snapshot.json | 118 ------------------ drizzle/meta/_journal.json | 9 +- 4 files changed, 28 insertions(+), 185 deletions(-) delete mode 100644 drizzle/0001_update_schema_valid_invalid_tables.sql delete mode 100644 drizzle/meta/0001_snapshot.json diff --git a/drizzle/0000_init.sql b/drizzle/0000_init.sql index f2adf16..23e0978 100644 --- a/drizzle/0000_init.sql +++ b/drizzle/0000_init.sql @@ -1,26 +1,31 @@ -CREATE TABLE "lexicons" ( - "id" varchar(317) NOT NULL, +-- Create valid_lexicons table +CREATE TABLE "valid_lexicons" ( + "nsid" varchar(317) NOT NULL, "cid" varchar(100) NOT NULL, + "repo_did" varchar(256) NOT NULL, + "repo_rev" varchar(256) NOT NULL, "data" jsonb NOT NULL, - "created_at" timestamp with time zone DEFAULT now() NOT NULL, - "updated_at" timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT "lexicons_id_cid_pk" PRIMARY KEY("id","cid") + "ingested_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "valid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did") ); --> statement-breakpoint -CREATE INDEX "lexicons_id_idx" ON "lexicons" USING btree ("id");--> statement-breakpoint -CREATE INDEX "lexicons_created_at_idx" ON "lexicons" USING btree ("created_at");--> statement-breakpoint -CREATE INDEX "lexicons_data_gin_idx" ON "lexicons" USING gin ("data");--> statement-breakpoint - - -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ language 'plpgsql'; - -CREATE TRIGGER update_lexicons_updated_at - BEFORE UPDATE ON "lexicons" - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); +-- Create invalid_lexicons table +CREATE TABLE "invalid_lexicons" ( + "nsid" varchar(317) NOT NULL, + "cid" varchar(100) NOT NULL, + "repo_did" varchar(256) NOT NULL, + "repo_rev" varchar(256) NOT NULL, + "raw_data" jsonb NOT NULL, + "validation_errors" jsonb NOT NULL, + "ingested_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "invalid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did") +); +--> statement-breakpoint +-- Create indexes for valid_lexicons +CREATE INDEX "valid_lexicons_nsid_idx" ON "valid_lexicons" USING btree ("nsid");--> statement-breakpoint +CREATE INDEX "valid_lexicons_repo_did_idx" ON "valid_lexicons" USING btree ("repo_did");--> statement-breakpoint +CREATE INDEX "valid_lexicons_data_gin_idx" ON "valid_lexicons" USING gin ("data");--> statement-breakpoint +-- Create indexes for invalid_lexicons +CREATE INDEX "invalid_lexicons_nsid_idx" ON "invalid_lexicons" USING btree ("nsid");--> statement-breakpoint +CREATE INDEX "invalid_lexicons_repo_did_idx" ON "invalid_lexicons" USING btree ("repo_did");--> statement-breakpoint +CREATE INDEX "invalid_lexicons_raw_data_gin_idx" ON "invalid_lexicons" USING gin ("raw_data"); diff --git a/drizzle/0001_update_schema_valid_invalid_tables.sql b/drizzle/0001_update_schema_valid_invalid_tables.sql deleted file mode 100644 index 131cd6f..0000000 --- a/drizzle/0001_update_schema_valid_invalid_tables.sql +++ /dev/null @@ -1,37 +0,0 @@ --- Custom SQL migration file, put your code below! -- - --- Drop old lexicons table -DROP TABLE IF EXISTS "lexicons"; - --- Create valid_lexicons table -CREATE TABLE "valid_lexicons" ( - "nsid" varchar(317) NOT NULL, - "cid" varchar(100) NOT NULL, - "repo_did" varchar(256) NOT NULL, - "repo_rev" varchar(256) NOT NULL, - "data" jsonb NOT NULL, - "ingested_at" timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT "valid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did") -); - --- Create invalid_lexicons table -CREATE TABLE "invalid_lexicons" ( - "nsid" varchar(317) NOT NULL, - "cid" varchar(100) NOT NULL, - "repo_did" varchar(256) NOT NULL, - "repo_rev" varchar(256) NOT NULL, - "raw_data" jsonb NOT NULL, - "validation_errors" jsonb NOT NULL, - "ingested_at" timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT "invalid_lexicons_nsid_cid_repo_did_pk" PRIMARY KEY("nsid","cid","repo_did") -); - --- Create indexes for valid_lexicons -CREATE INDEX "valid_lexicons_nsid_idx" ON "valid_lexicons" USING btree ("nsid"); -CREATE INDEX "valid_lexicons_repo_did_idx" ON "valid_lexicons" USING btree ("repo_did"); -CREATE INDEX "valid_lexicons_data_gin_idx" ON "valid_lexicons" USING gin ("data"); - --- Create indexes for invalid_lexicons -CREATE INDEX "invalid_lexicons_nsid_idx" ON "invalid_lexicons" USING btree ("nsid"); -CREATE INDEX "invalid_lexicons_repo_did_idx" ON "invalid_lexicons" USING btree ("repo_did"); -CREATE INDEX "invalid_lexicons_raw_data_gin_idx" ON "invalid_lexicons" USING gin ("raw_data"); diff --git a/drizzle/meta/0001_snapshot.json b/drizzle/meta/0001_snapshot.json deleted file mode 100644 index 8b81c6e..0000000 --- a/drizzle/meta/0001_snapshot.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "id": "9d4c7501-b55c-430c-9334-1c092fe00410", - "prevId": "1f9ba7ab-95a0-4e5b-b8d5-89a953fef8fb", - "version": "7", - "dialect": "postgresql", - "tables": { - "public.lexicons": { - "name": "lexicons", - "schema": "", - "columns": { - "id": { - "name": "id", - "type": "varchar(317)", - "primaryKey": false, - "notNull": true - }, - "cid": { - "name": "cid", - "type": "varchar(100)", - "primaryKey": false, - "notNull": true - }, - "data": { - "name": "data", - "type": "jsonb", - "primaryKey": false, - "notNull": true - }, - "created_at": { - "name": "created_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true, - "default": "now()" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": true, - "default": "now()" - } - }, - "indexes": { - "lexicons_id_idx": { - "name": "lexicons_id_idx", - "columns": [ - { - "expression": "id", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "with": {}, - "method": "btree", - "concurrently": false - }, - "lexicons_created_at_idx": { - "name": "lexicons_created_at_idx", - "columns": [ - { - "expression": "created_at", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "with": {}, - "method": "btree", - "concurrently": false - }, - "lexicons_data_gin_idx": { - "name": "lexicons_data_gin_idx", - "columns": [ - { - "expression": "data", - "isExpression": false, - "asc": true, - "nulls": "last" - } - ], - "isUnique": false, - "with": {}, - "method": "gin", - "concurrently": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "lexicons_id_cid_pk": { - "name": "lexicons_id_cid_pk", - "columns": [ - "id", - "cid" - ] - } - }, - "uniqueConstraints": {}, - "policies": {}, - "checkConstraints": {}, - "isRLSEnabled": false - } - }, - "enums": {}, - "schemas": {}, - "views": {}, - "sequences": {}, - "roles": {}, - "policies": {}, - "_meta": { - "columns": {}, - "schemas": {}, - "tables": {} - } -} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 6d05a99..760bbcf 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -5,16 +5,9 @@ { "idx": 0, "version": "7", - "when": 1763597119878, + "when": 1733093280000, "tag": "0000_init", "breakpoints": true - }, - { - "idx": 1, - "version": "7", - "when": 1764629271549, - "tag": "0001_update_schema_valid_invalid_tables", - "breakpoints": true } ] } \ No newline at end of file From de572ddb52d5f27a219dad3d99668b0a86de9be5 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 16:57:27 -0800 Subject: [PATCH 12/15] chore(dev): add compose.override.yaml for local development with custom Nexus and Postgres settings Provides a Docker Compose override for development, disabling lexhub in Docker, reducing resource usage for Nexus and Postgres, and enabling debug logging and SQL query logging. --- compose.override.yaml | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 compose.override.yaml diff --git a/compose.override.yaml b/compose.override.yaml new file mode 100644 index 0000000..5a6822d --- /dev/null +++ b/compose.override.yaml @@ -0,0 +1,51 @@ +# Development overrides for docker-compose +# This file is automatically loaded by docker-compose and overrides compose.yaml +# For production, use: docker-compose -f compose.yaml up +# +# In development, run lexhub locally with: npm run dev +# Only nexus and postgres run in Docker for development + +services: + lexhub: + # Disable lexhub service in development + # Run it locally instead with: npm run dev + scale: 0 + + nexus: + restart: no + network_mode: host + ports: [] + volumes: [] + environment: + # Reduced parallelism for lighter resource usage + - NEXUS_FIREHOSE_PARALLELISM=5 + - NEXUS_RESYNC_PARALLELISM=2 + - NEXUS_OUTBOX_PARALLELISM=2 + # Debug logging for development + - NEXUS_LOG_LEVEL=debug + - NEXUS_WEBHOOK_URL=http://host.docker.internal:10000/api/ingest + + postgres: + restart: no + command: + - "postgres" + - "-c" + - "shared_preload_libraries=pg_stat_statements" + - "-c" + - "pg_stat_statements.track=all" + # Reduced resources for development + - "-c" + - "max_connections=100" + - "-c" + - "shared_buffers=128MB" + - "-c" + - "effective_cache_size=512MB" + - "-c" + - "work_mem=8MB" + - "-c" + - "maintenance_work_mem=64MB" + # SQL query logging for debugging + - "-c" + - "log_statement=all" + - "-c" + - "log_duration=on" From 45910bd392d89c66093c8d9eb71d1df25769ca1c Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 17:03:12 -0800 Subject: [PATCH 13/15] fix(api): prevent duplicate valid lexicon inserts with onConflictDoNothing in ingest route --- src/app/api/ingest/route.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/app/api/ingest/route.ts b/src/app/api/ingest/route.ts index 487725d..9a8b4d3 100644 --- a/src/app/api/ingest/route.ts +++ b/src/app/api/ingest/route.ts @@ -100,13 +100,16 @@ export async function POST(request: NextRequest) { lexiconDoc = parseLexiconDoc(lexiconRecord); // Valid lexicon: store in valid_lexicons table - await db.insert(validLexicons).values({ - nsid: nsid, - cid: cid, - repoDid: commit.did, - repoRev: commit.rev, - data: lexiconDoc, - }); + await db + .insert(validLexicons) + .values({ + nsid: nsid, + cid: cid, + repoDid: commit.did, + repoRev: commit.rev, + data: lexiconDoc, + }) + .onConflictDoNothing(); console.log("Valid lexicon ingested:", { eventId: body.id, From b6b650911927b1f566c4e5c0960b937f50fad092 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 17:24:18 -0800 Subject: [PATCH 14/15] fix(api): improve ZodError detection to catch validation errors properly - Check for error shape (has 'issues' array) instead of instanceof check - parseLexiconDoc throws errors that may not pass instanceof z.ZodError check - Add onConflictDoNothing to invalid_lexicons insert - Ensures invalid lexicons are now properly stored for debugging --- src/app/api/ingest/route.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/app/api/ingest/route.ts b/src/app/api/ingest/route.ts index 9a8b4d3..59b8a95 100644 --- a/src/app/api/ingest/route.ts +++ b/src/app/api/ingest/route.ts @@ -121,7 +121,8 @@ export async function POST(request: NextRequest) { return ackEvent("Valid lexicon ingested successfully"); } catch (error) { - if (error instanceof z.ZodError) { + // Check if error has ZodError shape (has 'issues' array) + if (error && typeof error === 'object' && 'issues' in error && Array.isArray(error.issues)) { // Invalid lexicon: store in invalid_lexicons table for debugging await db.insert(invalidLexicons).values({ nsid: nsid, @@ -130,7 +131,8 @@ export async function POST(request: NextRequest) { repoRev: commit.rev, rawData: lexiconRecord, validationErrors: error.issues, - }); + }) + .onConflictDoNothing(); console.warn("Invalid lexicon ingested:", { eventId: body.id, From 74e9a2af2685c7cc6c3f51900b0abde2c3305f75 Mon Sep 17 00:00:00 2001 From: Caidan Williams Date: Mon, 1 Dec 2025 17:27:33 -0800 Subject: [PATCH 15/15] refactor(api): extract isZodError helper for clearer validation error handling in ingest route --- src/app/api/ingest/route.ts | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/app/api/ingest/route.ts b/src/app/api/ingest/route.ts index 59b8a95..dcf15bb 100644 --- a/src/app/api/ingest/route.ts +++ b/src/app/api/ingest/route.ts @@ -47,6 +47,18 @@ function isNexusEvent(obj: any): obj is NexusEvent { ); } +function isZodError(error: any): error is z.ZodError { + if (error instanceof z.ZodError) return true; + + // Check for ZodError shape, since instanceof may fail when ZodError is nested in another error + return ( + error && + typeof error === "object" && + "issues" in error && + Array.isArray(error.issues) + ); +} + /** * Acknowledges receipt of a Nexus event. * Nexus considers events 'acked' when it receives a 200 response. @@ -121,18 +133,19 @@ export async function POST(request: NextRequest) { return ackEvent("Valid lexicon ingested successfully"); } catch (error) { - // Check if error has ZodError shape (has 'issues' array) - if (error && typeof error === 'object' && 'issues' in error && Array.isArray(error.issues)) { + if (isZodError(error)) { // Invalid lexicon: store in invalid_lexicons table for debugging - await db.insert(invalidLexicons).values({ - nsid: nsid, - cid: cid, - repoDid: commit.did, - repoRev: commit.rev, - rawData: lexiconRecord, - validationErrors: error.issues, - }) - .onConflictDoNothing(); + await db + .insert(invalidLexicons) + .values({ + nsid: nsid, + cid: cid, + repoDid: commit.did, + repoRev: commit.rev, + rawData: lexiconRecord, + validationErrors: error.issues, + }) + .onConflictDoNothing(); console.warn("Invalid lexicon ingested:", { eventId: body.id,