From cac5ce591a474d11a9216b388ab4f95d3bb828aa Mon Sep 17 00:00:00 2001 From: Gerard Wilkinson Date: Mon, 24 Mar 2025 15:05:20 +0000 Subject: [PATCH 1/5] feat: collection hashing for multi instance support --- src/module.ts | 10 +++++----- src/types/collection.ts | 1 + src/types/database.ts | 3 ++- src/utils/collection.ts | 16 +++++++++++---- src/utils/database.ts | 13 ++++++++++-- test/basic.test.ts | 7 ++++--- test/empty.test.ts | 7 ++++--- test/unit/generateCollectionInsert.test.ts | 8 ++++---- .../generateCollectionTableDefinition.test.ts | 20 +++++++++---------- test/unit/resolveCollection.test.ts | 16 +++++++++++++++ 10 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/module.ts b/src/module.ts index 7ade14d9a..1d450c829 100644 --- a/src/module.ts +++ b/src/module.ts @@ -250,14 +250,14 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio let cachedFilesCount = 0 let parsedFilesCount = 0 - // Remove all existing content collections to start with a clean state - db.dropContentTables() + // Remove all out of date collections and only create new ones + const { upToDateTables } = await db.dropOldContentTables(collections) + const newCollections = collections.filter(c => !upToDateTables.includes(c.name)) // Create database dump - for await (const collection of collections) { + for await (const collection of newCollections) { if (collection.name === 'info') { continue } - const collectionHash = hash(collection) const collectionQueries = generateCollectionTableDefinition(collection, { drop: true }) .split('\n').map(q => `${q} -- structure`) @@ -295,7 +295,7 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio try { const content = await source.getItem?.(key) || '' - const checksum = getContentChecksum(configHash + collectionHash + content) + const checksum = getContentChecksum(configHash + collection.hash + content) let parsedContent if (cache && cache.checksum === checksum) { diff --git a/src/types/collection.ts b/src/types/collection.ts index e1cf7853a..906fc476d 100644 --- a/src/types/collection.ts +++ b/src/types/collection.ts @@ -73,6 +73,7 @@ export interface ResolvedCollection { * Private collections will not be available in the runtime. */ private: boolean + hash: string } export interface CollectionInfo { diff --git a/src/types/database.ts b/src/types/database.ts index 3b008acec..5d58f36da 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -1,4 +1,5 @@ import type { Primitive, Connector } from 'db0' +import type { ResolvedCollection } from '../module' export type CacheEntry = { id: string, checksum: string, parsedContent: string } @@ -19,7 +20,7 @@ export interface LocalDevelopmentDatabase { fetchDevelopmentCacheForKey(key: string): Promise insertDevelopmentCache(id: string, checksum: string, parsedContent: string): void deleteDevelopmentCache(id: string): void - dropContentTables(): void + dropOldContentTables(collections: ResolvedCollection[]): Promise<{ upToDateTables: string[] }> exec(sql: string): void close(): void database?: Connector diff --git a/src/utils/collection.ts b/src/utils/collection.ts index 2bea0ea7a..349736d64 100644 --- a/src/utils/collection.ts +++ b/src/utils/collection.ts @@ -11,8 +11,9 @@ import { logger } from './dev' const JSON_FIELDS_TYPES = ['ZodObject', 'ZodArray', 'ZodRecord', 'ZodIntersection', 'ZodUnion', 'ZodAny', 'ZodMap'] -export function getTableName(name: string) { - return `_content_${name}` +export function getTableName(name: string, collectionHash: string) { + const tableNameSafeHash = collectionHash.split('-').join('').toLowerCase().substring(0, 4) + return `_content_${name}_${tableNameSafeHash}` } export function defineCollection(collection: Collection): DefinedCollection { @@ -75,13 +76,20 @@ export function resolveCollection(name: string, collection: DefinedCollection): return undefined } - return { + const resolvedCollection: Omit = { ...collection, name, type: collection.type || 'page', - tableName: getTableName(name), private: name === 'info', } + + const collectionHash = hash(resolvedCollection) + + return { + ...resolvedCollection, + tableName: getTableName(name, collectionHash), + hash: collectionHash, + } } export function resolveCollections(collections: Record): ResolvedCollection[] { diff --git a/src/utils/database.ts b/src/utils/database.ts index 632181fdd..9f24440f2 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -6,6 +6,7 @@ import { isAbsolute, join, dirname } from 'pathe' import { isWebContainer } from '@webcontainer/env' import type { CacheEntry, D1DatabaseConfig, LocalDevelopmentDatabase, SqliteDatabaseConfig } from '../types' import type { ModuleOptions } from '../types/module' +import type { ResolvedCollection } from '../module' import { logger } from './dev' export async function refineDatabaseConfig(database: ModuleOptions['database'], opts: { rootDir: string, updateSqliteFileName?: boolean }) { @@ -85,12 +86,20 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa db.prepare(`DELETE FROM _development_cache WHERE id = ?`).run(id) } - const dropContentTables = async () => { + const dropOldContentTables = async (collections: ResolvedCollection[]) => { const tables = await db.prepare('SELECT name FROM sqlite_master WHERE type = ? AND name LIKE ?') .all('table', '_content_%') as { name: string }[] + const upToDateTables = new Set() for (const { name } of tables) { + if (collections.some(c => c.tableName === name)) { + upToDateTables.add(name) + continue + } db.exec(`DROP TABLE ${name}`) } + return { + upToDateTables: upToDateTables.values().toArray(), + } } return { @@ -105,7 +114,7 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa fetchDevelopmentCacheForKey, insertDevelopmentCache, deleteDevelopmentCache, - dropContentTables, + dropOldContentTables, } } diff --git a/test/basic.test.ts b/test/basic.test.ts index 354cae534..829319183 100644 --- a/test/basic.test.ts +++ b/test/basic.test.ts @@ -62,12 +62,13 @@ describe('basic', async () => { }) test('content table is created', async () => { - const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?;`) - .all(getTableName('content')) as { name: string }[] + const tableNameNoHash = getTableName('content', 'xxxx').slice(0, -4) + const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? || '%';`) + .all(tableNameNoHash) as { name: string }[] expect(cache).toBeDefined() expect(cache).toHaveLength(1) - expect(cache![0].name).toBe(getTableName('content')) + expect(cache![0].name.slice(0, -4)).toBe(tableNameNoHash) }) }) diff --git a/test/empty.test.ts b/test/empty.test.ts index 59ee78585..154c4a965 100644 --- a/test/empty.test.ts +++ b/test/empty.test.ts @@ -67,12 +67,13 @@ describe('empty', async () => { }) test('content table is created', async () => { - const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?;`) - .all(getTableName('content')) as { name: string }[] + const tableNameNoHash = getTableName('content', 'xxxx').slice(0, -4) + const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? || '%';`) + .all(tableNameNoHash) as { name: string }[] expect(cache).toBeDefined() expect(cache).toHaveLength(1) - expect(cache![0]!.name).toBe(getTableName('content')) + expect(cache![0].name.slice(0, -4)).toBe(tableNameNoHash) }) }) diff --git a/test/unit/generateCollectionInsert.test.ts b/test/unit/generateCollectionInsert.test.ts index ee4a61ea5..2e54d4c90 100644 --- a/test/unit/generateCollectionInsert.test.ts +++ b/test/unit/generateCollectionInsert.test.ts @@ -22,7 +22,7 @@ describe('generateCollectionInsert', () => { }) expect(sql[0]).toBe([ - `INSERT INTO ${getTableName('content')}`, + `INSERT INTO ${getTableName('content', collection.hash)}`, ' VALUES', ' (\'foo.md\', 13, \'2022-01-01T00:00:00.000Z\', \'md\', \'{}\', \'untitled\', true, \'foo\', \'vPdICyZ7sjhw1YY4ISEATbCTIs_HqNpMVWHnBWhOOYY\');', ].join('')) @@ -51,7 +51,7 @@ describe('generateCollectionInsert', () => { }) expect(sql[0]).toBe([ - `INSERT INTO ${getTableName('content')}`, + `INSERT INTO ${getTableName('content', collection.hash)}`, ' VALUES', ' (\'foo.md\', 42, \'2022-01-02T00:00:00.000Z\', \'md\', \'{}\', \'foo\', false, \'foo\', \'R5zX5zuyfvCtvXPcgINuEjEoHmZnse8kATeDd4V7I-c\');', ].join('')) @@ -88,14 +88,14 @@ describe('generateCollectionInsert', () => { expect(content).toEqual(querySlices.join('')) expect(sql[0]).toBe([ - `INSERT INTO ${getTableName('content')}`, + `INSERT INTO ${getTableName('content', collection.hash)}`, ' VALUES', ` ('foo.md', '${querySlices[0]}', 'md', '{}', 'foo', 'QMyFxMru9gVfaNx0fzjs5is7SvAZMEy3tNDANjkdogg');`, ].join('')) let index = 1 while (index < sql.length - 1) { expect(sql[index]).toBe([ - `UPDATE ${getTableName('content')}`, + `UPDATE ${getTableName('content', collection.hash)}`, ' SET', ` content = CONCAT(content, '${querySlices[index]}')`, ' WHERE id = \'foo.md\';', diff --git a/test/unit/generateCollectionTableDefinition.test.ts b/test/unit/generateCollectionTableDefinition.test.ts index a6603a0ee..1c7968b2a 100644 --- a/test/unit/generateCollectionTableDefinition.test.ts +++ b/test/unit/generateCollectionTableDefinition.test.ts @@ -11,7 +11,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "title" VARCHAR,', ' "body" TEXT,', @@ -38,7 +38,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "title" VARCHAR,', ' "body" TEXT,', @@ -66,7 +66,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" VARCHAR,', ' "extension" VARCHAR,', @@ -89,7 +89,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" VARCHAR(64) DEFAULT \'foo\',', ' "extension" VARCHAR,', @@ -111,7 +111,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" INT DEFAULT 13,', ' "extension" VARCHAR,', @@ -133,7 +133,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" BOOLEAN DEFAULT false,', ' "extension" VARCHAR,', @@ -155,7 +155,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" DATE,', ' "extension" VARCHAR,', @@ -180,7 +180,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" TEXT,', ' "extension" VARCHAR,', @@ -205,7 +205,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" TEXT,', ' "extension" VARCHAR,', @@ -237,7 +237,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "extension" VARCHAR,', ' "f1" BOOLEAN NULL,', diff --git a/test/unit/resolveCollection.test.ts b/test/unit/resolveCollection.test.ts index 10f1a2f2d..375c595de 100644 --- a/test/unit/resolveCollection.test.ts +++ b/test/unit/resolveCollection.test.ts @@ -10,4 +10,20 @@ describe('resolveCollection', () => { const resolvedCollection = resolveCollection('invalid-name', collection) expect(resolvedCollection).toBeUndefined() }) + + test('Collection hash changes with content', () => { + const collectionA = defineCollection({ + type: 'page', + source: '**', + }) + const collectionB = defineCollection({ + type: 'page', + source: 'someEmpty/**', + }) + const resolvedCollectionA = resolveCollection('collection', collectionA) + const resolvedCollectionB = resolveCollection('collection', collectionB) + expect(resolvedCollectionA?.hash).toBeDefined() + expect(resolvedCollectionB?.hash).toBeDefined() + expect(resolvedCollectionA?.hash).not.toBe(resolvedCollectionB?.hash) + }) }) From 7d48db1ce39d20952dea8de4fdb985a68625d699 Mon Sep 17 00:00:00 2001 From: Gerard Wilkinson Date: Mon, 24 Mar 2025 16:28:15 +0000 Subject: [PATCH 2/5] remove toArray --- src/utils/database.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/database.ts b/src/utils/database.ts index 9f24440f2..ad742dd14 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -98,7 +98,7 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa db.exec(`DROP TABLE ${name}`) } return { - upToDateTables: upToDateTables.values().toArray(), + upToDateTables: [...upToDateTables.values()], } } From d7b0a81da0f261655cc9b9fbd42acdc1417290c9 Mon Sep 17 00:00:00 2001 From: Gerard Wilkinson Date: Tue, 25 Mar 2025 09:54:44 +0000 Subject: [PATCH 3/5] fix tests --- src/runtime/internal/security.ts | 2 +- test/unit/assertSafeQuery.test.ts | 44 +++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/runtime/internal/security.ts b/src/runtime/internal/security.ts index 7c5310679..071dcc0f0 100644 --- a/src/runtime/internal/security.ts +++ b/src/runtime/internal/security.ts @@ -46,7 +46,7 @@ export function assertSafeQuery(sql: string, collection: string) { } // FROM - if (from !== `_content_${collection}`) { + if (!from.match(`^_content_${collection}_[a-z0-9]{4}$`)) { throw new Error('Invalid query') } diff --git a/test/unit/assertSafeQuery.test.ts b/test/unit/assertSafeQuery.test.ts index aea0d7e94..5335e7931 100644 --- a/test/unit/assertSafeQuery.test.ts +++ b/test/unit/assertSafeQuery.test.ts @@ -5,7 +5,7 @@ import { collectionQueryBuilder } from '../../src/runtime/internal/query' // Mock tables from manifest vi.mock('#content/manifest', () => ({ tables: { - test: '_content_test', + test: '_content_test_xxxx', }, })) const mockFetch = vi.fn().mockResolvedValue(Promise.resolve([{}])) @@ -21,29 +21,29 @@ describe('decompressSQLDump', () => { 'SELECT * FROM sqlite_master': false, 'INSERT INTO _test VALUES (\'abc\')': false, 'CREATE TABLE _test (id TEXT PRIMARY KEY)': false, - 'select * from _content_test ORDER BY id DESC': false, - ' SELECT * FROM _content_test ORDER BY id DESC': false, - 'SELECT * FROM _content_test ORDER BY id DESC ': false, - 'SELECT * FROM _content_test ORDER BY id DESC': true, - 'SELECT * FROM _content_test ORDER BY id ASC,stem DESC': false, - 'SELECT * FROM _content_test ORDER BY id ASC, stem DESC': true, - 'SELECT * FROM _content_test ORDER BY id ASC, publishedAt DESC': true, - 'SELECT "PublishedAt" FROM _content_test ORDER BY id ASC, PublishedAt DESC': true, - 'SELECT * FROM _content_test ORDER BY id DESC -- comment is not allowed': false, - 'SELECT * FROM _content_test ORDER BY id DESC; SELECT * FROM _content_test ORDER BY id DESC': false, - 'SELECT * FROM _content_test ORDER BY id DESC LIMIT 10': true, - 'SELECT * FROM _content_test ORDER BY id DESC LIMIT 10 OFFSET 10': true, + 'select * from _content_test_xxxx ORDER BY id DESC': false, + ' SELECT * FROM _content_test_xxxx ORDER BY id DESC': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC ': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id ASC,stem DESC': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id ASC, stem DESC': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id ASC, publishedAt DESC': true, + 'SELECT "PublishedAt" FROM _content_test_xxxx ORDER BY id ASC, PublishedAt DESC': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC -- comment is not allowed': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC; SELECT * FROM _content_test_xxxx ORDER BY id DESC': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC LIMIT 10': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC LIMIT 10 OFFSET 10': true, // Where clause should follow query builder syntax - 'SELECT * FROM _content_test WHERE id = 1 ORDER BY id DESC LIMIT 10 OFFSET 10': false, - 'SELECT * FROM _content_test WHERE (id = 1) ORDER BY id DESC LIMIT 10 OFFSET 10': true, - 'SELECT * FROM _content_test WHERE (id = \'");\'); select * from ((SELECT * FROM sqlite_master where 1 <> "") as t) ORDER BY type DESC': false, - 'SELECT "body" FROM _content_test ORDER BY body ASC': true, + 'SELECT * FROM _content_test_xxxx WHERE id = 1 ORDER BY id DESC LIMIT 10 OFFSET 10': false, + 'SELECT * FROM _content_test_xxxx WHERE (id = 1) ORDER BY id DESC LIMIT 10 OFFSET 10': true, + 'SELECT * FROM _content_test_xxxx WHERE (id = \'");\'); select * from ((SELECT * FROM sqlite_master where 1 <> "") as t) ORDER BY type DESC': false, + 'SELECT "body" FROM _content_test_xxxx ORDER BY body ASC': true, // Advanced - 'SELECT COUNT(*) UNION SELECT name /**/FROM sqlite_master-- FROM _content_test WHERE (1=1) ORDER BY id ASC': false, - 'SELECT * FROM _content_test WHERE (id /*\'*/IN (SELECT id FROM _content_test) /*\'*/) ORDER BY id ASC': false, - 'SELECT * FROM _content_test WHERE (1=\' \\\' OR id IN (SELECT id FROM _content_docs) OR 1!=\'\') ORDER BY id ASC': false, - 'SELECT "id", "id" FROM _content_docs WHERE (1=\' \\\') UNION SELECT tbl_name,tbl_name FROM sqlite_master-- \') ORDER BY id ASC': false, - 'SELECT "id" FROM _content_test WHERE (x=$\'$ OR x IN (SELECT BLAH) OR x=$\'$) ORDER BY id ASC': false, + 'SELECT COUNT(*) UNION SELECT name /**/FROM sqlite_master-- FROM _content_test_xxxx WHERE (1=1) ORDER BY id ASC': false, + 'SELECT * FROM _content_test_xxxx WHERE (id /*\'*/IN (SELECT id FROM _content_test_xxxx) /*\'*/) ORDER BY id ASC': false, + 'SELECT * FROM _content_test_xxxx WHERE (1=\' \\\' OR id IN (SELECT id FROM _content_docs_xxxx) OR 1!=\'\') ORDER BY id ASC': false, + 'SELECT "id", "id" FROM _content_docs_xxxx WHERE (1=\' \\\') UNION SELECT tbl_name,tbl_name FROM sqlite_master-- \') ORDER BY id ASC': false, + 'SELECT "id" FROM _content_test_xxxx WHERE (x=$\'$ OR x IN (SELECT BLAH) OR x=$\'$) ORDER BY id ASC': false, } Object.entries(queries).forEach(([query, isValid]) => { From 0d94aebfbca81fc1b8c92fc2e4d41c3ad3a242dc Mon Sep 17 00:00:00 2001 From: Gerard Wilkinson Date: Tue, 25 Mar 2025 10:54:13 +0000 Subject: [PATCH 4/5] fix test --- src/runtime/internal/collection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/internal/collection.ts b/src/runtime/internal/collection.ts index 3b14e8968..a4f7d6cbe 100644 --- a/src/runtime/internal/collection.ts +++ b/src/runtime/internal/collection.ts @@ -32,5 +32,5 @@ function findCollectionFields(sql: string): Record Date: Tue, 25 Mar 2025 16:16:28 +0000 Subject: [PATCH 5/5] resolve potential multiple instance cleanup --- src/utils/database.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/database.ts b/src/utils/database.ts index ad742dd14..4413014df 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -95,7 +95,7 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa upToDateTables.add(name) continue } - db.exec(`DROP TABLE ${name}`) + db.exec(`DROP TABLE IF EXISTS ${name}`) } return { upToDateTables: [...upToDateTables.values()],