diff --git a/src/module.ts b/src/module.ts index 7ade14d9a..1d450c829 100644 --- a/src/module.ts +++ b/src/module.ts @@ -250,14 +250,14 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio let cachedFilesCount = 0 let parsedFilesCount = 0 - // Remove all existing content collections to start with a clean state - db.dropContentTables() + // Remove all out of date collections and only create new ones + const { upToDateTables } = await db.dropOldContentTables(collections) + const newCollections = collections.filter(c => !upToDateTables.includes(c.name)) // Create database dump - for await (const collection of collections) { + for await (const collection of newCollections) { if (collection.name === 'info') { continue } - const collectionHash = hash(collection) const collectionQueries = generateCollectionTableDefinition(collection, { drop: true }) .split('\n').map(q => `${q} -- structure`) @@ -295,7 +295,7 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio try { const content = await source.getItem?.(key) || '' - const checksum = getContentChecksum(configHash + collectionHash + content) + const checksum = getContentChecksum(configHash + collection.hash + content) let parsedContent if (cache && cache.checksum === checksum) { diff --git a/src/runtime/internal/collection.ts b/src/runtime/internal/collection.ts index 3b14e8968..a4f7d6cbe 100644 --- a/src/runtime/internal/collection.ts +++ b/src/runtime/internal/collection.ts @@ -32,5 +32,5 @@ function findCollectionFields(sql: string): Record { * Private collections will not be available in the runtime. */ private: boolean + hash: string } export interface CollectionInfo { diff --git a/src/types/database.ts b/src/types/database.ts index 3b008acec..5d58f36da 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -1,4 +1,5 @@ import type { Primitive, Connector } from 'db0' +import type { ResolvedCollection } from '../module' export type CacheEntry = { id: string, checksum: string, parsedContent: string } @@ -19,7 +20,7 @@ export interface LocalDevelopmentDatabase { fetchDevelopmentCacheForKey(key: string): Promise insertDevelopmentCache(id: string, checksum: string, parsedContent: string): void deleteDevelopmentCache(id: string): void - dropContentTables(): void + dropOldContentTables(collections: ResolvedCollection[]): Promise<{ upToDateTables: string[] }> exec(sql: string): void close(): void database?: Connector diff --git a/src/utils/collection.ts b/src/utils/collection.ts index 2bea0ea7a..349736d64 100644 --- a/src/utils/collection.ts +++ b/src/utils/collection.ts @@ -11,8 +11,9 @@ import { logger } from './dev' const JSON_FIELDS_TYPES = ['ZodObject', 'ZodArray', 'ZodRecord', 'ZodIntersection', 'ZodUnion', 'ZodAny', 'ZodMap'] -export function getTableName(name: string) { - return `_content_${name}` +export function getTableName(name: string, collectionHash: string) { + const tableNameSafeHash = collectionHash.split('-').join('').toLowerCase().substring(0, 4) + return `_content_${name}_${tableNameSafeHash}` } export function defineCollection(collection: Collection): DefinedCollection { @@ -75,13 +76,20 @@ export function resolveCollection(name: string, collection: DefinedCollection): return undefined } - return { + const resolvedCollection: Omit = { ...collection, name, type: collection.type || 'page', - tableName: getTableName(name), private: name === 'info', } + + const collectionHash = hash(resolvedCollection) + + return { + ...resolvedCollection, + tableName: getTableName(name, collectionHash), + hash: collectionHash, + } } export function resolveCollections(collections: Record): ResolvedCollection[] { diff --git a/src/utils/database.ts b/src/utils/database.ts index 632181fdd..4413014df 100644 --- a/src/utils/database.ts +++ b/src/utils/database.ts @@ -6,6 +6,7 @@ import { isAbsolute, join, dirname } from 'pathe' import { isWebContainer } from '@webcontainer/env' import type { CacheEntry, D1DatabaseConfig, LocalDevelopmentDatabase, SqliteDatabaseConfig } from '../types' import type { ModuleOptions } from '../types/module' +import type { ResolvedCollection } from '../module' import { logger } from './dev' export async function refineDatabaseConfig(database: ModuleOptions['database'], opts: { rootDir: string, updateSqliteFileName?: boolean }) { @@ -85,11 +86,19 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa db.prepare(`DELETE FROM _development_cache WHERE id = ?`).run(id) } - const dropContentTables = async () => { + const dropOldContentTables = async (collections: ResolvedCollection[]) => { const tables = await db.prepare('SELECT name FROM sqlite_master WHERE type = ? AND name LIKE ?') .all('table', '_content_%') as { name: string }[] + const upToDateTables = new Set() for (const { name } of tables) { - db.exec(`DROP TABLE ${name}`) + if (collections.some(c => c.tableName === name)) { + upToDateTables.add(name) + continue + } + db.exec(`DROP TABLE IF EXISTS ${name}`) + } + return { + upToDateTables: [...upToDateTables.values()], } } @@ -105,7 +114,7 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa fetchDevelopmentCacheForKey, insertDevelopmentCache, deleteDevelopmentCache, - dropContentTables, + dropOldContentTables, } } diff --git a/test/basic.test.ts b/test/basic.test.ts index 354cae534..829319183 100644 --- a/test/basic.test.ts +++ b/test/basic.test.ts @@ -62,12 +62,13 @@ describe('basic', async () => { }) test('content table is created', async () => { - const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?;`) - .all(getTableName('content')) as { name: string }[] + const tableNameNoHash = getTableName('content', 'xxxx').slice(0, -4) + const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? || '%';`) + .all(tableNameNoHash) as { name: string }[] expect(cache).toBeDefined() expect(cache).toHaveLength(1) - expect(cache![0].name).toBe(getTableName('content')) + expect(cache![0].name.slice(0, -4)).toBe(tableNameNoHash) }) }) diff --git a/test/empty.test.ts b/test/empty.test.ts index 59ee78585..154c4a965 100644 --- a/test/empty.test.ts +++ b/test/empty.test.ts @@ -67,12 +67,13 @@ describe('empty', async () => { }) test('content table is created', async () => { - const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?;`) - .all(getTableName('content')) as { name: string }[] + const tableNameNoHash = getTableName('content', 'xxxx').slice(0, -4) + const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? || '%';`) + .all(tableNameNoHash) as { name: string }[] expect(cache).toBeDefined() expect(cache).toHaveLength(1) - expect(cache![0]!.name).toBe(getTableName('content')) + expect(cache![0].name.slice(0, -4)).toBe(tableNameNoHash) }) }) diff --git a/test/unit/assertSafeQuery.test.ts b/test/unit/assertSafeQuery.test.ts index aea0d7e94..5335e7931 100644 --- a/test/unit/assertSafeQuery.test.ts +++ b/test/unit/assertSafeQuery.test.ts @@ -5,7 +5,7 @@ import { collectionQueryBuilder } from '../../src/runtime/internal/query' // Mock tables from manifest vi.mock('#content/manifest', () => ({ tables: { - test: '_content_test', + test: '_content_test_xxxx', }, })) const mockFetch = vi.fn().mockResolvedValue(Promise.resolve([{}])) @@ -21,29 +21,29 @@ describe('decompressSQLDump', () => { 'SELECT * FROM sqlite_master': false, 'INSERT INTO _test VALUES (\'abc\')': false, 'CREATE TABLE _test (id TEXT PRIMARY KEY)': false, - 'select * from _content_test ORDER BY id DESC': false, - ' SELECT * FROM _content_test ORDER BY id DESC': false, - 'SELECT * FROM _content_test ORDER BY id DESC ': false, - 'SELECT * FROM _content_test ORDER BY id DESC': true, - 'SELECT * FROM _content_test ORDER BY id ASC,stem DESC': false, - 'SELECT * FROM _content_test ORDER BY id ASC, stem DESC': true, - 'SELECT * FROM _content_test ORDER BY id ASC, publishedAt DESC': true, - 'SELECT "PublishedAt" FROM _content_test ORDER BY id ASC, PublishedAt DESC': true, - 'SELECT * FROM _content_test ORDER BY id DESC -- comment is not allowed': false, - 'SELECT * FROM _content_test ORDER BY id DESC; SELECT * FROM _content_test ORDER BY id DESC': false, - 'SELECT * FROM _content_test ORDER BY id DESC LIMIT 10': true, - 'SELECT * FROM _content_test ORDER BY id DESC LIMIT 10 OFFSET 10': true, + 'select * from _content_test_xxxx ORDER BY id DESC': false, + ' SELECT * FROM _content_test_xxxx ORDER BY id DESC': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC ': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id ASC,stem DESC': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id ASC, stem DESC': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id ASC, publishedAt DESC': true, + 'SELECT "PublishedAt" FROM _content_test_xxxx ORDER BY id ASC, PublishedAt DESC': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC -- comment is not allowed': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC; SELECT * FROM _content_test_xxxx ORDER BY id DESC': false, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC LIMIT 10': true, + 'SELECT * FROM _content_test_xxxx ORDER BY id DESC LIMIT 10 OFFSET 10': true, // Where clause should follow query builder syntax - 'SELECT * FROM _content_test WHERE id = 1 ORDER BY id DESC LIMIT 10 OFFSET 10': false, - 'SELECT * FROM _content_test WHERE (id = 1) ORDER BY id DESC LIMIT 10 OFFSET 10': true, - 'SELECT * FROM _content_test WHERE (id = \'");\'); select * from ((SELECT * FROM sqlite_master where 1 <> "") as t) ORDER BY type DESC': false, - 'SELECT "body" FROM _content_test ORDER BY body ASC': true, + 'SELECT * FROM _content_test_xxxx WHERE id = 1 ORDER BY id DESC LIMIT 10 OFFSET 10': false, + 'SELECT * FROM _content_test_xxxx WHERE (id = 1) ORDER BY id DESC LIMIT 10 OFFSET 10': true, + 'SELECT * FROM _content_test_xxxx WHERE (id = \'");\'); select * from ((SELECT * FROM sqlite_master where 1 <> "") as t) ORDER BY type DESC': false, + 'SELECT "body" FROM _content_test_xxxx ORDER BY body ASC': true, // Advanced - 'SELECT COUNT(*) UNION SELECT name /**/FROM sqlite_master-- FROM _content_test WHERE (1=1) ORDER BY id ASC': false, - 'SELECT * FROM _content_test WHERE (id /*\'*/IN (SELECT id FROM _content_test) /*\'*/) ORDER BY id ASC': false, - 'SELECT * FROM _content_test WHERE (1=\' \\\' OR id IN (SELECT id FROM _content_docs) OR 1!=\'\') ORDER BY id ASC': false, - 'SELECT "id", "id" FROM _content_docs WHERE (1=\' \\\') UNION SELECT tbl_name,tbl_name FROM sqlite_master-- \') ORDER BY id ASC': false, - 'SELECT "id" FROM _content_test WHERE (x=$\'$ OR x IN (SELECT BLAH) OR x=$\'$) ORDER BY id ASC': false, + 'SELECT COUNT(*) UNION SELECT name /**/FROM sqlite_master-- FROM _content_test_xxxx WHERE (1=1) ORDER BY id ASC': false, + 'SELECT * FROM _content_test_xxxx WHERE (id /*\'*/IN (SELECT id FROM _content_test_xxxx) /*\'*/) ORDER BY id ASC': false, + 'SELECT * FROM _content_test_xxxx WHERE (1=\' \\\' OR id IN (SELECT id FROM _content_docs_xxxx) OR 1!=\'\') ORDER BY id ASC': false, + 'SELECT "id", "id" FROM _content_docs_xxxx WHERE (1=\' \\\') UNION SELECT tbl_name,tbl_name FROM sqlite_master-- \') ORDER BY id ASC': false, + 'SELECT "id" FROM _content_test_xxxx WHERE (x=$\'$ OR x IN (SELECT BLAH) OR x=$\'$) ORDER BY id ASC': false, } Object.entries(queries).forEach(([query, isValid]) => { diff --git a/test/unit/generateCollectionInsert.test.ts b/test/unit/generateCollectionInsert.test.ts index ee4a61ea5..2e54d4c90 100644 --- a/test/unit/generateCollectionInsert.test.ts +++ b/test/unit/generateCollectionInsert.test.ts @@ -22,7 +22,7 @@ describe('generateCollectionInsert', () => { }) expect(sql[0]).toBe([ - `INSERT INTO ${getTableName('content')}`, + `INSERT INTO ${getTableName('content', collection.hash)}`, ' VALUES', ' (\'foo.md\', 13, \'2022-01-01T00:00:00.000Z\', \'md\', \'{}\', \'untitled\', true, \'foo\', \'vPdICyZ7sjhw1YY4ISEATbCTIs_HqNpMVWHnBWhOOYY\');', ].join('')) @@ -51,7 +51,7 @@ describe('generateCollectionInsert', () => { }) expect(sql[0]).toBe([ - `INSERT INTO ${getTableName('content')}`, + `INSERT INTO ${getTableName('content', collection.hash)}`, ' VALUES', ' (\'foo.md\', 42, \'2022-01-02T00:00:00.000Z\', \'md\', \'{}\', \'foo\', false, \'foo\', \'R5zX5zuyfvCtvXPcgINuEjEoHmZnse8kATeDd4V7I-c\');', ].join('')) @@ -88,14 +88,14 @@ describe('generateCollectionInsert', () => { expect(content).toEqual(querySlices.join('')) expect(sql[0]).toBe([ - `INSERT INTO ${getTableName('content')}`, + `INSERT INTO ${getTableName('content', collection.hash)}`, ' VALUES', ` ('foo.md', '${querySlices[0]}', 'md', '{}', 'foo', 'QMyFxMru9gVfaNx0fzjs5is7SvAZMEy3tNDANjkdogg');`, ].join('')) let index = 1 while (index < sql.length - 1) { expect(sql[index]).toBe([ - `UPDATE ${getTableName('content')}`, + `UPDATE ${getTableName('content', collection.hash)}`, ' SET', ` content = CONCAT(content, '${querySlices[index]}')`, ' WHERE id = \'foo.md\';', diff --git a/test/unit/generateCollectionTableDefinition.test.ts b/test/unit/generateCollectionTableDefinition.test.ts index a6603a0ee..1c7968b2a 100644 --- a/test/unit/generateCollectionTableDefinition.test.ts +++ b/test/unit/generateCollectionTableDefinition.test.ts @@ -11,7 +11,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "title" VARCHAR,', ' "body" TEXT,', @@ -38,7 +38,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "title" VARCHAR,', ' "body" TEXT,', @@ -66,7 +66,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" VARCHAR,', ' "extension" VARCHAR,', @@ -89,7 +89,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" VARCHAR(64) DEFAULT \'foo\',', ' "extension" VARCHAR,', @@ -111,7 +111,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" INT DEFAULT 13,', ' "extension" VARCHAR,', @@ -133,7 +133,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" BOOLEAN DEFAULT false,', ' "extension" VARCHAR,', @@ -155,7 +155,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" DATE,', ' "extension" VARCHAR,', @@ -180,7 +180,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" TEXT,', ' "extension" VARCHAR,', @@ -205,7 +205,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "customField" TEXT,', ' "extension" VARCHAR,', @@ -237,7 +237,7 @@ describe('generateCollectionTableDefinition', () => { const sql = generateCollectionTableDefinition(collection) expect(sql).toBe([ - `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`, + `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`, 'id TEXT PRIMARY KEY,', ' "extension" VARCHAR,', ' "f1" BOOLEAN NULL,', diff --git a/test/unit/resolveCollection.test.ts b/test/unit/resolveCollection.test.ts index 10f1a2f2d..375c595de 100644 --- a/test/unit/resolveCollection.test.ts +++ b/test/unit/resolveCollection.test.ts @@ -10,4 +10,20 @@ describe('resolveCollection', () => { const resolvedCollection = resolveCollection('invalid-name', collection) expect(resolvedCollection).toBeUndefined() }) + + test('Collection hash changes with content', () => { + const collectionA = defineCollection({ + type: 'page', + source: '**', + }) + const collectionB = defineCollection({ + type: 'page', + source: 'someEmpty/**', + }) + const resolvedCollectionA = resolveCollection('collection', collectionA) + const resolvedCollectionB = resolveCollection('collection', collectionB) + expect(resolvedCollectionA?.hash).toBeDefined() + expect(resolvedCollectionB?.hash).toBeDefined() + expect(resolvedCollectionA?.hash).not.toBe(resolvedCollectionB?.hash) + }) })