feat: collection hashing for multi instance support

src/module.ts

@@ -250,14 +250,14 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
   let cachedFilesCount = 0
   let parsedFilesCount = 0
 
-  // Remove all existing content collections to start with a clean state
-  db.dropContentTables()
+  // Remove all out of date collections and only create new ones
+  const { upToDateTables } = await db.dropOldContentTables(collections)
+  const newCollections = collections.filter(c => !upToDateTables.includes(c.name))
   // Create database dump
-  for await (const collection of collections) {
+  for await (const collection of newCollections) {
     if (collection.name === 'info') {
       continue
     }
-    const collectionHash = hash(collection)
     const collectionQueries = generateCollectionTableDefinition(collection, { drop: true })
       .split('\n').map(q => `${q} -- structure`)
 
@@ -295,7 +295,7 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
 
           try {
             const content = await source.getItem?.(key) || ''
-            const checksum = getContentChecksum(configHash + collectionHash + content)
+            const checksum = getContentChecksum(configHash + collection.hash + content)
 
             let parsedContent
             if (cache && cache.checksum === checksum) {

src/runtime/internal/collection.ts

@@ -32,5 +32,5 @@ function findCollectionFields(sql: string): Record<string, 'string' | 'number' |
 }
 
 function getCollectionName(table: string) {
-  return table.replace(/^_content_/, '')
+  return table.replace(/^_content_/, '').replace(/_[a-z0-9]{4}$/, '')
 }

src/runtime/internal/security.ts

@@ -46,7 +46,7 @@ export function assertSafeQuery(sql: string, collection: string) {
   }
 
   // FROM
-  if (from !== `_content_${collection}`) {
+  if (!from.match(`^_content_${collection}_[a-z0-9]{4}$`)) {
     throw new Error('Invalid query')
   }
 

src/types/collection.ts

@@ -73,6 +73,7 @@ export interface ResolvedCollection<T extends ZodRawShape = ZodRawShape> {
    * Private collections will not be available in the runtime.
    */
   private: boolean
+  hash: string
 }
 
 export interface CollectionInfo {

src/types/database.ts

@@ -1,4 +1,5 @@
 import type { Primitive, Connector } from 'db0'
+import type { ResolvedCollection } from '../module'
 
 export type CacheEntry = { id: string, checksum: string, parsedContent: string }
 
@@ -19,7 +20,7 @@ export interface LocalDevelopmentDatabase {
   fetchDevelopmentCacheForKey(key: string): Promise<CacheEntry | undefined>
   insertDevelopmentCache(id: string, checksum: string, parsedContent: string): void
   deleteDevelopmentCache(id: string): void
-  dropContentTables(): void
+  dropOldContentTables(collections: ResolvedCollection[]): Promise<{ upToDateTables: string[] }>
   exec(sql: string): void
   close(): void
   database?: Connector

src/utils/collection.ts

@@ -11,8 +11,9 @@ import { logger } from './dev'
 
 const JSON_FIELDS_TYPES = ['ZodObject', 'ZodArray', 'ZodRecord', 'ZodIntersection', 'ZodUnion', 'ZodAny', 'ZodMap']
 
-export function getTableName(name: string) {
-  return `_content_${name}`
+export function getTableName(name: string, collectionHash: string) {
+  const tableNameSafeHash = collectionHash.split('-').join('').toLowerCase().substring(0, 4)
+  return `_content_${name}_${tableNameSafeHash}`
 }
 
 export function defineCollection<T extends ZodRawShape>(collection: Collection<T>): DefinedCollection {
@@ -75,13 +76,20 @@ export function resolveCollection(name: string, collection: DefinedCollection):
     return undefined
   }
 
-  return {
+  const resolvedCollection: Omit<ResolvedCollection, 'hash' | 'tableName'> = {
     ...collection,
     name,
     type: collection.type || 'page',
-    tableName: getTableName(name),
     private: name === 'info',
   }
+
+  const collectionHash = hash(resolvedCollection)
+
+  return {
+    ...resolvedCollection,
+    tableName: getTableName(name, collectionHash),
+    hash: collectionHash,
+  }
 }
 
 export function resolveCollections(collections: Record<string, DefinedCollection>): ResolvedCollection[] {

src/utils/database.ts

@@ -6,6 +6,7 @@ import { isAbsolute, join, dirname } from 'pathe'
 import { isWebContainer } from '@webcontainer/env'
 import type { CacheEntry, D1DatabaseConfig, LocalDevelopmentDatabase, SqliteDatabaseConfig } from '../types'
 import type { ModuleOptions } from '../types/module'
+import type { ResolvedCollection } from '../module'
 import { logger } from './dev'
 
 export async function refineDatabaseConfig(database: ModuleOptions['database'], opts: { rootDir: string, updateSqliteFileName?: boolean }) {
@@ -85,11 +86,19 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa
     db.prepare(`DELETE FROM _development_cache WHERE id = ?`).run(id)
   }
 
-  const dropContentTables = async () => {
+  const dropOldContentTables = async (collections: ResolvedCollection[]) => {
     const tables = await db.prepare('SELECT name FROM sqlite_master WHERE type = ? AND name LIKE ?')
       .all('table', '_content_%') as { name: string }[]
+    const upToDateTables = new Set<string>()
     for (const { name } of tables) {
-      db.exec(`DROP TABLE ${name}`)
+      if (collections.some(c => c.tableName === name)) {
+        upToDateTables.add(name)
+        continue
+      }
+      db.exec(`DROP TABLE IF EXISTS ${name}`)
+    }
+    return {
+      upToDateTables: [...upToDateTables.values()],
     }
   }
 
@@ -105,7 +114,7 @@ export async function getLocalDatabase(database: SqliteDatabaseConfig | D1Databa
     fetchDevelopmentCacheForKey,
     insertDevelopmentCache,
     deleteDevelopmentCache,
-    dropContentTables,
+    dropOldContentTables,
   }
 }
 

test/basic.test.ts

@@ -62,12 +62,13 @@ describe('basic', async () => {
     })
 
     test('content table is created', async () => {
-      const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?;`)
-        .all(getTableName('content')) as { name: string }[]
+      const tableNameNoHash = getTableName('content', 'xxxx').slice(0, -4)
+      const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? || '%';`)
+        .all(tableNameNoHash) as { name: string }[]
 
       expect(cache).toBeDefined()
       expect(cache).toHaveLength(1)
-      expect(cache![0].name).toBe(getTableName('content'))
+      expect(cache![0].name.slice(0, -4)).toBe(tableNameNoHash)
     })
   })
 

test/empty.test.ts

@@ -67,12 +67,13 @@ describe('empty', async () => {
     })
 
     test('content table is created', async () => {
-      const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?;`)
-        .all(getTableName('content')) as { name: string }[]
+      const tableNameNoHash = getTableName('content', 'xxxx').slice(0, -4)
+      const cache = await db.database?.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ? || '%';`)
+        .all(tableNameNoHash) as { name: string }[]
 
       expect(cache).toBeDefined()
       expect(cache).toHaveLength(1)
-      expect(cache![0]!.name).toBe(getTableName('content'))
+      expect(cache![0].name.slice(0, -4)).toBe(tableNameNoHash)
     })
   })
 

test/unit/assertSafeQuery.test.ts

@@ -5,7 +5,7 @@ import { collectionQueryBuilder } from '../../src/runtime/internal/query'
 // Mock tables from manifest
 vi.mock('#content/manifest', () => ({
   tables: {
-    test: '_content_test',
+    test: '_content_test_xxxx',
   },
 }))
 const mockFetch = vi.fn().mockResolvedValue(Promise.resolve([{}]))
@@ -21,29 +21,29 @@ describe('decompressSQLDump', () => {
     'SELECT * FROM sqlite_master': false,
     'INSERT INTO _test VALUES (\'abc\')': false,
     'CREATE TABLE _test (id TEXT PRIMARY KEY)': false,
-    'select * from _content_test ORDER BY id DESC': false,
-    ' SELECT * FROM _content_test ORDER BY id DESC': false,
-    'SELECT * FROM _content_test ORDER BY id DESC ': false,
-    'SELECT * FROM _content_test ORDER BY id DESC': true,
-    'SELECT * FROM _content_test ORDER BY id ASC,stem DESC': false,
-    'SELECT * FROM _content_test ORDER BY id ASC, stem DESC': true,
-    'SELECT * FROM _content_test ORDER BY id ASC, publishedAt DESC': true,
-    'SELECT "PublishedAt" FROM _content_test ORDER BY id ASC, PublishedAt DESC': true,
-    'SELECT * FROM _content_test ORDER BY id DESC -- comment is not allowed': false,
-    'SELECT * FROM _content_test ORDER BY id DESC; SELECT * FROM _content_test ORDER BY id DESC': false,
-    'SELECT * FROM _content_test ORDER BY id DESC LIMIT 10': true,
-    'SELECT * FROM _content_test ORDER BY id DESC LIMIT 10 OFFSET 10': true,
+    'select * from _content_test_xxxx ORDER BY id DESC': false,
+    ' SELECT * FROM _content_test_xxxx ORDER BY id DESC': false,
+    'SELECT * FROM _content_test_xxxx ORDER BY id DESC ': false,
+    'SELECT * FROM _content_test_xxxx ORDER BY id DESC': true,
+    'SELECT * FROM _content_test_xxxx ORDER BY id ASC,stem DESC': false,
+    'SELECT * FROM _content_test_xxxx ORDER BY id ASC, stem DESC': true,
+    'SELECT * FROM _content_test_xxxx ORDER BY id ASC, publishedAt DESC': true,
+    'SELECT "PublishedAt" FROM _content_test_xxxx ORDER BY id ASC, PublishedAt DESC': true,
+    'SELECT * FROM _content_test_xxxx ORDER BY id DESC -- comment is not allowed': false,
+    'SELECT * FROM _content_test_xxxx ORDER BY id DESC; SELECT * FROM _content_test_xxxx ORDER BY id DESC': false,
+    'SELECT * FROM _content_test_xxxx ORDER BY id DESC LIMIT 10': true,
+    'SELECT * FROM _content_test_xxxx ORDER BY id DESC LIMIT 10 OFFSET 10': true,
     // Where clause should follow query builder syntax
-    'SELECT * FROM _content_test WHERE id = 1 ORDER BY id DESC LIMIT 10 OFFSET 10': false,
-    'SELECT * FROM _content_test WHERE (id = 1) ORDER BY id DESC LIMIT 10 OFFSET 10': true,
-    'SELECT * FROM _content_test WHERE (id = \'");\'); select * from ((SELECT * FROM sqlite_master where 1 <> "") as t) ORDER BY type DESC': false,
-    'SELECT "body" FROM _content_test ORDER BY body ASC': true,
+    'SELECT * FROM _content_test_xxxx WHERE id = 1 ORDER BY id DESC LIMIT 10 OFFSET 10': false,
+    'SELECT * FROM _content_test_xxxx WHERE (id = 1) ORDER BY id DESC LIMIT 10 OFFSET 10': true,
+    'SELECT * FROM _content_test_xxxx WHERE (id = \'");\'); select * from ((SELECT * FROM sqlite_master where 1 <> "") as t) ORDER BY type DESC': false,
+    'SELECT "body" FROM _content_test_xxxx ORDER BY body ASC': true,
     // Advanced
-    'SELECT COUNT(*) UNION SELECT name /**/FROM sqlite_master-- FROM _content_test WHERE (1=1) ORDER BY id ASC': false,
-    'SELECT * FROM _content_test WHERE (id /*\'*/IN (SELECT id FROM _content_test) /*\'*/) ORDER BY id ASC': false,
-    'SELECT * FROM _content_test WHERE (1=\' \\\' OR id IN (SELECT id FROM _content_docs) OR 1!=\'\') ORDER BY id ASC': false,
-    'SELECT "id", "id" FROM _content_docs WHERE (1=\' \\\') UNION SELECT tbl_name,tbl_name FROM sqlite_master-- \') ORDER BY id ASC': false,
-    'SELECT "id" FROM _content_test WHERE (x=$\'$ OR x IN (SELECT BLAH) OR x=$\'$) ORDER BY id ASC': false,
+    'SELECT COUNT(*) UNION SELECT name /**/FROM sqlite_master-- FROM _content_test_xxxx WHERE (1=1) ORDER BY id ASC': false,
+    'SELECT * FROM _content_test_xxxx WHERE (id /*\'*/IN (SELECT id FROM _content_test_xxxx) /*\'*/) ORDER BY id ASC': false,
+    'SELECT * FROM _content_test_xxxx WHERE (1=\' \\\' OR id IN (SELECT id FROM _content_docs_xxxx) OR 1!=\'\') ORDER BY id ASC': false,
+    'SELECT "id", "id" FROM _content_docs_xxxx WHERE (1=\' \\\') UNION SELECT tbl_name,tbl_name FROM sqlite_master-- \') ORDER BY id ASC': false,
+    'SELECT "id" FROM _content_test_xxxx WHERE (x=$\'$ OR x IN (SELECT BLAH) OR x=$\'$) ORDER BY id ASC': false,
   }
 
   Object.entries(queries).forEach(([query, isValid]) => {

test/unit/generateCollectionInsert.test.ts

@@ -22,7 +22,7 @@ describe('generateCollectionInsert', () => {
     })
 
     expect(sql[0]).toBe([
-      `INSERT INTO ${getTableName('content')}`,
+      `INSERT INTO ${getTableName('content', collection.hash)}`,
       ' VALUES',
       ' (\'foo.md\', 13, \'2022-01-01T00:00:00.000Z\', \'md\', \'{}\', \'untitled\', true, \'foo\', \'vPdICyZ7sjhw1YY4ISEATbCTIs_HqNpMVWHnBWhOOYY\');',
     ].join(''))
@@ -51,7 +51,7 @@ describe('generateCollectionInsert', () => {
     })
 
     expect(sql[0]).toBe([
-      `INSERT INTO ${getTableName('content')}`,
+      `INSERT INTO ${getTableName('content', collection.hash)}`,
       ' VALUES',
       ' (\'foo.md\', 42, \'2022-01-02T00:00:00.000Z\', \'md\', \'{}\', \'foo\', false, \'foo\', \'R5zX5zuyfvCtvXPcgINuEjEoHmZnse8kATeDd4V7I-c\');',
     ].join(''))
@@ -88,14 +88,14 @@ describe('generateCollectionInsert', () => {
     expect(content).toEqual(querySlices.join(''))
 
     expect(sql[0]).toBe([
-      `INSERT INTO ${getTableName('content')}`,
+      `INSERT INTO ${getTableName('content', collection.hash)}`,
       ' VALUES',
       ` ('foo.md', '${querySlices[0]}', 'md', '{}', 'foo', 'QMyFxMru9gVfaNx0fzjs5is7SvAZMEy3tNDANjkdogg');`,
     ].join(''))
     let index = 1
     while (index < sql.length - 1) {
       expect(sql[index]).toBe([
-        `UPDATE ${getTableName('content')}`,
+        `UPDATE ${getTableName('content', collection.hash)}`,
         ' SET',
         ` content = CONCAT(content, '${querySlices[index]}')`,
         ' WHERE id = \'foo.md\';',

test/unit/generateCollectionTableDefinition.test.ts

@@ -11,7 +11,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "title" VARCHAR,',
       ' "body" TEXT,',
@@ -38,7 +38,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "title" VARCHAR,',
       ' "body" TEXT,',
@@ -66,7 +66,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" VARCHAR,',
       ' "extension" VARCHAR,',
@@ -89,7 +89,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" VARCHAR(64) DEFAULT \'foo\',',
       ' "extension" VARCHAR,',
@@ -111,7 +111,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" INT DEFAULT 13,',
       ' "extension" VARCHAR,',
@@ -133,7 +133,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" BOOLEAN DEFAULT false,',
       ' "extension" VARCHAR,',
@@ -155,7 +155,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" DATE,',
       ' "extension" VARCHAR,',
@@ -180,7 +180,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" TEXT,',
       ' "extension" VARCHAR,',
@@ -205,7 +205,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "customField" TEXT,',
       ' "extension" VARCHAR,',
@@ -237,7 +237,7 @@ describe('generateCollectionTableDefinition', () => {
     const sql = generateCollectionTableDefinition(collection)
 
     expect(sql).toBe([
-      `CREATE TABLE IF NOT EXISTS ${getTableName('content')} (`,
+      `CREATE TABLE IF NOT EXISTS ${getTableName('content', collection.hash)} (`,
       'id TEXT PRIMARY KEY,',
       ' "extension" VARCHAR,',
       ' "f1" BOOLEAN NULL,',

test/unit/resolveCollection.test.ts

@@ -10,4 +10,20 @@ describe('resolveCollection', () => {
     const resolvedCollection = resolveCollection('invalid-name', collection)
     expect(resolvedCollection).toBeUndefined()
   })
+
+  test('Collection hash changes with content', () => {
+    const collectionA = defineCollection({
+      type: 'page',
+      source: '**',
+    })
+    const collectionB = defineCollection({
+      type: 'page',
+      source: 'someEmpty/**',
+    })
+    const resolvedCollectionA = resolveCollection('collection', collectionA)
+    const resolvedCollectionB = resolveCollection('collection', collectionB)
+    expect(resolvedCollectionA?.hash).toBeDefined()
+    expect(resolvedCollectionB?.hash).toBeDefined()
+    expect(resolvedCollectionA?.hash).not.toBe(resolvedCollectionB?.hash)
+  })
 })