a more scalable full text search via SQLite

package.json

@@ -20,17 +20,23 @@
     "path-parser": "^6.1.0",
     "process": "^0.11.10",
     "stream-browserify": "^3.0.0",
+    "wa-sqlite": "rhashimoto/wa-sqlite#buildless",
     "warcio": "^1.5.1"
   },
   "devDependencies": {
     "@titelmedia/node-fetch": "^3.0.1",
+    "@types/better-sqlite3": "^7.5.0",
     "ava": "^3.14.0",
+    "better-sqlite3": "^7.5.1",
     "eslint": "^7.23.0",
     "esm": "^3.2.25",
     "fake-indexeddb": "^3.0.0",
     "nyc": "^15.0.1",
     "raw-loader": "^4.0.2",
     "test-listen": "^1.1.0",
+    "ts-loader": "^9.2.8",
+    "ts-node": "^10.7.0",
+    "typescript": "^4.6.3",
     "web-streams-node": "^0.4.0",
     "webpack": "^5.53.0",
     "webpack-cli": "^4.8.0",

src/api.js

@@ -62,6 +62,7 @@ class API {
       "curated": "c/:coll/curated/:list",
       "pages": "c/:coll/pages",
       "textIndex": "c/:coll/textIndex",
+      "sqliteFtsSearch": "c/:coll/sqliteFtsSearch",
       "deletePage": ["c/:coll/page/:page", "DELETE"],
     };
   }
@@ -188,6 +189,17 @@ class API {
         return {};
       }
     }
+    case "sqliteFtsSearch": {
+      const coll = await this.collections.getColl(params.coll);
+      if (!coll) {
+        return {error: "collection_not_found"};
+      }
+      if (coll.store.sqliteFtsSearch) {
+        return await coll.store.sqliteFtsSearch(Object.fromEntries(params._query));
+      } else {
+        return {};
+      }
+    }
 
     case "curated": {
       const coll = await this.collections.getColl(params.coll);

src/sqlite-fts/.gitignore

@@ -0,0 +1 @@
+*.js

src/sqlite-fts/HttpVFS.ts

@@ -0,0 +1,129 @@
+import SQLiteESMFactory from "wa-sqlite/dist/wa-sqlite-async.mjs";
+import * as SQLite from "wa-sqlite";
+import { Base } from "wa-sqlite/src/VFS.js";
+import * as SQLITE from "wa-sqlite/src/sqlite-constants.js";
+import { LazyUint8Array, HttpVfsProgressEvent } from "./LazyUint8Array";
+type SqliteFtsConfig = {
+  url: string;
+  startOffset: number;
+  length: number;
+  pageSize: number;
+};
+
+export class HttpVFS extends Base {
+  name: string;
+  config: SqliteFtsConfig;
+  dbFileId: number = 0;
+  dbFile: LazyUint8Array;
+
+  constructor(
+    name: string,
+    config: SqliteFtsConfig,
+    progressCallback: (p: HttpVfsProgressEvent) => void
+  ) {
+    super();
+    this.name = name;
+    this.config = config;
+    this.dbFile = new LazyUint8Array({
+      fileLength: config.length,
+      requestChunkSize: config.pageSize,
+      progressCallback,
+      rangeMapper: (from, to) => ({
+        url: config.url,
+        fromByte: config.startOffset + from,
+        toByte: config.startOffset + to,
+      }),
+    });
+  }
+  xClose(fileId: number): number | Promise<number> {
+    if (fileId !== this.dbFileId) throw Error("unknown file id " + fileId);
+    return SQLITE.SQLITE_OK;
+  }
+  xRead(
+    fileId: number,
+    pData: { size: number; value: Int8Array },
+    iOffset: number
+  ): number | Promise<number> {
+    return this.handleAsync(async () => {
+      if (fileId !== this.dbFileId)
+        throw Error("xRead: invalid file id " + fileId);
+      const uint8Array = new Uint8Array(
+        pData.value.buffer,
+        pData.value.byteOffset,
+        pData.value.length
+      );
+      await this.dbFile.copyInto(uint8Array, 0, pData.size, iOffset);
+
+      return SQLITE.SQLITE_OK;
+    });
+  }
+  xWrite(
+    fileId: number,
+    pData: { size: number; value: Int8Array },
+    iOffset: number
+  ): number | Promise<number> {
+    throw new Error("xWrite not implemented.");
+  }
+  xTruncate(fileId: number, iSize: number): number | Promise<number> {
+    throw new Error("xTruncate not implemented.");
+  }
+  xSync(fileId: number, flags: number): number | Promise<number> {
+    throw new Error("xSync not implemented.");
+  }
+  xFileSize(
+    fileId: number,
+    pSize64: { set(value: number): void }
+  ): number | Promise<number> {
+    if (fileId !== this.dbFileId)
+      throw new Error(`xFileSize: invalid file id ${fileId}`);
+    pSize64.set(this.config.length);
+    return SQLITE.SQLITE_OK;
+  }
+  xLock(fileId: number, flags: number): number | Promise<number> {
+    return SQLITE.SQLITE_OK;
+  }
+  xUnlock(fileId: number, flags: number): number | Promise<number> {
+    return SQLITE.SQLITE_OK;
+  }
+  xCheckReservedLock(
+    fileId: number,
+    pResOut: { set(value: number): void }
+  ): number | Promise<number> {
+    throw new Error("xCheckReservedLockMethod not implemented.");
+  }
+  xFileControl(
+    fileId: number,
+    flags: number,
+    pOut: { value: Int8Array }
+  ): number | Promise<number> {
+    return SQLITE.SQLITE_NOTFOUND;
+  }
+  xDeviceCharacteristics(fileId: number): number | Promise<number> {
+    return SQLITE.SQLITE_OK;
+  }
+  xOpen(
+    name: string | null,
+    fileId: number,
+    flags: number,
+    pOutFlags: { set(value: number): void }
+  ): number | Promise<number> {
+    if (name !== "dummy") throw Error("file name must be dummy");
+    this.dbFileId = fileId;
+    pOutFlags.set(flags);
+    return SQLITE.SQLITE_OK;
+  }
+  xDelete(name: string, syncDir: number): number | Promise<number> {
+    throw new Error("xDelete not implemented.");
+  }
+  xAccess(
+    name: string,
+    flags: number,
+    pResOut: { set(value: any): void }
+  ): number | Promise<number> {
+    if (["dummy-journal", "dummy-wal"].includes(name)) {
+      pResOut.set(0);
+      return SQLITE.SQLITE_OK;
+    }
+    throw new Error(`xAccess(${name}, ${flags}) not implemented.`);
+  }
+}