Merge pull request #532 from powersync-ja/node-commonjs

Compile node package as CommonJS module

Author: simolus3

.changeset/clean-roses-lick.md

@@ -0,0 +1,5 @@
+---
+'@powersync/node': patch
+---
+
+Include CommonJS distribution for this package.

demos/example-node/src/main.ts

@@ -2,12 +2,13 @@ import repl_factory from 'node:repl';
 import { once } from 'node:events';
 
 import { PowerSyncDatabase, SyncStreamConnectionMethod } from '@powersync/node';
-import Logger from 'js-logger';
+import { default as Logger } from 'js-logger';
 import { AppSchema, DemoConnector } from './powersync.js';
 import { exit } from 'node:process';
 
 const main = async () => {
-  Logger.useDefaults({ defaultLevel: Logger.WARN });
+  const logger = Logger.get('PowerSyncDemo');
+  Logger.useDefaults({ defaultLevel: logger.WARN });
 
   if (!('BACKEND' in process.env) || !('SYNC_SERVICE' in process.env)) {
     console.warn(
@@ -21,7 +22,7 @@ const main = async () => {
     database: {
       dbFilename: 'test.db'
     },
-    logger: Logger
+    logger
   });
   console.log(await db.get('SELECT powersync_rs_version();'));
 

demos/example-node/src/powersync.ts

@@ -33,8 +33,8 @@ export class DemoConnector implements PowerSyncBackendConnector {
 
     const response = await fetch(`${process.env.BACKEND}/api/data/`, {
       method: 'POST',
-      headers: {'Content-Type': 'application/json'},
-      body: JSON.stringify({batch: entries}),
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ batch: entries })
     });
     if (response.status !== 200) {
       throw new Error(`Server returned HTTP ${response.status}: ${await response.text()}`);

demos/example-node/tsconfig.json

@@ -4,7 +4,9 @@
     "baseUrl": ".",
     "rootDir": "src",
     "outDir": "lib",
-    "strictNullChecks": true
+    "strictNullChecks": true,
+    "moduleResolution": "nodenext",
+    "module": "NodeNext"
   },
   "references": [
     {

packages/node/package.json

@@ -6,23 +6,32 @@
     "access": "public"
   },
   "description": "PowerSync Node.js SDK. Sync Postgres, MongoDB or MySQL with SQLite in your Node.js app",
-  "main": "./lib/index.js",
-  "module": "./lib/index.js",
-  "types": "./lib/index.d.ts",
   "files": [
     "lib",
     "dist",
     "download_core.js"
   ],
   "scripts": {
     "install": "node download_core.js",
-    "build": "tsc -b",
+    "build": "tsc -b && rollup --config",
     "build:prod": "tsc -b --sourceMap false",
     "clean": "rm -rf lib dist tsconfig.tsbuildinfo dist",
     "watch": "tsc -b -w",
     "test": "vitest"
   },
   "type": "module",
+  "exports": {
+    ".": {
+      "import": "./lib/index.js",
+      "require": "./dist/bundle.cjs",
+      "types": "./lib/index.d.ts"
+    },
+    "./worker.js": {
+      "import": "./lib/worker.js",
+      "require": "./dist/worker.cjs",
+      "types": "./lib/worker.d.ts"
+    }
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/powersync-ja/powersync-js.git"
@@ -34,17 +43,18 @@
   },
   "homepage": "https://docs.powersync.com/",
   "peerDependencies": {
-    "@powersync/common": "workspace:^1.22.0",
-    "@powersync/better-sqlite3": "^0.1.0"
+    "@powersync/common": "workspace:^1.22.0"
   },
   "dependencies": {
+    "@powersync/better-sqlite3": "^0.1.0",
     "@powersync/common": "workspace:*",
     "async-lock": "^1.4.0",
     "bson": "^6.6.0",
     "comlink": "^4.4.2"
   },
   "devDependencies": {
     "@types/async-lock": "^1.4.0",
+    "rollup": "4.14.3",
     "typescript": "^5.5.3",
     "vitest": "^3.0.5"
   },

packages/node/rollup.config.mjs

@@ -0,0 +1,42 @@
+const plugin = () => {
+  return {
+    name: 'mark-as-commonjs',
+    resolveImportMeta: (property) => {
+      if (property == 'isBundlingToCommonJs') {
+        return 'true';
+      }
+
+      return null;
+    },
+  };
+};
+
+export default [
+  {
+    input: 'lib/index.js',
+    plugins: [plugin()],
+    output: {
+      file: 'dist/bundle.cjs',
+      format: 'cjs',
+      sourcemap: true
+    }
+  },
+  {
+    input: 'lib/db/DefaultWorker.js',
+    plugins: [plugin()],
+    output: {
+      file: 'dist/DefaultWorker.cjs',
+      format: 'cjs',
+      sourcemap: true
+    }
+  },
+  {
+    input: 'lib/worker.js',
+    plugins: [plugin()],
+    output: {
+      file: 'dist/worker.cjs',
+      format: 'cjs',
+      sourcemap: true
+    }
+  }
+];

packages/node/src/db/BetterSQLite3DBAdapter.ts

@@ -10,13 +10,13 @@ import {
   LockContext,
   Transaction,
   DBLockOptions,
-  QueryResult,
-  SQLOpenOptions
+  QueryResult
 } from '@powersync/common';
 import { Remote } from 'comlink';
 import { AsyncResource } from 'node:async_hooks';
 import { AsyncDatabase, AsyncDatabaseOpener } from './AsyncDatabase.js';
 import { RemoteConnection } from './RemoteConnection.js';
+import { NodeSQLOpenOptions } from './options.js';
 
 export type BetterSQLite3LockContext = LockContext & {
   executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
@@ -30,7 +30,7 @@ const READ_CONNECTIONS = 5;
  * Adapter for better-sqlite3
  */
 export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> implements DBAdapter {
-  private readonly options: SQLOpenOptions;
+  private readonly options: NodeSQLOpenOptions;
   public readonly name: string;
 
   private readConnections: RemoteConnection[];
@@ -39,9 +39,13 @@ export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> impl
   private readonly readQueue: Array<(connection: RemoteConnection) => void> = [];
   private readonly writeQueue: Array<() => void> = [];
 
-  constructor(options: SQLOpenOptions) {
+  constructor(options: NodeSQLOpenOptions) {
     super();
 
+    if (options.readWorkerCount != null && options.readWorkerCount < 1) {
+      throw `Needs at least one worker for reads, got ${options.readWorkerCount}`;
+    }
+
     this.options = options;
     this.name = options.dbFilename;
   }
@@ -53,7 +57,17 @@ export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> impl
     }
 
     const openWorker = async (isWriter: boolean) => {
-      const worker = new Worker(new URL('./SqliteWorker.js', import.meta.url), {name: isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`});
+      const isCommonJsModule = import.meta.isBundlingToCommonJs ?? false;
+      let worker: Worker;
+      const workerName = isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`;
+
+      const workerFactory = this.options.openWorker ?? ((...args) => new Worker(...args));
+      if (isCommonJsModule) {
+        worker = workerFactory(path.resolve(__dirname, 'DefaultWorker.cjs'), { name: workerName });
+      } else {
+        worker = workerFactory(new URL('./DefaultWorker.js', import.meta.url), { name: workerName});
+      }
+
       const listeners = new WeakMap<EventListenerOrEventListenerObject, (e: any) => void>();
 
       const comlink = Comlink.wrap<AsyncDatabaseOpener>({
@@ -94,7 +108,8 @@ export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> impl
     // Open the writer first to avoid multiple threads enabling WAL concurrently (causing "database is locked" errors).
     this.writeConnection = await openWorker(true);
     const createWorkers: Promise<RemoteConnection>[] = [];
-    for (let i = 0; i < READ_CONNECTIONS; i++) {
+    const amountOfReaders = this.options.readWorkerCount ?? READ_CONNECTIONS;
+    for (let i = 0; i < amountOfReaders; i++) {
       createWorkers.push(openWorker(false));
     }
     this.readConnections = await Promise.all(createWorkers);

packages/node/src/db/DefaultWorker.ts

@@ -0,0 +1,3 @@
+import { startPowerSyncWorker } from './SqliteWorker.js';
+
+startPowerSyncWorker();

packages/node/src/db/PowerSyncDatabase.ts

@@ -4,14 +4,21 @@ import {
   BucketStorageAdapter,
   DBAdapter,
   PowerSyncBackendConnector,
+  PowerSyncDatabaseOptions,
   PowerSyncDatabaseOptionsWithSettings,
-  SqliteBucketStorage
+  SqliteBucketStorage,
+  SQLOpenFactory
 } from '@powersync/common';
 
 import { NodeRemote } from '../sync/stream/NodeRemote.js';
 import { NodeStreamingSyncImplementation } from '../sync/stream/NodeStreamingSyncImplementation.js';
 
 import { BetterSQLite3DBAdapter } from './BetterSQLite3DBAdapter.js';
+import { NodeSQLOpenOptions } from './options.js';
+
+export type NodePowerSyncDatabaseOptions = PowerSyncDatabaseOptions & {
+  database: DBAdapter | SQLOpenFactory | NodeSQLOpenOptions;
+};
 
 /**
  * A PowerSync database which provides SQLite functionality
@@ -28,6 +35,10 @@ import { BetterSQLite3DBAdapter } from './BetterSQLite3DBAdapter.js';
  * ```
  */
 export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
+  constructor(options: NodePowerSyncDatabaseOptions) {
+    super(options);
+  }
+
   async _initialize(): Promise<void> {
     await (this.database as BetterSQLite3DBAdapter).initialize();
   }

packages/node/src/db/SqliteWorker.ts

@@ -1,3 +1,4 @@
+import * as path from 'node:path';
 import BetterSQLite3Database, { Database } from '@powersync/better-sqlite3';
 import * as Comlink from 'comlink';
 import { parentPort, threadId } from 'node:worker_threads';
@@ -81,10 +82,16 @@ class BlockingAsyncDatabase implements AsyncDatabase {
 }
 
 class BetterSqliteWorker implements AsyncDatabaseOpener {
+  options: PowerSyncWorkerOptions;
+
+  constructor(options: PowerSyncWorkerOptions) {
+    this.options = options;
+  }
+
   async open(path: string, isWriter: boolean): Promise<AsyncDatabase> {
     const baseDB = new BetterSQLite3Database(path);
     baseDB.pragma('journal_mode = WAL');
-    loadExtension(baseDB);
+    baseDB.loadExtension(this.options.extensionPath(), 'sqlite3_powersync_init');
     if (!isWriter) {
       baseDB.pragma('query_only = true');
     }
@@ -96,21 +103,43 @@ class BetterSqliteWorker implements AsyncDatabaseOpener {
   }
 }
 
-const loadExtension = (db: Database) => {
-  const platform = OS.platform();
-  let extensionPath: string;
-  if (platform === 'win32') {
-    extensionPath = 'powersync.dll';
-  } else if (platform === 'linux') {
-    extensionPath = 'libpowersync.so';
-  } else if (platform === 'darwin') {
-    extensionPath = 'libpowersync.dylib';
-  } else {
-    throw 'Unknown platform, PowerSync for Node.js currently supports Windows, Linux and macOS.';
-  }
+export interface PowerSyncWorkerOptions {
+  /**
+   * A function responsible for finding the powersync DLL/so/dylib file.
+   *
+   * @returns The absolute path of the PowerSync SQLite core extensions library.
+   */
+  extensionPath: () => string;
+}
 
-  const resolved = url.fileURLToPath(new URL(`../${extensionPath}`, import.meta.url));
-  db.loadExtension(resolved, 'sqlite3_powersync_init');
-};
+export function startPowerSyncWorker(options?: Partial<PowerSyncWorkerOptions>) {
+  const resolvedOptions: PowerSyncWorkerOptions = {
+    extensionPath() {
+      const isCommonJsModule = import.meta.isBundlingToCommonJs ?? false;
+
+      const platform = OS.platform();
+      let extensionPath: string;
+      if (platform === 'win32') {
+        extensionPath = 'powersync.dll';
+      } else if (platform === 'linux') {
+        extensionPath = 'libpowersync.so';
+      } else if (platform === 'darwin') {
+        extensionPath = 'libpowersync.dylib';
+      } else {
+        throw 'Unknown platform, PowerSync for Node.js currently supports Windows, Linux and macOS.';
+      }
+
+      let resolved: string;
+      if (isCommonJsModule) {
+        resolved = path.resolve(__dirname, '../lib/', extensionPath);
+      } else {
+        resolved = url.fileURLToPath(new URL(`../${extensionPath}`, import.meta.url));
+      }
 
-Comlink.expose(new BetterSqliteWorker(), parentPort! as Comlink.Endpoint);
+      return resolved;
+    },
+    ...options,
+  };
+
+  Comlink.expose(new BetterSqliteWorker(resolvedOptions), parentPort! as Comlink.Endpoint);
+}

packages/node/src/db/options.ts

@@ -0,0 +1,24 @@
+import { type Worker } from 'node:worker_threads';
+import { SQLOpenOptions } from '@powersync/common';
+
+export type WorkerOpener = (...args: ConstructorParameters<typeof Worker>) => InstanceType<typeof Worker>;
+
+/**
+ * The {@link SQLOpenOptions} available across all PowerSync SDKs for JavaScript extended with
+ * Node.JS-specific options.
+ */
+export interface NodeSQLOpenOptions extends SQLOpenOptions {
+  /**
+   * The Node.JS SDK will use one worker to run writing queries and additional workers to run reads.
+   * This option controls how many workers to use for reads.
+   */
+  readWorkerCount?: number;
+  /**
+   * A callback to allow customizing how the Node.JS SDK loads workers. This can be customized to
+   * use workers at different paths.
+   *
+   * @param args The arguments that would otherwise be passed to the {@link Worker} constructor.
+   * @returns the resolved worker.
+   */
+  openWorker?: WorkerOpener;
+}

packages/node/src/node.d.ts

@@ -0,0 +1,3 @@
+interface ImportMeta {
+  isBundlingToCommonJs?: boolean; // This property is set by our rollup configuration
+}

packages/node/src/worker.ts

@@ -0,0 +1 @@
+export * from './db/SqliteWorker.js';