docuement deserialization errors

Author: stevensJourney

docs/collections/powersync-collection.md

@@ -99,9 +99,19 @@ const documentsCollection = createCollection(
     table: APP_SCHEMA.props.documents,
   })
 )
-```
 
-#### Option 2: SQLite Types with Schema Validation
+/** Note: The types for input and output are defined as this */
+// Used for mutations like `insert` or `update`
+type DocumentCollectionInput = {
+  id: string
+  name: string | null
+  author: string | null
+  created_at: string | null // SQLite TEXT
+  archived: number | null // SQLite integer
+}
+// The type of query/data results
+type DocumentCollectionOutput = DocumentCollectionInput
+```
 
 The standard PowerSync SQLite types map to these TypeScript types:
 
@@ -111,13 +121,17 @@ The standard PowerSync SQLite types map to these TypeScript types:
 | `column.integer`      | `number \| null` | Integer values, also used for booleans (0/1)                         |
 | `column.real`         | `number \| null` | Floating point numbers                                               |
 
-Note: All PowerSync column types are nullable by default, as SQLite allows null values in any column. Your schema should always handle null values appropriately by using `.nullable()` in your Zod schemas and handling null cases in your transformations.
+Note: All PowerSync column types are nullable by default.
+
+#### Option 2: SQLite Types with Schema Validation
 
 Additional validations for collection mutations can be performed with a custom schema. The Schema below asserts that
 the `name`, `author` and `created_at` fields are required as input. `name` also has an additional string length check.
 
 Note: The input and output types specified in this example still satisfy the underlying SQLite types. An additional `deserializationSchema` is required if the typing differs. See the examples below for more details.
 
+The application logic (including the backend) should enforce that all incoming synced data passes validation with the `deserializationSchema`. Failing to validate data will result in inconsistency of the collection data. This is a fatal error! An `onDeserializationError` handler must be provided to react to this case.
+
 ```ts
 import { createCollection } from "@tanstack/react-db"
 import { powerSyncCollectionOptions } from "@tanstack/powersync-db-collection"
@@ -137,6 +151,9 @@ const documentsCollection = createCollection(
     database: db,
     table: APP_SCHEMA.props.documents,
     schema,
+    onDeserializationError: (error) => {
+      // Present fatal error
+    },
   })
 )
 
@@ -161,6 +178,10 @@ Note: The Transformed types are provided by TanStackDB to the PowerSync SQLite p
 order to be persisted to SQLite. Most types are converted by default. For custom types, override the serialization by providing a
 `serializer` param.
 
+The example below uses `nullable` columns, this is not a requirement.
+
+The application logic (including the backend) should enforce that all incoming synced data passes validation with the `deserializationSchema`. Failing to validate data will result in inconsistency of the collection data. This is a fatal error! An `onDeserializationError` handler must be provided to react to this case.
+
 ```ts
 const schema = z.object({
   id: z.string(),
@@ -180,6 +201,9 @@ const documentsCollection = createCollection(
     database: db,
     table: APP_SCHEMA.props.documents,
     schema,
+    onDeserializationError: (error) => {
+      // Present fatal error
+    },
     // Optional: custom column serialization
     serializer: {
       // Dates are serialized by default, this is just an example

packages/powersync-db-collection/src/definitions.ts

@@ -85,6 +85,13 @@ export type SerializerConfig<
    * ```
    */
   serializer?: CustomSQLiteSerializer<TOutput, TSQLite>
+
+  /**
+   * Application logic should ensure that incoming synced data is always valid.
+   * Failing to deserialize and apply incoming changes results in data inconsistency - which is a fatal error.
+   * Use this callback to react to deserialization errors.
+   */
+  onDeserializationError: (error: StandardSchemaV1.FailureResult) => void
 }
 
 /**
@@ -154,13 +161,6 @@ export type ConfigWithArbitraryCollectionTypes<
     ExtractedTable<TTable>,
     StandardSchemaV1.InferOutput<TSchema>
   >
-
-  /**
-   * Application logic should ensure that incoming synced data is always valid.
-   * Failing to deserialize and apply incoming changes results in data inconsistency - which is a fatal error.
-   * Use this callback to react to deserialization errors.
-   */
-  onDeserializationError: (error: StandardSchemaV1.FailureResult) => void
 }
 export type BasePowerSyncCollectionConfig<
   TTable extends Table = Table,

packages/powersync-db-collection/src/powersync.ts

@@ -248,23 +248,21 @@ export function powerSyncCollectionOptions<
    * Deserializes data from the incoming sync stream
    */
   const deserializeSyncRow = (value: TableType): OutputType => {
-    if (deserializationSchema) {
-      const validation = deserializationSchema[`~standard`].validate(value)
-      if (`value` in validation) {
-        return validation.value
-      } else if (`issues` in validation) {
-        const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
-        database.logger.error(issueMessage)
-        onDeserializationError!(validation)
-        throw new Error(issueMessage)
-      } else {
-        const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
-        database.logger.error(unknownErrorMessage)
-        onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
-        throw new Error(unknownErrorMessage)
-      }
+    const validationSchema = deserializationSchema || schema
+    const validation = validationSchema[`~standard`].validate(value)
+    if (`value` in validation) {
+      return validation.value
+    } else if (`issues` in validation) {
+      const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
+      database.logger.error(issueMessage)
+      onDeserializationError!(validation)
+      throw new Error(issueMessage)
+    } else {
+      const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
+      database.logger.error(unknownErrorMessage)
+      onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
+      throw new Error(unknownErrorMessage)
     }
-    return value as OutputType
   }
 
   // We can do basic runtime validations for columns if not explicit schema has been provided

packages/powersync-db-collection/tests/collection-schema.test.ts

@@ -2,9 +2,10 @@ import { randomUUID } from "node:crypto"
 import { tmpdir } from "node:os"
 import { PowerSyncDatabase, Schema, Table, column } from "@powersync/node"
 import { SchemaValidationError, createCollection } from "@tanstack/db"
-import { describe, expect, it, onTestFinished } from "vitest"
+import { describe, expect, it, onTestFinished, vi } from "vitest"
 import { z } from "zod"
 import { powerSyncCollectionOptions } from "../src"
+import type { StandardSchemaV1 } from "@standard-schema/spec"
 
 const APP_SCHEMA = new Schema({
   documents: new Table({
@@ -108,6 +109,7 @@ describe(`PowerSync Schema Integration`, () => {
           database: db,
           table: APP_SCHEMA.props.documents,
           schema,
+          onDeserializationError: () => {},
         })
       )
       onTestFinished(() => collection.cleanup())
@@ -167,6 +169,7 @@ describe(`PowerSync Schema Integration`, () => {
           database: db,
           table: APP_SCHEMA.props.documents,
           schema,
+          onDeserializationError: () => {},
         })
       )
       onTestFinished(() => collection.cleanup())
@@ -306,5 +309,54 @@ describe(`PowerSync Schema Integration`, () => {
       expect(item?.name instanceof MyDataClass).true
       expect(item?.name?.options.value).eq(`document`)
     })
+
+    /**
+     * We sync data which cannot be validated by the schema. This is a fatal error.
+     */
+    it(`should catch deserialization errors`, async () => {
+      const db = await createDatabase()
+
+      /**
+       * Here name is stored as a Buffer. We can't serialize this to SQLite automatically.
+       * We need to provide a serializer.
+       */
+      const schema = z.object({
+        id: z.string(),
+        name: z.string(),
+        archived: z.number(),
+        author: z.string(),
+        created_at: z.string(),
+      })
+
+      const onError = vi.fn((() => {}) as (
+        error: StandardSchemaV1.FailureResult
+      ) => void)
+
+      const collection = createCollection(
+        powerSyncCollectionOptions({
+          database: db,
+          table: APP_SCHEMA.props.documents,
+          schema,
+          onDeserializationError: onError,
+        })
+      )
+      onTestFinished(() => collection.cleanup())
+
+      await collection.stateWhenReady()
+
+      // The columns are not nullable in the schema
+      // Write invalid data to SQLite, this simulates a sync
+      await db.execute(`INSERT INTO documents(id) VALUES(uuid())`)
+
+      await vi.waitFor(
+        () => {
+          const issues = onError.mock.lastCall?.[0]?.issues
+          expect(issues).toBeDefined()
+          // Each column which should have been defined
+          expect(issues?.length).eq(4)
+        },
+        { timeout: 1000 }
+      )
+    })
   })
 })