Merge pull request #3210 from perspective-dev/virtual-server-arrow-jamboree

Add `STRUCT` and `LIST` support for Arrow/JSON, fix DuckDB Arrow coercion

Author: texodus

docs/md/how_to/javascript/events.md

@@ -88,8 +88,31 @@ elem.addEventListener("perspective-global-filter-update", function (event) {
 });
 ```
 
+## Layout events
+
+A multi-panel `<perspective-viewer>` reports changes to its panel _collection_
+on two separate channels. They are distinct facts — which panels exist, and
+which one is selected — so neither event implies the other.
+
+- `perspective-layout-update` fires when a panel is added to or removed from
+  the layout. Its `detail.panels` is the placed panel ids in insertion order,
+  identical to what [`getPanelNames()`](#) returns.
+- `perspective-active-panel-update` fires when the active panel changes, with
+  a `detail.panel` of the new panel's id — or `null` at zero panels.
+
+```javascript
+elem.addEventListener("perspective-layout-update", function (event) {
+    console.log("Panels are now", event.detail.panels);
+});
+```
+
+Geometry changes — dragging a split divider, reordering tabs — do **not** fire
+these events, because they change the layout tree without changing the panel
+set. Use `saveWorkspace()` to read the current geometry.
+
 <div class="warning">The <code>workspace-layout-update</code> and
 <code>workspace-new-view</code> events from the removed
-<code>@perspective-dev/workspace</code> package no longer exist. Use
-<code>perspective-config-update</code> and
-<code>perspective-global-filter-update</code>.</div>
+<code>@perspective-dev/workspace</code> package no longer exist.
+<code>perspective-layout-update</code> is the closest replacement for the
+former; for per-panel config changes use
+<code>perspective-config-update</code>.</div>

docs/md/how_to/javascript/virtual_server/duckdb.md

@@ -17,31 +17,60 @@ npm install @perspective-dev/client @perspective-dev/viewer @duckdb/duckdb-wasm
 
 Initialize DuckDB-WASM, load data, and connect it to a Perspective viewer:
 
+`DuckDBHandler` is an optional submodule and is _not_ exported from the package
+root, so it must be imported by path. It takes an `AsyncDuckDBConnection` — the
+result of `db.connect()` — not the `AsyncDuckDB` itself.
+
 ```javascript
-import perspective from "@perspective-dev/client";
+import perspective, { createMessageHandler } from "@perspective-dev/client";
 import "@perspective-dev/viewer";
 import * as duckdb from "@duckdb/duckdb-wasm";
+import { DuckDBHandler } from "@perspective-dev/client/dist/esm/virtual_servers/duckdb.js";
 
 // Initialize DuckDB-WASM
 const DUCKDB_BUNDLES = duckdb.getJsDelivrBundles();
 const bundle = await duckdb.selectBundle(DUCKDB_BUNDLES);
-const worker = await duckdb.createWorker(bundle.mainWorker);
+const worker_url = URL.createObjectURL(
+    new Blob([`importScripts("${bundle.mainWorker}");`], {
+        type: "text/javascript",
+    }),
+);
+
+const worker = new Worker(worker_url);
 const logger = new duckdb.ConsoleLogger();
 const db = new duckdb.AsyncDuckDB(logger, worker);
-await db.instantiate(bundle.mainModule);
+await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+URL.revokeObjectURL(worker_url);
 
-// Load data into DuckDB
+// Load data into DuckDB. This pragma is required to match Perspective's
+// sort-null semantics.
 const conn = await db.connect();
+await conn.query(`SET default_null_order=NULLS_FIRST_ON_ASC_LAST_ON_DESC;`);
 await conn.query(`CREATE TABLE my_table AS SELECT * FROM 'data.parquet'`);
 
 // Create a Perspective virtual server backed by DuckDB
-const handler = perspective.DuckDBHandler(db);
-const messageHandler = perspective.createMessageHandler(handler);
+const messageHandler = await createMessageHandler(new DuckDBHandler(conn));
 
-// Connect a viewer
+// Connect a viewer. Table ids are database-qualified, so a table created as
+// `my_table` is hosted as `memory.my_table`.
 const client = await perspective.worker(messageHandler);
-const table = await client.open_table("my_table");
-document.getElementById("viewer").load(table);
+const viewer = document.getElementById("viewer");
+viewer.load(client);
+viewer.restore({ table: "memory.my_table" });
+```
+
+<div class="warning">In the browser, <code>DuckDBHandler</code> resolves
+Perspective's WASM module from the registered
+<code>&lt;perspective-viewer&gt;</code> custom element, so it cannot be
+constructed until that element has been defined. Off-browser, pass the module
+explicitly as the second constructor argument.</div>
+
+Perspective never intercepts your SQL — it only discovers what `SHOW ALL
+TABLES` reports — so DuckDB's own remote-data features are available directly:
+
+```javascript
+await conn.query(`CREATE SECRET (TYPE s3, KEY_ID '...', SECRET '...', REGION 'us-east-1')`);
+await conn.query(`CREATE TABLE trades AS SELECT * FROM read_parquet('s3://bucket/trades/*.parquet')`);
 ```
 
 ## Examples

docs/md/how_to/python/table_data.md

@@ -36,6 +36,56 @@ with open("data.arrow", "rb") as f:
     table = perspective.table(f.read())
 ```
 
+### Nested columns
+
+Perspective's data model is flat, so Arrow `struct` and `list` columns are
+normalized on ingest.
+
+A `struct` column is hoisted into one dotted column per leaf, recursively. A
+null parent nulls every descendant leaf:
+
+```python
+arrow_table = pa.table({
+    "id": pa.array([1, 2], type=pa.int64()),
+    "s": pa.array([{"a": 10}, {"a": 20}], type=pa.struct([("a", pa.int64())])),
+})
+
+# Schema is `{"id": "integer", "s.a": "integer"}`
+table = perspective.table(arrow_table)
+```
+
+Because the flattened names are ordinary columns, a `Table` created from an
+explicit schema accepts nested updates with no further configuration:
+
+```python
+table = perspective.table({"id": "integer", "s.a": "integer"})
+table.update(arrow_table)
+```
+
+A `list` column is controlled by the `list_flatten` argument:
+
+-   `"zip"` (default) expands a row into one row per list element, repeating
+    its non-list siblings. An empty or null list yields a single row with a
+    null in that column, rather than dropping the row. When a row has more than
+    one list column, their non-empty lengths must match.
+-   `"cartesian"` expands a row into the product of its list columns' lengths,
+    with an empty or null list counting as a single null element.
+-   `"stringify"` encodes each list as a JSON array in a single string column,
+    leaving the row count unchanged.
+
+```python
+arrow_table = pa.table({
+    "x": pa.array([1, 2], type=pa.int64()),
+    "y": pa.array([[10, 20], [30]], type=pa.list_(pa.int64())),
+})
+
+# `{"x": [1, 1, 2], "y": [10, 20, 30]}`
+perspective.table(arrow_table)
+
+# `{"x": [1, 2], "y": ["[10,20]", "[30]"]}`
+perspective.table(arrow_table, list_flatten="stringify")
+```
+
 ## Polars
 
 ```python

docs/md/how_to/python/virtual_server/duckdb.md

@@ -48,6 +48,19 @@ const table = await websocket.open_table("my_table");
 document.getElementById("viewer").load(table);
 ```
 
+## Window functions
+
+Window columns are DuckDB's own functions, under their DuckDB names — the
+advertised name is emitted into the `OVER` clause verbatim.
+
+| | |
+| --- | --- |
+| Aggregating | `sum` `avg` `count` `min` `max` `product` `median` |
+| Deviation / variance | `stddev_samp` `stddev_pop` `var_samp` `var_pop` |
+| Navigation | `first_value` `last_value` `nth_value` `lag` `lead` |
+| Ranking | `row_number` `rank` `dense_rank` `percent_rank` `cume_dist` `ntile` |
+| Perspective's own | `diff` `rate` |
+
 ## Examples
 
 - [Python DuckDB example](https://github.com/perspective-dev/perspective/tree/master/examples/python-duckdb-virtual)

packages/react/README.md

@@ -0,0 +1,148 @@
+# `@perspective-dev/react`
+
+[![npm](https://img.shields.io/npm/v/@perspective-dev/react.svg?style=for-the-badge)](https://www.npmjs.com/package/@perspective-dev/react)
+
+React bindings for [Perspective](https://perspective-dev.github.io/), an
+interactive analytics and data visualization component for large, real-time
+and streaming datasets. This package wraps the
+[`<perspective-viewer>`](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html)
+Custom Element in an idiomatic, declarative React component,
+`<PerspectiveViewer>`, which manages the element's imperative
+`load()`/`restore()`/`delete()` lifecycle for you.
+
+## Installation
+
+```bash
+npm install @perspective-dev/react
+```
+
+`@perspective-dev/client` and `@perspective-dev/viewer` are installed as
+dependencies, but you'll also want at least one plugin package for the
+visualizations themselves:
+
+```bash
+npm install @perspective-dev/viewer-datagrid @perspective-dev/viewer-charts
+```
+
+## Setup
+
+Perspective's engine and UI are WebAssembly binaries which must be initialized
+once, before the first `<PerspectiveViewer>` renders. Plugins register
+themselves via import side effects. See the
+[User Guide's bundling section](https://perspective-dev.github.io/guide/how_to/javascript/importing.html)
+for bundler configuration details.
+
+```tsx
+import perspective from "@perspective-dev/client";
+import perspective_viewer from "@perspective-dev/viewer";
+import "@perspective-dev/viewer-datagrid";
+import "@perspective-dev/viewer-charts";
+import "@perspective-dev/viewer/dist/css/themes.css";
+
+import SERVER_WASM from "@perspective-dev/server/dist/wasm/perspective-server.wasm";
+import CLIENT_WASM from "@perspective-dev/viewer/dist/wasm/perspective-viewer.wasm";
+
+await Promise.all([
+    perspective.init_server(fetch(SERVER_WASM)),
+    perspective_viewer.init_client(fetch(CLIENT_WASM)),
+]);
+```
+
+## Usage
+
+Create a `Table` (here in a Web Worker `Client`) and pass it — or a `Promise`
+of it — to `<PerspectiveViewer>`:
+
+```tsx
+import * as React from "react";
+import { PerspectiveViewer } from "@perspective-dev/react";
+
+const WORKER = await perspective.worker();
+
+const TABLE = WORKER.table(
+    fetch("superstore.lz4.arrow").then((resp) => resp.arrayBuffer()),
+    { name: "superstore" },
+);
+
+const App: React.FC = () => (
+    <PerspectiveViewer
+        client={TABLE}
+        config={{ group_by: ["State"], plugin: "Y Bar" }}
+    />
+);
+```
+
+## Props
+
+| Prop             | Type                                                          | Description                                                     |
+| :--------------- | :------------------------------------------------------------ | :-------------------------------------------------------------- |
+| `client`         | `Client \| Table \| Promise<Client> \| Promise<Table>`        | Data source. When `undefined`, the viewer `eject()`s.           |
+| `config`         | `ViewerConfigUpdate \| WorkspaceConfigUpdate`                 | Declarative viewer state, applied via `restore()`.              |
+| `onConfigUpdate` | `(config: ViewerConfigUpdate) => void`                        | Called when the user reconfigures the viewer through its UI.    |
+| `onClick`        | `(detail: PerspectiveClickEventDetail) => void`               | Called when the user clicks a datapoint.                        |
+| `onSelect`       | `(detail: PerspectiveSelectEventDetail) => void`              | Called when the user selects (or deselects) a datapoint or row. |
+
+A subset of standard HTML attributes — `className`, `id`, `style`, `hidden`,
+`slot`, `tabIndex` and `title` — is forwarded to the underlying element.
+
+### `client`
+
+The viewer's data source, forwarded to
+[`viewer.load()`](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html)
+whenever it changes:
+
+-   A `Table` (or `Promise<Table>`) displays that table directly.
+-   A `Client` (e.g. from `perspective.worker()` or a WebSocket connection to a
+    remote server) connects the viewer to every table hosted by that client;
+    the table each panel displays is chosen by `config` or interactively by
+    the user.
+-   `undefined` ejects the viewer, returning it to an unloaded state without
+    unmounting it.
+
+The component does not take ownership of the `Table` — delete it yourself
+when it is no longer needed (e.g. `table.delete({ lazy: true })`).
+
+### `config`
+
+Declarative viewer state — group-bys, splits, filters, sorts, expressions,
+plugin and plugin config — applied with `restore()` whenever it (or `client`)
+changes. A config with a `panels` property is treated as a multi-panel
+workspace layout and applied with `restoreWorkspace()` instead. Configs are
+compared structurally, so passing a fresh-but-equal object literal on each
+render does not re-apply.
+
+Combine `config` with `onConfigUpdate` to make the viewer a controlled
+component — store the user's latest configuration in state (or persist it) and
+pass it back down:
+
+```tsx
+const App: React.FC = () => {
+    const [config, setConfig] = React.useState<pspViewer.ViewerConfigUpdate>({
+        group_by: ["Category"],
+    });
+
+    return (
+        <PerspectiveViewer
+            client={TABLE}
+            config={config}
+            onConfigUpdate={setConfig}
+        />
+    );
+};
+```
+
+## Lifecycle
+
+On unmount, the component calls the element's `delete()` method, freeing the
+viewer's WebAssembly resources. Tables and clients are created outside the
+component and are yours to manage; a `Table` passed as `client` survives
+unmount and can be shown again by a later mount.
+
+## See also
+
+-   [`react-example`](https://github.com/perspective-dev/perspective/tree/master/examples/react-example)
+    — a complete bundler-configured project using this package, including a
+    multi-panel workspace config.
+-   [Perspective User Guide](https://perspective-dev.github.io/guide/)
+-   [`<perspective-viewer>` API documentation](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html)
+-   [`@perspective-dev/client` API documentation](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html)

packages/react/src/index.tsx

@@ -11,11 +11,60 @@
 // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
 
 /**
+ * React bindings for [Perspective](https://perspective-dev.github.io/).
+ *
+ * This module exports {@link PerspectiveViewer}, a declarative React wrapper
+ * for the `<perspective-viewer>` Custom Element. The component manages the
+ * element's imperative lifecycle — `load()` and `restore()` in response to
+ * prop changes, `delete()` on unmount — and exposes the element's Custom
+ * Events as React-style callback props.
+ *
+ * Perspective's WebAssembly engine and UI must be initialized (and at least
+ * one plugin package imported) before the first `<PerspectiveViewer>`
+ * renders:
+ *
+ * ```tsx
+ * import perspective from "@perspective-dev/client";
+ * import perspective_viewer from "@perspective-dev/viewer";
+ * import "@perspective-dev/viewer-datagrid";
+ * import "@perspective-dev/viewer-charts";
+ *
+ * import SERVER_WASM from "@perspective-dev/server/dist/wasm/perspective-server.wasm";
+ * import CLIENT_WASM from "@perspective-dev/viewer/dist/wasm/perspective-viewer.wasm";
+ *
+ * await Promise.all([
+ *     perspective.init_server(fetch(SERVER_WASM)),
+ *     perspective_viewer.init_client(fetch(CLIENT_WASM)),
+ * ]);
+ * ```
+ *
+ * Then pass a `Table` (or `Client`, or a `Promise` of either) and an optional
+ * config:
+ *
+ * ```tsx
+ * import { PerspectiveViewer } from "@perspective-dev/react";
+ *
+ * const WORKER = await perspective.worker();
+ * const TABLE = WORKER.table(
+ *     fetch("superstore.lz4.arrow").then((resp) => resp.arrayBuffer()),
+ *     { name: "superstore" },
+ * );
+ *
+ * const App: React.FC = () => (
+ *     <PerspectiveViewer
+ *         client={TABLE}
+ *         config={{ group_by: ["State"], plugin: "Y Bar" }}
+ *     />
+ * );
+ * ```
  *
  * # See Also
  *
- * [`react-example`](https://github.com/perspective-dev/perspective/tree/master/examples/react-example)
- * project from the Perspective GitHub repo.
+ * - [`react-example`](https://github.com/perspective-dev/perspective/tree/master/examples/react-example)
+ *   project from the Perspective GitHub repo, a complete bundler-configured
+ *   application including a multi-panel workspace config.
+ * - [Perspective User Guide](https://perspective-dev.github.io/guide/)
+ * - [`<perspective-viewer>` API documentation](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html)
  *
  * @module
  */