Merge pull request #3232 from perspective-dev/fix-exports

Add `zstd` support to `View::to_arrow`

Author: texodus

.github/workflows/build.yaml

@@ -847,7 +847,12 @@ jobs:
                     - ubuntu-22.04
                 python-version:
                     - 3.11
-                node-version: [22.x]
+                node-version: [22.x, 26.x]
+                is-release:
+                    - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full }}
+                exclude:
+                    - node-version: 26.x
+                      is-release: false
 
         steps:
             - name: Checkout
@@ -890,7 +895,7 @@ jobs:
               if: ${{ failure() && steps.run_tests.outcome == 'failure' }}
               uses: actions/upload-artifact@v4
               with:
-                  name: perspective-js-test-results
+                  name: perspective-js-test-results-node${{ matrix.node-version }}
                   path: tools/test/dist/results
                   if-no-files-found: ignore
                   overwrite: true

docs/md/how_to/javascript/serializing.md

@@ -19,3 +19,12 @@ console.log(await view.to_ndjson());
 // ArrayBuffer
 console.log(await view.to_arrow());
 ```
+
+`to_arrow()` writes an uncompressed Arrow IPC stream by default; pass
+`compression` to apply LZ4 or ZSTD body compression, which `Client::table` reads
+back transparently:
+
+```javascript
+const compressed = await view.to_arrow({ compression: "zstd" });
+const table2 = await client.table(compressed);
+```

docs/md/how_to/javascript/viewer.md

@@ -153,8 +153,11 @@ never deletes the result.
 | `copy(options?)` | Copy a panel to the clipboard |
 
 `export()`, `download()` and `copy()` all take a `method`, one of `"csv"`,
-`"json"`, `"ndjson"` or `"arrow"` — each with `-all` and `-selected` variants
-(e.g. `"csv-selected"`) — plus `"html"`, `"json-config"`, and `"plugin"`.
+`"json"`, `"ndjson"`, `"arrow"`, `"arrow-lz4"` or `"arrow-zstd"` — each with
+`-all` and `-selected` variants (e.g. `"csv-selected"`, `"arrow-zstd-all"`) —
+plus `"html"`, `"json-config"`, and `"plugin"`. The `"arrow-lz4"` and
+`"arrow-zstd"` methods write the Arrow IPC stream with LZ4 or ZSTD body
+compression.
 The `"plugin"` method asks the plugin to render itself, which produces a PNG
 for charts and text for the datagrid.
 | `getSelection(options?)` / `setSelection(...)` | Get or set the selected region |

package.json

@@ -18,7 +18,7 @@
     "llvm": "17.0.6",
     "pyodide": "0.29.4",
     "engines": {
-        "node": ">=16 <24"
+        "node": ">=16 <27"
     },
     "workspaces": [
         "tools/test",

pnpm-lock.yaml

@@ -47,8 +47,8 @@ catalog:
     "@fontsource/roboto-mono": "4.5.10"
     "@iarna/toml": "3.0.0"
     "@jupyterlab/builder": "^4"
-    "@playwright/experimental-ct-react": "=1.58.0"
-    "@playwright/test": "=1.58.0"
+    "@playwright/experimental-ct-react": "=1.62.0"
+    "@playwright/test": "=1.62.0"
     "lightningcss": "^1.29.0"
     "@types/lodash": "^4.17.20"
     "@types/node": ">=22"

pnpm-workspace.yaml

@@ -105,8 +105,10 @@ pub struct ViewWindow {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub formatted: Option<bool>,
 
-    /// Only impacts [`View::to_arrow`]
+    /// Arrow IPC body compression for [`View::to_arrow`], `"lz4"` or `"zstd"`
+    /// (uncompressed when omitted).
     #[ts(optional)]
+    #[ts(type = "\"lz4\" | \"zstd\"")]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub compression: Option<String>,
 
@@ -423,7 +425,8 @@ impl View {
         }
     }
 
-    /// Serializes a [`View`] to the Apache Arrow data format.
+    /// Serializes a [`View`] to the Apache Arrow data format, with IPC body
+    /// compression per [`ViewWindow::compression`].
     pub async fn to_arrow(&self, window: ViewWindow) -> ClientResult<Bytes> {
         let msg = self.client_message(ClientReq::ViewToArrowReq(ViewToArrowReq {
             viewport: Some(window.clone().into()),

rust/perspective-client/src/rust/view.rs

@@ -195,6 +195,17 @@ impl View {
     }
 
     /// Serializes a [`View`] to the Apache Arrow data format.
+    ///
+    /// # Arguments
+    ///
+    /// - `window` - a [`ViewWindow`]; its `compression` key selects Arrow IPC
+    ///   body compression, `"lz4"` or `"zstd"` (uncompressed when omitted).
+    ///
+    /// # JavaScript Examples
+    ///
+    /// ```javascript
+    /// const arrow = await view.to_arrow({ compression: "zstd" });
+    /// ```
     #[wasm_bindgen]
     pub async fn to_arrow(&self, window: Option<JsViewWindow>) -> ApiResult<ArrayBuffer> {
         let window = window.into_serde_ext::<Option<ViewWindow>>()?;

rust/perspective-js/src/rust/view.rs

@@ -181,6 +181,7 @@ async function compile_server_module(wasm_path: string) {
     const bytes = await load_wasm_stage_0(buffer.buffer as ArrayBuffer);
     return await compile_perspective(bytes.buffer as ArrayBuffer, {
         make_disk_bridge: make_node_disk_bridge,
+        env: process.env,
     });
 }
 

rust/perspective-js/src/ts/perspective.node.ts

@@ -24,6 +24,8 @@ export interface DiskBridgeHelpers {
 }
 
 export interface CompileOptions {
+    env?: Record<string, string | undefined>;
+
     make_disk_bridge?: (helpers: DiskBridgeHelpers) => {
         store(
             namePtr: number | bigint,
@@ -49,6 +51,7 @@ export async function compile_perspective(
             return x;
         },
         make_disk_bridge: opts?.make_disk_bridge,
+        env: opts?.env,
         instantiateWasm: async (
             imports: any,
             receive: (_: WebAssembly.Instance) => void,