Skip to content

Commit 82ae796

Browse files
mbostockmbostock
committed
data loaders!
1 parent 77dff68 commit 82ae796

21 files changed

+629
-42
lines changed

‎docs/data-loader-test.html

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
<!doctype html>
2+
<notebook theme="midnight">
3+
<title>Observable Notebooks Data loader test</title>
4+
<script id="1" type="text/markdown">
5+
# Observable Notebooks<br> <span style="color: var(--theme-foreground-faint);">Data loader test</span>
6+
7+
<link rel="stylesheet" href="./style.css">
8+
</script>
9+
<script type="module" pinned="">
10+
Plot.plot({
11+
width: 960,
12+
projection: {
13+
type: "transverse-mercator",
14+
domain: land,
15+
rotate: [4, 0]
16+
},
17+
color: {
18+
domain: d3.range(n * n),
19+
range: d3.cross(d3.schemeBlues[n], d3.schemeOranges[n]).map(mixblend)
20+
},
21+
marks: [
22+
Plot.raster({length: ppt.length}, {
23+
x: lon,
24+
y: lat,
25+
fill: 0,
26+
interpolate: interpolateBivariate(n, ppt, temp),
27+
clip: land
28+
}),
29+
Plot.geo(land, {strokeWidth: 0.5}),
30+
(index, {scales}) =>
31+
Plot.plot({
32+
color: scales.color,
33+
axis: null,
34+
inset: 18,
35+
width: 136,
36+
height: 136,
37+
padding: 0,
38+
y: {reverse: true},
39+
style: "transform-origin: 68px 68px; transform: rotate(-45deg) translate(0px, 20px)",
40+
marks: [
41+
Plot.cell(d3.cross(d3.range(n), d3.range(n)), {
42+
fill: ([a, b]) => n * a + b,
43+
inset: -0.5
44+
}),
45+
Plot.text(["Precipitation →"], {
46+
frameAnchor: "bottom-right",
47+
fontWeight: 600,
48+
dx: -18,
49+
dy: -6
50+
}),
51+
Plot.text(["← Temperature"], {
52+
frameAnchor: "top-left",
53+
fontWeight: 600,
54+
rotate: 90,
55+
dx: 12,
56+
dy: 18
57+
})
58+
]
59+
})
60+
]
61+
})
62+
</script>
63+
<script type="module" pinned="">
64+
function interpolateBivariate(n, v1, v2) {
65+
const r = d3.range(n);
66+
const s1 = d3.scaleQuantile(v1, r);
67+
const s2 = d3.scaleQuantile(v2, r);
68+
const interpolate = Plot.interpolatorBarycentric();
69+
return (I, w, h, X, Y) => {
70+
I = I.filter((i) => !isNaN(v1[i]) && !isNaN(v2[i]));
71+
const V1 = interpolate(I, w, h, X, Y, v1);
72+
const V2 = interpolate(I, w, h, X, Y, v2);
73+
return Uint8Array.from(V1, (_, i) => n * s1(V1[i]) + s2(V2[i]));
74+
};
75+
}
76+
</script>
77+
<script type="module" pinned="">
78+
const n = 7; // number of color classes
79+
</script>
80+
<script type="module" pinned="">
81+
function mixblend([a, b]) {
82+
a = d3.rgb(a);
83+
b = d3.rgb(b);
84+
const l = Math.min(255, b.r + b.g + b.b);
85+
a.r *= b.r / l;
86+
a.g *= b.g / l;
87+
a.b *= b.b / l;
88+
return a;
89+
}
90+
</script>
91+
<script type="module" pinned="">
92+
import {NetCDFReader} from "npm:netcdfjs";
93+
94+
const tmaxReader = new NetCDFReader(tmaxBuffer);
95+
const tminReader = new NetCDFReader(tminBuffer);
96+
const pptReader = new NetCDFReader(pptBuffer);
97+
const tmax = Float32Array.from(tmaxReader.getDataVariable("tmax"), d => d !== -32768 ? d * 0.01 - 99 : NaN);
98+
const tmin = Float32Array.from(tminReader.getDataVariable("tmin"), d => d !== -32768 ? d * 0.01 - 99 : NaN);
99+
const ppt = Float32Array.from(pptReader.getDataVariable("ppt"), d => d !== -2147483648 ? d * 0.1 : NaN).map(d => d < 10 ? NaN : d);
100+
const lx = tmaxReader.getDataVariable("lon");
101+
const ly = tmaxReader.getDataVariable("lat");
102+
const l = lx.length;
103+
const temp = tmax.map((max, i) => (max + tmin[i]) / 2);
104+
const lon = (d, i) => lx[i % l];
105+
const lat = (d, i) => ly[i/ l | 0];
106+
</script>
107+
<script type="application/vnd.node.javascript" pinned="" hidden="" output="tmaxBuffer">
108+
import {Readable} from "node:stream";
109+
110+
fetch("http://thredds.northwestknowledge.net:8080/thredds/ncss/agg_terraclimate_tmax_1958_CurrentYear_GLOBE.nc?var=tmax&south=49.674&north=61.061&west=-14.015517&east=2.0919117&disableProjSubset=on&addLatLon=true&horizStride=1&accept=netcdf")
111+
.then((response) => response.ok ? response.body : Promise.reject(response.status))
112+
.then((body) => Readable.from(body).pipe(process.stdout));
113+
</script>
114+
<script type="application/vnd.node.javascript" pinned="" hidden="" output="tminBuffer">
115+
import {Readable} from "node:stream";
116+
117+
fetch("http://thredds.northwestknowledge.net:8080/thredds/ncss/agg_terraclimate_tmin_1958_CurrentYear_GLOBE.nc?var=tmin&south=49.674&north=61.061&west=-14.015517&east=2.0919117&disableProjSubset=on&addLatLon=true&horizStride=1&accept=netcdf")
118+
.then((response) => response.ok ? response.body : Promise.reject(response.status))
119+
.then((body) => Readable.from(body).pipe(process.stdout));
120+
</script>
121+
<script type="application/vnd.node.javascript" pinned="" hidden="" output="pptBuffer">
122+
import {Readable} from "node:stream";
123+
124+
fetch("http://thredds.northwestknowledge.net:8080/thredds/ncss/agg_terraclimate_ppt_1958_CurrentYear_GLOBE.nc?var=ppt&south=49.674&north=61.061&west=-14.015517&east=2.0919117&disableProjSubset=on&addLatLon=true&horizStride=1&accept=netcdf")
125+
.then((response) => response.ok ? response.body : Promise.reject(response.status))
126+
.then((body) => Readable.from(body).pipe(process.stdout));
127+
</script>
128+
<script type="module" pinned="">
129+
const land = fetch("https://cdn.jsdelivr.net/npm/world-atlas@2/countries-10m.json")
130+
.then((response) => response.ok ? response.json() : Promise.reject(response.status))
131+
.then((world) => topojson.feature(world, {type: "GeometryCollection", geometries: world.objects.countries.geometries.filter(({id}) => id === "826" || id === "372")}));
132+
</script>
133+
</notebook>

‎docs/data-loaders.html

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<!doctype html>
2+
<notebook theme="slate">
3+
<title>Observable Notebooks Data loaders</title>
4+
<script id="1" type="text/markdown">
5+
# Observable Notebooks<br> <span style="color: var(--theme-foreground-faint);">Data loaders</span>
6+
7+
<link rel="stylesheet" href="./style.css">
8+
</script>
9+
<script type="text/markdown">
10+
Data loaders are special cells that run at build time via an interpreter rather than in the browser at runtime. Data loader cells are useful for preparing static data, ensuring consistency and stability, and greatly improving runtime performance. Think of data loaders as a generalization of [database connectors](./databases) that allow languages besides SQL.
11+
</script>
12+
<script type="text/markdown">
13+
For example, here is a Node.js data loader that says hello and reports the current version of Node.js:
14+
</script>
15+
<script type="application/vnd.node.javascript" format="text" pinned="">
16+
process.stdout.write(`Hello from Node ${process.version}!`);
17+
</script>
18+
<script type="text/markdown">
19+
<aside>The cell above is JavaScript that runs in Node.js, unlike normal JavaScript cells that run in the browser.</aside>
20+
21+
The output of a data loader cell is automatically saved to a `.observable/cache` directory on your local file system alongside your notebooks.
22+
</script>
23+
<script type="text/markdown">
24+
In Observable Desktop, you can re-run a data loader cell by clicking the **Play** button, by hitting <span style="font-family: var(--sans-serif);">**shift-return**</span>, or by clicking on the query age in the cell toolbar. In Notebook Kit, delete the corresponding file from the `.observable/cache` directory; you can also use continuous deployment, such as GitHub Actions, to refresh data automatically.
25+
</script>
26+
<script type="text/markdown">
27+
Currently, Notebook Kit only supports the Node.js interpreter for data loader cells, but we plan on adding other interpreters, notably Python and R.
28+
</script>
29+
<script type="text/markdown">
30+
To improve security, the Node.js interpreter uses [process-based permissions](https://nodejs.org/api/permissions.html). Node.js cells are only allowed to read files in the same directory as the notebook, with no other permissions. (We may offer a way to relax permissions in the future, but want to encourage safety; let us know if you run into issues.)
31+
</script>
32+
<script type="text/markdown">
33+
Due to the above security restrictions, if you wish to import installed packages, they must be installed within the same directory as the notebook (_e.g._, if your notebook is in `docs`, packages must be installed in `docs/node_modules` with a `docs/package.json`).
34+
</script>
35+
</notebook>

‎src/databases/index.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import {createReadStream} from "node:fs";
22
import {dirname, join} from "node:path";
33
import {json} from "node:stream/consumers";
44
import {isEnoent} from "../lib/error.js";
5-
import {hash as getQueryHash, nameHash as getNameHash} from "../lib/hash.js";
5+
import {hash, nameHash} from "../lib/hash.js";
66
import type {ColumnSchema, QueryParam} from "../runtime/index.js";
77
import type {BigQueryConfig} from "./bigquery.js";
88
import type {DatabricksConfig} from "./databricks.js";
@@ -93,6 +93,6 @@ export async function getQueryCachePath(
9393
...params: unknown[]
9494
): Promise<string> {
9595
const sourceDir = dirname(sourcePath);
96-
const cacheName = `${await getNameHash(databaseName)}-${await getQueryHash(strings, ...params)}.json`;
96+
const cacheName = `${await nameHash(databaseName)}-${await hash(strings, ...params)}.json`;
9797
return join(sourceDir, ".observable", "cache", cacheName);
9898
}

‎src/interpreters/index.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import {dirname, join} from "node:path";
2+
import {nameHash, stringHash} from "../lib/hash.js";
3+
import {getInterpreterExtension} from "../lib/interpreters.js";
4+
import type {Cell} from "../lib/notebook.js";
5+
6+
export async function getInterpreterCachePath(
7+
sourcePath: string,
8+
interpreter: string,
9+
format: Cell["format"],
10+
input: string
11+
): Promise<string> {
12+
const sourceDir = dirname(sourcePath);
13+
const cacheName = `${await nameHash(interpreter)}-${await stringHash(input)}${getInterpreterExtension(format)}`; // TODO avoid conflict with database cache?
14+
return join(sourceDir, ".observable", "cache", cacheName);
15+
}

‎src/javascript/__snapshots__/template.test.ts.snap

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,24 @@ Node {
123123
}
124124
`;
125125

126-
exports[`transpiles a simple template 1`] = `"\`Hello, world!\`"`;
126+
exports[`transpiles a html template with an interpolated expression 1`] = `"html\`Hello, \${"world"}!\`"`;
127127

128-
exports[`transpiles a template with an interpolated expression 1`] = `"\`Hello, \${"world"}!\`"`;
128+
exports[`transpiles a html template with backquotes 1`] = `"html\`Hello, \\\`world\\\`!\`"`;
129129

130-
exports[`transpiles a template with backquotes 1`] = `"\`Hello, \\\`world\\\`!\`"`;
130+
exports[`transpiles a html template with backslashes 1`] = `"html\`Hello, \\world\\!\`"`;
131131

132-
exports[`transpiles a template with backslashes 1`] = `"\`Hello, \\\\world\\\\!\`"`;
132+
exports[`transpiles a markdown template with an interpolated expression 1`] = `"md\`Hello, \${"world"}!\`"`;
133133

134-
exports[`transpiles an empty template 1`] = `""`;
134+
exports[`transpiles a markdown template with backquotes 1`] = `"md\`Hello, \\\`world\\\`!\`"`;
135+
136+
exports[`transpiles a markdown template with backslashes 1`] = `"md\`Hello, \\\\world\\\\!\`"`;
137+
138+
exports[`transpiles a node template with backslashes 1`] = `"Interpreter("node", {id: 1, format: "buffer"}).run(\`Hello, \\\\world\\\\!\`).then((file) => file.arrayBuffer())"`;
139+
140+
exports[`transpiles a simple html template 1`] = `"html\`Hello, world!\`"`;
141+
142+
exports[`transpiles a simple markdown template 1`] = `"md\`Hello, world!\`"`;
143+
144+
exports[`transpiles an empty html template 1`] = `""`;
145+
146+
exports[`transpiles an empty markdown template 1`] = `""`;

‎src/javascript/__snapshots__/transpile.test.ts.snap

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,91 @@ return (
293293
}
294294
`;
295295
296+
exports[`transpiles node cells 1`] = `
297+
{
298+
"autodisplay": true,
299+
"autoview": false,
300+
"body": "(Interpreter) => {
301+
return (
302+
Interpreter("node", {id: -1, format: "buffer"}).run(\`process.stdout.write(\\\`Node $\\{process.version}\\\`);\`).then((file) => file.arrayBuffer())
303+
)
304+
}",
305+
"inputs": [
306+
"Interpreter",
307+
],
308+
"output": undefined,
309+
"outputs": [],
310+
}
311+
`;
312+
313+
exports[`transpiles node cells 2`] = `
314+
{
315+
"autodisplay": true,
316+
"autoview": false,
317+
"body": "(Interpreter) => {
318+
return (
319+
Interpreter("node", {id: -1, format: "buffer"}).run(\`process.stdout.write(\\\`Node \\\\$\\{process.version}\\\`);\`).then((file) => file.arrayBuffer())
320+
)
321+
}",
322+
"inputs": [
323+
"Interpreter",
324+
],
325+
"output": undefined,
326+
"outputs": [],
327+
}
328+
`;
329+
330+
exports[`transpiles node cells 3`] = `
331+
{
332+
"autodisplay": true,
333+
"autoview": false,
334+
"body": "(Interpreter) => {
335+
return (
336+
Interpreter("node", {id: -1, format: "buffer"}).run(\`process.stdout.write(\\\`Node \\\\\\\\$\\{process.version}\\\`);\`).then((file) => file.arrayBuffer())
337+
)
338+
}",
339+
"inputs": [
340+
"Interpreter",
341+
],
342+
"output": undefined,
343+
"outputs": [],
344+
}
345+
`;
346+
347+
exports[`transpiles node cells 4`] = `
348+
{
349+
"autodisplay": true,
350+
"autoview": false,
351+
"body": "(Interpreter) => {
352+
return (
353+
Interpreter("node", {id: -1, format: "buffer"}).run(\`process.stdout.write(\\\`Node $\\\\{process.version}\\\`);\`).then((file) => file.arrayBuffer())
354+
)
355+
}",
356+
"inputs": [
357+
"Interpreter",
358+
],
359+
"output": undefined,
360+
"outputs": [],
361+
}
362+
`;
363+
364+
exports[`transpiles node cells 5`] = `
365+
{
366+
"autodisplay": true,
367+
"autoview": false,
368+
"body": "(Interpreter) => {
369+
return (
370+
Interpreter("node", {id: -1, format: "buffer"}).run(\`process.stdout.write(\\\`Node \\\\$\\\\{process.version}\\\`);\`).then((file) => file.arrayBuffer())
371+
)
372+
}",
373+
"inputs": [
374+
"Interpreter",
375+
],
376+
"output": undefined,
377+
"outputs": [],
378+
}
379+
`;
380+
296381
exports[`transpiles static imports with {type: 'observable'} 1`] = `
297382
{
298383
"autodisplay": false,

0 commit comments

Comments
 (0)