From 95453f225feabdc685c9b842099e3a037e668a35 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 9 Mar 2022 08:03:53 -0800 Subject: [PATCH 01/12] Add flake --- .envrc | 5 ++++ dependencies.nix | 61 ++++++++++++++++++++++++++---------------------- flake.lock | 41 ++++++++++++++++++++++++++++++++ flake.nix | 22 +++++++++++++++++ shell.nix | 2 +- 5 files changed, 102 insertions(+), 29 deletions(-) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..c325083 --- /dev/null +++ b/.envrc @@ -0,0 +1,5 @@ +if ! has nix_direnv_version || ! nix_direnv_version 1.4.0; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/1.4.0/direnvrc" "sha256-4XfVDjv75eHMWN4G725VW7BoOV4Vl3vAabK4YXIfPyE=" +fi + +use flake diff --git a/dependencies.nix b/dependencies.nix index ee597fb..b35fad6 100644 --- a/dependencies.nix +++ b/dependencies.nix @@ -1,29 +1,34 @@ -rec { - pkgs = import (builtins.fetchTarball { - name = "nixos-21.11"; - url = "https://github.com/NixOS/nixpkgs/archive/21.11.tar.gz"; - sha256 = "162dywda2dvfj1248afxc45kcrg83appjd0nmdb541hl7rnncf02"; - }) {}; +{ system ? builtins.currentSystem }: rec { + pkgs = import + (builtins.fetchTarball { + name = "nixos-21.11"; + url = "https://github.com/NixOS/nixpkgs/archive/21.11.tar.gz"; + sha256 = "162dywda2dvfj1248afxc45kcrg83appjd0nmdb541hl7rnncf02"; + }) + { inherit system; }; + + zig = pkgs.stdenv.mkDerivation { + name = "zig"; + src = fetchTarball ( + if (pkgs.system == "x86_64-linux") then { + url = "https://ziglang.org/download/0.9.0/zig-linux-x86_64-0.9.0.tar.xz"; + sha256 = "1vagp72wxn6i9qscji6k3a1shy76jg4d6crmx9ijpch9kyn71c96"; + } else if (pkgs.system == "aarch64-linux") then { + url = "https://ziglang.org/download/0.9.0/zig-linux-aarch64-0.9.0.tar.xz"; + sha256 = "00m6nxp64nf6pwq407by52l8i0f2m4mw6hj17jbjdjd267b6sgri"; + } else if (pkgs.system == "aarch64-darwin") then { + url = "https://ziglang.org/download/0.9.0/zig-macos-aarch64-0.9.0.tar.xz"; + sha256 = "sha256:0irr2b8nvj43d7f3vxnz0x70m8jlz71mv3756hx49p5d7ramdrp7"; + } else throw ("Unknown system " ++ pkgs.system) + ); + dontConfigure = true; + dontBuild = true; + installPhase = '' + mkdir -p $out + mv ./* $out/ + mkdir -p $out/bin + mv $out/zig $out/bin + ''; + }; +} - zig = pkgs.stdenv.mkDerivation { - name = "zig"; - src = fetchTarball ( - if (pkgs.system == "x86_64-linux") then { - url = "https://ziglang.org/download/0.9.0/zig-linux-x86_64-0.9.0.tar.xz"; - sha256 = "1vagp72wxn6i9qscji6k3a1shy76jg4d6crmx9ijpch9kyn71c96"; - } else if (pkgs.system == "aarch64-linux") then { - url = "https://ziglang.org/download/0.9.0/zig-linux-aarch64-0.9.0.tar.xz"; - sha256 = "00m6nxp64nf6pwq407by52l8i0f2m4mw6hj17jbjdjd267b6sgri"; - } else - throw ("Unknown system " ++ pkgs.system) - ); - dontConfigure = true; - dontBuild = true; - installPhase = '' - mkdir -p $out - mv ./* $out/ - mkdir -p $out/bin - mv $out/zig $out/bin - ''; - }; -} \ No newline at end of file diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..80c7a88 --- /dev/null +++ b/flake.lock @@ -0,0 +1,41 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1646826225, + "narHash": "sha256-UbocGllbHFDJddhAey+MzYKzPcMWyP082ub/XjEAfzQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b99542dff8373721834d4c3458398ca8e1319953", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..a1e10c2 --- /dev/null +++ b/flake.nix @@ -0,0 +1,22 @@ +{ + description = "A very basic flake"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { system = system; }; + deps = (import ./dependencies.nix { inherit system; }); + + in + { + packages.hello = pkgs.hello; + defaultPackage = self.packages.${system}.hello; + devShell = + pkgs.mkShell rec { + buildInputs = [ + deps.zig + ]; + }; + }); +} diff --git a/shell.nix b/shell.nix index 1252c44..281158f 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ -with (import ./dependencies.nix); +with (import ./dependencies.nix { }); pkgs.mkShell rec { buildInputs = [ From 7c5721b682d2c5128d189baa0a099ed41266fd2c Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 9 Mar 2022 15:01:32 -0800 Subject: [PATCH 02/12] Add Deno+wasm example --- bindings/wasm/abi-with-bytes.js | 166 ++++++++++++++++++++++++++++++++ bindings/wasm/abi.js | 4 +- bindings/wasm/codegen.zig | 15 +++ examples/core.html | 145 ++++++++++++++-------------- examples/wasm-deno.js | 87 +++++++++++++++++ flake.nix | 7 ++ 6 files changed, 350 insertions(+), 74 deletions(-) create mode 100644 bindings/wasm/abi-with-bytes.js create mode 100644 examples/wasm-deno.js diff --git a/bindings/wasm/abi-with-bytes.js b/bindings/wasm/abi-with-bytes.js new file mode 100644 index 0000000..c04e34a --- /dev/null +++ b/bindings/wasm/abi-with-bytes.js @@ -0,0 +1,166 @@ +export default async function Abi(dida_wasm_bytes) { + // id >= 0 for values on stack + // id < 0 for values in ref_counted + + var stack = []; + var ref_counted = {}; + var next_ref_counted_id = -1; + + function stackPush(value) { + stack.push(value); + return stack.length - 1; + } + + function stackRead(ix) { + return stack[ix]; + } + + function stackGetLength() { + return stack.length; + } + + function stackReset(length) { + stack.length = length; + } + + function RefCounted(value, refcount) { + this.value = value; + this.refcount = refcount; + } + + function createRefCounted(value_id, refcount) { + const ref_counted_id = next_ref_counted_id; + next_ref_counted_id -= 1; + ref_counted[ref_counted_id] = new RefCounted(stackRead(value_id), refcount); + return ref_counted_id + } + + function getRefCounted(ref_counted_id) { + return stackPush(ref_counted[ref_counted_id].value); + } + + // Return values must be kept in sync with js_common.JsType + function jsTypeOf(value_ix) { + const value = stackRead(value_ix); + const typ = typeof (value); + if (typ == 'undefined') return 0; + // typeof(null) == 'object' :| + if (value == null) return 1; + if (typ == 'boolean') return 2; + if (typ == 'number') return 3; + if (typ == 'string') return 4; + if (typ == 'object') return 5; + if (typ == 'function') return 6; + throw (typ + ' is not a type that the abi understands'); + } + + function createUndefined() { + return stackPush(undefined); + } + + function createString(address, length) { + let bytes = new Uint8Array(wasm.instance.exports.memory.buffer); + let string = new TextDecoder().decode(bytes.slice(address, address + length)); + return stackPush(string); + } + + function createObject() { + return stackPush({}); + } + + function createArray(len) { + return stackPush(new Array(len)); + } + + function getStringLength(string_id) { + return stackRead(string_id).length; + } + + function getStringInto(string_id, address, max_len) { + const string = stackRead(string_id); + const encoded = new TextEncoder().encode(string); + const bytes = new Uint8Array(wasm.instance.exports.memory.buffer); + const len = Math.min(string.length, max_len); + bytes.set(encoded.subarray(0, len), address); + return len; + } + + function getArrayLength(array_id) { + return stackRead(array_id).length; + } + + function getElement(array_id, ix) { + return stackPush(stackRead(array_id)[ix]); + } + + function setElement(array_id, ix, value_id) { + stackRead(array_id)[ix] = stackRead(value_id); + } + + function getProperty(object_id, name_id) { + return stackPush(stackRead(object_id)[stackRead(name_id)]); + } + + function setProperty(object_id, name_id, value_id) { + stackRead(object_id)[stackRead(name_id)] = stackRead(value_id); + } + + function callFunction(function_id, args_id) { + return stackPush(stackRead(function_id).apply(null, stackRead(args_id))); + } + + function consoleLog(message_id) { + console.log(stackRead(message_id)); + } + + function consoleError(message_id) { + console.error(stackRead(message_id)); + } + + function throwException(value_ix) { + throw stackRead(value_ix); + } + + const wasm = await WebAssembly.instantiate( + dida_wasm_bytes, + { + env: { + jsTypeOf: jsTypeOf, + createUndefined: createUndefined, + createBool: stackPush, + createU32: stackPush, + createI32: stackPush, + createI64: stackPush, + createF64: stackPush, + createString: createString, + createObject: createObject, + createArray: createArray, + createRefCounted: createRefCounted, + getU32: stackRead, + getI32: stackRead, + getI64: stackRead, + getF64: stackRead, + getStringLength: getStringLength, + getStringInto: getStringInto, + getRefCounted: getRefCounted, + getArrayLength: getArrayLength, + getElement: getElement, + setElement: setElement, + getProperty: getProperty, + setProperty: setProperty, + callFunction: callFunction, + consoleLog: consoleLog, + consoleError: consoleError, + throwException: throwException, + } + } + ); + + return { + wasm: wasm, + stackGetLength: stackGetLength, + stackReset: stackReset, + stackRead: stackRead, + stackPush: stackPush, + }; +} \ No newline at end of file diff --git a/bindings/wasm/abi.js b/bindings/wasm/abi.js index 0ee648e..883f10f 100644 --- a/bindings/wasm/abi.js +++ b/bindings/wasm/abi.js @@ -1,7 +1,7 @@ async function Abi(dida_url) { // id >= 0 for values on stack // id < 0 for values in ref_counted - + var stack = []; var ref_counted = {}; var next_ref_counted_id = -1; @@ -42,7 +42,7 @@ async function Abi(dida_url) { // Return values must be kept in sync with js_common.JsType function jsTypeOf(value_ix) { const value = stackRead(value_ix); - const typ = typeof(value); + const typ = typeof (value); if (typ == 'undefined') return 0; // typeof(null) == 'object' :| if (value == null) return 1; diff --git a/bindings/wasm/codegen.zig b/bindings/wasm/codegen.zig index 578b91a..e619f5b 100644 --- a/bindings/wasm/codegen.zig +++ b/bindings/wasm/codegen.zig @@ -15,6 +15,21 @@ pub fn main() !void { try std.fmt.format(writer, "this.{s} = {s};\n", .{ T, T }); } try writer.writeAll("}"); + // try writer.writeAll("export default Dida"); + + const file2 = try std.fs.cwd().createFile("zig-out/lib/dida.mjs", .{ .read = false, .truncate = true }); + defer file2.close(); + writer = file2.writer(); + try writer.writeAll("function Dida(abi) {\n\n"); + inline for (js_common.types_with_js_constructors) |T| { + try generateConstructor(writer, T); + } + try writer.writeAll("\n\n"); + inline for (js_common.types_with_js_constructors) |T| { + try std.fmt.format(writer, "this.{s} = {s};\n", .{ T, T }); + } + try writer.writeAll("}\n\n"); + try writer.writeAll("export default Dida"); } fn generateConstructor(writer: anytype, comptime Type: type) !void { diff --git a/examples/core.html b/examples/core.html index 21fba16..dc13b4f 100644 --- a/examples/core.html +++ b/examples/core.html @@ -2,91 +2,92 @@ - - - - + + + run(); + - + + \ No newline at end of file diff --git a/examples/wasm-deno.js b/examples/wasm-deno.js new file mode 100644 index 0000000..66a7d28 --- /dev/null +++ b/examples/wasm-deno.js @@ -0,0 +1,87 @@ +import Abi from "../bindings/wasm/abi-with-bytes.js" +import Dida from "../bindings/wasm/zig-out/lib/dida.mjs" +import { Buffer } from "https://deno.land/std/io/mod.ts"; + +async function run() { + const file = await Deno.open('./bindings/wasm/zig-out/lib/dida.wasm'); + const b = new Buffer(); + await b.readFrom(file); + console.log(typeof b, typeof b.bytes()) + + const abi = await Abi(b.bytes()); + const dida = new Dida(abi); + + var graph_builder = new dida.GraphBuilder(); + const subgraph_0 = new dida.Subgraph(0); + const subgraph_1 = graph_builder.addSubgraph(subgraph_0); + const edges = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Input()); + const edges_1 = graph_builder.addNode(subgraph_1, new dida.NodeSpec.TimestampPush(edges)); + const reach_future = graph_builder.addNode(subgraph_1, new dida.NodeSpec.TimestampIncrement(null)); + const reach_index = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Index(reach_future)); + const distinct_reach_index = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Distinct(reach_index)); + const swapped_edges = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Map(edges_1, input => [input[1], input[0]])); + const swapped_edges_index = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Index(swapped_edges)); + const joined = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Join([distinct_reach_index, swapped_edges_index], 1)); + const without_middle = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Map(joined, input => [input[2], input[1]])); + const reach = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Union([edges_1, without_middle])); + graph_builder.connectLoop(reach, reach_future); + const reach_pop = graph_builder.addNode(subgraph_0, new dida.NodeSpec.TimestampPop(distinct_reach_index)); + const reach_out = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Output(reach_pop)); + + const reach_summary = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Reduce( + distinct_reach_index, + 1, + "", + function (reduced_value, row, count) { + for (var i = 0; i < count; i++) { + reduced_value += row[1]; + } + return reduced_value; + } + )); + const reach_summary_out = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Output(reach_summary)); + + const graph = graph_builder.finishAndReset(); + + var shard = new dida.Shard(graph); + + shard.pushInput(edges, new dida.Change(["a", "b"], [0], 1)); + shard.pushInput(edges, new dida.Change(["b", "c"], [0], 1)); + shard.pushInput(edges, new dida.Change(["b", "d"], [0], 1)); + shard.pushInput(edges, new dida.Change(["c", "a"], [0], 1)); + shard.pushInput(edges, new dida.Change(["b", "c"], [1], -1)); + shard.flushInput(edges); + + shard.advanceInput(edges, [1]); + while (shard.hasWork()) { + shard.doWork(); + while (true) { + const change_batch = shard.popOutput(reach_out); + if (change_batch == undefined) break; + console.log(change_batch); + } + while (true) { + const change_batch = shard.popOutput(reach_summary_out); + if (change_batch == undefined) break; + console.log(change_batch); + } + } + + console.log("Advancing!"); + + shard.advanceInput(edges, [2]); + while (shard.hasWork()) { + shard.doWork(); + while (true) { + const change_batch = shard.popOutput(reach_out); + if (change_batch == undefined) break; + console.log(change_batch); + } + while (true) { + const change_batch = shard.popOutput(reach_summary_out); + if (change_batch == undefined) break; + console.log(change_batch); + } + } +} +run(); \ No newline at end of file diff --git a/flake.nix b/flake.nix index a1e10c2..7bdc971 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,10 @@ let pkgs = import nixpkgs { system = system; }; deps = (import ./dependencies.nix { inherit system; }); + buildWasmBindings = pkgs.writeScriptBin "buildWasmBindings" '' + cd bindings/wasm + ${deps.zig}/bin/zig build install && ${deps.zig}/bin/zig build run-codegen + ''; in { @@ -15,7 +19,10 @@ devShell = pkgs.mkShell rec { buildInputs = [ + buildWasmBindings deps.zig + pkgs.nodejs-17_x + pkgs.deno ]; }; }); From 28534294ed10deaa08b3dea2b566bcad91744b1f Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 16 Mar 2022 22:27:38 -0700 Subject: [PATCH 03/12] Deno example --- examples/core.html | 4 ++++ examples/sugar.zig | 27 ++++++++++++++------------- examples/wasm-deno.js | 22 ++++++++++++++++++++++ lib/dida/sugar.zig | 2 +- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/examples/core.html b/examples/core.html index dc13b4f..9a2ba54 100644 --- a/examples/core.html +++ b/examples/core.html @@ -57,11 +57,13 @@ while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; + console.log("reach_out") console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; + console.log("reach_summary_out") console.log(change_batch); } } @@ -74,11 +76,13 @@ while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; + console.log("reach_out") console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; + console.log("reach_summary_out") console.log(change_batch); } } diff --git a/examples/sugar.zig b/examples/sugar.zig index cea8d95..821c47d 100644 --- a/examples/sugar.zig +++ b/examples/sugar.zig @@ -3,18 +3,19 @@ const std = @import("std"); const dida = @import("../lib/dida.zig"); -var gpa = std.heap.GeneralPurposeAllocator(.{ - .safety = true, - .never_unmap = true, -}){}; -var arena = std.heap.ArenaAllocator.init(&gpa.allocator); -const allocator = &arena.allocator; +// var gpa = std.heap.GeneralPurposeAllocator(.{ +// .safety = true, +// .never_unmap = true, +// }){}; +// var arena = std.heap.ArenaAllocator.init(&gpa.allocator); +// const allocator = &arena.allocator; +pub const allocator = std.testing.allocator; pub fn main() !void { - defer { - arena.deinit(); - _ = gpa.detectLeaks(); - } + // defer { + // arena.deinit(); + // _ = gpa.detectLeaks(); + // } var sugar = dida.sugar.Sugar.init(allocator); @@ -42,7 +43,7 @@ pub fn main() !void { try edges.advance(.{1}); try sugar.doAllWork(); while (out.pop()) |change_batch| { - dida.common.dump(change_batch); + dida.util.dump(change_batch); } std.debug.print("Advancing!\n", .{}); @@ -50,8 +51,8 @@ pub fn main() !void { try edges.advance(.{2}); try sugar.doAllWork(); while (out.pop()) |change_batch| { - dida.common.dump(change_batch); + dida.util.dump(change_batch); } - //dida.common.dump(sugar); + //dida.util.dump(sugar); } diff --git a/examples/wasm-deno.js b/examples/wasm-deno.js index 66a7d28..b49ad20 100644 --- a/examples/wasm-deno.js +++ b/examples/wasm-deno.js @@ -58,11 +58,13 @@ async function run() { while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; + console.log("reach_out") console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; + console.log("reach_out_summary") console.log(change_batch); } } @@ -75,11 +77,31 @@ async function run() { while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; + console.log("reach_out") console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; + console.log("reach_out_summary") + console.log(change_batch); + } + } + + console.log("Advancing!"); + shard.advanceInput(edges, [3]); + while (shard.hasWork()) { + shard.doWork(); + while (true) { + const change_batch = shard.popOutput(reach_out); + if (change_batch == undefined) break; + console.log("reach_out") + console.log(change_batch); + } + while (true) { + const change_batch = shard.popOutput(reach_summary_out); + if (change_batch == undefined) break; + console.log("reach_out_summary") console.log(change_batch); } } diff --git a/lib/dida/sugar.zig b/lib/dida/sugar.zig index dde813a..27049f3 100644 --- a/lib/dida/sugar.zig +++ b/lib/dida/sugar.zig @@ -116,7 +116,7 @@ pub const Subgraph = struct { } }; -pub fn Node(comptime tag_: std.meta.TagType(dida.core.NodeSpec)) type { +pub fn Node(comptime tag_: std.meta.Tag(dida.core.NodeSpec)) type { return struct { sugar: *Sugar, inner: dida.core.Node, From 6a707ec4817e2cc327827d345a6eca2a2c509cf4 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 25 Mar 2022 19:25:39 -0700 Subject: [PATCH 04/12] Export typescript types --- .gitignore | 3 +- bindings/wasm/codegen.zig | 25 +- flake.lock | 33 ++- flake.nix | 61 +++- tsc/default.nix | 17 ++ tsc/node-env.nix | 588 ++++++++++++++++++++++++++++++++++++++ tsc/node-packages.nix | 53 ++++ tsc/package-lock.json | 31 ++ tsc/package.json | 16 ++ 9 files changed, 806 insertions(+), 21 deletions(-) create mode 100644 tsc/default.nix create mode 100644 tsc/node-env.nix create mode 100644 tsc/node-packages.nix create mode 100644 tsc/package-lock.json create mode 100644 tsc/package.json diff --git a/.gitignore b/.gitignore index 44dbeed..afa4b46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ zig-cache -zig-out \ No newline at end of file +zig-out +result \ No newline at end of file diff --git a/bindings/wasm/codegen.zig b/bindings/wasm/codegen.zig index e619f5b..d234292 100644 --- a/bindings/wasm/codegen.zig +++ b/bindings/wasm/codegen.zig @@ -20,11 +20,11 @@ pub fn main() !void { const file2 = try std.fs.cwd().createFile("zig-out/lib/dida.mjs", .{ .read = false, .truncate = true }); defer file2.close(); writer = file2.writer(); - try writer.writeAll("function Dida(abi) {\n\n"); inline for (js_common.types_with_js_constructors) |T| { try generateConstructor(writer, T); } try writer.writeAll("\n\n"); + try writer.writeAll("function Dida(abi) {\n\n"); inline for (js_common.types_with_js_constructors) |T| { try std.fmt.format(writer, "this.{s} = {s};\n", .{ T, T }); } @@ -119,6 +119,7 @@ fn generateConstructor(writer: anytype, comptime Type: type) !void { if (union_info.tag_type) |_| { // TODO name payload args instead of using `arguments[i]` try std.fmt.format(writer, "const {s} = {{\n", .{Type}); + var functionsReferenced = std.ArrayList(u8).init(js_common.allocator); inline for (union_info.fields) |field_info| { const payload = switch (field_info.field_type) { []const u8, f64 => "arguments[0]", @@ -133,18 +134,30 @@ fn generateConstructor(writer: anytype, comptime Type: type) !void { }); }, }; + try std.fmt.format( + functionsReferenced.writer(), + \\function {s}_{s}_tag() {{ + \\ this.tag = "{s}"; + \\ this.payload = {s}; + \\}} + \\ + \\ + , + .{ Type, field_info.name, field_info.name, payload }, + ); + try std.fmt.format( writer, - \\ {s}: function () {{ - \\ this.tag = "{s}"; - \\ this.payload = {s}; - \\ }}, + \\ {s}: {s}_{s}_tag, \\ , - .{ field_info.name, field_info.name, payload }, + .{ field_info.name, Type, field_info.name }, ); } try writer.writeAll("};\n\n"); + try writer.writeAll(functionsReferenced.items); + try writer.writeAll("\n\n"); + } else { dida.util.compileError("Don't know how to make constructor for non-tagged union type {}", .{Type}); } diff --git a/flake.lock b/flake.lock index 80c7a88..448ecdd 100644 --- a/flake.lock +++ b/flake.lock @@ -17,22 +17,41 @@ }, "nixpkgs": { "locked": { - "lastModified": 1646826225, - "narHash": "sha256-UbocGllbHFDJddhAey+MzYKzPcMWyP082ub/XjEAfzQ=", - "owner": "NixOS", + "lastModified": 1648250841, + "narHash": "sha256-JSZLpMDDhDZq2KHWkbJc7VAkOrNIexVHQn7pfsjCQbA=", + "owner": "nixos", "repo": "nixpkgs", - "rev": "b99542dff8373721834d4c3458398ca8e1319953", + "rev": "e80f8f4d8336f5249d475d5c671a4e53b9d36634", "type": "github" }, "original": { - "id": "nixpkgs", - "type": "indirect" + "owner": "nixos", + "ref": "release-21.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1648097358, + "narHash": "sha256-GMoTKP/po2Nbkh1tvPvP8Ww6NyFW8FFst1Z3nfzffZc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "4d60081494259c0785f7e228518fee74e0792c1b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" } }, "root": { "inputs": { "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" } } }, diff --git a/flake.nix b/flake.nix index 7bdc971..bdf800b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,28 +1,75 @@ { description = "A very basic flake"; + inputs.nixpkgs.url = "github:nixos/nixpkgs/release-21.11"; + inputs.nixpkgs-unstable.url = "github:nixos/nixpkgs/nixpkgs-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; + # inputs.zls = { + # url = "github:zigtools/zls?submodules=1"; + # flake = false; + # fetchSubmodules = true; + # }; - outputs = { self, nixpkgs, flake-utils }: + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { system = system; }; + pkgs-unstable = import nixpkgs-unstable { system = system; }; deps = (import ./dependencies.nix { inherit system; }); - buildWasmBindings = pkgs.writeScriptBin "buildWasmBindings" '' - cd bindings/wasm - ${deps.zig}/bin/zig build install && ${deps.zig}/bin/zig build run-codegen - ''; in { packages.hello = pkgs.hello; + # packages.zls = (import zls {inherit system pkgs;}); + packages.zls = pkgs.stdenvNoCC.mkDerivation { + name = "zls"; + version = "master"; + src = pkgs.fetchFromGitHub { + owner = "zigtools"; + repo = "zls"; + # rev = "4e6564d7daec95b4fb51cadfe25973a87cac181a"; + rev = "0.9.0"; + fetchSubmodules = true; + sha256 = "sha256-MVo21qNCZop/HXBqrPcosGbRY+W69KNCc1DfnH47GsI="; + }; + nativeBuildInputs = [ deps.zig ]; + dontConfigure = true; + dontInstall = true; + buildPhase = '' + mkdir -p $out + ls -lha . + pwd + echo "!!!" + zig build install -Drelease-safe=true -Ddata_version=master --prefix $out + ''; + XDG_CACHE_HOME = ".cache"; + }; defaultPackage = self.packages.${system}.hello; + packages.buildWasmBindings = pkgs.writeScriptBin "buildWasmBindings" '' + cd $(git rev-parse --show-toplevel) + cd bindings/wasm + ${deps.zig}/bin/zig build install && ${deps.zig}/bin/zig build run-codegen + cd zig-out/lib + ${self.packages.${system}.tsc}/bin/tsc 'dida.mjs' --declaration --allowJs --emitDeclarationOnly --outDir types + ''; + # packages.tsc = pkgs.lib.mkDerivation{ + # name = "typescript"; + # packageName = "typescript"; + # version = "4.4.4"; + # src = fetchurl { + # url = "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz"; + # sha512 = "DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA=="; + # }; + # }; + packages.tsc = (import ./tsc { inherit system pkgs; }).nodeDependencies; devShell = pkgs.mkShell rec { buildInputs = [ - buildWasmBindings + self.packages.${system}.buildWasmBindings deps.zig pkgs.nodejs-17_x - pkgs.deno + pkgs-unstable.deno + self.packages.${system}.zls + self.packages.${system}.tsc ]; }; }); diff --git a/tsc/default.nix b/tsc/default.nix new file mode 100644 index 0000000..53bdef1 --- /dev/null +++ b/tsc/default.nix @@ -0,0 +1,17 @@ +# This file has been generated by node2nix 1.9.0. Do not edit! + +{pkgs ? import { + inherit system; + }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-12_x"}: + +let + nodeEnv = import ./node-env.nix { + inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript; + inherit pkgs nodejs; + libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null; + }; +in +import ./node-packages.nix { + inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit; + inherit nodeEnv; +} diff --git a/tsc/node-env.nix b/tsc/node-env.nix new file mode 100644 index 0000000..5f05578 --- /dev/null +++ b/tsc/node-env.nix @@ -0,0 +1,588 @@ +# This file originates from node2nix + +{lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile, writeShellScript}: + +let + # Workaround to cope with utillinux in Nixpkgs 20.09 and util-linux in Nixpkgs master + utillinux = if pkgs ? utillinux then pkgs.utillinux else pkgs.util-linux; + + python = if nodejs ? python then nodejs.python else python2; + + # Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise + tarWrapper = runCommand "tarWrapper" {} '' + mkdir -p $out/bin + + cat > $out/bin/tar <> $out/nix-support/hydra-build-products + ''; + }; + + # Common shell logic + installPackage = writeShellScript "install-package" '' + installPackage() { + local packageName=$1 src=$2 + + local strippedName + + local DIR=$PWD + cd $TMPDIR + + unpackFile $src + + # Make the base dir in which the target dependency resides first + mkdir -p "$(dirname "$DIR/$packageName")" + + if [ -f "$src" ] + then + # Figure out what directory has been unpacked + packageDir="$(find . -maxdepth 1 -type d | tail -1)" + + # Restore write permissions to make building work + find "$packageDir" -type d -exec chmod u+x {} \; + chmod -R u+w "$packageDir" + + # Move the extracted tarball into the output folder + mv "$packageDir" "$DIR/$packageName" + elif [ -d "$src" ] + then + # Get a stripped name (without hash) of the source directory. + # On old nixpkgs it's already set internally. + if [ -z "$strippedName" ] + then + strippedName="$(stripHash $src)" + fi + + # Restore write permissions to make building work + chmod -R u+w "$strippedName" + + # Move the extracted directory into the output folder + mv "$strippedName" "$DIR/$packageName" + fi + + # Change to the package directory to install dependencies + cd "$DIR/$packageName" + } + ''; + + # Bundle the dependencies of the package + # + # Only include dependencies if they don't exist. They may also be bundled in the package. + includeDependencies = {dependencies}: + lib.optionalString (dependencies != []) ( + '' + mkdir -p node_modules + cd node_modules + '' + + (lib.concatMapStrings (dependency: + '' + if [ ! -e "${dependency.name}" ]; then + ${composePackage dependency} + fi + '' + ) dependencies) + + '' + cd .. + '' + ); + + # Recursively composes the dependencies of a package + composePackage = { name, packageName, src, dependencies ? [], ... }@args: + builtins.addErrorContext "while evaluating node package '${packageName}'" '' + installPackage "${packageName}" "${src}" + ${includeDependencies { inherit dependencies; }} + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + ''; + + pinpointDependencies = {dependencies, production}: + let + pinpointDependenciesFromPackageJSON = writeTextFile { + name = "pinpointDependencies.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function resolveDependencyVersion(location, name) { + if(location == process.env['NIX_STORE']) { + return null; + } else { + var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json"); + + if(fs.existsSync(dependencyPackageJSON)) { + var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON)); + + if(dependencyPackageObj.name == name) { + return dependencyPackageObj.version; + } + } else { + return resolveDependencyVersion(path.resolve(location, ".."), name); + } + } + } + + function replaceDependencies(dependencies) { + if(typeof dependencies == "object" && dependencies !== null) { + for(var dependency in dependencies) { + var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency); + + if(resolvedVersion === null) { + process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n"); + } else { + dependencies[dependency] = resolvedVersion; + } + } + } + } + + /* Read the package.json configuration */ + var packageObj = JSON.parse(fs.readFileSync('./package.json')); + + /* Pinpoint all dependencies */ + replaceDependencies(packageObj.dependencies); + if(process.argv[2] == "development") { + replaceDependencies(packageObj.devDependencies); + } + replaceDependencies(packageObj.optionalDependencies); + + /* Write the fixed package.json file */ + fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); + ''; + }; + in + '' + node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"} + + ${lib.optionalString (dependencies != []) + '' + if [ -d node_modules ] + then + cd node_modules + ${lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies} + cd .. + fi + ''} + ''; + + # Recursively traverses all dependencies of a package and pinpoints all + # dependencies in the package.json file to the versions that are actually + # being used. + + pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args: + '' + if [ -d "${packageName}" ] + then + cd "${packageName}" + ${pinpointDependencies { inherit dependencies production; }} + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + fi + ''; + + # Extract the Node.js source code which is used to compile packages with + # native bindings + nodeSources = runCommand "node-sources" {} '' + tar --no-same-owner --no-same-permissions -xf ${nodejs.src} + mv node-* $out + ''; + + # Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty) + addIntegrityFieldsScript = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + function augmentDependencies(baseDir, dependencies) { + for(var dependencyName in dependencies) { + var dependency = dependencies[dependencyName]; + + // Open package.json and augment metadata fields + var packageJSONDir = path.join(baseDir, "node_modules", dependencyName); + var packageJSONPath = path.join(packageJSONDir, "package.json"); + + if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored + console.log("Adding metadata fields to: "+packageJSONPath); + var packageObj = JSON.parse(fs.readFileSync(packageJSONPath)); + + if(dependency.integrity) { + packageObj["_integrity"] = dependency.integrity; + } else { + packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads. + } + + if(dependency.resolved) { + packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided + } else { + packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories. + } + + if(dependency.from !== undefined) { // Adopt from property if one has been provided + packageObj["_from"] = dependency.from; + } + + fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2)); + } + + // Augment transitive dependencies + if(dependency.dependencies !== undefined) { + augmentDependencies(packageJSONDir, dependency.dependencies); + } + } + } + + if(fs.existsSync("./package-lock.json")) { + var packageLock = JSON.parse(fs.readFileSync("./package-lock.json")); + + if(![1, 2].includes(packageLock.lockfileVersion)) { + process.stderr.write("Sorry, I only understand lock file versions 1 and 2!\n"); + process.exit(1); + } + + if(packageLock.dependencies !== undefined) { + augmentDependencies(".", packageLock.dependencies); + } + } + ''; + }; + + # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes + reconstructPackageLock = writeTextFile { + name = "addintegrityfields.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var lockObj = { + name: packageObj.name, + version: packageObj.version, + lockfileVersion: 1, + requires: true, + dependencies: {} + }; + + function augmentPackageJSON(filePath, dependencies) { + var packageJSON = path.join(filePath, "package.json"); + if(fs.existsSync(packageJSON)) { + var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + dependencies[packageObj.name] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: {} + }; + processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + } + } + + function processDependencies(dir, dependencies) { + if(fs.existsSync(dir)) { + var files = fs.readdirSync(dir); + + files.forEach(function(entry) { + var filePath = path.join(dir, entry); + var stats = fs.statSync(filePath); + + if(stats.isDirectory()) { + if(entry.substr(0, 1) == "@") { + // When we encounter a namespace folder, augment all packages belonging to the scope + var pkgFiles = fs.readdirSync(filePath); + + pkgFiles.forEach(function(entry) { + if(stats.isDirectory()) { + var pkgFilePath = path.join(filePath, entry); + augmentPackageJSON(pkgFilePath, dependencies); + } + }); + } else { + augmentPackageJSON(filePath, dependencies); + } + } + }); + } + } + + processDependencies("node_modules", lockObj.dependencies); + + fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); + ''; + }; + + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: + let + forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; + in + '' + # Pinpoint the versions of all dependencies to the ones that are actually being used + echo "pinpointing versions of dependencies..." + source $pinpointDependenciesScriptPath + + # Patch the shebangs of the bundled modules to prevent them from + # calling executables outside the Nix store as much as possible + patchShebangs . + + # Deploy the Node.js package by running npm install. Since the + # dependencies have been provided already by ourselves, it should not + # attempt to install them again, which is good, because we want to make + # it Nix's responsibility. If it needs to install any dependencies + # anyway (e.g. because the dependency parameters are + # incomplete/incorrect), it fails. + # + # The other responsibilities of NPM are kept -- version checks, build + # steps, postprocessing etc. + + export HOME=$TMPDIR + cd "${packageName}" + runHook preRebuild + + ${lib.optionalString bypassCache '' + ${lib.optionalString reconstructLock '' + if [ -f package-lock.json ] + then + echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!" + echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!" + rm package-lock.json + else + echo "No package-lock.json file found, reconstructing..." + fi + + node ${reconstructPackageLock} + ''} + + node ${addIntegrityFieldsScript} + ''} + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild + + if [ "''${dontNpmInstall-}" != "1" ] + then + # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. + rm -f npm-shrinkwrap.json + + npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install + fi + ''; + + # Builds and composes an NPM package including all its dependencies + buildNodePackage = + { name + , packageName + , version + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , preRebuild ? "" + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , meta ? {} + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" "meta" ]; + in + stdenv.mkDerivation ({ + name = "${name}-${version}"; + buildInputs = [ tarWrapper python nodejs ] + ++ lib.optional (stdenv.isLinux) utillinux + ++ lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit nodejs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall preRebuild unpackPhase buildPhase; + + compositionScript = composePackage args; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "compositionScript" "pinpointDependenciesScript" ]; + + installPhase = '' + source ${installPackage} + + # Create and enter a root node_modules/ folder + mkdir -p $out/lib/node_modules + cd $out/lib/node_modules + + # Compose the package and all its dependencies + source $compositionScriptPath + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Create symlink to the deployed executable folder, if applicable + if [ -d "$out/lib/node_modules/.bin" ] + then + ln -s $out/lib/node_modules/.bin $out/bin + fi + + # Create symlinks to the deployed manual page folders, if applicable + if [ -d "$out/lib/node_modules/${packageName}/man" ] + then + mkdir -p $out/share + for dir in "$out/lib/node_modules/${packageName}/man/"* + do + mkdir -p $out/share/man/$(basename "$dir") + for page in "$dir"/* + do + ln -s $page $out/share/man/$(basename "$dir") + done + done + fi + + # Run post install hook, if provided + runHook postInstall + ''; + + meta = { + # default to Node.js' platforms + platforms = nodejs.meta.platforms; + } // meta; + } // extraArgs); + + # Builds a node environment (a node_modules folder and a set of binaries) + buildNodeDependencies = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ]; + in + stdenv.mkDerivation ({ + name = "node-dependencies-${name}-${version}"; + + buildInputs = [ tarWrapper python nodejs ] + ++ lib.optional (stdenv.isLinux) utillinux + ++ lib.optional (stdenv.isDarwin) libtool + ++ buildInputs; + + inherit dontStrip; # Stripping may fail a build for some package deployments + inherit dontNpmInstall unpackPhase buildPhase; + + includeScript = includeDependencies { inherit dependencies; }; + pinpointDependenciesScript = pinpointDependenciesOfPackage args; + + passAsFile = [ "includeScript" "pinpointDependenciesScript" ]; + + installPhase = '' + source ${installPackage} + + mkdir -p $out/${packageName} + cd $out/${packageName} + + source $includeScriptPath + + # Create fake package.json to make the npm commands work properly + cp ${src}/package.json . + chmod 644 package.json + ${lib.optionalString bypassCache '' + if [ -f ${src}/package-lock.json ] + then + cp ${src}/package-lock.json . + fi + ''} + + # Go to the parent folder to make sure that all packages are pinpointed + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + ${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }} + + # Expose the executables that were installed + cd .. + ${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."} + + mv ${packageName} lib + ln -s $out/lib/node_modules/.bin $out/bin + ''; + } // extraArgs); + + # Builds a development shell + buildNodeShell = + { name + , packageName + , version + , src + , dependencies ? [] + , buildInputs ? [] + , production ? true + , npmFlags ? "" + , dontNpmInstall ? false + , bypassCache ? false + , reconstructLock ? false + , dontStrip ? true + , unpackPhase ? "true" + , buildPhase ? "true" + , ... }@args: + + let + nodeDependencies = buildNodeDependencies args; + in + stdenv.mkDerivation { + name = "node-shell-${name}-${version}"; + + buildInputs = [ python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux ++ buildInputs; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/shell <=4.2.0" + } + } + }, + "dependencies": { + "typescript": { + "version": "4.7.0-dev.20220325", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.0-dev.20220325.tgz", + "integrity": "sha512-3S7MAr3P2vkrpptkhoEuiATOU1kPa9+3FUrJAAYZ4/6mnI1SZWeVPeBIbhzIOUij27QrRMgSwN8aHrqtgGVqLA==" + } + } +} diff --git a/tsc/package.json b/tsc/package.json new file mode 100644 index 0000000..84a9aa4 --- /dev/null +++ b/tsc/package.json @@ -0,0 +1,16 @@ +{ + "name": "unused", + "type": "module", + "dependencies": { + "typescript": "^4.7.0-dev.20220325" + }, + "version": "1.0.0", + "main": "dida.js", + "devDependencies": {}, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "description": "" +} From 8390a401b60c66297da7df9480ea24da3cf66970 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 25 Mar 2022 20:01:04 -0700 Subject: [PATCH 05/12] TS bindings --- bindings/ts/.vscode/settings.json | 5 + bindings/ts/sugar.ts | 162 ++++++++++++++++++++++++++++++ bindings/ts/sugar_test.ts | 34 +++++++ 3 files changed, 201 insertions(+) create mode 100644 bindings/ts/.vscode/settings.json create mode 100644 bindings/ts/sugar.ts create mode 100644 bindings/ts/sugar_test.ts diff --git a/bindings/ts/.vscode/settings.json b/bindings/ts/.vscode/settings.json new file mode 100644 index 0000000..e40716f --- /dev/null +++ b/bindings/ts/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "deno.enable": true, + "deno.lint": true, + "deno.unstable": true +} diff --git a/bindings/ts/sugar.ts b/bindings/ts/sugar.ts new file mode 100644 index 0000000..b77c93a --- /dev/null +++ b/bindings/ts/sugar.ts @@ -0,0 +1,162 @@ +import Dida from '../wasm/zig-out/lib/types/dida' + + +// type Dida = { +// GraphBuilder: DidaBuilderClass +// } & any +type DidaBuilder = any +type DidaSubgraph = any +type DidaNode = any +type DidaBuilderClass = { + new(): DidaBuilder +} + +type SugarState = + { state: "building", builder: DidaBuilder, rootSubGraph: Subgraph } + | { state: "running" } + +export class Sugar { + state: SugarState + dida: Dida + + constructor(dida: Dida) { + const builder = new dida.GraphBuilder() + this.state = { state: "building", builder: new dida.GraphBuilder(), rootSubGraph: builder } + this.dida = dida + } + + get builder(): DidaBuilder { + if (this.state.state !== "building") { + throw new Error("Sugar is not in building state") + } + return this.state.builder + } + + get rootSubGraph(): Subgraph { + if (this.state.state !== "building") { + throw new Error("Sugar is not in building state") + } + return this.state.rootSubGraph + } + + + input(): InputNode { + const builder = this.builder() + const rootGraph = this.rootSubGraph() + + const node = builder.addNode(rootGraph, new this.dida.NodeSpec.Input()); + const edges = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Input()); + + const edges = this.s.addNode(subgraph_0, new dida.NodeSpec.Input()); + return new InputNode(this, null) + } + + loop(): Subgraph { + if (this.state.state !== "building") { + throw new Error("Cannot loop on a graph that is not being built") + } + + this.state.builder.add + const subgraph_1 = this.state.builder.addSubgraph(this.state.rootSubGraph); + return new Subgraph(this, subgraph_1) + } + + build() { } + doAllWork() { } +} + +export class Subgraph { + readonly sugar: Sugar + readonly inner: DidaSubgraph + + constructor(sugar: Sugar, inner: DidaSubgraph) { + this.sugar = sugar + this.inner = inner + } + + // loop(): Subgraph { + // return new Subgraph() + // } + + // loopNode(): Node { + // return new Node() + // } + + importNode(n: Node): Node { + if (this.sugar.state.state !== "building") { + throw new Error("Not in building state") + } + + // TODO add check for + // "Can only import from parent subgraph into child subgraph", + console.log("asdf") + console.log(this.inner) + console.log(this.inner.id) + + return new Node(this.sugar, this.inner) + } + + // exportNode(n: Node): Node { + // return new Node() + // } +} + +export class Node { + readonly sugar: Sugar + readonly inner: DidaNode + + constructor(sugar: Sugar, inner: DidaNode) { + this.sugar = sugar + this.inner = inner + } + + // index(): Node { + // return new Node(this.sugar) + // } + + map(f: any): this { + return this + } + reduce(): this { + return this + } + project(): this { + return this + } + union(): this { + return this + } + + // output(f: any): OutputNode { + // return new OutputNode() + // } +} + +export class InputNode extends Node { + + push() { } + flush() { } + advance() { } +} + +export class OutputNode extends Node { + pop() { } +} + +export class JoinedNode extends Node { } + +export class IndexedNode extends Node { + distinct(): this { + return this + } + join(): this { + return this + } +} + +export class DistinctNode extends IndexedNode { } +export class ReduceNode extends IndexedNode { } + +export class TimestampPushedNode extends Node { } + +// type HasIndex = T extends IndexedNode ? true : T extends DistinctNode ? T exte \ No newline at end of file diff --git a/bindings/ts/sugar_test.ts b/bindings/ts/sugar_test.ts new file mode 100644 index 0000000..76fedf1 --- /dev/null +++ b/bindings/ts/sugar_test.ts @@ -0,0 +1,34 @@ +import { assertEquals } from "https://deno.land/std@0.132.0/testing/asserts.ts"; +import Abi from "../wasm/abi-with-bytes.js"; +import Dida from "../wasm/zig-out/lib/dida.mjs"; +import { Buffer } from "https://deno.land/std/io/mod.ts"; +import { Sugar, Subgraph, Node, OutputNode } from "./sugar.ts"; + +Deno.test("sugar example", async () => { + const file = await Deno.open("../wasm/zig-out/lib/dida.wasm"); + const b = new Buffer(); + await b.readFrom(file); + console.log(typeof b, typeof b.bytes()); + file.close() + + const abi = await Abi(b.bytes()); + const dida = new Dida(abi); + + const sugar = new Sugar(dida); + + const edges = sugar.input() + const loop = sugar.loop() + const edges1 = loop.importNode(edges) + + // const edges = sugar.input() + +}); + +Deno.test("add test", () => { + const a = 1 + 1 + assertEquals(a, 2); + // test 2 + 2 = 4 + const b = 2 + 2 + assertEquals(b, 4); +}); + From 2536d3cf16443cf9c0844de3f6785cea18a855c3 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 25 Mar 2022 20:02:01 -0700 Subject: [PATCH 06/12] Update deno example --- examples/wasm-deno.js | 156 +++++++++++++++++++++++++++++++----------- 1 file changed, 116 insertions(+), 40 deletions(-) diff --git a/examples/wasm-deno.js b/examples/wasm-deno.js index b49ad20..082c04f 100644 --- a/examples/wasm-deno.js +++ b/examples/wasm-deno.js @@ -1,45 +1,106 @@ -import Abi from "../bindings/wasm/abi-with-bytes.js" -import Dida from "../bindings/wasm/zig-out/lib/dida.mjs" +import Abi from "../bindings/wasm/abi-with-bytes.js"; +import Dida from "../bindings/wasm/zig-out/lib/dida.mjs"; import { Buffer } from "https://deno.land/std/io/mod.ts"; -async function run() { - const file = await Deno.open('./bindings/wasm/zig-out/lib/dida.wasm'); - const b = new Buffer(); - await b.readFrom(file); - console.log(typeof b, typeof b.bytes()) +function probeHelper(dida, graph_builder, name, subgraph, node) { + const probe_out = graph_builder.addNode( + subgraph, + new dida.NodeSpec.Output(node), + ); - const abi = await Abi(b.bytes()); - const dida = new Dida(abi); + return function (shard) { + while (true) { + const change_batch = shard.popOutput(probe_out); + if (change_batch == undefined) break; + console.log(name); + console.log(change_batch); + } + } +} +function exampleQuery(dida) { + const probeFns = []; var graph_builder = new dida.GraphBuilder(); + const probe = (...args) => probeHelper(dida, graph_builder, ...args); + const subgraph_0 = new dida.Subgraph(0); const subgraph_1 = graph_builder.addSubgraph(subgraph_0); + const edges = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Input()); - const edges_1 = graph_builder.addNode(subgraph_1, new dida.NodeSpec.TimestampPush(edges)); - const reach_future = graph_builder.addNode(subgraph_1, new dida.NodeSpec.TimestampIncrement(null)); - const reach_index = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Index(reach_future)); - const distinct_reach_index = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Distinct(reach_index)); - const swapped_edges = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Map(edges_1, input => [input[1], input[0]])); - const swapped_edges_index = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Index(swapped_edges)); - const joined = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Join([distinct_reach_index, swapped_edges_index], 1)); - const without_middle = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Map(joined, input => [input[2], input[1]])); - const reach = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Union([edges_1, without_middle])); + const edges_1 = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.TimestampPush(edges), + ); + + probeFns.push(probe("edges", subgraph_0, edges)); + + const reach_future = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.TimestampIncrement(null), + ); + + probeFns.push(probe("reach_future", subgraph_1, reach_future)); + + const reach_index = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Index(reach_future), + ); + + const distinct_reach_index = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Distinct(reach_index), + ); + + const swapped_edges = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Map(edges_1, (input) => [input[1], input[0]]), + ); + const swapped_edges_index = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Index(swapped_edges), + ); + + const joined = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Join([distinct_reach_index, swapped_edges_index], 1), + ); + + const without_middle = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Map(joined, (input) => [input[2], input[1]]), + ); + const reach = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Union([edges_1, without_middle]), + ); graph_builder.connectLoop(reach, reach_future); - const reach_pop = graph_builder.addNode(subgraph_0, new dida.NodeSpec.TimestampPop(distinct_reach_index)); - const reach_out = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Output(reach_pop)); - - const reach_summary = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Reduce( - distinct_reach_index, - 1, - "", - function (reduced_value, row, count) { - for (var i = 0; i < count; i++) { - reduced_value += row[1]; - } - return reduced_value; - } - )); - const reach_summary_out = graph_builder.addNode(subgraph_1, new dida.NodeSpec.Output(reach_summary)); + const reach_pop = graph_builder.addNode( + subgraph_0, + new dida.NodeSpec.TimestampPop(distinct_reach_index), + ); + const reach_out = graph_builder.addNode( + subgraph_0, + new dida.NodeSpec.Output(reach_pop), + ); + + const reach_summary = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Reduce( + distinct_reach_index, + 1, + "", + function (reduced_value, row, count) { + for (var i = 0; i < count; i++) { + reduced_value += row[1]; + } + return reduced_value; + }, + ), + ); + const reach_summary_out = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Output(reach_summary), + ); const graph = graph_builder.finishAndReset(); @@ -58,13 +119,13 @@ async function run() { while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; - console.log("reach_out") + console.log("reach_out"); console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; - console.log("reach_out_summary") + console.log("reach_summary_out"); console.log(change_batch); } } @@ -72,18 +133,22 @@ async function run() { console.log("Advancing!"); shard.advanceInput(edges, [2]); + while (shard.hasWork()) { shard.doWork(); + for (const fn of probeFns) { + fn(shard); + } while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; - console.log("reach_out") + console.log("reach_out"); console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; - console.log("reach_out_summary") + console.log("reach_summary_out"); console.log(change_batch); } } @@ -95,15 +160,26 @@ async function run() { while (true) { const change_batch = shard.popOutput(reach_out); if (change_batch == undefined) break; - console.log("reach_out") + console.log("reach_out"); console.log(change_batch); } while (true) { const change_batch = shard.popOutput(reach_summary_out); if (change_batch == undefined) break; - console.log("reach_out_summary") + console.log("reach_out_summary"); console.log(change_batch); } } } -run(); \ No newline at end of file + +async function run() { + const file = await Deno.open("./bindings/wasm/zig-out/lib/dida.wasm"); + const b = new Buffer(); + await b.readFrom(file); + console.log(typeof b, typeof b.bytes()); + + const abi = await Abi(b.bytes()); + const dida = new Dida(abi); + exampleQuery(dida); +} +run(); From f7afb0b03dae45d63033d5e7d4624aec5f0ee163 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 28 Mar 2022 09:10:29 -0700 Subject: [PATCH 07/12] Continue TS sugar --- bindings/ts/sugar.ts | 267 +++++++++++++++++++++++++++----------- bindings/ts/sugar_test.ts | 51 +++++++- 2 files changed, 238 insertions(+), 80 deletions(-) diff --git a/bindings/ts/sugar.ts b/bindings/ts/sugar.ts index b77c93a..0d03810 100644 --- a/bindings/ts/sugar.ts +++ b/bindings/ts/sugar.ts @@ -1,30 +1,38 @@ -import Dida from '../wasm/zig-out/lib/types/dida' +import Dida from '../wasm/zig-out/lib/types/dida.d.mts' - -// type Dida = { -// GraphBuilder: DidaBuilderClass -// } & any -type DidaBuilder = any -type DidaSubgraph = any -type DidaNode = any -type DidaBuilderClass = { - new(): DidaBuilder -} +type DidaBuilder = InstanceType type SugarState = - { state: "building", builder: DidaBuilder, rootSubGraph: Subgraph } - | { state: "running" } + { state: "building", builder: InstanceType, rootSubGraph: Subgraph } + | { state: "running", shard: InstanceType } + +type UglyHackToPreserverType, G extends Array> = + A extends IndexedNode ? IndexedNode : + A extends Node ? Node : A + + +type Any1 = any + export class Sugar { state: SugarState dida: Dida constructor(dida: Dida) { + const rootID = 0 const builder = new dida.GraphBuilder() - this.state = { state: "building", builder: new dida.GraphBuilder(), rootSubGraph: builder } + const rootSubGraph = builder.addSubgraph(rootID) + this.state = { state: "building", builder: builder, rootSubGraph } this.dida = dida } + get shard(): InstanceType { + if (this.state.state !== "running") { + throw new Error("Sugar is not in building state") + } + return this.state.shard + } + get builder(): DidaBuilder { if (this.state.state !== "building") { throw new Error("Sugar is not in building state") @@ -40,15 +48,12 @@ export class Sugar { } - input(): InputNode { - const builder = this.builder() - const rootGraph = this.rootSubGraph() + input>(): InputNode { + const builder = this.builder + const rootGraph = this.rootSubGraph const node = builder.addNode(rootGraph, new this.dida.NodeSpec.Input()); - const edges = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Input()); - - const edges = this.s.addNode(subgraph_0, new dida.NodeSpec.Input()); - return new InputNode(this, null) + return new InputNode(this, node, rootGraph) } loop(): Subgraph { @@ -56,107 +61,223 @@ export class Sugar { throw new Error("Cannot loop on a graph that is not being built") } - this.state.builder.add const subgraph_1 = this.state.builder.addSubgraph(this.state.rootSubGraph); - return new Subgraph(this, subgraph_1) + return new Subgraph(this, this.rootSubGraph, subgraph_1) } - build() { } - doAllWork() { } + build() { + const graph = this.builder.finishAndReset(); + this.state = { state: "running", shard: new this.dida.Shard(graph) } + } + + doAllWork() { + while (this.shard.hasWork()) { + this.shard.doWork() + } + } } export class Subgraph { readonly sugar: Sugar - readonly inner: DidaSubgraph + readonly inner: InstanceType + readonly parent: Subgraph - constructor(sugar: Sugar, inner: DidaSubgraph) { + constructor(sugar: Sugar, parent: Subgraph["parent"], inner: Subgraph["inner"]) { this.sugar = sugar this.inner = inner + this.parent = parent } // loop(): Subgraph { // return new Subgraph() // } - // loopNode(): Node { - // return new Node() - // } + loopNode>(): TimestampIncrementedNode { + const builder = this.sugar.builder - importNode(n: Node): Node { - if (this.sugar.state.state !== "building") { - throw new Error("Not in building state") - } + const nodeInner = builder.addNode(this.inner, new this.sugar.dida.NodeSpec.TimestampIncrement()) + return new TimestampIncrementedNode(this.sugar, nodeInner, this) + } + + importNode>(n: Node): TimestampPushedNode { + const builder = this.sugar.builder // TODO add check for // "Can only import from parent subgraph into child subgraph", - console.log("asdf") - console.log(this.inner) - console.log(this.inner.id) - return new Node(this.sugar, this.inner) + const nodeInner = builder.addNode( + this.inner, + new this.sugar.dida.NodeSpec.TimestampPush(n.inner) + ) + + return new TimestampPushedNode(this.sugar, nodeInner, this) } - // exportNode(n: Node): Node { - // return new Node() - // } + exportNode>(n: Node): TimestampPoppedNode { + const builder = this.sugar.builder + // TODO add check for + // "Can only export from child subgraph into parent subgraph", + + const nodeInner = builder.addNode( + this.parent.inner, + new this.sugar.dida.NodeSpec.TimestampPush(n.inner) + ) + + return new TimestampPushedNode(this.sugar, nodeInner, this.parent) + } } -export class Node { +export class Node> { readonly sugar: Sugar - readonly inner: DidaNode + readonly inner: ReturnType + readonly subgraph: Subgraph - constructor(sugar: Sugar, inner: DidaNode) { + constructor(sugar: Sugar, inner: Node["inner"], subgraph: Node["subgraph"]) { this.sugar = sugar this.inner = inner + this.subgraph = subgraph } - // index(): Node { - // return new Node(this.sugar) - // } + index(): IndexedNode { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Index(this.inner) + ) - map(f: any): this { - return this - } - reduce(): this { - return this + return new IndexedNode(this.sugar, nodeInner, this.subgraph) } - project(): this { - return this + + mapInner>(f: (input: T) => G): Node { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Map(this.inner, f), + ); + + return new Node(this.sugar, nodeInner, this.subgraph) } - union(): this { - return this + + // Each subclass needs to implement this if they want to map into their own type + map>(f: (input: T) => G): Node { + return this.mapInner(f) } - // output(f: any): OutputNode { - // return new OutputNode() + // reduce(): this { + // return this // } + + reduce(keyColumns: number, reduceFn: (v: V, row: T, count: number) => V): Node { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Reduce( + this.inner, + keyColumns, + reduceFn + ), + ); + + return new Node(this.sugar, nodeInner, this.subgraph) + } + + union, G extends Array>(other: Node): Node { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Union([this.inner, other.inner]), + ); + + return new Node(this.sugar, nodeInner, this.subgraph) + } + + timestampPop(): Node { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.TimestampPop(this.inner), + ); + + return new Node(this.sugar, nodeInner, this.subgraph) + } + + output(): OutputNode { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Output(this.inner), + ); + + return new OutputNode(this.sugar, nodeInner, this.subgraph) + } } -export class InputNode extends Node { +export class InputNode> extends Node { + push(v: T, timestamp: Array, diff: number) { + this.sugar.shard.pushInput( + this.inner, + new this.sugar.dida.Change(v, timestamp, diff) + ) + } + flush() { + this.sugar.shard.flushInput(this.inner) + } - push() { } - flush() { } - advance() { } + advance(timestamp: Array) { + this.sugar.shard.advanceInput(this.inner, timestamp) + } } -export class OutputNode extends Node { - pop() { } +export class OutputNode> extends Node { + pop(): T | undefined { + return this.inner.pop() + } + + *[Symbol.iterator]() { + while (true) { + const change_batch = this.sugar.shard.popOutput(this.inner); + if (change_batch == undefined) break; + yield change_batch + } + } } -export class JoinedNode extends Node { } +export class JoinedNode> extends Node { } -export class IndexedNode extends Node { - distinct(): this { - return this +export class IndexedNode> extends Node { + map>(f: (input: T) => G): IndexedNode { + return this.mapInner(f) as IndexedNode } - join(): this { - return this + + distinct(): DistinctNode { + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Distinct(this.inner) + ) + + return new DistinctNode(this.sugar, nodeInner, this.subgraph) + } + + join, O extends Array>( + other: IndexedNode, + joinFirstNCols: number, + ): IndexedNode { + //TODO + const nodeInner = this.sugar.builder.addNode( + this.subgraph, + new this.sugar.dida.NodeSpec.Join([this.inner, other.inner], joinFirstNCols), + ) + + return new IndexedNode(this.sugar, nodeInner, this.subgraph) } } -export class DistinctNode extends IndexedNode { } -export class ReduceNode extends IndexedNode { } +export class DistinctNode> extends IndexedNode { } +export class ReduceNode> extends IndexedNode { } -export class TimestampPushedNode extends Node { } +export class TimestampPushedNode> extends Node { } +export class TimestampPoppedNode> extends Node { } + +export class TimestampIncrementedNode> extends Node { + // TODO fix type + fixpoint(future: IndexedNode) { + const builder = this.sugar.builder + builder.connectLoop(future.inner, this.inner) + } +} // type HasIndex = T extends IndexedNode ? true : T extends DistinctNode ? T exte \ No newline at end of file diff --git a/bindings/ts/sugar_test.ts b/bindings/ts/sugar_test.ts index 76fedf1..a14892b 100644 --- a/bindings/ts/sugar_test.ts +++ b/bindings/ts/sugar_test.ts @@ -4,23 +4,60 @@ import Dida from "../wasm/zig-out/lib/dida.mjs"; import { Buffer } from "https://deno.land/std/io/mod.ts"; import { Sugar, Subgraph, Node, OutputNode } from "./sugar.ts"; -Deno.test("sugar example", async () => { +async function loadDida(): Promise { const file = await Deno.open("../wasm/zig-out/lib/dida.wasm"); const b = new Buffer(); await b.readFrom(file); - console.log(typeof b, typeof b.bytes()); - file.close() const abi = await Abi(b.bytes()); - const dida = new Dida(abi); + return new Dida(abi); +} + +Deno.test("sugar example", async () => { + const dida = await loadDida() const sugar = new Sugar(dida); - const edges = sugar.input() + const edges = sugar.input<[string, string]>() const loop = sugar.loop() - const edges1 = loop.importNode(edges) - // const edges = sugar.input() + const edges1 = loop.importNode<[string, string]>(edges) + const reach = loop.loopNode<[string, string]>() + + reach.fixpoint( + reach + .index() + .join( + // Swap columns + edges1.map(v => [v[1], v[0]] + ).index(), + 1 + ) + // Without the first key, and flip order again + .map(v => [v[2], v[1]]) + .union(edges1) + .index() + .distinct() + ) + + + const out = loop.exportNode(reach).output() + + sugar.build() + + edges.push(["a", "b"], [0], 1); + edges.push(["b", "c"], [0], 1); + edges.push(["b", "d"], [0], 1); + edges.push(["c", "a"], [0], 1); + edges.push(["b", "c"], [1], -1); + edges.flush() + edges.advance([1]) + + for (const v of out) { + console.log("reach_out"); + console.log(v) + } + }); From 0edd570026ebfd695190209f8a7ef1f8ae8de1e0 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 28 Mar 2022 09:11:33 -0700 Subject: [PATCH 08/12] Rm console.log --- examples/wasm-deno.js | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/wasm-deno.js b/examples/wasm-deno.js index 082c04f..ee7245a 100644 --- a/examples/wasm-deno.js +++ b/examples/wasm-deno.js @@ -176,7 +176,6 @@ async function run() { const file = await Deno.open("./bindings/wasm/zig-out/lib/dida.wasm"); const b = new Buffer(); await b.readFrom(file); - console.log(typeof b, typeof b.bytes()); const abi = await Abi(b.bytes()); const dida = new Dida(abi); From 9963a3df618407b1d3de5f20f167f092cfcc6761 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 28 Mar 2022 09:11:46 -0700 Subject: [PATCH 09/12] Create vscode workspace --- .vscode/dida.code-workspace | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .vscode/dida.code-workspace diff --git a/.vscode/dida.code-workspace b/.vscode/dida.code-workspace new file mode 100644 index 0000000..94f5b8f --- /dev/null +++ b/.vscode/dida.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "path": ".." + }, + { + "path": "../bindings/ts" + } + ], + "settings": {} +} \ No newline at end of file From 6698d818a15a857a24b5624f24e9428c1bff6fea Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Mon, 28 Mar 2022 09:20:03 -0700 Subject: [PATCH 10/12] Revert generate types --- bindings/wasm/codegen.zig | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/bindings/wasm/codegen.zig b/bindings/wasm/codegen.zig index d234292..e619f5b 100644 --- a/bindings/wasm/codegen.zig +++ b/bindings/wasm/codegen.zig @@ -20,11 +20,11 @@ pub fn main() !void { const file2 = try std.fs.cwd().createFile("zig-out/lib/dida.mjs", .{ .read = false, .truncate = true }); defer file2.close(); writer = file2.writer(); + try writer.writeAll("function Dida(abi) {\n\n"); inline for (js_common.types_with_js_constructors) |T| { try generateConstructor(writer, T); } try writer.writeAll("\n\n"); - try writer.writeAll("function Dida(abi) {\n\n"); inline for (js_common.types_with_js_constructors) |T| { try std.fmt.format(writer, "this.{s} = {s};\n", .{ T, T }); } @@ -119,7 +119,6 @@ fn generateConstructor(writer: anytype, comptime Type: type) !void { if (union_info.tag_type) |_| { // TODO name payload args instead of using `arguments[i]` try std.fmt.format(writer, "const {s} = {{\n", .{Type}); - var functionsReferenced = std.ArrayList(u8).init(js_common.allocator); inline for (union_info.fields) |field_info| { const payload = switch (field_info.field_type) { []const u8, f64 => "arguments[0]", @@ -134,30 +133,18 @@ fn generateConstructor(writer: anytype, comptime Type: type) !void { }); }, }; - try std.fmt.format( - functionsReferenced.writer(), - \\function {s}_{s}_tag() {{ - \\ this.tag = "{s}"; - \\ this.payload = {s}; - \\}} - \\ - \\ - , - .{ Type, field_info.name, field_info.name, payload }, - ); - try std.fmt.format( writer, - \\ {s}: {s}_{s}_tag, + \\ {s}: function () {{ + \\ this.tag = "{s}"; + \\ this.payload = {s}; + \\ }}, \\ , - .{ field_info.name, Type, field_info.name }, + .{ field_info.name, field_info.name, payload }, ); } try writer.writeAll("};\n\n"); - try writer.writeAll(functionsReferenced.items); - try writer.writeAll("\n\n"); - } else { dida.util.compileError("Don't know how to make constructor for non-tagged union type {}", .{Type}); } From d1ee41d151e4b857b24f166bc3b0c80c7f68abf6 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 13 Apr 2022 17:54:45 -0700 Subject: [PATCH 11/12] Add TS Sugar --- bindings/ts/sugar.ts | 79 ++++++++++++++++----- bindings/ts/sugar_test.ts | 144 ++++++++++++++++++++++++++++++++++---- flake.nix | 3 +- 3 files changed, 192 insertions(+), 34 deletions(-) diff --git a/bindings/ts/sugar.ts b/bindings/ts/sugar.ts index 0d03810..243b2b6 100644 --- a/bindings/ts/sugar.ts +++ b/bindings/ts/sugar.ts @@ -1,9 +1,10 @@ import Dida from '../wasm/zig-out/lib/types/dida.d.mts' type DidaBuilder = InstanceType +type DidaGraph = any type SugarState = - { state: "building", builder: InstanceType, rootSubGraph: Subgraph } + { state: "building", builder: Builder, rootSubGraph: Subgraph } | { state: "running", shard: InstanceType } type UglyHackToPreserverType, G extends Array> = @@ -21,8 +22,10 @@ export class Sugar { constructor(dida: Dida) { const rootID = 0 const builder = new dida.GraphBuilder() - const rootSubGraph = builder.addSubgraph(rootID) - this.state = { state: "building", builder: builder, rootSubGraph } + const rootSubGraphInner = new dida.Subgraph(rootID); + const rootSubGraph = new Subgraph(this, null, rootSubGraphInner) + + this.state = { state: "building", builder: new Builder(this, builder), rootSubGraph } this.dida = dida } @@ -33,7 +36,7 @@ export class Sugar { return this.state.shard } - get builder(): DidaBuilder { + get builder(): Builder { if (this.state.state !== "building") { throw new Error("Sugar is not in building state") } @@ -44,6 +47,7 @@ export class Sugar { if (this.state.state !== "building") { throw new Error("Sugar is not in building state") } + return this.state.rootSubGraph } @@ -61,8 +65,7 @@ export class Sugar { throw new Error("Cannot loop on a graph that is not being built") } - const subgraph_1 = this.state.builder.addSubgraph(this.state.rootSubGraph); - return new Subgraph(this, this.rootSubGraph, subgraph_1) + return this.builder.addSubgraph(this.state.rootSubGraph) } build() { @@ -80,7 +83,7 @@ export class Sugar { export class Subgraph { readonly sugar: Sugar readonly inner: InstanceType - readonly parent: Subgraph + readonly parent: Subgraph | null constructor(sugar: Sugar, parent: Subgraph["parent"], inner: Subgraph["inner"]) { this.sugar = sugar @@ -88,14 +91,10 @@ export class Subgraph { this.parent = parent } - // loop(): Subgraph { - // return new Subgraph() - // } - loopNode>(): TimestampIncrementedNode { const builder = this.sugar.builder - const nodeInner = builder.addNode(this.inner, new this.sugar.dida.NodeSpec.TimestampIncrement()) + const nodeInner = builder.addNode(this, new this.sugar.dida.NodeSpec.TimestampIncrement(null)) return new TimestampIncrementedNode(this.sugar, nodeInner, this) } @@ -106,7 +105,7 @@ export class Subgraph { // "Can only import from parent subgraph into child subgraph", const nodeInner = builder.addNode( - this.inner, + this, new this.sugar.dida.NodeSpec.TimestampPush(n.inner) ) @@ -118,18 +117,22 @@ export class Subgraph { // TODO add check for // "Can only export from child subgraph into parent subgraph", + if (this.parent == null) { + throw new Error("Cannot export from root subgraph") + } + const nodeInner = builder.addNode( - this.parent.inner, - new this.sugar.dida.NodeSpec.TimestampPush(n.inner) + this.parent, + new this.sugar.dida.NodeSpec.TimestampPop(n.inner) ) - return new TimestampPushedNode(this.sugar, nodeInner, this.parent) + return new TimestampPoppedNode(this.sugar, nodeInner, this.parent) } } export class Node> { readonly sugar: Sugar - readonly inner: ReturnType + readonly inner: NodeInner readonly subgraph: Subgraph constructor(sugar: Sugar, inner: Node["inner"], subgraph: Node["subgraph"]) { @@ -223,6 +226,13 @@ export class InputNode> extends Node { } export class OutputNode> extends Node { + readonly inner: NodeInner & { pop: () => T | undefined } + + constructor(sugar: Sugar, inner: Node["inner"], subgraph: Node["subgraph"]) { + super(sugar, inner, subgraph) + this.inner = inner as any + } + pop(): T | undefined { return this.inner.pop() } @@ -249,6 +259,9 @@ export class IndexedNode> extends Node { new this.sugar.dida.NodeSpec.Distinct(this.inner) ) + console.log("distinct", nodeInner) + console.log("sg", this.subgraph.inner) + return new DistinctNode(this.sugar, nodeInner, this.subgraph) } @@ -276,7 +289,37 @@ export class TimestampIncrementedNode> extends Node { // TODO fix type fixpoint(future: IndexedNode) { const builder = this.sugar.builder - builder.connectLoop(future.inner, this.inner) + builder.connectLoop(future, this) + } +} + +// type NodeInner = ReturnType & {} +class NodeInner { } + +type NodeTag = InstanceType<(Dida["NodeSpec"][keyof Dida["NodeSpec"]])> +class Builder { + private readonly sugar: Sugar + private readonly inner: DidaBuilder + constructor(sugar: Sugar, inner: DidaBuilder) { + this.sugar = sugar + this.inner = inner + } + + addNode(subgraph: Subgraph, spec: NodeTag): NodeInner { + return this.inner.addNode(subgraph.inner, spec) + } + + addSubgraph(parentGraph: Subgraph): Subgraph { + const subgraph = this.inner.addSubgraph(parentGraph.inner); + return new Subgraph(this.sugar, parentGraph, subgraph) + } + + finishAndReset(): DidaGraph { + return this.inner.finishAndReset() + } + + connectLoop>(a: Node, b: Node) { + this.inner.connectLoop(a.inner, b.inner) } } diff --git a/bindings/ts/sugar_test.ts b/bindings/ts/sugar_test.ts index a14892b..8532e33 100644 --- a/bindings/ts/sugar_test.ts +++ b/bindings/ts/sugar_test.ts @@ -1,13 +1,14 @@ import { assertEquals } from "https://deno.land/std@0.132.0/testing/asserts.ts"; import Abi from "../wasm/abi-with-bytes.js"; import Dida from "../wasm/zig-out/lib/dida.mjs"; -import { Buffer } from "https://deno.land/std/io/mod.ts"; +import { Buffer } from "https://deno.land/std@0.128.0/io/mod.ts"; import { Sugar, Subgraph, Node, OutputNode } from "./sugar.ts"; async function loadDida(): Promise { const file = await Deno.open("../wasm/zig-out/lib/dida.wasm"); const b = new Buffer(); await b.readFrom(file); + file.close() const abi = await Abi(b.bytes()); return new Dida(abi); @@ -19,9 +20,11 @@ Deno.test("sugar example", async () => { const sugar = new Sugar(dida); const edges = sugar.input<[string, string]>() + const loop = sugar.loop() const edges1 = loop.importNode<[string, string]>(edges) + const reach = loop.loopNode<[string, string]>() reach.fixpoint( @@ -29,8 +32,7 @@ Deno.test("sugar example", async () => { .index() .join( // Swap columns - edges1.map(v => [v[1], v[0]] - ).index(), + edges1.map(v => [v[1], v[0]]).index(), 1 ) // Without the first key, and flip order again @@ -51,21 +53,133 @@ Deno.test("sugar example", async () => { edges.push(["c", "a"], [0], 1); edges.push(["b", "c"], [1], -1); edges.flush() - edges.advance([1]) - for (const v of out) { - console.log("reach_out"); - console.log(v) - } + const sugarChanges = [] + for (const i of [1, 2, 3]) { + edges.advance([i]) + sugar.doAllWork() + for (const v of out) { + console.log("reach_out"); + console.log(v) + sugarChanges.push(v) + } + } + assertEquals(await unsweetDidaOutput(), sugarChanges) }); -Deno.test("add test", () => { - const a = 1 + 1 - assertEquals(a, 2); - // test 2 + 2 = 4 - const b = 2 + 2 - assertEquals(b, 4); -}); +async function unsweetDidaOutput() { + const changes = [] + + const dida = await loadDida() + var graph_builder = new dida.GraphBuilder(); + + const subgraph_0 = new dida.Subgraph(0); + const subgraph_1 = graph_builder.addSubgraph(subgraph_0); + + const edges = graph_builder.addNode(subgraph_0, new dida.NodeSpec.Input()); + const edges_1 = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.TimestampPush(edges), + ); + + + const reach_future = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.TimestampIncrement(null), + ); + + + const reach_index = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Index(reach_future), + ); + + const distinct_reach_index = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Distinct(reach_index), + ); + + const swapped_edges = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Map(edges_1, (input: any) => [input[1], input[0]]), + ); + const swapped_edges_index = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Index(swapped_edges), + ); + + const joined = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Join([distinct_reach_index, swapped_edges_index], 1), + ); + + const without_middle = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Map(joined, (input: any) => [input[2], input[1]]), + ); + + const reach = graph_builder.addNode( + subgraph_1, + new dida.NodeSpec.Union([edges_1, without_middle]), + ); + + graph_builder.connectLoop(reach, reach_future); + + const reach_pop = graph_builder.addNode( + subgraph_0, + new dida.NodeSpec.TimestampPop(distinct_reach_index), + ); + const reach_out = graph_builder.addNode( + subgraph_0, + new dida.NodeSpec.Output(reach_pop), + ); + + const graph = graph_builder.finishAndReset(); + + var shard = new dida.Shard(graph); + + shard.pushInput(edges, new dida.Change(["a", "b"], [0], 1)); + shard.pushInput(edges, new dida.Change(["b", "c"], [0], 1)); + shard.pushInput(edges, new dida.Change(["b", "d"], [0], 1)); + shard.pushInput(edges, new dida.Change(["c", "a"], [0], 1)); + shard.pushInput(edges, new dida.Change(["b", "c"], [1], -1)); + shard.flushInput(edges); + + shard.advanceInput(edges, [1]); + while (shard.hasWork()) { + shard.doWork(); + while (true) { + const change_batch = shard.popOutput(reach_out); + if (change_batch == undefined) break; + changes.push(change_batch) + } + } + + console.log("Advancing!"); + shard.advanceInput(edges, [2]); + + while (shard.hasWork()) { + shard.doWork(); + while (true) { + const change_batch = shard.popOutput(reach_out); + if (change_batch == undefined) break; + changes.push(change_batch) + } + } + + console.log("Advancing!"); + shard.advanceInput(edges, [3]); + while (shard.hasWork()) { + shard.doWork(); + while (true) { + const change_batch = shard.popOutput(reach_out); + if (change_batch == undefined) break; + changes.push(change_batch) + } + } + + return changes +} diff --git a/flake.nix b/flake.nix index bdf800b..7a79777 100644 --- a/flake.nix +++ b/flake.nix @@ -49,7 +49,8 @@ cd bindings/wasm ${deps.zig}/bin/zig build install && ${deps.zig}/bin/zig build run-codegen cd zig-out/lib - ${self.packages.${system}.tsc}/bin/tsc 'dida.mjs' --declaration --allowJs --emitDeclarationOnly --outDir types + # Emit declarations. Disabled since it doesn't work + # ${self.packages.${system}.tsc}/bin/tsc 'dida.mjs' --declaration --allowJs --emitDeclarationOnly --outDir types ''; # packages.tsc = pkgs.lib.mkDerivation{ # name = "typescript"; From eb77c4b47c4030cf253a5e1a4005a2c5245cdb4e Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 13 Apr 2022 17:55:06 -0700 Subject: [PATCH 12/12] Add some more debugging information --- lib/dida/core.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dida/core.zig b/lib/dida/core.zig index 6a09d65..eb4b17b 100644 --- a/lib/dida/core.zig +++ b/lib/dida/core.zig @@ -1160,8 +1160,8 @@ pub const Graph = struct { const output_subgraph = u.last(Subgraph, self.node_subgraphs[node_id]); u.assert( input_subgraph.id == output_subgraph.id, - "Nodes (other than TimestampPop and TimestampPush) must be on the same subgraph as their inputs", - .{}, + "Nodes (other than TimestampPop and TimestampPush) must be on the same subgraph as their inputs. Mismatch On Node IDs (input) {d} and {d}, and graph IDs (input) {d} and {d}", + .{input_node.id, node_id, input_subgraph.id, output_subgraph.id}, ); }, }