Support additional precompiled dependencies

bin/build-forked-compiler

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -eu
+
+readonly repo_root=$(dirname $(dirname $(realpath $0)))
+
+# You must have cloned the `gleam-lang/gleam` git repository for this command.
+# This gives you a choice of compilers without having to go into git submodules.
+GLEAM_DIR=${GLEAM_DIR:?"Please clone gleam-lang/gleam and set the GLEAM_DIR env to the repository root"}
+
+rm -fr "${repo_root}/wasm-compiler"
+mkdir "${repo_root}/wasm-compiler"
+wasm-pack build --release --target web --out-dir "${repo_root}/wasm-compiler" "${GLEAM_DIR}/compiler-wasm"

gleam.toml

@@ -19,6 +19,8 @@ simplifile = ">= 2.2.0 and < 3.0.0"
 snag = "~> 0.2"
 htmb = "~> 1.1"
 filepath = ">= 1.0.0 and < 2.0.0"
+globlin = ">= 2.0.2 and < 3.0.0"
+globlin_fs = ">= 2.0.0 and < 3.0.0"
 
 [dev-dependencies]
 gleeunit = ">= 1.0.0 and < 2.0.0"

manifest.toml

@@ -5,6 +5,8 @@ packages = [
   { name = "filepath", version = "1.0.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "filepath", source = "hex", outer_checksum = "EFB6FF65C98B2A16378ABC3EE2B14124168C0CE5201553DE652E2644DCFDB594" },
   { name = "gleam_stdlib", version = "0.40.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "86606B75A600BBD05E539EB59FABC6E307EEEA7B1E5865AFB6D980A93BCB2181" },
   { name = "gleeunit", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "F7A7228925D3EE7D0813C922E062BFD6D7E9310F0BEE585D3A42F3307E3CFD13" },
+  { name = "globlin", version = "2.0.2", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "globlin", source = "hex", outer_checksum = "393E3421E4DA269B0E6025D69DA0F2D3DDD8517500F6BA2AE3C4024FA5F0B498" },
+  { name = "globlin_fs", version = "2.0.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib", "globlin", "simplifile"], otp_app = "globlin_fs", source = "hex", outer_checksum = "2A84CE81FD7958B967EF39CC234AFB64DAB20169D0EF9B9C3943CD3C5B561182" },
   { name = "htmb", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "htmb", source = "hex", outer_checksum = "30D448F0E15DFCF7283AAAC2F351D77B9D54E318219C9FDDB1877572B67C27B7" },
   { name = "simplifile", version = "2.2.0", build_tools = ["gleam"], requirements = ["filepath", "gleam_stdlib"], otp_app = "simplifile", source = "hex", outer_checksum = "0DFABEF7DC7A9E2FF4BB27B108034E60C81BEBFCB7AB816B9E7E18ED4503ACD8" },
   { name = "snag", version = "0.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "snag", source = "hex", outer_checksum = "54D32E16E33655346AA3E66CBA7E191DE0A8793D2C05284E3EFB90AD2CE92BCC" },
@@ -14,6 +16,8 @@ packages = [
 filepath = { version = ">= 1.0.0 and < 2.0.0" }
 gleam_stdlib = { version = ">= 0.34.0 and < 2.0.0" }
 gleeunit = { version = ">= 1.0.0 and < 2.0.0" }
+globlin = { version = ">= 2.0.2 and < 3.0.0" }
+globlin_fs = { version = ">= 2.0.0 and < 3.0.0" }
 htmb = { version = "~> 1.1" }
 simplifile = { version = ">= 2.2.0 and < 3.0.0" }
 snag = { version = "~> 0.2" }

src/playground.gleam

@@ -4,6 +4,8 @@ import gleam/list
 import gleam/result
 import gleam/string
 import gleam/string_builder
+import globlin
+import globlin_fs
 import htmb.{type Html, h}
 import playground/html.{
   ScriptOptions, html_dangerous_inline_script, html_link, html_meta,
@@ -23,6 +25,8 @@ const meta_image = "https://gleam.run/images/og-image.png"
 
 const meta_url = "https://play.gleam.run"
 
+const available_packages = ["filepath", "gleam_stdlib", "globlin"]
+
 // Paths
 
 const static = "static"
@@ -31,13 +35,7 @@ const public = "public"
 
 const public_precompiled = "public/precompiled"
 
-const prelude = "build/dev/javascript/prelude.mjs"
-
-const stdlib_compiled = "build/dev/javascript/gleam_stdlib/gleam"
-
-const stdlib_sources = "build/packages/gleam_stdlib/src/gleam"
-
-const stdlib_external = "build/packages/gleam_stdlib/src"
+const compiled_lib = "build/dev/javascript"
 
 const compiler_wasm = "./wasm-compiler"
 
@@ -53,16 +51,19 @@ pub fn main() {
 pub fn main() {
   let result = {
     use _ <- result.try(reset_output())
-    use _ <- result.try(make_prelude_available())
-    use _ <- result.try(make_stdlib_available())
+    use _ <- result.try(ensure_directory(public_precompiled))
+    use _ <- result.try(make_packages_available(available_packages))
+    use _ <- result.try(
+      simplifile.copy_directory(static, public)
+      |> file_error("Failed to copy static directory"),
+    )
     use _ <- result.try(copy_wasm_compiler())
 
     let page_html =
       home_page()
       |> htmb.render_page("html")
       |> string_builder.to_string
 
-    use _ <- result.try(ensure_directory(public))
     let path = filepath.join(public, "index.html")
 
     use _ <- result.try(write_text(path, page_html))
@@ -90,127 +91,92 @@ fn write_text(path: String, text: String) -> snag.Result(Nil) {
   |> file_error("Failed to write " <> path)
 }
 
-fn copy_wasm_compiler() -> snag.Result(Nil) {
-  use compiler_wasm_exists <- result.try(
-    simplifile.is_directory(compiler_wasm)
-    |> file_error("Failed to check compiler-wasm directory"),
-  )
-  use <- require(compiler_wasm_exists, "compiler-wasm must have been compiled")
-
-  simplifile.copy_directory(compiler_wasm, public <> "/compiler")
-  |> file_error("Failed to copy compiler-wasm")
-}
-
-fn make_prelude_available() -> snag.Result(Nil) {
-  use _ <- result.try(
-    simplifile.create_directory_all(public_precompiled)
-    |> file_error("Failed to make " <> public_precompiled),
-  )
-
-  simplifile.copy_file(prelude, public_precompiled <> "/gleam.mjs")
-  |> file_error("Failed to copy prelude.mjs")
-}
-
-fn make_stdlib_available() -> snag.Result(Nil) {
-  use files <- result.try(
-    simplifile.read_directory(stdlib_sources)
-    |> file_error("Failed to read stdlib directory"),
-  )
-
-  let modules =
-    files
-    |> list.filter(fn(file) { string.ends_with(file, ".gleam") })
-    |> list.map(string.replace(_, ".gleam", ""))
-
-  use _ <- result.try(
-    generate_stdlib_bundle(modules)
-    |> snag.context("Failed to generate stdlib.js bundle"),
-  )
-
+pub fn make_packages_available(packages: List(String)) -> snag.Result(Nil) {
+  // Set up prelude
   use _ <- result.try(
-    copy_compiled_stdlib(modules)
-    |> snag.context("Failed to copy precompiled stdlib modules"),
+    copy_lib_files(["prelude.mjs", "gleam_version"])
+    |> snag.context("Failed to copy lib prelude"),
   )
 
+  // Recursive directory copies for packages
   use _ <- result.try(
-    copy_stdlib_externals()
-    |> snag.context("Failed to copy stdlib external files"),
+    copy_lib_dirs(packages)
+    |> snag.context("Failed to copy lib packages"),
   )
 
-  Ok(Nil)
+  // Walk the lib directory to enumerate them in a manifest.
+  generate_lib_manifest(packages)
 }
 
-fn copy_stdlib_externals() -> snag.Result(Nil) {
-  use files <- result.try(
-    simplifile.read_directory(stdlib_external)
-    |> file_error("Failed to read stdlib external directory"),
-  )
-  let files = list.filter(files, string.ends_with(_, ".mjs"))
-
+fn copy_lib_files(files: List(String)) -> snag.Result(Nil) {
   list.try_each(files, fn(file) {
-    let from = stdlib_external <> "/" <> file
-    let to = public_precompiled <> "/" <> file
-    simplifile.copy_file(from, to)
-    |> file_error("Failed to copy stdlib external file " <> from)
+    simplifile.copy_file(
+      filepath.join(compiled_lib, file),
+      filepath.join(public_precompiled, file),
+    )
+    |> file_error("Failed to copy file " <> file)
   })
 }
 
-fn copy_compiled_stdlib(modules: List(String)) -> snag.Result(Nil) {
-  use stdlib_dir_exists <- result.try(
-    simplifile.is_directory(stdlib_compiled)
-    |> file_error("Failed to check stdlib directory"),
-  )
-  use <- require(
-    stdlib_dir_exists,
-    "Project must have been compiled for JavaScript",
-  )
+fn copy_lib_dirs(packages: List(String)) -> snag.Result(Nil) {
+  list.try_each(packages, fn(package) {
+    simplifile.copy_directory(
+      filepath.join(compiled_lib, package),
+      filepath.join(public_precompiled, package),
+    )
+    |> file_error("Failed to copy directory " <> package)
+  })
+}
 
-  let dest = public_precompiled <> "/gleam"
-  use _ <- result.try(
-    simplifile.create_directory_all(dest)
-    |> file_error("Failed to make " <> dest),
+fn generate_lib_manifest(packages: List(String)) -> snag.Result(Nil) {
+  let assert Ok(pattern) = globlin.new_pattern("**/*")
+
+  use cwd <- result.try(
+    simplifile.current_directory()
+    |> file_error("Finding current directory"),
   )
 
-  use _ <- result.try(
-    list.try_each(modules, fn(name) {
-      let from = stdlib_compiled <> "/" <> name <> ".mjs"
-      let to = dest <> "/" <> name <> ".mjs"
-      simplifile.copy_file(from, to)
-      |> file_error("Failed to copy stdlib module " <> from)
-    }),
+  let abs_dir = filepath.join(cwd, public_precompiled)
+  use files <- result.try(
+    globlin_fs.glob_from(
+      pattern,
+      directory: abs_dir,
+      returning: globlin_fs.RegularFiles,
+    )
+    |> file_error("Walking lib files"),
   )
 
-  Ok(Nil)
+  let files =
+    files
+    // Make sure we turn the matched absolute paths back to relative ones.
+    |> list.map(string.drop_left(_, string.length(abs_dir) + 1))
+    |> list.sort(string.compare)
+    // Export them as a const JS array literal.
+    |> string.join("',\n  '")
+    |> string.append("export const files = [\n  '", _)
+    |> string.append("'\n];\n")
+
+  let packages =
+    packages
+    |> list.sort(string.compare)
+    // Export them as a const JS array literal.
+    |> string.join("', '")
+    |> string.append("export const packages = ['", _)
+    |> string.append("'];\n")
+
+  simplifile.write(public_precompiled <> ".js", string.append(packages, files))
+  |> file_error("Failed to write lib manifest")
 }
 
-fn generate_stdlib_bundle(modules: List(String)) -> snag.Result(Nil) {
-  use entries <- result.try(
-    list.try_map(modules, fn(name) {
-      let path = stdlib_sources <> "/" <> name <> ".gleam"
-      use code <- result.try(
-        simplifile.read(path)
-        |> file_error("Failed to read stdlib module " <> path),
-      )
-      let name = string.replace(name, ".gleam", "")
-      let code =
-        code
-        |> string.replace("\\", "\\\\")
-        |> string.replace("`", "\\`")
-        |> string.split("\n")
-        |> list.filter(fn(line) { !string.starts_with(string.trim(line), "//") })
-        |> list.filter(fn(line) { line != "" })
-        |> string.join("\n")
-
-      Ok("  \"gleam/" <> name <> "\": `" <> code <> "`")
-    }),
+fn copy_wasm_compiler() -> snag.Result(Nil) {
+  use compiler_wasm_exists <- result.try(
+    simplifile.is_directory(compiler_wasm)
+    |> file_error("Failed to check compiler-wasm directory"),
   )
+  use <- require(compiler_wasm_exists, "compiler-wasm must have been compiled")
 
-  entries
-  |> string.join(",\n")
-  |> string.append("export default {\n", _)
-  |> string.append("\n}\n")
-  |> simplifile.write(public <> "/stdlib.js", _)
-  |> file_error("Failed to write stdlib.js")
+  simplifile.copy_directory(compiler_wasm, public <> "/compiler")
+  |> file_error("Failed to copy compiler-wasm")
 }
 
 fn reset_output() -> snag.Result(Nil) {
@@ -224,15 +190,10 @@ fn reset_output() -> snag.Result(Nil) {
     |> file_error("Failed to read public directory"),
   )
 
-  use _ <- result.try(
-    files
-    |> list.map(string.append(public <> "/", _))
-    |> simplifile.delete_all
-    |> file_error("Failed to delete public directory"),
-  )
-
-  simplifile.copy_directory(static, public)
-  |> file_error("Failed to copy static directory")
+  files
+  |> list.map(string.append(public <> "/", _))
+  |> simplifile.delete_all
+  |> file_error("Failed to delete public directory")
 }
 
 fn require(

static/compiler.js

@@ -53,6 +53,10 @@ class Project {
     return this.#id;
   }
 
+  writeFileBytes(fileName, content) {
+    compiler.wasm.write_file_bytes(this.#id, fileName, content);
+  }
+
   writeModule(moduleName, code) {
     compiler.wasm.write_module(this.#id, moduleName, code);
   }

static/worker.js

@@ -1,13 +1,47 @@
 import initGleamCompiler from "./compiler.js";
-import stdlib from "./stdlib.js";
+import {
+  files as libFiles,
+  packages as availablePackages,
+} from "./precompiled.js";
 
 const compiler = await initGleamCompiler();
 const project = compiler.newProject();
 
-for (const [name, code] of Object.entries(stdlib)) {
-  project.writeModule(name, code);
+function libUrl(file) {
+  const url = new URL(import.meta.url);
+  url.pathname = file ? `precompiled/${file}` : "precompiled";
+  url.hash = "";
+  url.search = "";
+  return url.toString();
 }
 
+// Write all files from /lib ahead of time.
+// Use binary because we also need capnp cache files here.
+for (const file of libFiles) {
+  const url = libUrl(file);
+  const res = await fetch(url);
+  const bytes = await res.bytes();
+  project.writeFileBytes(`/lib/${file}`, bytes);
+}
+
+// Ensures precompiled libraries are direct dependencies.
+// The compiler will warn about transitive dependencies, but not check the version requirements.
+function configWithDependencies(deps) {
+  const conf = `
+    name = "library"
+    version = "1.0.0"
+    [dependencies]
+    ${deps.map((dep) => `${dep} = "0.0.0"`).join("\n")}
+  `;
+  return new TextEncoder().encode(conf);
+}
+
+// TODO: packages could be filtered to restore transitive dependency warnings.
+project.writeFileBytes(
+  "/gleam.toml",
+  configWithDependencies(availablePackages)
+);
+
 // Monkey patch console.log to keep a copy of the output
 let logged = "";
 const log = console.log;
@@ -17,15 +51,12 @@ console.log = (...args) => {
 };
 
 async function loadProgram(js) {
-  const url = new URL(import.meta.url);
-  url.pathname = "";
-  url.hash = "";
-  url.search = "";
-  const href = url.toString();
-  const js1 = js.replaceAll(
-    /from\s+"\.\/(.+)"/g,
-    `from "${href}precompiled/$1"`,
-  );
+  const href = libUrl();
+  const js1 = js
+    // Importing a dependency uses `../{packageName}/{module}.mjs`
+    .replaceAll(/from\s+"\.\.\/(.+)"/g, `from "${href}/$1"`)
+    // The root package depending on prelude `./gleam.mjs`.
+    .replaceAll(/from\s+"\.\/gleam\.mjs\"/g, `from "${href}/prelude.mjs"`);
   const js2 = btoa(unescape(encodeURIComponent(js1)));
   const module = await import("data:text/javascript;base64," + js2);
   return module.main;