Move modularization code into js compiler

Prior to this change the mudularization was done as a post-processing
step in python.  This complicated things since acorn and closure passes
would only see the inner code and not the whole module.

One concrete benefit is that we can now use `await` in the body of the
factory function since closure no longer sees it as top level (which it
isn't).

Fixes: #23158

Author: sbc100

src/closure-externs/closure-externs.js

@@ -14,6 +14,7 @@
 // Special placeholder for `import.meta` and `await import`.
 var EMSCRIPTEN$IMPORT$META;
 var EMSCRIPTEN$AWAIT$IMPORT;
+var EMSCRIPTEN$EXPORT$DEFAULT;
 
 // Don't minify startWorker which we use to start workers once the runtime is ready.
 /**
@@ -157,9 +158,10 @@ var wakaUnknownBefore;
 // Module loaders externs, for AMD etc.
 
 /**
+ * @param {Object} deps
  * @param {Function} wrapper
  */
-var define = function (wrapper) {};
+var define = function (deps, wrapper) {};
 
 /**
  * @type {Worker}
@@ -228,20 +230,6 @@ var sampleRate;
  */
 var id;
 
-/**
- * Used in MODULARIZE mode as the name of the incoming module argument.
- * This is generated outside of the code we pass to closure so from closure's
- * POV this is "extern".
- */
-var moduleArg;
-
-/**
- * Used in MODULARIZE mode.
- * We need to access this after the code we pass to closure so from closure's
- * POV this is "extern".
- */
-var moduleRtn;
-
 /**
  * This was removed from upstream closure compiler in
  * https://github.com/google/closure-compiler/commit/f83322c1b.

src/closure-externs/modularize-externs.js

@@ -42,16 +42,17 @@ export function processMacros(text, filename) {
 // Simple #if/else/endif preprocessing for a file. Checks if the
 // ident checked is true in our global.
 // Also handles #include x.js (similar to C #include <file>)
-export function preprocess(filename) {
+export function preprocess(filename, closureFriendly = true) {
   let text = read(filename);
-  if (EXPORT_ES6 && USE_ES6_IMPORT_META) {
+  if (closureFriendly && EXPORT_ES6 && USE_ES6_IMPORT_META) {
     // `eval`, Terser and Closure don't support module syntax; to allow it,
     // we need to temporarily replace `import.meta` and `await import` usages
     // with placeholders during preprocess phase, and back after all the other ops.
     // See also: `phase_final_emitting` in emcc.py.
     text = text
       .replace(/\bimport\.meta\b/g, 'EMSCRIPTEN$IMPORT$META')
-      .replace(/\bawait import\b/g, 'EMSCRIPTEN$AWAIT$IMPORT');
+      .replace(/\bawait import\b/g, 'EMSCRIPTEN$AWAIT$IMPORT')
+      .replace(/\bexport default\b/g, 'EMSCRIPTEN$EXPORT$DEFAULT =');
   }
   // Remove windows line endings, if any
   text = text.replace(/\r\n/g, '\n');
@@ -1069,6 +1070,31 @@ function ENVIRONMENT_IS_WORKER_THREAD() {
   return '(' + envs.join('||') + ')';
 }
 
+function nodePthreadDetection() {
+  // Under node we detect that we are running in a pthread by checking the
+  // workerData property.
+  if (EXPORT_ES6) {
+    return "(await import('worker_threads')).workerData === 'em-pthread'";
+  } else {
+    return "require('worker_threads').workerData === 'em-pthread'";
+  }
+}
+
+function declareInstanceExports() {
+  const allExports = Array.from(EXPORTED_FUNCTIONS.keys()).concat(
+    Array.from(EXPORTED_RUNTIME_METHODS.keys()),
+  );
+  const mangledExports = allExports.map((e) => `__exp_${e}`);
+  const mangledExportsAs = allExports.map((e) => `__exp_${e} as ${e}`);
+  // Declare a top level var for each export so that code in the init function
+  // can assign to it and update the live module bindings.
+  if (allExports.length == 0) return '';
+  let rtn = 'var ' + mangledExports.join(', ') + ';\n';
+  // Export the functions with their original name.
+  rtn += 'export {' + mangledExportsAs.join(', ') + '};\n';
+  return rtn;
+}
+
 addToCompileTimeContext({
   ATEXITS,
   ATINITS,
@@ -1134,4 +1160,6 @@ addToCompileTimeContext({
   storeException,
   to64,
   toIndexType,
+  nodePthreadDetection,
+  declareInstanceExports,
 });

src/parseTools.mjs

@@ -6,18 +6,6 @@
 
 #if WASM_ASYNC_COMPILATION
 
-#if USE_READY_PROMISE
-moduleRtn = readyPromise;
-#else
-moduleRtn = {};
-#endif
-
-#else  // WASM_ASYNC_COMPILATION
-
-moduleRtn = Module;
-
-#endif // WASM_ASYNC_COMPILATION
-
 #if ASSERTIONS
 // Assertion for attempting to access module properties on the incoming
 // moduleArg.  In the past we used this object as the prototype of the module
@@ -35,3 +23,70 @@ for (const prop of Object.keys(Module)) {
   }
 }
 #endif
+
+#if USE_READY_PROMISE
+  return readyPromise;
+#else
+  return {};
+#endif
+#else  // WASM_ASYNC_COMPILATION
+  return Module;
+#endif // WASM_ASYNC_COMPILATION
+}; // End factory function
+
+#if ASSERTIONS && MODULARIZE != 'instance'
+(() => {
+  // Create a small, never-async wrapper around {{{ EXPORT_NAME }}} which
+  // checks for callers incorrectly using it with `new`.
+  var real_{{{ EXPORT_NAME }}} = {{{ EXPORT_NAME }}};
+  {{{ EXPORT_NAME }}} = function(arg) {
+    if (new.target) throw new Error("{{{ EXPORT_NAME }}}() should not be called with `new {{{ EXPORT_NAME }}}()`");
+    return real_{{{ EXPORT_NAME }}}(arg);
+  }
+})();
+#endif
+
+// Export using a UMD style export, or ES6 exports if selected
+#if EXPORT_ES6
+#if MODULARIZE == 'instance'
+{{{ declareInstanceExports() }}}
+#else
+export default {{{ EXPORT_NAME }}};
+#endif
+#else
+if (typeof exports === 'object' && typeof module === 'object') {
+  module.exports = {{{ EXPORT_NAME }}};
+  // This default export looks redundant, but it allows TS to import this
+  // commonjs style module.
+  module.exports.default = {{{ EXPORT_NAME }}};
+} else if (typeof define === 'function' && define['amd']) {
+  define([], () => {{{ EXPORT_NAME }}});
+}
+#endif
+
+
+#if PTHREADS
+
+// Create code for detecting if we are running in a pthread.
+// Normally this detection is done when the module is itself run but
+// when running in MODULARIZE mode we need use this to know if we should
+// run the module constructor on startup (true only for pthreads).
+#if ENVIRONMENT_MAY_BE_WEB || ENVIRONMENT_MAY_BE_WORKER
+var isPthread = globalThis.self?.name?.startsWith('em-pthread');
+#if ENVIRONMENT_MAY_BE_NODE
+// In order to support both web and node we also need to detect node here.
+var isNode = typeof globalThis.process?.versions?.node == 'string';
+if (isNode) isPthread = {{{ nodePthreadDetection() }}};
+#endif
+#elif ENVIRONMENT_MAY_BE_NODE
+var isPthread = {{{ nodePthreadDetection() }}};
+#endif ENVIRONMENT_MAY_BE_WEB || ENVIRONMENT_MAY_BE_WORKER
+
+// When running as a pthread, construct a new instance on startup
+#if MODULARIZE == 'instance'
+isPthread && init();
+#else
+isPthread && {{{ EXPORT_NAME }}}();
+#endif
+
+#endif // PTHREADS

src/postamble_modularize.js

@@ -0,0 +1,18 @@
+#if !MINIMAL_RUNTIME || PTHREADS
+#if EXPORT_ES6 && USE_ES6_IMPORT_META
+var _scriptName = import.meta.url;
+#else
+var _scriptName = typeof document != 'undefined' ? document.currentScript?.src : undefined;
+#if ENVIRONMENT_MAY_BE_NODE
+if (typeof __filename != 'undefined') _scriptName = _scriptName || __filename;
+#endif
+#endif
+#endif
+
+#if MODULARIZE == 'instance'
+export default {{{ asyncIf(WASM_ASYNC_COMPILATION || (EXPORT_ES6 && ENVIRONMENT_MAY_BE_NODE)) }}}function init(moduleArg = {}) {
+#else
+var {{{ EXPORT_NAME }}} = {{{ asyncIf(WASM_ASYNC_COMPILATION || (EXPORT_ES6 && ENVIRONMENT_MAY_BE_NODE)) }}}function(moduleArg = {}) {
+#endif
+
+var Module = moduleArg;

src/preamble_modularize.js

@@ -21,7 +21,7 @@
 // before the code. Then that object will be used in the code, and you
 // can continue to use Module afterwards as well.
 #if MODULARIZE
-var Module = moduleArg;
+#include "preamble_modularize.js"
 #elif USE_CLOSURE_COMPILER
 /** @type{Object} */
 var Module;
@@ -99,9 +99,8 @@ if (ENVIRONMENT_IS_PTHREAD) {
 #endif
 #endif
 
-#if ENVIRONMENT_MAY_BE_NODE
+#if ENVIRONMENT_MAY_BE_NODE && (PTHREADS || WASM_WORKERS)
 if (ENVIRONMENT_IS_NODE) {
-#if PTHREADS || WASM_WORKERS
 #if EXPORT_ES6
   var worker_threads = await import('worker_threads');
 #else
@@ -114,9 +113,8 @@ if (ENVIRONMENT_IS_NODE) {
   // is hosting a pthread.
   ENVIRONMENT_IS_PTHREAD = ENVIRONMENT_IS_WORKER && worker_threads['workerData'] == 'em-pthread'
 #endif // PTHREADS
-#endif // PTHREADS || WASM_WORKERS
 }
-#endif // ENVIRONMENT_MAY_BE_NODE
+#endif // ENVIRONMENT_MAY_BE_NODE && (PTHREADS || WASM_WORKERS)
 
 #if WASM_WORKERS
 var ENVIRONMENT_IS_WASM_WORKER = Module['$ww'];

src/shell.js

@@ -5,7 +5,7 @@
  */
 
 #if MODULARIZE
-var Module = moduleArg;
+#include "preamble_modularize.js"
 #elif USE_CLOSURE_COMPILER
 /** @type{Object} */
 var Module;

src/shell_minimal.js

@@ -1,10 +1,10 @@
 {
   "a.html": 454,
   "a.html.gz": 328,
-  "a.js": 4538,
-  "a.js.gz": 2320,
+  "a.js": 6293,
+  "a.js.gz": 3087,
   "a.wasm": 10206,
   "a.wasm.gz": 6663,
-  "total": 15198,
-  "total_gz": 9311
+  "total": 16953,
+  "total_gz": 10078
 }

test/code_size/hello_webgl2_wasm.json

@@ -1,8 +1,8 @@
 {
   "a.html": 346,
   "a.html.gz": 262,
-  "a.js": 22202,
-  "a.js.gz": 11604,
-  "total": 22548,
-  "total_gz": 11866
+  "a.js": 24118,
+  "a.js.gz": 12472,
+  "total": 24464,
+  "total_gz": 12734
 }

test/code_size/hello_webgl2_wasm2js.json

@@ -1,10 +1,10 @@
 {
   "a.html": 454,
   "a.html.gz": 328,
-  "a.js": 4076,
-  "a.js.gz": 2163,
+  "a.js": 5799,
+  "a.js.gz": 2911,
   "a.wasm": 10206,
   "a.wasm.gz": 6663,
-  "total": 14736,
-  "total_gz": 9154
+  "total": 16459,
+  "total_gz": 9902
 }

test/code_size/hello_webgl_wasm.json

@@ -1,8 +1,8 @@
 {
   "a.html": 346,
   "a.html.gz": 262,
-  "a.js": 21728,
-  "a.js.gz": 11435,
-  "total": 22074,
-  "total_gz": 11697
+  "a.js": 23615,
+  "a.js.gz": 12296,
+  "total": 23961,
+  "total_gz": 12558
 }

test/code_size/hello_webgl_wasm2js.json

@@ -5,6 +5,11 @@
 
 #if PTHREADS
 // Avoid instantiating the module on pthreads.
-if (!isPthread)
+#if EXPORT_ES6
+const isMainThread = (await import('worker_threads')).isMainThread;
+#else
+const { isMainThread } = require('worker_threads');
+#endif
+if (isMainThread)
 #endif
 {{{ EXPORT_NAME }}}();

test/modularize_post_js.js

@@ -1 +1 @@
-1509
+1496

test/other/codesize/test_codesize_minimal_esm.gzsize

@@ -1 +1 @@
-3157
+3085

test/other/codesize/test_codesize_minimal_esm.jssize

@@ -1 +1 @@
-53887
+53857

test/other/test_unoptimized_code_size.js.size

@@ -1 +1 @@
-29086
+29056

test/other/test_unoptimized_code_size_no_asserts.js.size

@@ -1 +1 @@
-52670
+52640

test/other/test_unoptimized_code_size_strict.js.size

@@ -889,10 +889,7 @@ function emitDCEGraph(ast) {
   // Scoping must balance out.
   assert(specialScopes === 0);
   // We must have found the info we need.
-  assert(
-    foundWasmImportsAssign,
-    'could not find the assignment to "wasmImports". perhaps --pre-js or --post-js code moved it out of the global scope? (things like that should be done after emcc runs, as they do not need to be run through the optimizer which is the special thing about --pre-js/--post-js code)',
-  );
+  assert(foundWasmImportsAssign, 'could not find the assignment to "wasmImports"');
   // Read exports that were declared in extraInfo
   if (extraInfo) {
     for (const exp of extraInfo.exports) {

tools/acorn-optimizer.mjs

@@ -554,7 +554,14 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None):
   CLOSURE_EXTERNS = [path_from_root('src/closure-externs/closure-externs.js')]
 
   if settings.MODULARIZE:
-    CLOSURE_EXTERNS += [path_from_root('src/closure-externs/modularize-externs.js')]
+    temp = shared.get_temp_files().get('.js', prefix='emcc_closure_externs_').name
+    utils.write_file(temp, f'''
+/**
+ * @suppress {{duplicate}}
+ */
+var {settings.EXPORT_NAME};
+''')
+    CLOSURE_EXTERNS += [temp]
 
   if settings.USE_WEBGPU:
     CLOSURE_EXTERNS += [path_from_root('src/closure-externs/webgpu-externs.js')]

tools/building.py

@@ -899,8 +899,7 @@ def can_use_await():
   # function.
   # However, because closure does not see this (it runs only on the inner code),
   # it sees this as a top-level-await, which it does not yet support.
-  # FIXME(https://github.com/emscripten-core/emscripten/issues/23158)
-  return settings.MODULARIZE and not settings.USE_CLOSURE_COMPILER
+  return settings.MODULARIZE
 
 
 def make_export_wrappers(function_exports):