file_packager: Cleanup file embedding (#17220)

- Simplify library helper code
- Avoid `--pre-js` completely where possible (it is still needed for --preload-file, but not for --embed-file)

Author: sbc100

ChangeLog.md

@@ -20,6 +20,9 @@ See docs/process.md for more on how version tagging works.
 
 3.2.0
 -----
+- `tools/file_packager` no longer generates (or requires) any "pre-js" code when
+  running in `--embed-file` mode.  Instead the embedded files are loaded at
+  static constructor time.
 - Emscripten now knows what minimum browser versions the `WASM_BIGINT` feature
   requires and will automatically set the defaults accordingly. (#17163)
 - Weak undefined symbols fixed in dynamic linking. (#17164)

emcc.py

@@ -1014,9 +1014,14 @@ def package_files(options, target):
     rtn.append(object_file)
 
   cmd = [shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args
-  file_code = shared.check_call(cmd, stdout=PIPE).stdout
-
-  options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code)
+  if options.preload_files:
+    # Preloading files uses --pre-js code that runs before the module is loaded.
+    file_code = shared.check_call(cmd, stdout=PIPE).stdout
+    options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code)
+  else:
+    # Otherwise, we are embedding files, which does not require --pre-js code,
+    # and instead relies on a static constrcutor to populate the filesystem.
+    shared.check_call(cmd)
 
   return rtn
 

src/library.js

@@ -3626,30 +3626,29 @@ mergeInto(LibraryManager.library, {
 #endif
 #endif
 
-  _emscripten_fs_load_embedded_files__deps: ['$FS'],
+  _emscripten_fs_load_embedded_files__deps: ['$FS', '$PATH'],
   _emscripten_fs_load_embedded_files__sig: 'vp',
   _emscripten_fs_load_embedded_files: function(ptr) {
-#if MEMORY64
-    var start64 = ptr >> 3;
-    do {
-      var name_addr = Number(HEAPU64[start64++]);
-      var len = HEAPU32[start64 << 1];
-      start64++;
-      var content = Number(HEAPU64[start64++]);
-      var name = UTF8ToString(name_addr)
-      // canOwn this data in the filesystem, it is a slice of wasm memory that will never change
-      FS.createDataFile(name, null, HEAP8.subarray(content, content + len), true, true, true);
-    } while (HEAPU64[start64]);
-#else
-    var start32 = ptr >> 2;
+#if RUNTIME_DEBUG
+    err('preloading data files');
+#endif
     do {
-      var name_addr = HEAPU32[start32++];
-      var len = HEAPU32[start32++];
-      var content = HEAPU32[start32++];
+      var name_addr = {{{ makeGetValue('ptr', '0', '*') }}};
+      ptr += {{{ POINTER_SIZE }}};
+      var len = {{{ makeGetValue('ptr', '0', '*') }}};
+      ptr += {{{ POINTER_SIZE }}};
+      var content = {{{ makeGetValue('ptr', '0', '*') }}};
+      ptr += {{{ POINTER_SIZE }}};
       var name = UTF8ToString(name_addr)
+#if RUNTIME_DEBUG
+      err('preloading files: ' + name);
+#endif
+      FS.createPath('/', PATH.dirname(name), true, true);
       // canOwn this data in the filesystem, it is a slice of wasm memory that will never change
       FS.createDataFile(name, null, HEAP8.subarray(content, content + len), true, true, true);
-    } while (HEAPU32[start32]);
+    } while ({{{ makeGetValue('ptr', '0', '*') }}});
+#if RUNTIME_DEBUG
+    err('done preloading data files');
 #endif
   },
 });

src/library_wasmfs.js

@@ -15,6 +15,7 @@ mergeInto(LibraryManager.library, {
     '$wasmFSPreloadedFiles',
     '$wasmFSPreloadedDirs',
     '$asyncLoad',
+    '$PATH',
   ],
   $FS : {
     // TODO: Clean up the following functions - currently copied from library_fs.js directly.
@@ -64,7 +65,15 @@ mergeInto(LibraryManager.library, {
     },
     createPath: (parent, path, canRead, canWrite) => {
       // Cache file path directory names.
-      wasmFSPreloadedDirs.push({parentPath: parent, childName: path});
+      var parts = path.split('/').reverse();
+      while (parts.length) {
+        var part = parts.pop();
+        if (!part) continue;
+        var current = PATH.join2(parent, part);
+        wasmFSPreloadedDirs.push({parentPath: parent, childName: part});
+        parent = current;
+      }
+      return current;
     },
     readFile: (path, opts) => {
       opts = opts || {};

tests/test_other.py

@@ -2756,7 +2756,7 @@ def test_file_packager_embed(self):
     err = self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--js-output=data.js'], stderr=PIPE).stderr
     self.assertContained('--obj-output is recommended when using --embed', err)
 
-    self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--obj-output=data.o', '--js-output=data.js'])
+    self.run_process([FILE_PACKAGER, 'test.data', '--embed', 'data.txt', '--obj-output=data.o'])
 
     create_file('test.c', '''
     #include <stdio.h>
@@ -2771,7 +2771,7 @@ def test_file_packager_embed(self):
       return 0;
     }
     ''')
-    self.run_process([EMCC, '--pre-js=data.js', 'test.c', 'data.o', '-sFORCE_FILESYSTEM'])
+    self.run_process([EMCC, 'test.c', 'data.o', '-sFORCE_FILESYSTEM'])
     output = self.run_js('a.out.js')
     self.assertContained('hello data', output)
 
@@ -5956,9 +5956,9 @@ def test_ld_library_path(self):
 }
 ''')
     create_file('pre.js', r'''
-Module['preRun'].push(function (){
+Module['preRun'] = function () {
   ENV['LD_LIBRARY_PATH']='/lib:/usr/lib';
-});
+};
 ''')
     create_file('main.c', r'''
 #include <stdio.h>
@@ -9909,19 +9909,19 @@ def test_files_and_module_assignment(self):
     # a pre-js can set Module to a new object or otherwise undo file preloading/
     # embedding changes to Module.preRun. we show an error to avoid confusion
     create_file('pre.js', 'Module = {};')
-    create_file('src.cpp', r'''
+    create_file('src.c', r'''
       #include <stdio.h>
       int main() {
-        printf("file exists: %d\n", !!fopen("src.cpp", "rb"));
+        printf("file exists: %p\n", fopen("src.cpp", "rb"));
       }
     ''')
-    self.run_process([EMXX, 'src.cpp', '--pre-js', 'pre.js', '--embed-file', 'src.cpp'])
+    self.run_process([EMCC, 'src.c', '--pre-js=pre.js', '--preload-file=src.c'])
     result = self.run_js('a.out.js', assert_returncode=NON_ZERO)
     self.assertContained('Module.preRun should exist because file support used it; did a pre-js delete it?', result)
 
     def test_error(pre):
       create_file('pre.js', pre)
-      self.run_process([EMXX, 'src.cpp', '--pre-js', 'pre.js', '--embed-file', 'src.cpp'])
+      self.run_process([EMXX, 'src.c', '--pre-js=pre.js', '--preload-file=src.c'])
       result = self.run_js('a.out.js', assert_returncode=NON_ZERO)
       self.assertContained('All preRun tasks that exist before user pre-js code should remain after; did you replace Module or modify Module.preRun?', result)
 

tools/file_packager.py

@@ -537,6 +537,8 @@ def was_seen(name):
       err('--obj-output is only applicable when embedding files')
       return 1
     generate_object_file(data_files)
+    if not options.has_preloaded:
+      return 0
 
   ret = generate_js(data_target, data_files, metadata)
 
@@ -680,13 +682,12 @@ def generate_js(data_target, data_files, metadata):
     dirname = os.path.dirname(filename)
     basename = os.path.basename(filename)
     if file_.mode == 'embed':
-      if not options.obj_output:
-        # Embed
-        data = base64_encode(utils.read_binary(file_.srcpath))
-        code += "      var fileData%d = '%s';\n" % (counter, data)
-        # canOwn this data in the filesystem (i.e. there is no need to create a copy in the FS layer).
-        code += ("      Module['FS_createDataFile']('%s', '%s', decodeBase64(fileData%d), true, true, true);\n"
-                 % (dirname, basename, counter))
+      # Embed
+      data = base64_encode(utils.read_binary(file_.srcpath))
+      code += "      var fileData%d = '%s';\n" % (counter, data)
+      # canOwn this data in the filesystem (i.e. there is no need to create a copy in the FS layer).
+      code += ("      Module['FS_createDataFile']('%s', '%s', decodeBase64(fileData%d), true, true, true);\n"
+               % (dirname, basename, counter))
     elif file_.mode == 'preload':
       # Preload
       metadata_el = {