Pyodide unzip/untar abilities + MicroPython unzip

Author: WebReflection

docs/index.js

@@ -7,6 +7,15 @@ export const registerJSModule = (interpreter, name, value) => {
     interpreter.registerJsModule(name, value);
 };
 
+export const getFormat = (path, url) => {
+    if (path.endsWith('/*')) {
+        if (/\.(zip|tar(?:\.gz)?)$/.test(url))
+            return RegExp.$1;
+        throw new Error(`Unsupported archive ${url}`);
+    }
+    return '';
+};
+
 export const run = (interpreter, code, ...args) => {
     try {
         return interpreter.runPython(dedent(code), ...args);

docs/index.js.map

@@ -11,7 +11,9 @@ export const RUNNING_IN_WORKER = !globalThis.window;
 // This should be the only helper needed for all Emscripten based FS exports
 export const writeFile = ({ FS, PATH, PATH_FS }, path, buffer) => {
     const absPath = PATH_FS.resolve(path);
-    FS.mkdirTree(PATH.dirname(absPath));
+    const dirPath = PATH.dirname(absPath);
+    if (FS.mkdirTree) FS.mkdirTree(dirPath);
+    else mkdirTree(FS, dirPath);
     return FS.writeFile(absPath, new Uint8Array(buffer), {
         canOwn: true,
     });
@@ -141,7 +143,12 @@ export const fetchFiles = (module, interpreter, config_files) =>
     all(
         calculateFilesPaths(config_files).map(({ url, path }) =>
             fetchBuffer(config_files, url)
-                .then((buffer) => module.writeFile(interpreter, path, buffer)),
+                .then((buffer) => module.writeFile(
+                    interpreter,
+                    path,
+                    buffer,
+                    url,
+                )),
         ),
     );
 

docs/zip-CVv62MiA.js

@@ -1,7 +1,9 @@
+// import fetch from '@webreflection/fetch';
 import { fetchFiles, fetchJSModules, fetchPaths, writeFile } from './_utils.js';
-import { registerJSModule, run, runAsync, runEvent } from './_python.js';
+import { getFormat, registerJSModule, run, runAsync, runEvent } from './_python.js';
 import { stdio } from './_io.js';
 import mip from '../python/mip.js';
+import zip from '../zip.js';
 
 const type = 'micropython';
 
@@ -33,7 +35,58 @@ export default {
     runAsync,
     runEvent,
     transform: (interpreter, value) => interpreter.PyProxy.toJs(value),
-    writeFile: ({ FS, _module: { PATH, PATH_FS } }, path, buffer) =>
-        writeFile({ FS, PATH, PATH_FS }, path, buffer),
+    writeFile: (interpreter, path, buffer, url) => {
+        const { FS, _module: { PATH, PATH_FS } } = interpreter;
+        const fs = { FS, PATH, PATH_FS };
+        const format = getFormat(path, url);
+        if (format) {
+            const extractDir = path.slice(0, -1);
+            if (extractDir !== './') FS.mkdir(extractDir);
+            switch (format) {
+                case 'zip': {
+                    const blob = new Blob([buffer], { type: 'application/zip' });
+                    return zip().then(async ({ BlobReader, Uint8ArrayWriter, ZipReader }) => {
+                        const zipFileReader = new BlobReader(blob);
+                        const zipReader = new ZipReader(zipFileReader);
+                        for (const entry of await zipReader.getEntries()) {
+                            const { directory, filename } = entry;
+                            if (directory) {
+                                FS.mkdir(extractDir + filename);
+                            }
+                            else {
+                                const buffer = await entry.getData(new Uint8ArrayWriter);
+                                FS.writeFile(extractDir + filename, buffer, {
+                                    canOwn: true,
+                                });
+                            }
+                        }
+                        zipReader.close();
+                    });
+                }
+                case 'tar.gz': {
+                    const TMP = './_.tar.gz';
+                    writeFile(fs, TMP, buffer);
+                    interpreter.runPython(`
+                        import os, gzip, tarfile
+                        tar = tarfile.TarFile(fileobj=gzip.GzipFile(fileobj=open("${TMP}", "rb")))
+                        for f in tar:
+                            name = f"${extractDir}{f.name[2:]}"
+                            if f.type == tarfile.DIRTYPE:
+                                if f.name != "./":
+                                    os.mkdir(name.strip("/"))
+                            else:
+                                source = tar.extractfile(f)
+                                with open(name, "wb") as dest:
+                                    dest.write(source.read())
+                                    dest.close()
+                        tar.close()
+                        os.remove("${TMP}")
+                    `);
+                    return;
+                }
+            }
+        }
+        return writeFile(fs, path, buffer);
+    },
 };
 /* c8 ignore stop */

docs/zip-CVv62MiA.js.map

@@ -1,7 +1,7 @@
 import { create } from 'gc-hook';
 
 import { RUNNING_IN_WORKER, fetchFiles, fetchJSModules, fetchPaths, writeFile } from './_utils.js';
-import { registerJSModule, run, runAsync, runEvent } from './_python.js';
+import { getFormat, registerJSModule, run, runAsync, runEvent } from './_python.js';
 import { stdio } from './_io.js';
 
 const type = 'pyodide';
@@ -109,7 +109,15 @@ export default {
             value.toJs(toJsOptions) :
             value
     ),
-    writeFile: ({ FS, PATH, _module: { PATH_FS } }, path, buffer) =>
-        writeFile({ FS, PATH, PATH_FS }, path, buffer),
+    writeFile: (interpreter, path, buffer, url) => {
+        const format = getFormat(path, url);
+        if (format) {
+            return interpreter.unpackArchive(buffer, format, {
+                extractDir: path.slice(0, -1)
+            });
+        }
+        const { FS, PATH, _module: { PATH_FS } } = interpreter;
+        return writeFile({ FS, PATH, PATH_FS }, path, buffer);
+    },
 };
 /* c8 ignore stop */

esm/interpreter/_python.js

@@ -0,0 +1,3 @@
+/* c8 ignore start */
+export default () => import(/* webpackIgnore: true */'./3rd-party/zip.js');
+/* c8 ignore stop */

esm/interpreter/_utils.js

@@ -88,6 +88,6 @@
         "to-json-callback": "^0.1.1"
     },
     "worker": {
-        "blob": "sha256-tV7jypF54M8P2vuzREc5ZRk1nPGj9AL1VmnxnE2az/s="
+        "blob": "sha256-jf8fAU4aXz2BNme/XOUMyIliOinwW2o+moYgdacP8qs="
     }
 }

esm/interpreter/micropython.js

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+</head>
+<body>
+  <ul>
+    <li><a href="pyodide.html">pyodide</a></li>
+    <li><a href="micropython.html">micropython</a></li>
+  </ul>
+</body>
+</html>

esm/interpreter/pyodide.js

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <script type="module" src="/dist/index.js"></script>
+  <style>
+    html.true::before {
+      content: "OK"
+    }
+  </style>
+</head>
+<body>
+  <script type="micropython" config="zip.toml" worker>
+    from polyscript import xworker
+    f = open("./package/index.html", "r")
+    lines = f.readlines()
+    xworker.window.document.documentElement.classList.add(
+      lines[0] == "<!DOCTYPE html>\n"
+    )
+  </script>
+  <script type="micropython" config="tar.gz.toml" worker>
+    from polyscript import xworker
+    f = open("./package/index.html", "r")
+    lines = f.readlines()
+    xworker.window.document.documentElement.classList.add(
+      lines[0] == "<!DOCTYPE html>\n"
+    )
+  </script>
+</body>
+</html>

esm/zip.js

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <script type="module" src="/dist/index.js"></script>
+  <style>
+    html.true::before {
+      content: "OK"
+    }
+  </style>
+</head>
+<body>
+  <script type="pyodide" config="zip.toml" worker>
+    from polyscript import xworker
+    f = open("./package/index.html", "r")
+    lines = f.readlines()
+    xworker.window.document.documentElement.classList.add(
+      lines[0] == "<!DOCTYPE html>\n"
+    )
+  </script>
+  <script type="pyodide" config="tar.gz.toml" worker>
+    from polyscript import xworker
+    f = open("./package/index.html", "r")
+    lines = f.readlines()
+    xworker.window.document.documentElement.classList.add(
+      lines[0] == "<!DOCTYPE html>\n"
+    )
+  </script>
+</body>
+</html>

package.json

@@ -0,0 +1,2 @@
+[files]
+"./index.tar.gz" = "./package/*"

test/zip/index.html

@@ -0,0 +1,2 @@
+[files]
+"./index.zip" = "./package/*"