From 26b1de54d184a4c424a4149113247e58fc69337f Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Thu, 11 Aug 2022 19:53:55 +0100 Subject: [PATCH 01/33] import modules parsing to support py-env dependencies The new module extends the ModuleFinder from find-imports (in graveyard) to detect all the dependencies of the file being converted. Currently the parser is able to collect either external packages (limited to those supported by Pyodide) and any local module to be included in `paths`. Local submodules (node.level > 0) not yet supported. --- src/pyscript/_node_parser.py | 116 +++++++++ src/pyscript/_supported_packages.py | 356 ++++++++++++++++++++++++++++ 2 files changed, 472 insertions(+) create mode 100644 src/pyscript/_node_parser.py create mode 100644 src/pyscript/_supported_packages.py diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py new file mode 100644 index 0000000..7f01f6c --- /dev/null +++ b/src/pyscript/_node_parser.py @@ -0,0 +1,116 @@ +""" +ast-based parser to gather modules/package dependencies of a Python module. +Code adapted from the find-imports project, currently in graveyard archive. +""" +import ast +from pathlib import Path +from collections import namedtuple +from threading import local + +from ._supported_packages import PACKAGE_RENAMES, STANDARD_LIBRARY, PYODIDE_PACKAGES + + +class UnsupportedFileType(Exception): + pass + + +Environment = namedtuple("Environment", ["packages", "paths"]) + +# https://stackoverflow.com/a/58847554 +class ModuleFinder(ast.NodeVisitor): + def __init__(self, *args, **kwargs): + self.packages = set() + self.other_modules = set() + super().__init__(*args, **kwargs) + + def visit_Import(self, node): + for name in node.names: + imported = name.name.split(".")[0] + self._import_name(imported) + + def visit_ImportFrom(self, node): + # if node.module is missing it's a "from . import ..." statement + # if level > 0 it's a "from .submodule import ..." statement + if node.module is not None and node.level == 0: + imported = node.module.split(".")[0] + self._import_name(imported) + + def _import_name(self, imported): + pkg_name = PACKAGE_RENAMES.get(imported, imported) + if pkg_name not in STANDARD_LIBRARY: + if pkg_name in PYODIDE_PACKAGES: + self.packages.add(pkg_name) + else: + self.other_modules.add(pkg_name) + + +def _find_modules(source: str, source_fpath: Path) -> Environment: + fname = source_fpath.name + # passing mode='exec' just in case defaults will change in the future + nodes = ast.parse(source, fname, mode="exec") + + finder = ModuleFinder() + finder.visit(nodes) + print("Found modules: ", finder.packages, finder.other_modules) + source_basefolder = source_fpath.parent + local_mods = set( + map( + lambda p: Path(*p.parts[1:]), + filter( + lambda d: d.stem in finder.other_modules + and ((d.is_file() and d.suffix == ".py") or d.is_dir()), + source_basefolder.iterdir(), + ), + ) + ) + print("local modules", local_mods) + print("external mods", finder.packages) + return Environment(packages=finder.packages, paths=local_mods) + + +def _convert_notebook(source_fpath: Path) -> str: + from nbconvert import ScriptExporter + + exporter = ScriptExporter() + source, _ = exporter.from_filename(source_fpath) + + return source + + +def find_imports(source_fpath: Path) -> Environment: + """ + Parse the input source, and returns its dependencies, as organised in + the sets of external packages, and local modules, respectively. + Any modules or package with the same name found in the local + + Parameters + ---------- + source_fpath : Path + Path to the input Python module to parse + + Returns + ------- + tuple[set[str], set[str]] + Pair of external modules, and local modules + """ + fname, extension = source_fpath.name, source_fpath.suffix + if extension == ".py": + with open(source_fpath, "rt") as f: + source = f.read() + + elif extension == ".ipynb": + try: + import nbconvert + except ImportError as e: # pragma no cover + raise ImportError( + "Please install nbconvert to serve Jupyter Notebooks." + ) from e + + source = _convert_notebook(source_fpath) + + else: + raise UnsupportedFileType( + "{} is neither a script (.py) nor a notebook (.ipynb)".format(fname) + ) + + return _find_modules(source, source_fpath) diff --git a/src/pyscript/_supported_packages.py b/src/pyscript/_supported_packages.py new file mode 100644 index 0000000..3efcf46 --- /dev/null +++ b/src/pyscript/_supported_packages.py @@ -0,0 +1,356 @@ +# extending find-import package names with Pyodide supported packages + +PACKAGE_RENAMES = { + "sklearn": "scikit-learn", + "umap": "umap-learn", # has numba dep. so not yet supported in PyScript/Pyodide +} + +# taken from https://pyodide.org/en/stable/usage/packages-in-pyodide.html +PYODIDE_PACKAGES = { + "asciitree", + "astropy", + "atomicwrites", + "attrs", + "autograd", + "beautifulsoup4", + "biopython", + "bitarray", + "bleach", + "bokeh", + "boost-histogram", + "brotli", + "certifi", + "cffi", + "cffi_example", + "cftime", + "CLAPACK", + "cloudpickle", + "cmyt", + "colorspacious", + "cryptography", + "cssselect", + "cycler", + "cytoolz", + "decorator", + "demes", + "distlib", + "docutils", + "fonttools", + "freesasa", + "future", + "galpy", + "geos", + "gmpy2", + "gsw", + "h5py", + "html5lib", + "imageio", + "iniconfig", + "jedi", + "Jinja2", + "joblib", + "jsonschema", + "kiwisolver", + "lazy-object-proxy", + "libmagic", + "logbook", + "lxml", + "MarkupSafe", + "matplotlib", + "micropip", + "mne", + "more-itertools", + "mpmath", + "msgpack", + "msprime", + "networkx", + "newick", + "nlopt", + "nltk", + "nose", + "numcodecs", + "numpy", + "opencv-python", + "openssl", + "optlang", + "packaging", + "pandas", + "parso", + "patsy", + "Pillow", + "pkgconfig", + "pluggy", + "py", + "pyb2d", + "pyclipper", + "pycparser", + "pydantic", + "pyerfa", + "Pygments", + "pyparsing", + "pyproj", + "pyrsistent", + "pytest", + "pytest-benchmark", + "python-dateutil", + "python-magic", + "python-sat", + "python_solvespace", + "pytz", + "pywavelets", + "pyyaml", + "rebound", + "reboundx", + "regex", + "retrying", + "RobotRaconteur", + "ruamel", + "scikit-image", + "scikit-learn", + "scipy", + "setuptools", + "shapely", + "six", + "soupsieve", + "sparseqr", + "sqlalchemy", + "ssl", + "statsmodels", + "suitesparse", + "svgwrite", + "swiglpk", + "sympy", + "tblib", + "termcolor", + "threadpoolctl", + "tomli", + "tomli-w", + "toolz", + "tqdm", + "traits", + "tskit", + "typing-extensions", + "uncertainties", + "unyt", + "webencodings", + "wrapt", + "xarray", + "xgboost", + "xlrd", + "yt", + "zarr", +} + +# taken from https://docs.python.org/3/py-modindex.html +STANDARD_LIBRARY = { + "__future__", + "__main__", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "cProfile", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} From 96db47dd2ad148fa6c0d1ce2c93b985c9472242a Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Thu, 11 Aug 2022 19:58:21 +0100 Subject: [PATCH 02/33] generator now includes import collection for files --- src/pyscript/_generator.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index a0eda3b..1c7114d 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -3,18 +3,29 @@ import jinja2 -_env = jinja2.Environment(loader=jinja2.PackageLoader("pyscript")) +from ._node_parser import find_imports, Environment +_env = jinja2.Environment(loader=jinja2.PackageLoader("pyscript"), trim_blocks=True, lstrip_blocks=True) -def string_to_html(input_str: str, title: str, output_path: Path) -> None: + +def string_to_html( + input_str: str, title: str, output_path: Path, env: Environment = None +) -> None: """Write a Python script string to an HTML file template.""" template = _env.get_template("basic.html") + if env is not None: + modules, paths = env.packages, env.paths + else: + modules = paths = () with output_path.open("w") as fp: - fp.write(template.render(code=input_str, title=title)) + fp.write( + template.render(code=input_str, title=title, modules=modules, paths=paths) + ) def file_to_html(input_path: Path, title: str, output_path: Optional[Path]) -> None: """Write a Python script string to an HTML file template.""" output_path = output_path or input_path.with_suffix(".html") + environment = find_imports(input_path) with input_path.open("r") as fp: - string_to_html(fp.read(), title, output_path) + string_to_html(fp.read(), title, output_path, env=environment) From 07ba09a86fcdc19cb9879c9b1d076a7b5a5f5feb Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Thu, 11 Aug 2022 19:58:38 +0100 Subject: [PATCH 03/33] basic template extended to support py-env --- src/pyscript/templates/basic.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/pyscript/templates/basic.html b/src/pyscript/templates/basic.html index 16c1a3a..8cd3049 100644 --- a/src/pyscript/templates/basic.html +++ b/src/pyscript/templates/basic.html @@ -4,6 +4,21 @@ {{ title }} + + {% if modules or paths %} + + {% for module in modules %} + - {{module}} + {% endfor %} + {% if paths %} + - paths: + {% for path in paths %} + - {{path}} + {% endfor %} + {% endif %} + + {% endif %} + From 39c7fc232d983285919dfddd8cf1c2d5a4aba767 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Fri, 12 Aug 2022 14:46:13 +0100 Subject: [PATCH 04/33] Improved parsing with local namespace inspection Code parsing has been completely refactored and improved to support the identification of corner import conditions that won't work in Pyscript. This includes unsupported packages, and local modules/packages import that won't work if not imported directly. As a result, ModuleFinder gathers all the packages via NamespaceInfo (populated via pkgutil) and collects results in a FinderResult object. This object will be later used to specialise the processing result. --- src/pyscript/_node_parser.py | 173 +++++++++++++++++++++++++++-------- 1 file changed, 135 insertions(+), 38 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 7f01f6c..e681460 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -3,9 +3,13 @@ Code adapted from the find-imports project, currently in graveyard archive. """ import ast +from inspect import Attribute +import os +import pkgutil from pathlib import Path -from collections import namedtuple -from threading import local +from typing import Union +from collections import namedtuple, defaultdict +from itertools import filterfalse, chain from ._supported_packages import PACKAGE_RENAMES, STANDARD_LIBRARY, PYODIDE_PACKAGES @@ -14,58 +18,146 @@ class UnsupportedFileType(Exception): pass -Environment = namedtuple("Environment", ["packages", "paths"]) +ImportInfo = namedtuple("ImportInfo", ["packages", "paths"]) + + +class NamespaceInfo: + def __init__(self, source_fpath: Path) -> None: + # expanding base_folder to absolute as pkgutils.FileFinder will do so - easier for later purging + self.base_folder = str(source_fpath.parent.absolute()) + self.source_mod_name = source_fpath.stem + self._collect() + # storing this as it will be useful for multiple lookups + self._all_namespace = set(chain(self.modules, self.packages)) + + def _collect(self): + iter_modules_paths = [self.base_folder] + for root, dirs, files in os.walk(self.base_folder): + for dirname in dirs: + iter_modules_paths.append(os.path.join(root, dirname)) + + # need to consume generator as I will iterate two times for packages, and modules + pkg_mods = tuple(pkgutil.iter_modules(iter_modules_paths)) + modules = map( + lambda mi: os.path.join(mi.module_finder.path, mi.name), + filterfalse( + lambda mi: mi.ispkg or mi.name == self.source_mod_name, pkg_mods + ), + ) + packages = map( + lambda mi: os.path.join(mi.module_finder.path, mi.name), + filter(lambda mi: mi.ispkg, pkg_mods), + ) + self.modules = set(map(self._dotted_path, modules)) + self.packages = set(map(self._dotted_path, packages)) + + def _dotted_path(self, p: str): + p = p.replace(self.base_folder, "").replace(os.path.sep, ".") + if p.startswith("."): + p = p[1:] + return p + + def __contains__(self, item: str) -> bool: + return item in self._all_namespace + + def __str__(self) -> str: + return f"NameSpace info for {self.base_folder} \n\t Modules: {self.modules} \n\t Packages: {self.packages}" + + def __repr__(self) -> str: + return str(self) + + +class FinderResult: + def __init__(self) -> None: + self.packages = set() + self.locals = set() + self.unsupported = defaultdict(set) + + def add_package(self, pkg_name: str) -> None: + self.packages.add(pkg_name) + + def add_locals(self, pkg_name: str) -> None: + self.locals.add(pkg_name) + + def add_unsupported_external_package(self, pkg_name: str) -> None: + self.unsupported["external"].add(pkg_name) + + def add_unsupported_local_package(self, pkg_name: str) -> None: + self.unsupported["local"].add(pkg_name) + + @property + def has_warnings(self): + return len(self.unsupported) > 0 + + @property + def unsupported_packages(self): + return self.unsupported["external"] + + @property + def unsupported_paths(self): + return self.unsupported["local"] + # https://stackoverflow.com/a/58847554 class ModuleFinder(ast.NodeVisitor): - def __init__(self, *args, **kwargs): - self.packages = set() - self.other_modules = set() + def __init__(self, context: NamespaceInfo, *args, **kwargs): + # list of all potential local imports + self.context = context + self.results = FinderResult() super().__init__(*args, **kwargs) def visit_Import(self, node): for name in node.names: - imported = name.name.split(".")[0] - self._import_name(imported) + # need to check for absolute module import here as they won't work in PyScript + # absolute package imports will be found later in _import_name + if len(name.name.split(".")) > 1 and name.name in self.context: + self.results.add_unsupported_local_package(name.name) + else: + self._import_name(name.name) def visit_ImportFrom(self, node): # if node.module is missing it's a "from . import ..." statement # if level > 0 it's a "from .submodule import ..." statement - if node.module is not None and node.level == 0: - imported = node.module.split(".")[0] - self._import_name(imported) + if node.module is not None: + self._import_name(node.module) def _import_name(self, imported): - pkg_name = PACKAGE_RENAMES.get(imported, imported) - if pkg_name not in STANDARD_LIBRARY: - if pkg_name in PYODIDE_PACKAGES: - self.packages.add(pkg_name) + if imported in self.context: + if imported not in self.context.packages: + self.results.add_locals(imported) else: - self.other_modules.add(pkg_name) - - -def _find_modules(source: str, source_fpath: Path) -> Environment: + self.results.add_unsupported_local_package(imported) + else: + imported = imported.split(".")[0] + pkg_name = PACKAGE_RENAMES.get(imported, imported) + if pkg_name not in STANDARD_LIBRARY: + if pkg_name in PYODIDE_PACKAGES: + self.results.add_package(pkg_name) + else: + self.results.add_unsupported_external_package(pkg_name) + + +def _find_modules(source: str, source_fpath: Path): fname = source_fpath.name + # importing all local modules from source_fpath + namespace_info = NamespaceInfo(source_fpath=source_fpath) # passing mode='exec' just in case defaults will change in the future nodes = ast.parse(source, fname, mode="exec") - finder = ModuleFinder() + finder = ModuleFinder(context=namespace_info) finder.visit(nodes) - print("Found modules: ", finder.packages, finder.other_modules) - source_basefolder = source_fpath.parent - local_mods = set( - map( - lambda p: Path(*p.parts[1:]), - filter( - lambda d: d.stem in finder.other_modules - and ((d.is_file() and d.suffix == ".py") or d.is_dir()), - source_basefolder.iterdir(), - ), - ) + report = finder.results + pyenv_paths = map( + lambda l: "{}.py".format(l.replace(".", os.path.sep)), report.locals + ) + pyenv = ImportInfo(packages=report.packages, paths=set(pyenv_paths)) + if not report.has_warnings: + return pyenv, None + + warnings = ImportInfo( + packages=report.unsupported_packages, paths=report.unsupported_paths ) - print("local modules", local_mods) - print("external mods", finder.packages) - return Environment(packages=finder.packages, paths=local_mods) + return pyenv, warnings def _convert_notebook(source_fpath: Path) -> str: @@ -77,9 +169,11 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports(source_fpath: Path) -> Environment: +def find_imports( + source_fpath: Path, +) -> Union[ImportInfo, tuple[ImportInfo, ImportInfo]]: """ - Parse the input source, and returns its dependencies, as organised in + Parse the input source, and returns its dependencies, as organised in the sets of external packages, and local modules, respectively. Any modules or package with the same name found in the local @@ -90,8 +184,11 @@ def find_imports(source_fpath: Path) -> Environment: Returns ------- - tuple[set[str], set[str]] - Pair of external modules, and local modules + Union[ImportInfo, tuple[ImportInfo, ImportInfo]] + The function returns an instance of `ImportInfo` containing the + environment with packages and paths to include in py-env. + Optionally, if the parsing detected unsupported packages and local modules, + this will be returned as well (still as `ImportInfo` instance) """ fname, extension = source_fpath.name, source_fpath.suffix if extension == ".py": From de00f059cf944e748a2ee783e2311fd26fb624eb Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Fri, 12 Aug 2022 15:09:13 +0100 Subject: [PATCH 05/33] Improved find_imports encapsulation find_imports functions now leverages directly on the FinderResults class to encapsualte parsing results. --- src/pyscript/_node_parser.py | 76 ++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index e681460..55d0a78 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -3,11 +3,9 @@ Code adapted from the find-imports project, currently in graveyard archive. """ import ast -from inspect import Attribute import os import pkgutil from pathlib import Path -from typing import Union from collections import namedtuple, defaultdict from itertools import filterfalse, chain @@ -18,9 +16,6 @@ class UnsupportedFileType(Exception): pass -ImportInfo = namedtuple("ImportInfo", ["packages", "paths"]) - - class NamespaceInfo: def __init__(self, source_fpath: Path) -> None: # expanding base_folder to absolute as pkgutils.FileFinder will do so - easier for later purging @@ -28,7 +23,7 @@ def __init__(self, source_fpath: Path) -> None: self.source_mod_name = source_fpath.stem self._collect() # storing this as it will be useful for multiple lookups - self._all_namespace = set(chain(self.modules, self.packages)) + self._all_namespace = set(chain(self.modules, self._packages)) def _collect(self): iter_modules_paths = [self.base_folder] @@ -36,7 +31,7 @@ def _collect(self): for dirname in dirs: iter_modules_paths.append(os.path.join(root, dirname)) - # need to consume generator as I will iterate two times for packages, and modules + # need to consume generator as I will iterate two times for _packages, and modules pkg_mods = tuple(pkgutil.iter_modules(iter_modules_paths)) modules = map( lambda mi: os.path.join(mi.module_finder.path, mi.name), @@ -44,12 +39,12 @@ def _collect(self): lambda mi: mi.ispkg or mi.name == self.source_mod_name, pkg_mods ), ) - packages = map( + _packages = map( lambda mi: os.path.join(mi.module_finder.path, mi.name), filter(lambda mi: mi.ispkg, pkg_mods), ) self.modules = set(map(self._dotted_path, modules)) - self.packages = set(map(self._dotted_path, packages)) + self._packages = set(map(self._dotted_path, _packages)) def _dotted_path(self, p: str): p = p.replace(self.base_folder, "").replace(os.path.sep, ".") @@ -61,7 +56,7 @@ def __contains__(self, item: str) -> bool: return item in self._all_namespace def __str__(self) -> str: - return f"NameSpace info for {self.base_folder} \n\t Modules: {self.modules} \n\t Packages: {self.packages}" + return f"NameSpace info for {self.base_folder} \n\t Modules: {self.modules} \n\t Packages: {self._packages}" def __repr__(self) -> str: return str(self) @@ -69,33 +64,44 @@ def __repr__(self) -> str: class FinderResult: def __init__(self) -> None: - self.packages = set() - self.locals = set() - self.unsupported = defaultdict(set) + self._packages = set() + self._locals = set() + self._unsupported = defaultdict(set) def add_package(self, pkg_name: str) -> None: - self.packages.add(pkg_name) + self._packages.add(pkg_name) def add_locals(self, pkg_name: str) -> None: - self.locals.add(pkg_name) + self._locals.add(pkg_name) def add_unsupported_external_package(self, pkg_name: str) -> None: - self.unsupported["external"].add(pkg_name) + self._unsupported["external"].add(pkg_name) def add_unsupported_local_package(self, pkg_name: str) -> None: - self.unsupported["local"].add(pkg_name) + self._unsupported["local"].add(pkg_name) @property def has_warnings(self): - return len(self.unsupported) > 0 + return len(self._unsupported) > 0 @property def unsupported_packages(self): - return self.unsupported["external"] + return self._unsupported["external"] @property def unsupported_paths(self): - return self.unsupported["local"] + return self._unsupported["local"] + + @property + def packages(self): + return self._packages + + @property + def paths(self): + pyenv_paths = map( + lambda l: "{}.py".format(l.replace(".", os.path.sep)), self._locals + ) + return set(pyenv_paths) # https://stackoverflow.com/a/58847554 @@ -123,7 +129,7 @@ def visit_ImportFrom(self, node): def _import_name(self, imported): if imported in self.context: - if imported not in self.context.packages: + if imported not in self.context._packages: self.results.add_locals(imported) else: self.results.add_unsupported_local_package(imported) @@ -146,18 +152,7 @@ def _find_modules(source: str, source_fpath: Path): finder = ModuleFinder(context=namespace_info) finder.visit(nodes) - report = finder.results - pyenv_paths = map( - lambda l: "{}.py".format(l.replace(".", os.path.sep)), report.locals - ) - pyenv = ImportInfo(packages=report.packages, paths=set(pyenv_paths)) - if not report.has_warnings: - return pyenv, None - - warnings = ImportInfo( - packages=report.unsupported_packages, paths=report.unsupported_paths - ) - return pyenv, warnings + return finder.results def _convert_notebook(source_fpath: Path) -> str: @@ -169,12 +164,10 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports( - source_fpath: Path, -) -> Union[ImportInfo, tuple[ImportInfo, ImportInfo]]: +def find_imports(source_fpath: Path,) -> FinderResult: """ Parse the input source, and returns its dependencies, as organised in - the sets of external packages, and local modules, respectively. + the sets of external _packages, and local modules, respectively. Any modules or package with the same name found in the local Parameters @@ -184,11 +177,10 @@ def find_imports( Returns ------- - Union[ImportInfo, tuple[ImportInfo, ImportInfo]] - The function returns an instance of `ImportInfo` containing the - environment with packages and paths to include in py-env. - Optionally, if the parsing detected unsupported packages and local modules, - this will be returned as well (still as `ImportInfo` instance) + FinderResults + Return the results of parsing as a `FinderResults` instance. + This instance provides reference to packages and paths to + include in the py-env, as well as any unsuppoted import. """ fname, extension = source_fpath.name, source_fpath.suffix if extension == ".py": From 0e625ceecd65143e5c30345e5e3ae5039e6960d1 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Fri, 12 Aug 2022 15:10:44 +0100 Subject: [PATCH 06/33] fixed typo in docstring --- src/pyscript/_node_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 55d0a78..ad321aa 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -177,8 +177,8 @@ def find_imports(source_fpath: Path,) -> FinderResult: Returns ------- - FinderResults - Return the results of parsing as a `FinderResults` instance. + FinderResult + Return the results of parsing as a `FinderResult` instance. This instance provides reference to packages and paths to include in the py-env, as well as any unsuppoted import. """ From 4862c57f04082c72133da4f5b9435df07b8191e3 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Fri, 12 Aug 2022 15:13:15 +0100 Subject: [PATCH 07/33] finderresults integration --- src/pyscript/_generator.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 1c7114d..90f2158 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -3,18 +3,20 @@ import jinja2 -from ._node_parser import find_imports, Environment +from ._node_parser import find_imports, FinderResult -_env = jinja2.Environment(loader=jinja2.PackageLoader("pyscript"), trim_blocks=True, lstrip_blocks=True) +_env = jinja2.Environment( + loader=jinja2.PackageLoader("pyscript"), trim_blocks=True, lstrip_blocks=True +) def string_to_html( - input_str: str, title: str, output_path: Path, env: Environment = None + input_str: str, title: str, output_path: Path, pyenv: FinderResult = None ) -> None: """Write a Python script string to an HTML file template.""" template = _env.get_template("basic.html") - if env is not None: - modules, paths = env.packages, env.paths + if pyenv is not None: + modules, paths = pyenv.packages, pyenv.paths else: modules = paths = () with output_path.open("w") as fp: @@ -23,9 +25,16 @@ def string_to_html( ) -def file_to_html(input_path: Path, title: str, output_path: Optional[Path]) -> None: - """Write a Python script string to an HTML file template.""" +def file_to_html( + input_path: Path, title: str, output_path: Optional[Path] +) -> Optional[FinderResult]: + """Write a Python script string to an HTML file template. + + Warnings will be returned when scanning for environment, if any. + """ output_path = output_path or input_path.with_suffix(".html") - environment = find_imports(input_path) + import_results = find_imports(input_path) with input_path.open("r") as fp: - string_to_html(fp.read(), title, output_path, env=environment) + string_to_html(fp.read(), title, output_path, pyenv=import_results) + if import_results.has_warnings: + return import_results From d94afdf8ea786a1479aba7b7c8965925ef3da89c Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Fri, 12 Aug 2022 15:17:30 +0100 Subject: [PATCH 08/33] Integration of parsing results and new Warning msg Warning message is now generated (in yellow) when the converted Python module contains unsupported packages or local modules (as in packages) --- src/pyscript/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/pyscript/cli.py b/src/pyscript/cli.py index d193dcd..005194f 100644 --- a/src/pyscript/cli.py +++ b/src/pyscript/cli.py @@ -63,6 +63,12 @@ def __init__(self, msg: str, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) +class Warning(typer.Exit): + def __init__(self, msg: str, *args: Any, **kwargs: Any): + console.print(msg, style="yellow") + super().__init__(*args, **kwargs) + + @app.command() def wrap( input_file: Optional[Path] = _input_file_argument, @@ -94,7 +100,18 @@ def wrap( raise Abort("Must provide an output file or use `--show` option") if input_file is not None: - file_to_html(input_file, title, output) + parsing_res = file_to_html(input_file, title, output) + if parsing_res is not None: + msg_template = "WARNING: The input file contains some imports which are not currently supported PyScript.\nTherefore the code might not work, or require some changes.{packages}{locals}" + msg = msg_template.format( + packages=f"\n{str(parsing_res.unsupported_packages)}" + if parsing_res.unsupported_packages + else "", + locals=f"\n{str(parsing_res.unsupported_paths)}" + if parsing_res.unsupported_paths + else "", + ) + raise Warning(msg=msg) if command: string_to_html(command, title, output) From 3fa0ecce699571701392e0ad378a70cc258018f7 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Fri, 12 Aug 2022 16:22:25 +0100 Subject: [PATCH 09/33] Jupyter notebook conversion supported --- src/pyscript/_generator.py | 30 ++++++++++++++++++++++++++++-- src/pyscript/_node_parser.py | 27 +++------------------------ 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 90f2158..283dee3 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -4,6 +4,12 @@ import jinja2 from ._node_parser import find_imports, FinderResult +from ._node_parser import _convert_notebook + + +class UnsupportedFileType(Exception): + pass + _env = jinja2.Environment( loader=jinja2.PackageLoader("pyscript"), trim_blocks=True, lstrip_blocks=True @@ -33,8 +39,28 @@ def file_to_html( Warnings will be returned when scanning for environment, if any. """ output_path = output_path or input_path.with_suffix(".html") - import_results = find_imports(input_path) + + fname, extension = input_path.name, input_path.suffix + if extension == ".py": + with open(input_path, "rt") as f: + source = f.read() + + elif extension == ".ipynb": + try: + import nbconvert + except ImportError as e: # pragma no cover + raise ImportError( + "Please install nbconvert to serve Jupyter Notebooks." + ) from e + source = _convert_notebook(input_path) + + else: + raise UnsupportedFileType( + "{} is neither a script (.py) nor a notebook (.ipynb)".format(fname) + ) + + import_results = find_imports(source, input_path) with input_path.open("r") as fp: - string_to_html(fp.read(), title, output_path, pyenv=import_results) + string_to_html(source, title, output_path, pyenv=import_results) if import_results.has_warnings: return import_results diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index ad321aa..92e3ad5 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -12,10 +12,6 @@ from ._supported_packages import PACKAGE_RENAMES, STANDARD_LIBRARY, PYODIDE_PACKAGES -class UnsupportedFileType(Exception): - pass - - class NamespaceInfo: def __init__(self, source_fpath: Path) -> None: # expanding base_folder to absolute as pkgutils.FileFinder will do so - easier for later purging @@ -164,7 +160,7 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports(source_fpath: Path,) -> FinderResult: +def find_imports(source: str, source_fpath: Path,) -> FinderResult: """ Parse the input source, and returns its dependencies, as organised in the sets of external _packages, and local modules, respectively. @@ -172,6 +168,8 @@ def find_imports(source_fpath: Path,) -> FinderResult: Parameters ---------- + source : str + Python source code to parse source_fpath : Path Path to the input Python module to parse @@ -182,24 +180,5 @@ def find_imports(source_fpath: Path,) -> FinderResult: This instance provides reference to packages and paths to include in the py-env, as well as any unsuppoted import. """ - fname, extension = source_fpath.name, source_fpath.suffix - if extension == ".py": - with open(source_fpath, "rt") as f: - source = f.read() - - elif extension == ".ipynb": - try: - import nbconvert - except ImportError as e: # pragma no cover - raise ImportError( - "Please install nbconvert to serve Jupyter Notebooks." - ) from e - - source = _convert_notebook(source_fpath) - - else: - raise UnsupportedFileType( - "{} is neither a script (.py) nor a notebook (.ipynb)".format(fname) - ) return _find_modules(source, source_fpath) From 94861f3b5183f5f9ca3afdfabd212688ca2b24f0 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 10:14:13 +0100 Subject: [PATCH 10/33] ignored spotlight index files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5202bcf..2bb435f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# OSx rubbish +.DS_Store + # Byte-compiled __pycache__/ *.py[cod] From 55407bf777dd892a92d82aa600c781ad713880f1 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 10:15:46 +0100 Subject: [PATCH 11/33] ignored PyCharm dev env --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2bb435f..bcbba9d 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ docs/_build/ .venv env/ venv/ +.idea # Poetry lock file poetry.lock From d28f625a6f3e32bff98a03052bf3f2ad17c604ba Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 10:45:41 +0100 Subject: [PATCH 12/33] nbconvert and pre-commit as extra deps nbconvert is an extra dependency required to support the conversion of notebooks into python files. Pre-commit on the other hand was a potentially missing dependency for dev. Last but not least, requests and six are also needed to be included otherwise those will be removed at every `poetry install` --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 522f8bf..74d7482 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,12 +32,16 @@ sphinx-autobuild = {version = "^2021.3.14", optional = true} sphinx-autodoc-typehints = {version = "^1.19.2", optional = true} myst-parser = {version = "^0.18.0", optional = true} pydata-sphinx-theme = {version = "^0.9.0", optional = true} +requests = "^2.28.1" +six = "^1.16.0" +nbconvert = "^7.0.0" [tool.poetry.dev-dependencies] coverage = "^6.3.2" mypy = "^0.950" pytest = "^7.1.2" types-toml = "^0.10.8" +pre-commit = "^2.20.0" [tool.poetry.extras] docs = [ @@ -48,5 +52,10 @@ docs = [ "pydata-sphinx-theme", ] +[tool.poetry.group.dev-dependencies.dependencies] +pre-commit = "^2.20.0" +requests = "^2.28.1" +six = "^1.16.0" + [tool.poetry.scripts] pyscript = "pyscript.cli:app" From 77b5d78c2c7558ee3847c72a1b70a9852be383c4 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 10:48:45 +0100 Subject: [PATCH 13/33] Specialised contribution instructions for dev --- CONTRIBUTING.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e7b9b97..8eb2e2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,45 @@ # Contribution guide for developers +## Development + +### Setting up the environment + +To get started, you will need to setup the development environment. This goes +through two main steps, namely [(1) Installing `pyscript-cli` dependencies](#install-pyscript-cli-dependencies) +and [(2) Installing `pyscript-cli` Dev dependencies](#install-the-development-dependencies). + +### Install `pyscript-cli` dependencies + +`pyscript-cli` requires [`poetry`](https://python-poetry.org/) to manage dependencies +and setup the environment. + +Therefore, the first thing to do is to make sure that you have `poetry` installed +in your current Python environment. Please refer to the official `poetry` +[documentation](https://python-poetry.org/docs/master/#installation) for installation +instructions. + +To install `pyscript-cli` dependencies, you will need to run the following command from the project root: + +```shell +poetry install +``` + +### Install the development dependencies + +There are a few extra dependencies that are solely required for development. +To install these packages, you will need to run the following command from the project root: + +```shell +poetry install --with dev-dependencies +``` + +Once all the dependencies are installed, you will only need to setup the git hooks +via [`pre-commit`](https://pre-commit.com/): + +```shell +pre-commit install +``` + ## Documentation ### Install the documentation dependencies From efae1d6058a1a038ae3c91136cca6df4df4239b5 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 11:22:30 +0100 Subject: [PATCH 14/33] Finder results are not optional anymore! Also added missing dependencies --- src/pyscript/_generator.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 0fcf115..2b99e7b 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -1,10 +1,11 @@ +import datetime from pathlib import Path from typing import Optional import jinja2 +import toml -from ._node_parser import find_imports, FinderResult -from ._node_parser import _convert_notebook +from ._node_parser import FinderResult, _convert_notebook, find_imports class UnsupportedFileType(Exception): @@ -33,7 +34,7 @@ def string_to_html( def file_to_html( input_path: Path, title: str, output_path: Optional[Path] -) -> Optional[FinderResult]: +) -> FinderResult: """Write a Python script string to an HTML file template. Warnings will be returned when scanning for environment, if any. @@ -47,12 +48,13 @@ def file_to_html( elif extension == ".ipynb": try: - import nbconvert + import nbconvert as _ # noqa except ImportError as e: # pragma no cover raise ImportError( "Please install nbconvert to serve Jupyter Notebooks." ) from e - source = _convert_notebook(input_path) + else: + source = _convert_notebook(input_path) else: raise UnsupportedFileType( @@ -60,17 +62,12 @@ def file_to_html( ) import_results = find_imports(source, input_path) - with input_path.open("r") as fp: - string_to_html(source, title, output_path, pyenv=import_results) - if import_results.has_warnings: - return import_results + string_to_html(source, title, output_path, pyenv=import_results) + return import_results def create_project( - app_name: str, - app_description: str, - author_name: str, - author_email: str, + app_name: str, app_description: str, author_name: str, author_email: str, ) -> None: """ New files created: From 95591a3d1c9b0e2897e2b293d9546270f17b8ccf Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 11:23:30 +0100 Subject: [PATCH 15/33] Finder results are not optional anymore! Also added missing dependencies --- src/pyscript/_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 2b99e7b..31d31c6 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -48,7 +48,7 @@ def file_to_html( elif extension == ".ipynb": try: - import nbconvert as _ # noqa + import nbconvert # noqa except ImportError as e: # pragma no cover raise ImportError( "Please install nbconvert to serve Jupyter Notebooks." From 2e76c499e65d476b5c0de224f51a38ee10c05128 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 11:25:09 +0100 Subject: [PATCH 16/33] docstring to new Warning typer class and removed useless code from prev. manual merge --- src/pyscript/cli.py | 60 +++------------------------------------------ 1 file changed, 4 insertions(+), 56 deletions(-) diff --git a/src/pyscript/cli.py b/src/pyscript/cli.py index 9e048a2..2f54bbd 100644 --- a/src/pyscript/cli.py +++ b/src/pyscript/cli.py @@ -32,6 +32,10 @@ def __init__(self, msg: str, *args: Any, **kwargs: Any): class Warning(typer.Exit): + """Warning with a consistent error message. + (Similar in essence to the Abort class - it only changes font color to yellow!) + """ + def __init__(self, msg: str, *args: Any, **kwargs: Any): console.print(msg, style="yellow") super().__init__(*args, **kwargs) @@ -53,62 +57,6 @@ def main( pm = PluginManager("pyscript") -# @app.command() -# def wrap( -# input_file: Optional[Path] = _input_file_argument, -# output: Optional[Path] = _output_file_option, -# command: Optional[str] = _command_option, -# show: Optional[bool] = _show_option, -# title: Optional[str] = _title_option, -# ) -> None: -# """Wrap a Python script inside an HTML file.""" -# title = title or "PyScript App" -# -# if not input_file and not command: -# raise Abort( -# "Must provide either an input '.py' file or a command with the '-c' option." -# ) -# if input_file and command: -# raise Abort("Cannot provide both an input '.py' file and '-c' option.") -# -# # Derive the output path if it is not provided -# remove_output = False -# if output is None: -# if command and show: -# output = Path("pyscript_tmp.html") -# remove_output = True -# elif not command: -# assert input_file is not None -# output = input_file.with_suffix(".html") -# else: -# raise Abort("Must provide an output file or use `--show` option") -# -# if input_file is not None: -# parsing_res = file_to_html(input_file, title, output) -# if parsing_res is not None: -# msg_template = "WARNING: The input file contains some imports which are not currently supported PyScript.\nTherefore the code might not work, or require some changes.{packages}{locals}" -# msg = msg_template.format( -# packages=f"\n{str(parsing_res.unsupported_packages)}" -# if parsing_res.unsupported_packages -# else "", -# locals=f"\n{str(parsing_res.unsupported_paths)}" -# if parsing_res.unsupported_paths -# else "", -# ) -# raise Warning(msg=msg) -# -# if command: -# string_to_html(command, title, output) -# -# assert output is not None -# -# if show: -# console.print("Opening in web browser!") -# webbrowser.open(f"file://{output.resolve()}") -# -# if remove_output: -# time.sleep(1) -# output.unlink() pm.add_hookspecs(hookspecs) for modname in DEFAULT_PLUGINS: From 953a8a31fec67361da4b978c5bb38171ffbd2cb5 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 11:25:43 +0100 Subject: [PATCH 17/33] Integrated the new imports/pyenv support in new wrap hook command --- src/pyscript/plugins/wrap.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/pyscript/plugins/wrap.py b/src/pyscript/plugins/wrap.py index e73190d..b06d9fe 100644 --- a/src/pyscript/plugins/wrap.py +++ b/src/pyscript/plugins/wrap.py @@ -52,9 +52,28 @@ def wrap( else: raise cli.Abort("Must provide an output file or use `--show` option") if input_file is not None: - file_to_html(input_file, title, output) + parsing_res = file_to_html(input_file, title, output) + if parsing_res.has_warnings: + warn_msg_template = ( + "WARNING: The input file has some dependencies that are not currently supported " + "in PyScript:{list}\n As a result, the wrapped code may not work." + ) + unsupported_deps = "" + if parsing_res.unsupported_packages: + unsupported_deps += "\n\t -" + "\n\t -".join( + parsing_res.unsupported_packages + ) + + if parsing_res.unsupported_paths: + unsupported_deps += "\n\t -" + "\n\t -".join( + parsing_res.unsupported_paths + ) + + raise cli.Warning(msg=warn_msg_template.format(list=unsupported_deps)) + if command: string_to_html(command, title, output) + if output: if show: console.print("Opening in web browser!") From 1635b2ee554908a692e1053bdb1a4bdfaa731250 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 13:27:55 +0100 Subject: [PATCH 18/33] New tests for wrapping code with imports this commit includes a few general abstractions to ease the testing of cli with multiple py-code snippets. In particular, the default py-code (no imports) has been wrapped into fixture, and all tests changed accordingly to avoid hard-coding py-code. Moreover, a new py_code_with_import fixture has been added to support testing of the new py-env integration feature. A new test has been added, having multiple dependencies dynamically injected via `Dependency` dataclass. --- tests/test_cli.py | 150 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 6 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 115bf94..6db75eb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ from __future__ import annotations import unittest.mock +from collections.abc import Sequence +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Callable, Optional @@ -17,6 +19,17 @@ CLIInvoker = Callable[[VarArg(str)], Result] +@dataclass +class Dependency: + """Data class to encapsulate code dependency dynamically injected in code under test. + This class is a general abstraction over both external and local module dependency.""" + + import_line: str # Import line to inject in code under test + filename: str = None # filename: to be used for tmp local modules, i.e. paths + code: str = None # code in the tmp local modules + inject: str = None # any code to inject in code under test + + @pytest.fixture() def invoke_cli(tmp_path: Path, monkeypatch: "MonkeyPatch") -> CLIInvoker: """Returns a function, which can be used to call the CLI from within a temporary directory.""" @@ -30,6 +43,35 @@ def f(*args: str) -> Result: return f +@pytest.fixture() +def py_code() -> str: + pycode = """ +from math import sqrt + + +def square_root(number: int) -> float: + return sqrt(number) + + +print(f"Square root of 25 is {square_root(25)}") + """ + return pycode + + +@pytest.fixture() +def py_code_with_import() -> str: + pycode = """ +import numpy as np + +from sklearn.datasets import load_iris + +SEED = 12345 +np.random.seed(SEED) +iris = load_iris() + """ + return pycode + + def test_version() -> None: runner = CliRunner() result = runner.invoke(app, "--version") @@ -69,14 +111,14 @@ def test_wrap_abort(invoke_cli: CLIInvoker, wrap_args: tuple[str]): def test_wrap_file( invoke_cli: CLIInvoker, tmp_path: Path, + py_code: str, wrap_args: tuple[str], expected_output_filename: str, ) -> None: - command = 'print("Hello World!")' input_file = tmp_path / "hello.py" with input_file.open("w") as fp: - fp.write(command) + fp.write(py_code) result = invoke_cli("wrap", str(input_file), *wrap_args) assert result.exit_code == 0 @@ -87,7 +129,101 @@ def test_wrap_file( with expected_html_path.open() as fp: html_text = fp.read() - assert f"\n{command}\n" in html_text + assert f"\n{py_code}\n" in html_text + + +@pytest.mark.parametrize( + "dependencies, expected_warnings", + [ + (None, False), + ( + ( + Dependency( + filename="preprocessor.py", + code="from sklearn.preprocessing import StandardScaler\n scaler = StandardScaler()", + import_line="from preprocessor import scaler", + inject="scaler.fit(iris.data)", + ), + ), + False, + ), + ( + ( + Dependency( + filename="preprocessor.py", + code="from sklearn.preprocessing import StandardScaler\n scaler = StandardScaler()", + import_line="import preprocessor", + inject="preprocessor.scaler.fit(iris.data)", + ), + ), + False, + ), + ((Dependency(import_line="from umap import UMAP",),), True,), + ], +) +def test_wrap_file_with_imports( + invoke_cli: CLIInvoker, + tmp_path: Path, + py_code_with_import, + dependencies: Optional[Sequence[Dependency]], + expected_warnings: bool, +) -> None: + + # A. Inject code dependencies, if any + if dependencies: + # inject dependency + for dep in dependencies: + if dep.filename and dep.code: + tmp_mod = tmp_path / dep.filename + with tmp_mod.open("w") as fp: + fp.write(dep.code) + # inject import line in code under test + py_code_with_import = dep.import_line + py_code_with_import + if dep.inject: + py_code_with_import += ( + "\n" + dep.inject + ) # append code to inject to code under test + + input_file = tmp_path / "ml.py" + with input_file.open("w") as fp: + fp.write(py_code_with_import) + + result = invoke_cli("wrap", str(input_file)) + assert result.exit_code == 0 + if expected_warnings: + assert "WARNING" in result.stdout + else: + assert "WARNING" not in result.stdout + + expected_html_path = tmp_path / "ml.html" + assert expected_html_path.exists() + + with expected_html_path.open() as fp: + html_text = fp.read() + + assert f"\n{py_code_with_import}\n" in html_text + assert "" in html_text and "" in html_text + + # get pyenv_tag content + pyenv_tag = html_text[html_text.find("") : html_text.find("")] + pyscript_tag = html_text[ + html_text.find("") : html_text.find("") + ] + + # default py-env dependency in fixture code + assert "numpy" in pyenv_tag + assert "scikit-learn" in pyenv_tag + + if dependencies: + for dep in dependencies: + if dep.filename: + assert "paths" in pyenv_tag + assert dep.filename in pyenv_tag + assert dep.import_line in py_code_with_import + assert dep.import_line in pyscript_tag + if dep.inject: + assert dep.inject in pyscript_tag + assert dep.inject in py_code_with_import @pytest.mark.parametrize( @@ -101,6 +237,7 @@ def test_wrap_file( def test_wrap_show( invoke_cli: CLIInvoker, tmp_path: Path, + py_code: str, input_filename: Optional[str], additional_args: tuple[str, ...], expected_output_filename: Optional[str], @@ -112,7 +249,7 @@ def test_wrap_show( if input_filename: input_file = tmp_path / input_filename with input_file.open("w") as fp: - fp.write('print("Hello World!")') + fp.write(py_code) args = (str(input_file), *additional_args) else: args = additional_args @@ -147,12 +284,13 @@ def test_wrap_show( ) def test_wrap_title( invoke_cli: CLIInvoker, + py_code: str, title: Optional[str], expected_title: str, tmp_path: Path, ) -> None: - command = 'print("Hello World!")' - args = ["wrap", "-c", command, "-o", "output.html"] + command = py_code + args = ["wrap", "-c", py_code, "-o", "output.html"] if title is not None: args.extend(["--title", title]) result = invoke_cli(*args) From 35f064bfac718a7e9f07472518dceb27bba8299f Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 14:26:10 +0100 Subject: [PATCH 19/33] Reformatted to not exceed line length --- tests/test_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6db75eb..6dabae4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -140,7 +140,8 @@ def test_wrap_file( ( Dependency( filename="preprocessor.py", - code="from sklearn.preprocessing import StandardScaler\n scaler = StandardScaler()", + code="from sklearn.preprocessing import StandardScaler\n " + "scaler = StandardScaler()", import_line="from preprocessor import scaler", inject="scaler.fit(iris.data)", ), @@ -151,7 +152,8 @@ def test_wrap_file( ( Dependency( filename="preprocessor.py", - code="from sklearn.preprocessing import StandardScaler\n scaler = StandardScaler()", + code="from sklearn.preprocessing import StandardScaler\n " + "scaler = StandardScaler()", import_line="import preprocessor", inject="preprocessor.scaler.fit(iris.data)", ), From d9c18a18823d6a939690dc446bdf8afa7854d3b3 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 16:41:01 +0100 Subject: [PATCH 20/33] Added ipykernel dep and example in README for notebooks conv. ipykernel dependency is required by nbconvert to convert notebooks into Python code files. --- README.md | 8 +++++++- pyproject.toml | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a4d048b..75e3ba0 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,18 @@ $ pip install pyscript ## Usage -### Embed a Python script into a PyScript HTML file +### Embed Python code into a PyScript HTML file ```shell $ pyscript wrap ``` +Alternatively you could also use a **Jupyter notebook** as input file: + +```shell +$ python wrap +``` + This will generate a file called `` by default. This can be overwritten with the `-o` or `--output` option: diff --git a/pyproject.toml b/pyproject.toml index 74d7482..25ddac8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ pydata-sphinx-theme = {version = "^0.9.0", optional = true} requests = "^2.28.1" six = "^1.16.0" nbconvert = "^7.0.0" +ipykernel = "^6.15.2" [tool.poetry.dev-dependencies] coverage = "^6.3.2" From ec403122372f5bbb9dd24d1068464ae424d9f1f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 15:56:45 +0000 Subject: [PATCH 21/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyscript/_generator.py | 5 ++++- src/pyscript/_node_parser.py | 15 +++++++++------ tests/test_cli.py | 9 ++++++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 31d31c6..31fbe85 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -67,7 +67,10 @@ def file_to_html( def create_project( - app_name: str, app_description: str, author_name: str, author_email: str, + app_name: str, + app_description: str, + author_name: str, + author_email: str, ) -> None: """ New files created: diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 92e3ad5..0434723 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -5,11 +5,11 @@ import ast import os import pkgutil +from collections import defaultdict, namedtuple +from itertools import chain, filterfalse from pathlib import Path -from collections import namedtuple, defaultdict -from itertools import filterfalse, chain -from ._supported_packages import PACKAGE_RENAMES, STANDARD_LIBRARY, PYODIDE_PACKAGES +from ._supported_packages import PACKAGE_RENAMES, PYODIDE_PACKAGES, STANDARD_LIBRARY class NamespaceInfo: @@ -160,7 +160,10 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports(source: str, source_fpath: Path,) -> FinderResult: +def find_imports( + source: str, + source_fpath: Path, +) -> FinderResult: """ Parse the input source, and returns its dependencies, as organised in the sets of external _packages, and local modules, respectively. @@ -176,8 +179,8 @@ def find_imports(source: str, source_fpath: Path,) -> FinderResult: Returns ------- FinderResult - Return the results of parsing as a `FinderResult` instance. - This instance provides reference to packages and paths to + Return the results of parsing as a `FinderResult` instance. + This instance provides reference to packages and paths to include in the py-env, as well as any unsuppoted import. """ diff --git a/tests/test_cli.py b/tests/test_cli.py index 6dabae4..79ee993 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -160,7 +160,14 @@ def test_wrap_file( ), False, ), - ((Dependency(import_line="from umap import UMAP",),), True,), + ( + ( + Dependency( + import_line="from umap import UMAP", + ), + ), + True, + ), ], ) def test_wrap_file_with_imports( From 9cefe98f3a30407ac04538141cf3f573b5fba92e Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 17:05:18 +0100 Subject: [PATCH 22/33] Removed unspotted unused import --- src/pyscript/_node_parser.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 92e3ad5..906ccf5 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -5,16 +5,17 @@ import ast import os import pkgutil +from collections import defaultdict +from itertools import chain, filterfalse from pathlib import Path -from collections import namedtuple, defaultdict -from itertools import filterfalse, chain -from ._supported_packages import PACKAGE_RENAMES, STANDARD_LIBRARY, PYODIDE_PACKAGES +from ._supported_packages import PACKAGE_RENAMES, PYODIDE_PACKAGES, STANDARD_LIBRARY class NamespaceInfo: def __init__(self, source_fpath: Path) -> None: - # expanding base_folder to absolute as pkgutils.FileFinder will do so - easier for later purging + # expanding base_folder to absolute as pkgutils. + # FileFinder will do so - easier for later purging self.base_folder = str(source_fpath.parent.absolute()) self.source_mod_name = source_fpath.stem self._collect() @@ -27,7 +28,8 @@ def _collect(self): for dirname in dirs: iter_modules_paths.append(os.path.join(root, dirname)) - # need to consume generator as I will iterate two times for _packages, and modules + # need to consume generator as I will iterate + # two times for _packages, and modules pkg_mods = tuple(pkgutil.iter_modules(iter_modules_paths)) modules = map( lambda mi: os.path.join(mi.module_finder.path, mi.name), @@ -52,7 +54,10 @@ def __contains__(self, item: str) -> bool: return item in self._all_namespace def __str__(self) -> str: - return f"NameSpace info for {self.base_folder} \n\t Modules: {self.modules} \n\t Packages: {self._packages}" + return ( + f"NameSpace info for {self.base_folder} \n\t " + f"Modules: {self.modules} \n\t Packages: {self._packages}" + ) def __repr__(self) -> str: return str(self) @@ -160,7 +165,10 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports(source: str, source_fpath: Path,) -> FinderResult: +def find_imports( + source: str, + source_fpath: Path, +) -> FinderResult: """ Parse the input source, and returns its dependencies, as organised in the sets of external _packages, and local modules, respectively. @@ -176,9 +184,9 @@ def find_imports(source: str, source_fpath: Path,) -> FinderResult: Returns ------- FinderResult - Return the results of parsing as a `FinderResult` instance. - This instance provides reference to packages and paths to - include in the py-env, as well as any unsuppoted import. + Return the results of parsing as a `FinderResult` instance. + This instance provides reference to packages and paths to + include in the py-env, as well as any unsupported import. """ return _find_modules(source, source_fpath) From 2ffb9eb4688748dcb1bcbee5e6a94627953c295b Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 17:06:45 +0100 Subject: [PATCH 23/33] merge and removed import unused --- src/pyscript/_node_parser.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 906ccf5..93667c2 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -165,10 +165,7 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports( - source: str, - source_fpath: Path, -) -> FinderResult: +def find_imports(source: str, source_fpath: Path,) -> FinderResult: """ Parse the input source, and returns its dependencies, as organised in the sets of external _packages, and local modules, respectively. From 686e44633247739cc1ed15a0b5c31a5c18f3abaa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 16:11:31 +0000 Subject: [PATCH 24/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyscript/_node_parser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 93667c2..906ccf5 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -165,7 +165,10 @@ def _convert_notebook(source_fpath: Path) -> str: return source -def find_imports(source: str, source_fpath: Path,) -> FinderResult: +def find_imports( + source: str, + source_fpath: Path, +) -> FinderResult: """ Parse the input source, and returns its dependencies, as organised in the sets of external _packages, and local modules, respectively. From 92ce5a0294140026bf1a987c518752f10ea7f520 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 17:36:30 +0100 Subject: [PATCH 25/33] ignored pyenv python version --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bcbba9d..56413ba 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ venv/ # Poetry lock file poetry.lock +.python-version # Unit test / coverage reports htmlcov/ From 945535a096fcee5eb7ea0724ee4d24e152f66947 Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 17:43:28 +0100 Subject: [PATCH 26/33] configuring isort to avoid conflicts with black --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 25ddac8..f8b0037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,9 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" +[tool.isort] +profile = "black" + [tool.poetry] name = "pyscript" version = "0.2.4" From 19d753f1a96cd7d13960795d3530f2aba87105be Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 17:55:34 +0100 Subject: [PATCH 27/33] Upgrading poetry-core dep to see if tests pass REF: https://github.com/python-poetry/poetry/issues/4983 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8b0037..e7ed160 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.1.0a6"] build-backend = "poetry.core.masonry.api" [tool.isort] From 072c2026f7007ce694c288a55d88250323529e6e Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 17:58:07 +0100 Subject: [PATCH 28/33] Upgrading poetry-core dep to see if tests pass REF: https://github.com/python-poetry/poetry/issues/4983 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7ed160..73ad698 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["poetry-core>=1.1.0a6"] +requires = ["poetry-core>=1.2.0"] build-backend = "poetry.core.masonry.api" [tool.isort] From 8466380a18d98a03db3c031246d852ce84faaf6e Mon Sep 17 00:00:00 2001 From: Valerio Maggio Date: Wed, 7 Sep 2022 18:01:47 +0100 Subject: [PATCH 29/33] Upgrading poetry-core dep to see if tests pass REF: https://github.com/python-poetry/poetry/issues/4983 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 73ad698..7fc8b8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["poetry-core>=1.2.0"] +requires = ["poetry-core>=1.1.0b3"] build-backend = "poetry.core.masonry.api" [tool.isort] From 1e9628086e0604d2c2da960f079eb395170d2019 Mon Sep 17 00:00:00 2001 From: Matt Kramer Date: Wed, 7 Sep 2022 12:40:32 -0500 Subject: [PATCH 30/33] Comment out invalid poetry dependency section --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7fc8b8c..ade96fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,10 @@ docs = [ "pydata-sphinx-theme", ] -[tool.poetry.group.dev-dependencies.dependencies] -pre-commit = "^2.20.0" -requests = "^2.28.1" -six = "^1.16.0" +# [tool.poetry.group.dev-dependencies.dependencies] +# pre-commit = "^2.20.0" +# requests = "^2.28.1" +# six = "^1.16.0" [tool.poetry.scripts] pyscript = "pyscript.cli:app" From 40751a9f563aaab5b5979aba136f4a91abd4a935 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:40:40 +0000 Subject: [PATCH 31/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ade96fc..be6803c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,5 @@ docs = [ "pydata-sphinx-theme", ] -# [tool.poetry.group.dev-dependencies.dependencies] -# pre-commit = "^2.20.0" -# requests = "^2.28.1" -# six = "^1.16.0" - [tool.poetry.scripts] pyscript = "pyscript.cli:app" From fa93160b422bf0d8036029469039b9c956a4b455 Mon Sep 17 00:00:00 2001 From: Matt Kramer Date: Wed, 7 Sep 2022 12:49:11 -0500 Subject: [PATCH 32/33] Fix type hings --- src/pyscript/_node_parser.py | 7 ++++--- tests/test_cli.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index 906ccf5..b1369da 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -8,6 +8,7 @@ from collections import defaultdict from itertools import chain, filterfalse from pathlib import Path +from typing import DefaultDict from ._supported_packages import PACKAGE_RENAMES, PYODIDE_PACKAGES, STANDARD_LIBRARY @@ -65,9 +66,9 @@ def __repr__(self) -> str: class FinderResult: def __init__(self) -> None: - self._packages = set() - self._locals = set() - self._unsupported = defaultdict(set) + self._packages: set[str] = set() + self._locals: set[str] = set() + self._unsupported: DefaultDict[str, set] = defaultdict(set) def add_package(self, pkg_name: str) -> None: self._packages.add(pkg_name) diff --git a/tests/test_cli.py b/tests/test_cli.py index 79ee993..7ed545e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,9 +25,9 @@ class Dependency: This class is a general abstraction over both external and local module dependency.""" import_line: str # Import line to inject in code under test - filename: str = None # filename: to be used for tmp local modules, i.e. paths - code: str = None # code in the tmp local modules - inject: str = None # any code to inject in code under test + filename: str | None = None # filename: to be used for tmp local modules, i.e. paths + code: str | None = None # code in the tmp local modules + inject: str | None = None # any code to inject in code under test @pytest.fixture() From 29bfdda55ca375c9ec8c9d5cde55886605ca92b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:49:20 +0000 Subject: [PATCH 33/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyscript/_node_parser.py | 5 +++-- tests/test_cli.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pyscript/_node_parser.py b/src/pyscript/_node_parser.py index b1369da..90fb48e 100644 --- a/src/pyscript/_node_parser.py +++ b/src/pyscript/_node_parser.py @@ -2,13 +2,14 @@ ast-based parser to gather modules/package dependencies of a Python module. Code adapted from the find-imports project, currently in graveyard archive. """ +from __future__ import annotations + import ast import os import pkgutil from collections import defaultdict from itertools import chain, filterfalse from pathlib import Path -from typing import DefaultDict from ._supported_packages import PACKAGE_RENAMES, PYODIDE_PACKAGES, STANDARD_LIBRARY @@ -68,7 +69,7 @@ class FinderResult: def __init__(self) -> None: self._packages: set[str] = set() self._locals: set[str] = set() - self._unsupported: DefaultDict[str, set] = defaultdict(set) + self._unsupported: defaultdict[str, set] = defaultdict(set) def add_package(self, pkg_name: str) -> None: self._packages.add(pkg_name) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7ed545e..055fd19 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,7 +25,9 @@ class Dependency: This class is a general abstraction over both external and local module dependency.""" import_line: str # Import line to inject in code under test - filename: str | None = None # filename: to be used for tmp local modules, i.e. paths + filename: str | None = ( + None # filename: to be used for tmp local modules, i.e. paths + ) code: str | None = None # code in the tmp local modules inject: str | None = None # any code to inject in code under test