CI: export env details (#16)

* export env details
* ci integrate
* minor format

Author: Borda

.actions/helpers.py

@@ -1,12 +1,17 @@
 import glob
 import os
 import shutil
+from datetime import datetime
 from pprint import pprint
 
 import fire
 import tqdm
 import yaml
+from pip._internal.operations import freeze
 
+PATH_HERE = os.path.dirname(__file__)
+PATH_ROOT = os.path.dirname(PATH_HERE)
+PATH_REQ_DEFAULT = os.path.join(PATH_ROOT, "requirements", "default.txt")
 REPO_NAME = "lightning-tutorials"
 DEFAULT_BRANCH = "main"
 TEMPLATE_HEADER = f"""
@@ -64,15 +69,20 @@
 class HelperCLI:
 
     SKIP_DIRS = (
-        "docs",
         ".actions",
         ".github",
+        ".notebooks",
+        "docs",
     )
     META_FILE = ".meta.yml"
     REQUIREMENTS_FILE = "requirements.txt"
 
     @staticmethod
-    def expand_script(fpath: str):
+    def enrich_script(fpath: str):
+        """Add template header and footer to the python base script.
+        Args:
+            fpath: path to python script
+        """
         with open(fpath, "r") as fp:
             py_file = fp.readlines()
         fpath_meta = os.path.join(os.path.dirname(fpath), HelperCLI.META_FILE)
@@ -94,7 +104,6 @@ def group_folders(
         fpath_drop_folders: str = "dropped-folders.txt",
     ) -> None:
         """Group changes by folders
-
         Args:
             fpath_gitdiff: raw git changes
 
@@ -125,8 +134,8 @@ def group_folders(
     @staticmethod
     def parse_requirements(dir_path: str):
         """Parse standard requirements from meta file
-
-        :param dir_path: path to the folder
+        Args:
+            dir_path: path to the folder
         """
         fpath = os.path.join(dir_path, HelperCLI.META_FILE)
         assert os.path.isfile(fpath)
@@ -141,6 +150,11 @@ def parse_requirements(dir_path: str):
 
     @staticmethod
     def copy_notebooks(path_root: str, path_docs_ipynb: str = "docs/source/notebooks"):
+        """Copy all notebooks from a folder to doc folder.
+        Args:
+            path_root: source path to the project root in this tutorials
+            path_docs_ipynb: destination path to the notebooks location
+        """
         ls_ipynb = []
         for sub in (['*.ipynb'], ['**', '*.ipynb']):
             ls_ipynb += glob.glob(os.path.join(path_root, '.notebooks', *sub))
@@ -159,8 +173,8 @@ def copy_notebooks(path_root: str, path_docs_ipynb: str = "docs/source/notebooks
     @staticmethod
     def valid_accelerator(dir_path: str):
         """Parse standard requirements from meta file
-
-        :param dir_path: path to the folder
+        Args:
+            dir_path: path to the folder
         """
         fpath = os.path.join(dir_path, HelperCLI.META_FILE)
         assert os.path.isfile(fpath)
@@ -170,6 +184,35 @@ def valid_accelerator(dir_path: str):
         os_acc = os.environ.get("ACCLERATOR", 'cpu')
         return int(os_acc in accels)
 
+    @staticmethod
+    def update_env_details(dir_path: str):
+        """Export the actial packages used in runtime
+        Args:
+             dir_path: path to the folder
+        """
+        fpath = os.path.join(dir_path, HelperCLI.META_FILE)
+        assert os.path.isfile(fpath)
+        meta = yaml.safe_load(open(fpath))
+        # default is COU runtime
+        with open(PATH_REQ_DEFAULT) as fp:
+            req = fp.readlines()
+        req += meta.get('requirements', [])
+        req = [r.strip() for r in req]
+
+        def _parse(pkg: str, keys: str = " <=>") -> str:
+            """Parsing just the package name"""
+            if any(c in pkg for c in keys):
+                ix = min([pkg.index(c) for c in keys if c in pkg])
+                pkg = pkg[:ix]
+            return pkg
+
+        require = set([_parse(r) for r in req if r])
+        env = {_parse(p): p for p in freeze.freeze()}
+        meta['environment'] = [env[r] for r in require]
+        meta['updated'] = datetime.now().isoformat()
+
+        yaml.safe_dump(meta, stream=open(fpath, 'w'))
+
 
 if __name__ == '__main__':
     fire.Fire(HelperCLI)

.actions/ipynb-generate.sh

@@ -9,6 +9,6 @@ python -c "import os ; assert(os.path.isfile(os.path.join('$1', '.meta.yml')))"
 py_file=( $(ls "$1"/*.py) )
 echo $py_file
 
-python .actions/helpers.py expand_script $py_file
+python .actions/helpers.py enrich-script $py_file
 
 jupytext --set-formats "ipynb,py:percent" $py_file

.actions/ipynb-render.sh

@@ -17,13 +17,15 @@ pip install --quiet --requirement requirements.txt
 cat "$1/requirements.txt"
 pip install --requirement "$1/requirements.txt"
 
-accel=$(python .actions/helpers.py valid_accelerator $1 2>&1)
+accel=$(python .actions/helpers.py valid-accelerator $1 2>&1)
 if [ $accel -eq 1 ]
 then
   papermill $ipynb_file $pub_file
+  python .actions/helpers.py update-env-details $1
 else
   echo "WARNING: not valid accelerator so no outputs will be generated"
   cp $ipynb_file $pub_file
 fi
 
+git add "$1/.meta.yml"
 git add $pub_file

.actions/requirements.txt

@@ -1,3 +1,4 @@
 Fire
 tqdm
 PyYAML
+pip

.pre-commit-config.yaml

@@ -13,7 +13,6 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
       - id: check-yaml
-      - id: pretty-format-json
 
   - repo: https://github.com/PyCQA/isort
     rev: 5.8.0

requirements.txt

@@ -1,13 +1,5 @@
-virtualenv
-ipython[notebook]
-jupytext  # converting
-treon  # testing
-papermill  # render
-yapf
-flake8
-isort
-
+-r requirements/devel.txt
 -r .actions/requirements.txt
 
 # default for all examples
-pytorch-lightning>=1.2
+-r requirements/default.txt

requirements/default.txt

@@ -0,0 +1,2 @@
+pytorch-lightning>=1.2
+torch

requirements/devel.txt

@@ -0,0 +1,8 @@
+virtualenv
+ipython[notebook]
+jupytext  # converting
+treon  # testing
+papermill  # render
+yapf
+flake8
+isort

starters/datamodules/.meta.yml

@@ -1,7 +1,7 @@
 title: PyTorch Lightning DataModules
 author: PL team
 created: 2021-01-31
-updated: 2021-02-22
+updated: 2021-02-21
 license: CC
 description: |
   This notebook will walk you through how to start using Datamodules.