Merge remote-tracking branch 'origin/main' into publication

Author: Azure Pipelines

.actions/assistant.py

@@ -113,7 +113,7 @@ def get_running_cuda_version() -> str:
         return ""
 
 
-def get_running_torch_version():
+def get_running_torch_version() -> str:
     """Extract the version of actual PyTorch for this runtime."""
     try:
         import torch
@@ -154,7 +154,7 @@ class AssistantCLI:
         "lightning_examples": "Lightning-Examples",
         "flash_tutorials": "Kaggle",
     }
-    _BASH_SCRIPT_BASE = ("#!/bin/bash", "set -e", "")
+    _BASH_SCRIPT_BASE = ("#!/bin/bash", "set -ex", "")
     _EXT_ARCHIVE_ZIP = (".zip",)
     _EXT_ARCHIVE_TAR = (".tar", ".gz")
     _EXT_ARCHIVE = _EXT_ARCHIVE_ZIP + _EXT_ARCHIVE_TAR
@@ -167,6 +167,7 @@ def _find_meta(folder: str) -> str:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         files = glob.glob(os.path.join(folder, AssistantCLI._META_FILE_REGEX), flags=glob.BRACE)
         if len(files) == 1:
@@ -180,6 +181,7 @@ def _load_meta(folder: str, strict: bool = False) -> Optional[dict]:
         Args:
             folder: path to the folder with python script, meta and artefacts
             strict: raise error if meta is missing required fields
+
         """
         fpath = AssistantCLI._find_meta(folder)
         assert fpath, f"Missing meta file in folder: {folder}"
@@ -197,6 +199,7 @@ def _valid_conf_folder(folder: str) -> Tuple[str, str]:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta_files = [os.path.join(folder, f".meta.{ext}") for ext in ("yml", "yaml")]
         meta_files = [pf for pf in meta_files if os.path.isfile(pf)]
@@ -217,6 +220,7 @@ def _valid_folder(folder: str, ext: str) -> Tuple[str, str, str]:
         Args:
             folder: path to the folder with python script, meta and artefacts
             ext: extension determining the stage - ".py" for python script nad ".ipynb" for notebook
+
         """
         files = glob.glob(os.path.join(folder, f"*{ext}"))
         if len(files) != 1:
@@ -231,6 +235,7 @@ def _valid_accelerator(folder: str) -> bool:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta = AssistantCLI._load_meta(folder)
         meta_accels = [acc.lower() for acc in meta.get("accelerator", AssistantCLI._META_ACCEL_DEFAULT)]
@@ -243,6 +248,7 @@ def _parse_requirements(folder: str) -> Tuple[str, str]:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta = AssistantCLI._load_meta(folder)
         reqs = meta.get("requirements", [])
@@ -268,6 +274,7 @@ def _bash_download_data(folder: str) -> List[str]:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta = AssistantCLI._load_meta(folder)
         datasets = meta.get("datasets", {})
@@ -299,6 +306,7 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
 
         Returns:
             string with nash script content
+
         """
         cmd = list(AssistantCLI._BASH_SCRIPT_BASE) + [f"# Rendering: {folder}"]
         if not AssistantCLI.DRY_RUN:
@@ -314,7 +322,13 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             # dry run does not execute the notebooks just takes them as they are
             cmd.append(f"cp {ipynb_file} {pub_ipynb}")
             # copy and add meta config
-            cmd += [f"cp {meta_file} {pub_meta}", f"cat {pub_meta}", f"git add {pub_meta}"]
+            cmd += [
+                f"cp {meta_file} {pub_meta}",
+                'echo "#====== START OF YAML FILE ======#"',
+                f"cat {pub_meta}",
+                'echo "#======= END OF YAML FILE =======#"',
+                f"git add {pub_meta}",
+            ]
         else:
             pip_req, pip_args = AssistantCLI._parse_requirements(folder)
             cmd += [f"pip install {pip_req} --quiet {pip_args}", "pip list"]
@@ -327,7 +341,13 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             # Export the actual packages used in runtime
             cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder})")
             # copy and add to version the enriched meta config
-            cmd += ["echo $meta_file", "cat $meta_file", "git add $meta_file"]
+            cmd += [
+                "echo $meta_file",
+                'echo "#====== START OF YAML FILE ======#"',
+                "cat $meta_file",
+                'echo "#======= END OF YAML FILE =======#"',
+                "git add $meta_file",
+            ]
         # if thumb image is linked to the notebook, copy and version it too
         if thumb_file:
             cmd += [f"cp {thumb_file} {pub_thumb}", f"git add {pub_thumb}"]
@@ -339,7 +359,7 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             fopen.write(os.linesep.join(cmd))
 
     @staticmethod
-    def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]:
+    def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST, virtualenv: bool = False) -> Optional[str]:
         """Prepare bash script for running tests of a particular notebook.
 
         Args:
@@ -348,18 +368,20 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]
 
         Returns:
             string with nash script content
+
         """
         cmd = list(AssistantCLI._BASH_SCRIPT_BASE) + [f"# Testing: {folder}"]
         cmd += AssistantCLI._bash_download_data(folder)
         ipynb_file, meta_file, _ = AssistantCLI._valid_folder(folder, ext=".ipynb")
 
         # prepare isolated environment with inheriting the global packages
         path_venv = os.path.join(folder, "venv")
-        cmd += [
-            f"python -m virtualenv --system-site-packages {path_venv}",
-            f"source {os.path.join(path_venv, 'bin', 'activate')}",
-            "pip --version",
-        ]
+        if virtualenv:
+            cmd += [
+                f"python -m virtualenv --system-site-packages {path_venv}",
+                f"source {os.path.join(path_venv, 'bin', 'activate')}",
+                "pip --version",
+            ]
 
         cmd.append(f"# available: {AssistantCLI.DEVICE_ACCELERATOR}")
         if AssistantCLI._valid_accelerator(folder):
@@ -369,21 +391,30 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]
             # Export the actual packages used in runtime
             cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder} --base_path .)")
             # show created meta config
-            cmd += ["echo $meta_file", "cat $meta_file"]
-            cmd.append(f"python -m pytest {ipynb_file} -v --nbval --nbval-cell-timeout=300")
+            cmd += [
+                "echo $meta_file",
+                'echo "#====== START OF YAML FILE ======#"',
+                "cat $meta_file",
+                'echo "#======= END OF YAML FILE =======#"',
+            ]
+            # use standard jupyter's executable via CMD
+            cmd.append(f"jupyter execute {ipynb_file} --inplace")
         else:
             pub_ipynb = os.path.join(DIR_NOTEBOOKS, f"{folder}.ipynb")
             pub_meta = pub_ipynb.replace(".ipynb", ".yaml")
             # copy and add meta config
             cmd += [
                 f"mkdir -p {os.path.dirname(pub_meta)}",
                 f"cp {meta_file} {pub_meta}",
+                'echo "#====== START OF YAML FILE ======#"',
                 f"cat {pub_meta}",
+                'echo "#======= END OF YAML FILE =======#"',
                 f"git add {pub_meta}",
             ]
             warn("Invalid notebook's accelerator for this device. So no tests will be run!!!", RuntimeWarning)
         # deactivate and clean local environment
-        cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"]
+        if virtualenv:
+            cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"]
         if not output_file:
             return os.linesep.join(cmd)
         with open(output_file, "w") as fopen:
@@ -395,6 +426,7 @@ def convert_ipynb(folder: str) -> None:
 
         Args:
             folder: folder with python script
+
         """
         fpath, _, _ = AssistantCLI._valid_folder(folder, ext=".py")
         with open(fpath) as fopen:
@@ -426,6 +458,7 @@ def _replace_images(lines: list, local_dir: str) -> list:
         Args:
             lines: string lines from python script
             local_dir: relative path to the folder with script
+
         """
         md = os.linesep.join([ln.rstrip() for ln in lines])
         p_imgs = []
@@ -488,6 +521,7 @@ def group_folders(
         Example:
             $ python assistant.py group-folders ../target-diff.txt \
                 --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
+
         """
         with open(fpath_gitdiff) as fopen:
             changed = [ln.strip() for ln in fopen.readlines()]
@@ -534,6 +568,7 @@ def generate_matrix(fpath_change_folders: str, json_indent: Optional[int] = None
         Args:
             fpath_change_folders: output of previous ``group_folders``
             json_indent: makes the json more readable, recommendation is 4
+
         """
         with open(fpath_change_folders) as fopen:
             folders = [ln.strip() for ln in fopen.readlines()]
@@ -623,6 +658,7 @@ def copy_notebooks(
             path_docs_images: destination path to the images' location relative to ``docs_root``
             patterns: patterns to use when glob-ing notebooks
             ignore: ignore some specific notebooks even when the given string is in path
+
         """
         all_ipynb = []
         for pattern in patterns:
@@ -693,7 +729,11 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str:
 
         Args:
              folder: path to the folder
-             base_path:
+             base_path: base path with notebooks
+
+        Returns:
+            path the updated YAML file
+
         """
         meta = AssistantCLI._load_meta(folder)
         # default is COU runtime

.azure/ipynb-publish.yml

@@ -24,6 +24,7 @@ jobs:
           printf "commit hash:\n $(COMMIT_HASH)\n"
           printf "commit message:\n $(COMMIT_MSG)\n"
         displayName: "Set Git user"
+        timeoutInMinutes: "5"
       - bash: |
           set -e
           git fetch --all
@@ -59,14 +60,18 @@ jobs:
           git commit -m "prune: $(COMMIT_HASH)"
         condition: gt(variables['dropped.folders'], 0)
         displayName: "Prune notebook"
+        timeoutInMinutes: "10"
 
       - bash: |
+          set -ex
           git status
           git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH)
-        displayName: "Finish push"
+        displayName: "Finish with push"
 
   - job: papermill
     dependsOn: sync_pub
+    # run if the initial job succeeded and the strategy matrix is not empty
+    condition: and(succeeded(), ne(dependencies.sync_pub.outputs['mtrx.dirs'], ''))
     strategy:
       # generated matrix with changed notebooks, include fields: "notebook", "agent-pool" and "docker-image"
       matrix: $[ dependencies.sync_pub.outputs['mtrx.dirs'] ]
@@ -94,8 +99,6 @@ jobs:
       COMMIT_HASH: "$(Build.SourceVersion)"
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
 
-    condition: ne(dependencies.sync_pub.outputs['mtrx.dirs'], '')
-
     steps:
       - bash: |
           echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
@@ -118,6 +121,7 @@ jobs:
           printf "commit hash:\n $(COMMIT_HASH)\n"
           printf "commit message:\n $(COMMIT_MSG)\n"
         displayName: "Set Git user"
+        timeoutInMinutes: "5"
       - bash: |
           set -e
           git fetch --all
@@ -128,6 +132,7 @@ jobs:
           git show-ref $(PUB_BRANCH)
           git pull
         displayName: "Git check & switch branch"
+        timeoutInMinutes: "5"
 
       - bash: |
           set -e
@@ -136,6 +141,7 @@ jobs:
           # todo: adjust torch ecosystem versions
           pip install -r requirements.txt -r _requirements/data.txt
         displayName: "Install dependencies"
+        timeoutInMinutes: "15"
 
       - bash: |
           set -e
@@ -146,6 +152,7 @@ jobs:
 
       - bash: python .actions/assistant.py convert-ipynb $(notebook)
         displayName: "Generate notebook"
+        timeoutInMinutes: "5"
 
       - bash: |
           set -e
@@ -161,7 +168,8 @@ jobs:
         displayName: "Render notebook"
 
       - bash: |
+          set -ex
           git status
           git show-ref $(PUB_BRANCH)
           git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH)
-        displayName: "Finish push"
+        displayName: "Finish with push"

.azure/ipynb-tests.yml .azure/ipynb-validate.yml

@@ -14,14 +14,18 @@ jobs:
       vmImage: "Ubuntu-20.04"
     steps:
       - bash: |
+          set -ex
           pip install -r .actions/requires.txt
           pip list
         displayName: "Install dependencies"
 
       - bash: |
-          head=$(git rev-parse origin/main)
-          printf "Head: $head\n"
-          git diff --name-only $head --output=target-diff.txt
+          git fetch --all  # some issues with missing main :/
+          # head=$(git rev-parse origin/main)
+          # printf "Head: $head\n"  # this shall be commit hash
+          # git diff --name-only $head --output=target-diff.txt
+          git diff --name-only origin/main HEAD --output=target-diff.txt
+          cat target-diff.txt
           python .actions/assistant.py group-folders --fpath_gitdiff=target-diff.txt
           printf "Changed folders:\n"
           cat changed-folders.txt
@@ -35,8 +39,10 @@ jobs:
       - bash: echo '$(mtrx.dirs)' | python -m json.tool
         displayName: "Show matrix"
 
-  - job: nbval
+  - job: ipython
     dependsOn: check_diff
+    # run if the initial job succeeded and the strategy matrix is not empty
+    condition: and(succeeded(), ne(dependencies.check_diff.outputs['mtrx.dirs'], ''))
     strategy:
       matrix: $[ dependencies.check_diff.outputs['mtrx.dirs'] ]
     # how long to run the job before automatically cancelling
@@ -55,8 +61,6 @@ jobs:
       PATH_DATASETS: "$(Build.Repository.LocalPath)/.datasets"
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
 
-    condition: ne(dependencies.check_diff.outputs['mtrx.dirs'], '')
-
     steps:
       - bash: |
           echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
@@ -96,4 +100,4 @@ jobs:
         env:
           KAGGLE_USERNAME: $(KAGGLE_USERNAME)
           KAGGLE_KEY: $(KAGGLE_KEY)
-        displayName: "PyTest notebook"
+        displayName: "Execute notebook"

.github/dependabot.yml

@@ -16,7 +16,7 @@ updates:
     # Allow up to 5 open pull requests for pip dependencies
     open-pull-requests-limit: 5
     reviewers:
-      - "Lightning-AI/teams/core-lightning"
+      - "Borda"
 
   - package-ecosystem: "github-actions"
     directory: "/"
@@ -27,4 +27,4 @@ updates:
       separator: "-"
     open-pull-requests-limit: 5
     reviewers:
-      - "Lightning-AI/core-lightning"
+      - "Borda"

.github/label-change.yml

@@ -1,8 +1,12 @@
-documentation:
-  - _docs/**/*
+"topic: documentation":
+  - changed-files:
+      - any-glob-to-any-file:
+          - _docs/**/*
 
-CI/CD:
-  - .actions/**/*
-  - .azure-*/**/*
-  - .github/**/*
-  - _dockers/**/*
+"topic: CI/CD":
+  - changed-files:
+      - any-glob-to-any-file:
+          - .actions/**/*
+          - .azure-*/**/*
+          - .github/**/*
+          - _dockers/**/*