Add tutorial view support (#72)

ethanwharris · pre-commit-ci[bot] · web-flow · commit 274e39c6e57a · 2021-08-27T15:06:34.000Z
* Try something
* Make tags no longer required
* Remove extensions
* Clean

Co-authored-by: pre-commit-ci[bot] &lt;66853113+pre-commit-ci[bot]@users.noreply.github.com&gt;
diff --git a/.actions/helpers.py b/.actions/helpers.py
@@ -1,10 +1,11 @@
 import base64
+import json
 import os
 import re
-import shutil
 from datetime import datetime
 from pprint import pprint
-from typing import Sequence
+from textwrap import wrap
+from typing import Any, Dict, Sequence
 from warnings import warn
 
 import fire
@@ -80,6 +81,12 @@
 # ![Pytorch Lightning](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/docs/source/_static/images/logo.png){height="60px" width="240px"}
 
 """
+TEMPLATE_CARD_ITEM = """
+.. customcarditem::
+   :header: %(title)s
+   :card_description: %(short_description)s
+   :tags: %(tags)s
+"""
 
 
 def default_requirements(path_req: str = PATH_REQ_DEFAULT) -> list:
@@ -163,6 +170,7 @@ def augment_script(fpath: str):
             dict(local_ipynb=f"{os.path.dirname(fpath)}.ipynb"),
             generated=datetime.now().isoformat(),
         )
+
         meta['description'] = meta['description'].replace(os.linesep, f"{os.linesep}# ")
 
         header = TEMPLATE_HEADER % meta
@@ -311,6 +319,43 @@ def parse_requirements(dir_path: str):
         with open(fname, "w") as fp:
             fp.write(" ".join(cmd_args))
 
+    @staticmethod
+    def _get_card_item_cell(path_ipynb: str) -> Dict[str, Any]:
+        """Build the card item cell for the given notebook path."""
+        fpath_meta = path_ipynb.replace(".ipynb", ".yaml")
+        meta = yaml.safe_load(open(fpath_meta))
+
+        # Clamp description length
+        wrapped_description = wrap(
+            meta.get('short_description', meta['description']).strip().replace(os.linesep, " "), 175
+        )
+        suffix = "..." if len(wrapped_description) > 1 else ""
+        meta['short_description'] = wrapped_description[0] + suffix
+
+        # Resolve some default tags based on accelerators and directory name
+        meta['tags'] = meta.get('tags', [])
+
+        accelerators = meta.get("accelerator", ('CPU', ))
+        if ('GPU' in accelerators) or ('TPU' in accelerators):
+            meta['tags'].append('GPU/TPU')
+
+        dirname = os.path.basename(os.path.dirname(path_ipynb))
+        if dirname != ".notebooks":
+            meta['tags'].append(dirname)
+
+        meta['tags'] = ",".join(meta['tags'])
+
+        # Build the notebook cell
+        rst_cell = TEMPLATE_CARD_ITEM % meta
+
+        return {
+            "cell_type": "raw",
+            "metadata": {
+                "raw_mimetype": "text/restructuredtext"
+            },
+            "source": rst_cell.strip().splitlines(True)
+        }
+
     @staticmethod
     def copy_notebooks(path_root: str, path_docs_ipynb: str = "docs/source/notebooks"):
         """Copy all notebooks from a folder to doc folder.
@@ -331,7 +376,15 @@ def copy_notebooks(path_root: str, path_docs_ipynb: str = "docs/source/notebooks
             new_ipynb = os.path.join(path_docs_ipynb, sub_ipynb)
             os.makedirs(os.path.dirname(new_ipynb), exist_ok=True)
             print(f'{path_ipynb} -> {new_ipynb}')
-            shutil.copy(path_ipynb, new_ipynb)
+
+            with open(path_ipynb) as f:
+                ipynb = json.load(f)
+
+            ipynb["cells"].append(HelperCLI._get_card_item_cell(path_ipynb))
+
+            with open(new_ipynb, 'w') as f:
+                json.dump(ipynb, f)
+
             ipynb_content.append(os.path.join('notebooks', sub_ipynb))
 
     @staticmethod
diff --git a/.actions/ipynb-render.sh b/.actions/ipynb-render.sh
@@ -7,9 +7,16 @@ python -c "import glob ; assert(len(glob.glob('$1/*.ipynb')) == 1)"
 ipynb_file=( $(ls "$1"/*.ipynb) )
 printf $ipynb_file
 
+python -c "import glob ; assert(len(glob.glob('$1/.meta.{yaml,yml}')) == 1)"
+meta_file=( $(ls "$1"/.meta.*) )
+printf $meta_file
+
 pub_file=".notebooks/$1.ipynb"
 printf $pub_file
 
+pub_meta_file=".notebooks/$1.yaml"
+printf $pub_meta_file
+
 pub_dir=$(dirname "$pub_file")
 mkdir -p $pub_dir
 
@@ -27,10 +34,12 @@ then
   printf "Processing: $ipynb_file\n"
   python -m papermill.cli $ipynb_file $pub_file --kernel python
   python .actions/helpers.py update-env-details $1
-  git add ".notebooks/$1.yaml"
 else
   printf "WARNING: not valid accelerator so no outputs will be generated.\n"
   cp $ipynb_file $pub_file
 fi
 
+cp $meta_file $pub_meta_file
+git add ".notebooks/$1.yaml"
+
 git add $pub_file
diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml
@@ -62,12 +62,14 @@ jobs:
           while IFS= read -r line; do
               bash .actions/ipynb-generate.sh $line
               ipynb=( $(ls "$line"/*.ipynb) )
+              meta=( $(ls "$line"/.meta.*) )
               # create folder of it is missing
               pub_dir=$(dirname ".notebooks/$line")
               printf "Dir: $pub_dir"
               mkdir -p $pub_dir
               # simulate render as somple move
               mv $ipynb ".notebooks/$line.ipynb"
+              mv $meta ".notebooks/$line.yaml"
           done <<< $(cat changed-folders.txt)
         shell: bash
 
@@ -76,3 +78,11 @@ jobs:
         run: |
           # First run the same pipeline as Read-The-Docs
           make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going"
+
+      - name: Upload built docs
+        uses: actions/upload-artifact@v2
+        with:
+          name: docs-results-${{ github.sha }}
+          path: docs/build/html/
+        # Use always() to always run this step to publish test results when there are test failures
+        if: success()
diff --git a/.gitignore b/.gitignore
@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+.idea/
diff --git a/course_UvA-DL/autoregressive-image-modeling/.meta.yml b/course_UvA-DL/autoregressive-image-modeling/.meta.yml
@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-07-12
 updated: 2021-07-12
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we implement an autoregressive likelihood model for the task of image modeling.
   Autoregressive models are naturally strong generative models that constitute one of the current
diff --git a/course_UvA-DL/deep-autoencoders/.meta.yml b/course_UvA-DL/deep-autoencoders/.meta.yml
@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-07-12
 updated: 2021-07-12
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we will take a closer look at autoencoders (AE).
   Autoencoders are trained on encoding input data such as images into a smaller feature vector,
diff --git a/course_UvA-DL/deep-energy-based-generative-models/.meta.yml b/course_UvA-DL/deep-energy-based-generative-models/.meta.yml
@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-07-12
 updated: 2021-07-12
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we will look at energy-based deep learning models, and focus on their application as generative models.
   Energy models have been a popular tool before the huge deep learning hype around 2012 hit.
diff --git a/course_UvA-DL/graph-neural-networks/.meta.yml b/course_UvA-DL/graph-neural-networks/.meta.yml
@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-06-07
 updated: 2021-06-16
 license: CC BY-SA
+tags:
+  - Graph
 description: |
   In this tutorial, we will discuss the application of neural networks on graphs.
   Graph Neural Networks (GNNs) have recently gained increasing popularity in both applications and research,
diff --git a/course_UvA-DL/normalizing-flows/.meta.yml b/course_UvA-DL/normalizing-flows/.meta.yml
@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-06-07
 updated: 2021-06-16
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we will take a closer look at complex, deep normalizing flows.
   The most popular, current application of deep normalizing flows is to model datasets of images.
diff --git a/course_UvA-DL/transformers-and-MH-attention/.meta.yml b/course_UvA-DL/transformers-and-MH-attention/.meta.yml
@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-06-30
 updated: 2021-06-30
 license: CC BY-SA
+tags:
+  - Text
 description: |
   In this tutorial, we will discuss one of the most impactful architectures of the last 2 years: the Transformer model.
   Since the paper Attention Is All You Need by Vaswani et al. had been published in 2017,
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -66,6 +66,7 @@
     'nbsphinx',
     'myst_parser',
     'sphinx_paramlinks',
+    'pt_lightning_sphinx_theme.extensions.lightning_tutorials',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -5,6 +5,13 @@
 
 Lightning-Sandbox documentation
 ===============================
+
+.. tutoriallist::
+
+.. raw:: html
+
+   <div style="display:none">
+
 .. toctree::
    :maxdepth: 1
    :name: start
@@ -14,6 +21,9 @@ Lightning-Sandbox documentation
    notebooks/*
    notebooks/**/*
 
+.. raw:: html
+
+   </div>
 
 Indices and tables
 ==================
diff --git a/lightning_examples/augmentation_kornia/.meta.yml b/lightning_examples/augmentation_kornia/.meta.yml
@@ -4,6 +4,8 @@ created: 2021-06-11
 updated: 2021-06-16
 license: CC BY-SA
 build: 0
+tags:
+  - Image
 description: |
   In this tutorial we will show how to combine both Kornia.org and PyTorch Lightning
   to perform efficient data augmentation to train a simpple model using the GPU in batch
diff --git a/lightning_examples/basic-gan/.meta.yaml b/lightning_examples/basic-gan/.meta.yaml
@@ -4,6 +4,8 @@ created: 2020-12-21
 updated: 2021-06-16
 license: CC BY-SA
 build: 0
+tags:
+  - Image
 description: |
   How to train a GAN!
 
diff --git a/lightning_examples/cifar10-baseline/.meta.yml b/lightning_examples/cifar10-baseline/.meta.yml
@@ -4,6 +4,8 @@ created: 2020-12-21
 updated: 2021-06-16
 license: CC BY-SA
 build: 0
+tags:
+  - Image
 description: >
   Train a Resnet to 94% accuracy on Cifar10!
 requirements:
diff --git a/lightning_examples/datamodules/.meta.yml b/lightning_examples/datamodules/.meta.yml
@@ -10,7 +10,7 @@ description: This notebook will walk you through how to start using Datamodules.
   The most up to date documentation on datamodules can be found
   [here](https://pytorch-lightning.readthedocs.io/en/latest/extensions/datamodules.html).
 requirements:
-- torchvision
+  - torchvision
 accelerator:
-- CPU
-- GPU
+  - CPU
+  - GPU
diff --git a/lightning_examples/mnist-hello-world/.meta.yml b/lightning_examples/mnist-hello-world/.meta.yml
@@ -4,10 +4,12 @@ created: 2020-12-21
 updated: 2021-06-16
 license: CC BY-SA
 build: 0
+tags:
+  - Image
 description: In this notebook, we'll go over the basics of lightning by preparing
   models to train on the [MNIST Handwritten Digits dataset](https://en.wikipedia.org/wiki/MNIST_database).
 requirements:
-- torchvision
+  - torchvision
 accelerator:
-- CPU
-- GPU
+  - CPU
+  - GPU
diff --git a/lightning_examples/mnist-tpu-training/.meta.yml b/lightning_examples/mnist-tpu-training/.meta.yml
@@ -3,10 +3,12 @@ author: PL team
 created: 2020-12-21
 updated: 2021-06-25
 license: CC BY-SA
+tags:
+  - Image
 description: In this notebook, we'll train a model on TPUs. Updating one Trainer flag is all you need for that.
   The most up to documentation related to TPU training can be found
   [here](https://pytorch-lightning.readthedocs.io/en/latest/advanced/tpu.html).
 requirements:
-- torchvision
+  - torchvision
 accelerator:
-- TPU
+  - TPU
diff --git a/lightning_examples/reinforce-learning-DQN/.meta.yml b/lightning_examples/reinforce-learning-DQN/.meta.yml
@@ -4,6 +4,8 @@ created: 2021-01-31
 updated: 2021-06-17
 license: CC BY-SA
 build: 0
+tags:
+  - RL
 description: |
   Main takeaways:
 
diff --git a/lightning_examples/text-transformers/.meta.yml b/lightning_examples/text-transformers/.meta.yml
@@ -4,6 +4,8 @@ created: 2021-01-31
 updated: 2021-06-21
 license: CC BY-SA
 build: 0
+tags:
+  - Text
 description: |
   This notebook will use HuggingFace's `datasets` library to get data, which will be wrapped in a `LightningDataModule`.
   Then, we write a class to perform text classification on any dataset from the [GLUE Benchmark](https://gluebenchmark.com/).

-Original file line number
+Diff line change
 # Pyre type checker
 .pyre/
++
 +.idea/
Original file line number	Diff line number	Diff line change
`@@ -66,6 +66,7 @@`
`66`	`66`	`'nbsphinx',`
`67`	`67`	`'myst_parser',`
`68`	`68`	`'sphinx_paramlinks',`
	`69`	`+ 'pt_lightning_sphinx_theme.extensions.lightning_tutorials',`
`69`	`70`	`]`
`70`	`71`
`71`	`72`	`# Add any paths that contain templates here, relative to this directory.`