Add tutorial view support (#72)

* Try something
* Make tags no longer required
* Remove extensions
* Clean

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: ethanwharris

.actions/helpers.py

@@ -1,10 +1,11 @@
 import base64
+import json
 import os
 import re
-import shutil
 from datetime import datetime
 from pprint import pprint
-from typing import Sequence
+from textwrap import wrap
+from typing import Any, Dict, Sequence
 from warnings import warn
 
 import fire
@@ -80,6 +81,12 @@
 # ![Pytorch Lightning](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/docs/source/_static/images/logo.png){height="60px" width="240px"}
 
 """
+TEMPLATE_CARD_ITEM = """
+.. customcarditem::
+   :header: %(title)s
+   :card_description: %(short_description)s
+   :tags: %(tags)s
+"""
 
 
 def default_requirements(path_req: str = PATH_REQ_DEFAULT) -> list:
@@ -163,6 +170,7 @@ def augment_script(fpath: str):
             dict(local_ipynb=f"{os.path.dirname(fpath)}.ipynb"),
             generated=datetime.now().isoformat(),
         )
+
         meta['description'] = meta['description'].replace(os.linesep, f"{os.linesep}# ")
 
         header = TEMPLATE_HEADER % meta
@@ -311,6 +319,43 @@ def parse_requirements(dir_path: str):
         with open(fname, "w") as fp:
             fp.write(" ".join(cmd_args))
 
+    @staticmethod
+    def _get_card_item_cell(path_ipynb: str) -> Dict[str, Any]:
+        """Build the card item cell for the given notebook path."""
+        fpath_meta = path_ipynb.replace(".ipynb", ".yaml")
+        meta = yaml.safe_load(open(fpath_meta))
+
+        # Clamp description length
+        wrapped_description = wrap(
+            meta.get('short_description', meta['description']).strip().replace(os.linesep, " "), 175
+        )
+        suffix = "..." if len(wrapped_description) > 1 else ""
+        meta['short_description'] = wrapped_description[0] + suffix
+
+        # Resolve some default tags based on accelerators and directory name
+        meta['tags'] = meta.get('tags', [])
+
+        accelerators = meta.get("accelerator", ('CPU', ))
+        if ('GPU' in accelerators) or ('TPU' in accelerators):
+            meta['tags'].append('GPU/TPU')
+
+        dirname = os.path.basename(os.path.dirname(path_ipynb))
+        if dirname != ".notebooks":
+            meta['tags'].append(dirname)
+
+        meta['tags'] = ",".join(meta['tags'])
+
+        # Build the notebook cell
+        rst_cell = TEMPLATE_CARD_ITEM % meta
+
+        return {
+            "cell_type": "raw",
+            "metadata": {
+                "raw_mimetype": "text/restructuredtext"
+            },
+            "source": rst_cell.strip().splitlines(True)
+        }
+
     @staticmethod
     def copy_notebooks(path_root: str, path_docs_ipynb: str = "docs/source/notebooks"):
         """Copy all notebooks from a folder to doc folder.
@@ -331,7 +376,15 @@ def copy_notebooks(path_root: str, path_docs_ipynb: str = "docs/source/notebooks
             new_ipynb = os.path.join(path_docs_ipynb, sub_ipynb)
             os.makedirs(os.path.dirname(new_ipynb), exist_ok=True)
             print(f'{path_ipynb} -> {new_ipynb}')
-            shutil.copy(path_ipynb, new_ipynb)
+
+            with open(path_ipynb) as f:
+                ipynb = json.load(f)
+
+            ipynb["cells"].append(HelperCLI._get_card_item_cell(path_ipynb))
+
+            with open(new_ipynb, 'w') as f:
+                json.dump(ipynb, f)
+
             ipynb_content.append(os.path.join('notebooks', sub_ipynb))
 
     @staticmethod

.actions/ipynb-render.sh

@@ -7,9 +7,16 @@ python -c "import glob ; assert(len(glob.glob('$1/*.ipynb')) == 1)"
 ipynb_file=( $(ls "$1"/*.ipynb) )
 printf $ipynb_file
 
+python -c "import glob ; assert(len(glob.glob('$1/.meta.{yaml,yml}')) == 1)"
+meta_file=( $(ls "$1"/.meta.*) )
+printf $meta_file
+
 pub_file=".notebooks/$1.ipynb"
 printf $pub_file
 
+pub_meta_file=".notebooks/$1.yaml"
+printf $pub_meta_file
+
 pub_dir=$(dirname "$pub_file")
 mkdir -p $pub_dir
 
@@ -27,10 +34,12 @@ then
   printf "Processing: $ipynb_file\n"
   python -m papermill.cli $ipynb_file $pub_file --kernel python
   python .actions/helpers.py update-env-details $1
-  git add ".notebooks/$1.yaml"
 else
   printf "WARNING: not valid accelerator so no outputs will be generated.\n"
   cp $ipynb_file $pub_file
 fi
 
+cp $meta_file $pub_meta_file
+git add ".notebooks/$1.yaml"
+
 git add $pub_file

.github/workflows/ci_docs.yml

@@ -62,12 +62,14 @@ jobs:
           while IFS= read -r line; do
               bash .actions/ipynb-generate.sh $line
               ipynb=( $(ls "$line"/*.ipynb) )
+              meta=( $(ls "$line"/.meta.*) )
               # create folder of it is missing
               pub_dir=$(dirname ".notebooks/$line")
               printf "Dir: $pub_dir"
               mkdir -p $pub_dir
               # simulate render as somple move
               mv $ipynb ".notebooks/$line.ipynb"
+              mv $meta ".notebooks/$line.yaml"
           done <<< $(cat changed-folders.txt)
         shell: bash
 
@@ -76,3 +78,11 @@ jobs:
         run: |
           # First run the same pipeline as Read-The-Docs
           make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going"
+
+      - name: Upload built docs
+        uses: actions/upload-artifact@v2
+        with:
+          name: docs-results-${{ github.sha }}
+          path: docs/build/html/
+        # Use always() to always run this step to publish test results when there are test failures
+        if: success()

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+.idea/

course_UvA-DL/autoregressive-image-modeling/.meta.yml

@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-07-12
 updated: 2021-07-12
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we implement an autoregressive likelihood model for the task of image modeling.
   Autoregressive models are naturally strong generative models that constitute one of the current

course_UvA-DL/deep-autoencoders/.meta.yml

@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-07-12
 updated: 2021-07-12
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we will take a closer look at autoencoders (AE).
   Autoencoders are trained on encoding input data such as images into a smaller feature vector,

course_UvA-DL/deep-energy-based-generative-models/.meta.yml

@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-07-12
 updated: 2021-07-12
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we will look at energy-based deep learning models, and focus on their application as generative models.
   Energy models have been a popular tool before the huge deep learning hype around 2012 hit.

course_UvA-DL/graph-neural-networks/.meta.yml

@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-06-07
 updated: 2021-06-16
 license: CC BY-SA
+tags:
+  - Graph
 description: |
   In this tutorial, we will discuss the application of neural networks on graphs.
   Graph Neural Networks (GNNs) have recently gained increasing popularity in both applications and research,

course_UvA-DL/normalizing-flows/.meta.yml

@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-06-07
 updated: 2021-06-16
 license: CC BY-SA
+tags:
+  - Image
 description: |
   In this tutorial, we will take a closer look at complex, deep normalizing flows.
   The most popular, current application of deep normalizing flows is to model datasets of images.

course_UvA-DL/transformers-and-MH-attention/.meta.yml

@@ -3,6 +3,8 @@ author: Phillip Lippe
 created: 2021-06-30
 updated: 2021-06-30
 license: CC BY-SA
+tags:
+  - Text
 description: |
   In this tutorial, we will discuss one of the most impactful architectures of the last 2 years: the Transformer model.
   Since the paper Attention Is All You Need by Vaswani et al. had been published in 2017,