fix changed & data cache & update Azure triggers (#39)

* update Azure triggers

* fix render

* update cli

* long lines

* data cache

* fix unique

* cleaning

Author: Borda

.actions/helpers.py

@@ -128,6 +128,7 @@ class HelperCLI:
         ".datasets",
         ".github",
         "docs",
+        "requirements",
         DIR_NOTEBOOKS,
     )
     META_FILE_REGEX = ".meta.{yaml,yml}"
@@ -158,8 +159,8 @@ def augment_script(fpath: str):
 
         first_empty = min([i for i, ln in enumerate(py_file) if not ln.startswith("#")])
         header = TEMPLATE_HEADER % meta
-        requests = default_requirements() + meta["requirements"]
-        setup = TEMPLATE_SETUP % dict(requirements=" ".join(requests))
+        requires = set(default_requirements() + meta["requirements"])
+        setup = TEMPLATE_SETUP % dict(requirements=" ".join(requires))
         py_file[first_empty] = header + setup
         py_file.append(TEMPLATE_FOOTER)
 
@@ -212,9 +213,7 @@ def group_folders(
         """
         with open(fpath_gitdiff, "r") as fp:
             changed = [ln.strip() for ln in fp.readlines()]
-
-        # unique folders
-        dirs = set([os.path.dirname(ln) for ln in changed])
+        dirs = [os.path.dirname(ln) for ln in changed]
         # not empty paths
         dirs = [ln for ln in dirs if ln]
 
@@ -225,6 +224,8 @@ def group_folders(
             # get only different
             dirs += list(set.union(*dir_sets) - set.intersection(*dir_sets))
 
+        # unique folders
+        dirs = set(dirs)
         # drop folder with skip folder
         dirs = [pd for pd in dirs if not any(nd in HelperCLI.SKIP_DIRS for nd in pd.split(os.path.sep))]
         # valid folder has meta
@@ -256,7 +257,7 @@ def parse_requirements(dir_path: str):
 
         req = meta.get('requirements', [])
         fname = os.path.join(dir_path, HelperCLI.REQUIREMENTS_FILE)
-        print(f"Requirements: {fname}")
+        print(f"File for requirements: {fname}")
         with open(fname, "w") as fp:
             fp.write(os.linesep.join(req))
 
@@ -273,7 +274,7 @@ def parse_requirements(dir_path: str):
                 cmd_args.append(f"--{pip_key} {arg}")
 
         fname = os.path.join(dir_path, HelperCLI.PIP_ARGS_FILE)
-        print(f"PIP arguments: {fname}")
+        print(f"File for PIP arguments: {fname}")
         with open(fname, "w") as fp:
             fp.write(" ".join(cmd_args))
 

.actions/ipynb-render.sh

@@ -17,8 +17,8 @@ python .actions/helpers.py parse-requirements $1
 pip install --quiet --requirement requirements.txt --upgrade-strategy only-if-needed
 cat "$1/requirements.txt"
 pip_args=$(cat "$1/pip_arguments.txt")
-printf "pip arguments: $pip_args"
-pip install --requirement "$1/requirements.txt"$pip_args
+printf "pip arguments:\n $pip_args\n\n"
+pip install --requirement "$1/requirements.txt" $pip_args
 
 printf "available: $ACCELERATOR\n"
 accel=$(python .actions/helpers.py valid-accelerator $1 2>&1)

.actions/ipynb-test.sh

@@ -18,7 +18,7 @@ source "$1/venv/bin/activate"
 pip --version
 pip install --quiet --requirement requirements.txt --upgrade-strategy only-if-needed
 pip_args=$(cat "$1/pip_arguments.txt")
-printf "pip arguments: $pip_args\n"
+printf "pip arguments:\n $pip_args\n\n"
 pip install --requirement "$1/requirements.txt" $pip_args
 pip list
 

.azure-pipelines/ipynb-publish.yml

@@ -1,7 +1,7 @@
 trigger:
+  batch: true
   branches:
-    include:
-      - main
+    include: [ main ]
 pr: none
 #stages:
 #- stage: testing

.azure-pipelines/ipynb-tests.yml

@@ -1,9 +1,9 @@
 trigger: none
 pr:
-  - main
-
-#stages:
-#- stage: testing
+  branches:
+    include: [ main ]
+  autoCancel: true
+  drafts: true
 
 jobs:
 
@@ -59,7 +59,15 @@ jobs:
         echo "##vso[task.setvariable variable=folders;isOutput=true]$COUNT"
       name: changed
 
+    - task: Cache@2
+      inputs:
+        key: data | .actions/data-download.sh
+        restoreKeys: data
+        path: $(PATH_DATASETS)
+        cacheHitVar: DATA_RESTORED
+
     - script: bash .actions/data-download.sh $(PATH_DATASETS)
+      condition: ne(variables.DATA_RESTORED, 'true')
       displayName: 'Pull datasets'
 
     - bash: |

.github/workflows/ci_testing.yml

@@ -41,7 +41,16 @@ jobs:
           pip list
         shell: bash
 
+      - name: Cache Data
+        id: cache-data
+        uses: actions/cache@v2
+        with:
+          path: $PATH_DATASETS
+          key: ${{ runner.os }}-datasets-${{ hashFiles('.actions/data-download.sh') }}
+          restore-keys: ${{ runner.os }}-datasets-
+
       - name: Download Data
+        if: steps.cache-data.outputs.cache-hit != 'true'
         run: bash .actions/data-download.sh $PATH_DATASETS
 
       - name: Process folders

course_UvA-DL/graph-neural-networks/.meta.yml

@@ -5,9 +5,12 @@ updated: 2021-06-07
 license: CC
 description: |
   In this tutorial, we will discuss the application of neural networks on graphs.
-  Graph Neural Networks (GNNs) have recently gained increasing popularity in both applications and research, including domains such as social networks, knowledge graphs, recommender systems, and bioinformatics.
-  While the theory and math behind GNNs might first seem complicated, the implementation of those models is quite simple and helps in understanding the methodology.
-  Therefore, we will discuss the implementation of basic network layers of a GNN, namely graph convolutions, and attention layers.
+  Graph Neural Networks (GNNs) have recently gained increasing popularity in both applications and research,
+  including domains such as social networks, knowledge graphs, recommender systems, and bioinformatics.
+  While the theory and math behind GNNs might first seem complicated,
+  the implementation of those models is quite simple and helps in understanding the methodology.
+  Therefore, we will discuss the implementation of basic network layers of a GNN,
+  namely graph convolutions, and attention layers.
   Finally, we will apply a GNN on semi-supervised node classification and molecule categorization.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
@@ -17,7 +20,6 @@ requirements:
   - torch-cluster
   - torch-spline-conv
   - torch-geometric
-  - pytorch-lightning
 pip__find-link:
   # - https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
   - https://pytorch-geometric.com/whl/torch-%(TORCH_MAJOR_DOT_MINOR)s.0+%(DEVICE)s.html

requirements/default.txt

@@ -1,3 +1,3 @@
 pytorch-lightning>=1.3
 torchmetrics>=0.3
-torch>=1.7
+torch>=1.6, <1.9