Convert nmaci to a pip-installable Python package

.all-contributorsrc

@@ -7,6 +7,16 @@
   "commitType": "docs",
   "commitConvention": "angular",
   "contributors": [
+    {
+      "login": "OleBialas",
+      "name": "Ole Bialas",
+      "avatar_url": "https://avatars.githubusercontent.com/u/38684453?v=4",
+      "profile": "https://github.com/OleBialas",
+      "contributions": [
+        "code",
+        "maintenance"
+      ]
+    },
     {
       "login": "iamzoltan",
       "name": "Zoltan",

.github/workflows/ci.yaml

@@ -19,15 +19,15 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: actions/setup-python@v4
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: 3.9
+          python-version: "3.9"
 
       - name: Install dependencies
         run: |
-          pip install pytest -r requirements.txt
-          jupyter kernelspec list
+          uv sync --extra dev
+          uv run jupyter kernelspec list
 
       - name: Execute tests
-        run: pytest tests/
+        run: uv run pytest tests/ src/nmaci/

README.md

@@ -1,9 +1,45 @@
 # nmaci
-[![CI](https://github.com/neuromatch/nmaci/actions/workflows/ci.yaml/badge.svg)](https://github.com/neuromatch/nmaci/actions/workflows/ci.yaml)
+[![CI](https://github.com/OleBialas/nmaci/actions/workflows/ci.yaml/badge.svg)](https://github.com/OleBialas/nmaci/actions/workflows/ci.yaml)
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
-Automated tools for building and verifying NMA tutorial materials
+Automated tools for building and verifying NMA tutorial materials.
+
+## Installation
+
+```bash
+pip install git+https://github.com/OleBialas/nmaci@main
+```
+
+## Usage
+
+```
+nmaci <command> [args]
+```
+
+| Command | Description |
+|---|---|
+| `process-notebooks` | Execute notebooks, extract solutions, create student/instructor versions |
+| `verify-exercises` | Check exercise cells match solution cells |
+| `lint-tutorial` | Run flake8/pyflakes over notebook code cells |
+| `generate-readmes` | Auto-generate tutorial `README.md` files |
+| `generate-book` | Build Jupyter Book from `materials.yml` |
+| `generate-book-dl` | Build Jupyter Book (Deep Learning variant) |
+| `generate-book-precourse` | Build Jupyter Book (Precourse variant) |
+| `select-notebooks` | Filter which notebooks to process |
+| `make-pr-comment` | Generate PR comment with Colab badges and lint report |
+| `find-unreferenced` | Identify unused solution images/scripts |
+| `extract-links` | Extract video/slide links from notebooks |
+| `parse-html` | Check HTML build output for errors |
+
+## Development
+
+```bash
+git clone https://github.com/OleBialas/nmaci
+cd nmaci
+uv sync --extra dev
+uv run pytest tests/
+```
 
 ## Contributors ✨
 
@@ -16,6 +52,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
   <tbody>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/iamzoltan"><img src="https://avatars.githubusercontent.com/u/21369773?v=4?s=100" width="100px;" alt="Zoltan"/><br /><sub><b>Zoltan</b></sub></a><br /><a href="https://github.com/neuromatch/nmaci/commits?author=iamzoltan" title="Code">💻</a> <a href="https://github.com/neuromatch/nmaci/commits?author=iamzoltan" title="Tests">⚠️</a> <a href="#maintenance-iamzoltan" title="Maintenance">🚧</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/OleBialas"><img src="https://avatars.githubusercontent.com/u/38684453?v=4?s=100" width="100px;" alt="Ole Bialas"/><br /><sub><b>Ole Bialas</b></sub></a><br /><a href="https://github.com/neuromatch/nmaci/commits?author=OleBialas" title="Code">💻</a> <a href="#maintenance-OleBialas" title="Maintenance">🚧</a></td>
     </tr>
   </tbody>
 </table>

pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "nmaci"
+version = "0.1.0"
+requires-python = ">=3.9"
+dependencies = [
+    "nbformat",
+    "nbconvert",
+    "notebook",
+    "pillow",
+    "flake8",
+    "fuzzywuzzy[speedup]",
+    "pyyaml",
+    "beautifulsoup4",
+    "decorator==5.0.9",
+    "Jinja2==3.0.0",
+    "jupyter-client",
+]
+
+[project.optional-dependencies]
+dev = ["pytest"]
+
+[project.scripts]
+nmaci = "nmaci.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nmaci"]

scripts/extract_links.py

@@ -1,138 +1,4 @@
-"""
-Neuromatch Academy
-
-Extract slide and video links from notebooks
-"""
-import argparse
-import ast
-import collections
-import json
-import os
-from urllib.request import urlopen, Request
-from urllib.error import HTTPError
+from nmaci.extract_links import main
 import sys
-
-import nbformat
-
-
-def bilibili_url(video_id):
-    return f"https://www.bilibili.com/video/{video_id}"
-
-
-def youtube_url(video_id):
-    return f"https://youtube.com/watch?v={video_id}"
-
-
-def osf_url(link_id):
-    return f"https://osf.io/download/{link_id}"
-
-def tutorial_order(fname):
-    fname = os.path.basename(fname)
-    try:
-        first, last = fname.split("_")
-    except ValueError:
-        return (99, 99, fname)
-    if first.startswith("Bonus"):
-        week, day = 9, 9
-    else:
-        try:
-            week, day = int(first[1]), int(first[3])
-        except ValueError:
-            week, day = 9, 9
-    if last.startswith("Intro"):
-        order = 0
-    elif last.startswith("Tutorial"):
-        order = int(last[8])
-    elif last.startswith("Outro"):
-        order = 10
-    elif last.startswith("DaySummary"):
-        order = 20
-    else:
-        order = 30
-    return (week, day, order)
-
-def main(arglist):
-    """Process IPython notebooks from a list of files."""
-    args = parse_args(arglist)
-
-    nb_paths = [arg for arg in args.files
-                if arg.endswith(".ipynb") and
-                   'student' not in arg and
-                   'instructor' not in arg]
-    if not nb_paths:
-        print("No notebook files found")
-        sys.exit(0)
-
-    videos = collections.defaultdict(list)
-    slides = collections.defaultdict(list)
-
-    for nb_path in sorted(nb_paths, key=tutorial_order):
-        # Load the notebook structure
-        with open(nb_path) as f:
-            nb = nbformat.read(f, nbformat.NO_CONVERT)
-
-        # Extract components of the notebook path
-        nb_dir, nb_fname = os.path.split(nb_path)
-        nb_name, _ = os.path.splitext(nb_fname)
-
-        # Loop through the cells and find video and slide ids
-        for cell in nb.get("cells", []):
-            for line in cell.get("source", "").split("\n"):
-                l = line.strip()
-                if l.startswith("video_ids = "):
-                    rhs = l.split("=")[1].strip()
-                    video_dict = dict(ast.literal_eval(rhs))
-                    try:
-                        if args.noyoutube:
-                            url = bilibili_url(video_dict["Bilibili"])
-                        else:
-                            url = youtube_url(video_dict["Youtube"])
-                    except KeyError:
-                        print(f"Malformed video id in {nb_name}? '{rhs}'")
-                        continue
-                    if url not in videos[nb_name]:
-                        videos[nb_name].append(url)
-                elif l.startswith("link_id = "):
-                    rhs = l.split("=")[1].strip()
-                    url = osf_url(ast.literal_eval(rhs))
-                    # Slides are sometimes used in multiple notebooks, so we
-                    # just store the filename and the link
-                    if url not in slides:
-                        api_request = f"https://api.osf.io/v2/files/{ast.literal_eval(rhs)}/"
-                        httprequest = Request(api_request,
-                                              headers={"Accept": "application/json"})
-                        try:
-                            with urlopen(httprequest) as response:
-                                data = json.load(response)
-                                filename = data["data"]["attributes"]["name"]
-                        except HTTPError as e:
-                            sys.stderr.write(str(e) + "\n")
-                            sys.stderr.write(f"Skipping slide {url}\n")
-                            continue
-                        if 'DaySummary' in nb_name:
-                            filename = os.path.splitext(filename.replace("_", ""))[0] + '_DaySummary.pdf'
-                        slides[url] = filename
-
-    print(json.dumps({"videos": videos, "slides": slides}, indent=4))
-
-
-def parse_args(arglist):
-    """Handle the command-line arguments."""
-    parser = argparse.ArgumentParser(
-        description="Process neuromatch tutorial notebooks"
-    )
-    parser.add_argument(
-        "--noyoutube",
-        action="store_true",
-        help="Extract Bilibili links instead of youtube",
-    )
-    parser.add_argument(
-        "files",
-        nargs="+",
-        help="File name(s) to process. Will filter for .ipynb extension.",
-    )
-    return parser.parse_args(arglist)
-
-
 if __name__ == "__main__":
     main(sys.argv[1:])

scripts/find_unreferenced_content.py

@@ -1,24 +1,4 @@
-"""Print names of derivative files that are no longer used in the notebooks."""
-from glob import glob
-
+from nmaci.find_unreferenced_content import main
+import sys
 if __name__ == "__main__":
-
-    day_paths = glob("tutorials/W?D?_*")
-    for day_path in sorted(day_paths):
-
-        # Read all of the text for this day's student notebooks into one string
-        student_notebooks = glob(f"{day_path}/student/*.ipynb")
-        notebook_text = ""
-        for nb_path in student_notebooks:
-            with open(nb_path) as f:
-                notebook_text += f.read()
-
-        # Find solution images and scripts
-        solution_pattern = "W?D?_*_Solution*"
-        static_paths = glob(f"{day_path}/static/{solution_pattern}")
-        script_paths = glob(f"{day_path}/solutions/{solution_pattern}")
-
-        # Print paths that are not referenced in the notebooks
-        for path in sorted(static_paths + script_paths):
-            if path not in notebook_text:
-                print(path)
+    main(sys.argv[1:])