ci: add notebook execution to coverage report

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   check:
-    uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@main
+    uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@notebooks-in-coverage
 
     strategy:
       fail-fast: false
@@ -28,7 +28,7 @@ jobs:
   # Checks the library using minimum version resolution
   # `uv` has this feature built-in, c.f. https://github.com/astral-sh/uv
   check-min-version:
-    uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@main
+    uses: geo-engine/geoengine-python/.github/workflows/test-python.yml@notebooks-in-coverage
 
     with:
       python-version: "3.10"

.github/workflows/test-python.yml

@@ -103,6 +103,15 @@ jobs:
         env:
           GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
           GEOENGINE_TEST_BUILD_TYPE: "release"
+      - name: Examples
+        run: |
+          ${{ steps.vars.outputs.VENV_CALL }}
+          ${{ steps.vars.outputs.PIP_INSTALL }} -e .[examples]
+          python test_all_notebooks.py
+        env:
+          GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
+          GEOENGINE_TEST_BUILD_TYPE: "release"
+          COVERAGE_COMMAND: ${{ steps.vars.outputs.COVERAGE_COMMAND }}
       - name: Report coverage to Coveralls
         if: ${{ inputs.coverage }}
         # 1. We need to adjust the paths in the lcov file to match the repository structure.
@@ -114,11 +123,3 @@ jobs:
         working-directory: library/geoengine
         env:
           COVERALLS_REPO_TOKEN: ${{ github.token }}
-      - name: Examples
-        run: |
-          ${{ steps.vars.outputs.VENV_CALL }}
-          ${{ steps.vars.outputs.PIP_INSTALL }} -e .[examples]
-          python test_all_notebooks.py
-        env:
-          GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
-          GEOENGINE_TEST_BUILD_TYPE: "release"

examples/data_usage.ipynb

@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -329,21 +329,25 @@
     }
    ],
    "source": [
-    "df = ge.data_usage_summary(ge.UsageSummaryGranularity.MINUTES)\n",
-    "if df.empty:\n",
-    "    print(\"No data usage found\")\n",
-    "    exit()\n",
-    "df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"]).dt.tz_localize(None)\n",
+    "def plot_summary():\n",
+    "    df = ge.data_usage_summary(ge.UsageSummaryGranularity.MINUTES)\n",
+    "    if df.empty:\n",
+    "        print(\"No data usage found\")\n",
+    "        return\n",
+    "    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"]).dt.tz_localize(None)\n",
+    "\n",
+    "    pivot_df = df.pivot(index=\"timestamp\", columns=\"data\", values=\"count\").fillna(0)\n",
+    "    pivot_df.plot(kind=\"bar\", figsize=(10, 6))\n",
+    "\n",
+    "    plt.title(\"Data Usage by Data over time\")\n",
+    "    plt.xlabel(\"Timestamp\")\n",
+    "    plt.ylabel(\"Count\")\n",
+    "    plt.xticks(rotation=45)\n",
+    "    plt.legend(title=\"Data\")\n",
+    "    plt.show()\n",
     "\n",
-    "pivot_df = df.pivot(index=\"timestamp\", columns=\"data\", values=\"count\").fillna(0)\n",
-    "pivot_df.plot(kind=\"bar\", figsize=(10, 6))\n",
     "\n",
-    "plt.title(\"Data Usage by Data over time\")\n",
-    "plt.xlabel(\"Timestamp\")\n",
-    "plt.ylabel(\"Count\")\n",
-    "plt.xticks(rotation=45)\n",
-    "plt.legend(title=\"Data\")\n",
-    "plt.show()"
+    "plot_summary()"
    ]
   }
  ],

pyproject.toml

@@ -96,3 +96,8 @@ select = [
 "tests/__init__.py" = [
     "F401", # module imported but unused
 ]
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests",
+]

test_all_notebooks.py

@@ -7,22 +7,28 @@
 import subprocess
 import sys
 
+COVERAGE_COMMAND_ENV_VAR = "COVERAGE_COMMAND"
+
 
 def eprint(*args, **kwargs):
     """Print to stderr."""
     print(*args, file=sys.stderr, **kwargs)
 
 
-def run_test_notebook(notebook_path) -> bool:
+def run_test_notebook(notebook_path: str, coverage_args: list[str]) -> bool:
     """Run test_notebook.py for the given notebook."""
 
-    python_bin = shutil.which("python3")
+    pytest_bin = shutil.which("pytest")
 
-    if python_bin is None:
+    if pytest_bin is None:
         raise RuntimeError("Python 3 not found")
 
     result = subprocess.run(
-        [python_bin, "test_notebook.py", notebook_path],
+        [pytest_bin, "--ignore=test", *coverage_args, "test_notebook.py"],
+        env={
+            **os.environ,
+            "INPUT_FILE": notebook_path,
+        },
         capture_output=True,
         text=True,
         check=False,
@@ -36,6 +42,14 @@ def run_test_notebook(notebook_path) -> bool:
         return False
 
 
+def parse_coverage_command() -> list[str]:
+    """Get coverage command from environment variable."""
+    coverage_cmd = os.getenv(COVERAGE_COMMAND_ENV_VAR)
+    if not coverage_cmd:
+        return []
+    return [*coverage_cmd.split(), "--cov-append"]
+
+
 def main() -> int:
     """Run all Jupyter Notebooks and check for errors."""
 
@@ -45,13 +59,19 @@ def main() -> int:
         eprint(f"The folder {example_folder} does not exist.")
         return -1
 
+    coverage_args = parse_coverage_command()
+    if coverage_args:
+        eprint(f"Using coverage args: {' '.join(coverage_args)}")
+    else:
+        eprint(f"No coverage args in env {COVERAGE_COMMAND_ENV_VAR} provided.")
+
     for root, _dirs, files in os.walk(example_folder):
         for file in files:
             if not file.endswith(".ipynb"):
                 eprint(f"Skipping non-notebook file {file}")
                 continue
             notebook_path = os.path.join(root, file)
-            if not run_test_notebook(notebook_path):
+            if not run_test_notebook(notebook_path, coverage_args):
                 return -1
 
         break  # skip subdirectories

test_notebook.py

@@ -4,6 +4,7 @@
 
 import argparse
 import ast
+import os
 import sys
 import warnings
 
@@ -55,6 +56,8 @@ def run_script(script: str) -> bool:
         with warnings.catch_warnings(record=True):
             # pylint: disable-next=exec-used
             exec(code, {})
+            # pytest.main(["-p", "no:warnings", "-c", "pytest.ini",
+            #              "--tb=short", "-"], plugins=[], args=[], obj=code)
 
         eprint("SUCCESS")
         return True
@@ -64,22 +67,40 @@ def run_script(script: str) -> bool:
         return False
 
 
+def setup_geoengine_and_run_script(input_file: str) -> bool:
+    """Setup Geo Engine test instance and run the script."""
+    python_script = convert_to_python(input_file)
+
+    eprint(f"Running script `{input_file}`", end=": ")
+
+    with GeoEngineTestInstance(port=3030) as ge_instance:
+        ge_instance.wait_for_ready()
+
+        return run_script(python_script)
+
+
 def main():
     """Main entry point."""
 
     input_file = parse_args()
 
-    python_script = convert_to_python(input_file)
+    if setup_geoengine_and_run_script(input_file):
+        sys.exit(0)
+    else:
+        sys.exit(1)
 
-    eprint(f"Running script `{input_file}`", end=": ")
 
-    with GeoEngineTestInstance(port=3030) as ge_instance:
-        ge_instance.wait_for_ready()
+def test_main():
+    """Run main function with pytest"""
+    input_file = os.getenv("INPUT_FILE")
+
+    if not input_file:
+        raise AssertionError("INPUT_FILE environment variable not set")
 
-        if run_script(python_script):
-            sys.exit(0)
-        else:
-            sys.exit(1)
+    if setup_geoengine_and_run_script(input_file):
+        assert True, "Notebook ran successfully"
+    else:
+        raise AssertionError("Notebook failed")
 
 
 if __name__ == "__main__":