Caching and other improvements for test.yml

.github/workflows/test.yml

@@ -2,24 +2,115 @@ name: ANTsPyNet Unit Tests
 
 on:
   push:
+    branches: [master]
   pull_request:
+    branches: [master]
+  workflow_dispatch:
 
 env:
-  USE_SOURCE_BUILD: false  # Set to 'false' to use PyPI version of ANTsPy
+  USE_ANTSPY_SOURCE_BUILD: false
 
 jobs:
+
+  ###########################################
+  # Job 1 — Build or Restore ANTsXNet Data Cache
+  #
+  # This job downloads all ANTsXNet data and pretrained models and uploads as an artifact
+  # The data is too big for Github's cache and runner disk space to handle
+  #
+  # But we still avoid independent downloads from figshare for each build matrix job
+  #
+  ###########################################
+  prepare-data:
+    name: Prepare ANTsXNet data and models
+    runs-on: ubuntu-22.04
+
+    outputs:
+      cache-key: ${{ steps.compute-key.outputs.cache-key }}
+      cache-hit: ${{ steps.check-cache.outputs.cache-hit }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Compute cache key
+        id: compute-key
+        run: |
+          KEY="antsxnet-${{ hashFiles(
+            'antspynet/utilities/get_antsxnet_data.py',
+            'antspynet/utilities/get_pretrained_network.py'
+          ) }}"
+          echo "cache-key=$KEY" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install ANTsPy for data download
+        run: |
+          echo "Installing ANTsPy from PyPI for cache preparation"
+          pip install antspyx
+
+      # Try to restore an existing artifact for this key
+      - name: Restore ANTsXNet data artifact (if present)
+        id: restore-cache
+        uses: actions/download-artifact@v4
+        continue-on-error: true # don't fail if correct cached data is not found
+        with:
+          name: ANTSXNet-data-${{ steps.compute-key.outputs.cache-key }}
+          path: ${{ runner.temp }}/ANTsXNet
+
+      # Convert the restore step outcome into a boolean output
+      - name: Check cache hit
+        id: check-cache
+        run: |
+          if [ "${{ steps.restore-cache.outcome }}" = "success" ]; then
+            echo "cache-hit=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "cache-hit=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Download from figshare if we didn't restore an artifact
+      - name: Download ANTsXNet data and models
+        if: steps.restore-cache.outcome != 'success'
+        run: |
+          pip install -e .
+          python download_all_data.py --strict --cache-dir ${{ runner.temp }}/ANTsXNet
+
+      # Upload if we did a download above
+      - name: Upload data to artifact
+        if: steps.restore-cache.outcome != 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ANTSXNet-data-${{ steps.compute-key.outputs.cache-key }}
+          path: ${{ runner.temp }}/ANTsXNet
+          retention-days: 14
+
+
+  ###########################################
+  # Job 2 — Test Matrix
+  ###########################################
   test:
     name: Test on Python ${{ matrix.python-version }}
     runs-on: ubuntu-22.04
+    needs: prepare-data
 
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.11]
+        python-version: ["3.11", "3.12", "3.13"]
 
     steps:
-      - name: Checkout ANTsPyNet
+      - name: Checkout repository
         uses: actions/checkout@v3
 
+      - name: Download ANTsXNet data and models artifact
+        uses: actions/download-artifact@v4
+        with:
+         name: ANTSXNet-data-${{ needs.prepare-data.outputs.cache-key }}
+         path: ~/.keras/ANTsXNet
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -30,35 +121,21 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y cmake build-essential git
 
-      - name: Install ANTsPy (source or PyPI)
+      - name: Install ANTsPy
         run: |
-          if [ "$USE_SOURCE_BUILD" = "true" ]; then
-            echo "🔧 Installing ANTsPy from GitHub source..."
+          if [ "$USE_ANTSPY_SOURCE_BUILD" = "true" ]; then
+            echo "Installing ANTsPy from source..."
             git clone https://github.com/ANTsX/ANTsPy.git
-            pip install scikit-build-core pybind11 nanobind # 🔑 install build backend dependencies
+            pip install scikit-build-core pybind11 nanobind
             pip install ./ANTsPy --no-build-isolation --no-deps
           else
-            echo "📦 Installing ANTsPy from PyPI..."
+            echo "Installing ANTsPy from PyPI..."
             pip install antspyx
           fi
 
-      - name: Cache ANTsXNet model/data downloads
-        uses: actions/cache@v4
-        with:
-          path: ~/.keras/ANTsXNet
-          key: antsxnet-${{ runner.os }}-${{ hashFiles('**/download_all_data.py') }}
-          restore-keys: |
-            antsxnet-${{ runner.os }}-
-
-      - name: Install ANTsPyNet and dependencies
+      - name: Install ANTsPyNet
         run: |
           pip install -e .
-          sed -i '/antspyx==/d' requirements.txt
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Pre-download ANTsXNet models and data
-        run: |
-          python download_all_data.py --strict
 
       - name: Run tests (individually via pytest)
         run: |

download_all_data.py

@@ -2,8 +2,11 @@
 import argparse
 import sys
 
-def download_all_data(strict=False):
+def download_all_data(strict=False, cache_dir=None):
     print("Downloading data files from get_antsxnet_data...")
+    if cache_dir is not None:
+        antspynet.set_antsxnet_cache_directory(cache_dir)
+        print(f"Using custom cache directory: {cache_dir}")
     try:
         data_keys = antspynet.get_antsxnet_data("show")
         for key in data_keys:
@@ -44,10 +47,11 @@ def download_all_data(strict=False):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--strict", action="store_true", help="Exit on first failed download.")
+    parser.add_argument("--cache-dir", type=str, help="Custom cache directory for downloads.", default=None)
     args = parser.parse_args()
 
     try:
-        download_all_data(strict=args.strict)
+        download_all_data(strict=args.strict, cache_dir=args.cache_dir)
     except Exception as e:
         print(f"\nAborted due to error: {e}")
         sys.exit(1)