🚸 Add where-to-find information to missing resource nodeblock connection errors

.circleci/main.yml

@@ -48,7 +48,7 @@ commands:
             sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13
             curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
             sudo apt-get update
-            sudo apt-get install git -y
+            sudo apt-get install git openssh-client -y
             git config --global user.email "[email protected]"
             git config --global user.name "Theodore (machine user) @ CircleCI"
   create-docker-test-container:
@@ -64,11 +64,6 @@ commands:
             mkdir -p ~/project/test-results
             docker pull ${DOCKER_TAG}
             docker run -v /etc/passwd:/etc/passwd --user=$(id -u):c-pac -dit -P -e COVERAGE_FILE=<< parameters.coverage-file >> -v /home/circleci/project/test-results:/code/test-results -v /home/circleci:/home/circleci -v /home/circleci/project/CPAC/resources/configs/test_configs:/test_configs -v $PWD:/code -v $PWD/dev/circleci_data:$PWD/dev/circleci_data --workdir=/home/circleci/project --entrypoint=/bin/bash --name docker_test ${DOCKER_TAG}
-  get-sample-bids-data:
-    steps:
-      - run:
-          name: Getting Sample BIDS Data
-          command: git clone https://github.com/bids-standard/bids-examples.git
   get-singularity:
     parameters:
       version:
@@ -231,7 +226,6 @@ jobs:
       - set-up-variant:
           variant: "<< parameters.variant >>"
       - set-python-version
-      - get-sample-bids-data
       - run-pytest-docker
       - store_test_results:
           path: test-results

CHANGELOG.md

@@ -24,12 +24,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Required positional parameter "wf" in input and output of `ingress_pipeconfig_paths` function, where a node to reorient templates is added to the `wf`.
 - Required positional parameter "orientation" to `resolve_resolution`.
 - Optional positional argument "cfg" to `create_lesion_preproc`.
+- `resource_inventory` utility to inventory NodeBlock function inputs and outputs.
 
 ### Changed
 
 - Moved `pygraphviz` from requirements to `graphviz` optional dependencies group.
 - Automatically tag untagged `subject_id` and `unique_id` as `!!str` when loading data config files.
 - Made orientation configurable (was hard-coded as "RPI").
+- Resource-not-found errors now include information about where to source those resources.
 
 ### Fixed
 

CPAC/_entrypoints/run.py

@@ -795,7 +795,7 @@ def run_main():
                 args.data_config_file, args.participant_label, args.aws_input_creds
             )
         sub_list = sub_list_filter_by_labels(
-            sub_list, {"T1w": args.T1w_label, "bold": args.bold_label}
+            list(sub_list), {"T1w": args.T1w_label, "bold": args.bold_label}
         )
 
         # C-PAC only handles single anatomical images (for now)

CPAC/anat_preproc/anat_preproc.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012-2023  C-PAC Developers
+# Copyright (C) 2012-2025  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -2572,7 +2572,7 @@ def brain_mask_acpc_niworkflows_ants_T2(wf, cfg, strat_pool, pipe_num, opt=None)
     config=["anatomical_preproc", "brain_extraction"],
     option_key="using",
     option_val="UNet",
-    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet_model"],
+    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet-model"],
     outputs=["space-T2w_desc-brain_mask"],
 )
 def brain_mask_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None):
@@ -2586,7 +2586,7 @@ def brain_mask_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None):
     config=["anatomical_preproc", "brain_extraction"],
     option_key="using",
     option_val="UNet",
-    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet_model"],
+    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet-model"],
     outputs=["space-T2w_desc-acpcbrain_mask"],
 )
 def brain_mask_acpc_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None):

CPAC/conftest.py

@@ -0,0 +1,34 @@
+# Copyright (C) 2025  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+"""Global fixtures for C-PAC tests."""
+
+from pathlib import Path
+
+from _pytest.tmpdir import TempPathFactory
+from git import Repo
+import pytest
+
+
+@pytest.fixture(scope="session")
+def bids_examples(tmp_path_factory: TempPathFactory) -> Path:
+    """Get the BIDS examples dataset."""
+    example_dir = tmp_path_factory.mktemp("bids-examples")
+    if not example_dir.exists() or not any(example_dir.iterdir()):
+        Repo.clone_from(
+            "https://github.com/bids-standard/bids-examples.git", str(example_dir)
+        )
+    return example_dir

CPAC/pipeline/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2021-2024  C-PAC Developers
+# Copyright (C) 2021-2025  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -17,13 +17,15 @@
 import ast
 import copy
 import hashlib
+from importlib.resources import files
 from itertools import chain
 import json
 import os
 import re
 from typing import Optional
 import warnings
 
+import pandas as pd
 from nipype import config, logging
 from nipype.interfaces import afni
 from nipype.interfaces.utility import Rename
@@ -418,10 +420,12 @@ def get(
             if report_fetched:
                 return (None, None)
             return None
+        from CPAC.pipeline.resource_inventory import where_to_find
+
         msg = (
             "\n\n[!] C-PAC says: None of the listed resources are in "
-            f"the resource pool:\n\n  {resource}\n\nOptions:\n- You "
-            "can enable a node block earlier in the pipeline which "
+            f"the resource pool:\n\n  {where_to_find(resource)}\n\nOptions:\n"
+            "- You can enable a node block earlier in the pipeline which "
             "produces these resources. Check the 'outputs:' field in "
             "a node block's documentation.\n- You can directly "
             "provide this required data by pulling it from another "
@@ -456,7 +460,9 @@ def copy_resource(self, resource, new_name):
         try:
             self.rpool[new_name] = self.rpool[resource]
         except KeyError:
-            msg = f"[!] {resource} not in the resource pool."
+            from CPAC.pipeline.resource_inventory import where_to_find
+
+            msg = f"[!] Not in the resource pool:\n{where_to_find(resource)}"
             raise Exception(msg)
 
     def update_resource(self, resource, new_name):
@@ -628,11 +634,13 @@ def get_strats(self, resources, debug=False):
             total_pool.append(sub_pool)
 
         if not total_pool:
+            from CPAC.pipeline.resource_inventory import where_to_find
+
             raise LookupError(
                 "\n\n[!] C-PAC says: None of the listed "
                 "resources in the node block being connected "
                 "exist in the resource pool.\n\nResources:\n"
-                "%s\n\n" % resource_list
+                "%s\n\n" % where_to_find(resource_list)
             )
 
         # TODO: right now total_pool is:
@@ -1007,6 +1015,19 @@ def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x, outs)
             for label_con_tpl in post_labels:
                 label = label_con_tpl[0]
                 connection = (label_con_tpl[1], label_con_tpl[2])
+                if "desc-" not in label:
+                    if "space-template" in label:
+                        new_label = label.replace(
+                            "space-template", "space-template_desc-zstd"
+                        )
+                    else:
+                        new_label = f"desc-zstd_{label}"
+                else:
+                    for tag in label.split("_"):
+                        if "desc-" in tag:
+                            newtag = f"{tag}-zstd"
+                            new_label = label.replace(tag, newtag)
+                            break
                 if label in Outputs.to_zstd:
                     zstd = z_score_standardize(f"{label}_zstd_{pipe_x}", input_type)
 
@@ -1015,20 +1036,6 @@ def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x, outs)
                     node, out = self.get_data(mask, pipe_idx=mask_idx)
                     wf.connect(node, out, zstd, "inputspec.mask")
 
-                    if "desc-" not in label:
-                        if "space-template" in label:
-                            new_label = label.replace(
-                                "space-template", "space-template_desc-zstd"
-                            )
-                        else:
-                            new_label = f"desc-zstd_{label}"
-                    else:
-                        for tag in label.split("_"):
-                            if "desc-" in tag:
-                                newtag = f"{tag}-zstd"
-                                new_label = label.replace(tag, newtag)
-                                break
-
                     post_labels.append((new_label, zstd, "outputspec.out_file"))
 
                     self.set_data(
@@ -2408,15 +2415,17 @@ def strip_template(data_label, dir_path, filename):
     return data_label, json
 
 
+def template_dataframe() -> pd.DataFrame:
+    """Return the template dataframe."""
+    template_csv = files("CPAC").joinpath("resources/cpac_templates.csv")
+    return pd.read_csv(str(template_csv), keep_default_na=False)
+
+
 def ingress_pipeconfig_paths(wf, cfg, rpool, unique_id, creds_path=None):
     # ingress config file paths
     # TODO: may want to change the resource keys for each to include one level up in the YAML as well
 
-    import pandas as pd
-    import pkg_resources as p
-
-    template_csv = p.resource_filename("CPAC", "resources/cpac_templates.csv")
-    template_df = pd.read_csv(template_csv, keep_default_na=False)
+    template_df = template_dataframe()
     desired_orientation = cfg.pipeline_setup["desired_orientation"]
 
     for row in template_df.itertuples():