Rename histology to anat and move to session level. (#229)

JoeZiminski · web-flow · commit c02a68fd294c · 2023-10-24T15:25:54.000+01:00
* Update all tests to remove use_datatype flag.

* Fix documentation.

* Update all tests to remove use_datatype flag.

* Rename histology to anat.

* Change position of anat.

* Fix tests.

* Fix documentation.

* Fix documentation.

* Fix linting after rebase.

* Fix documentation again.
diff --git a/README.md b/README.md
@@ -36,32 +36,32 @@ See the SWC-BIDS [specification](https://swc-bids.neuroinformatics.dev/) for mor
     └── raw_data/
         ├── sub-001/
         │   └── ses-001/
-        │   │   ├── ephys/
-        │   │   └── behav/
-        │   └── histology/
+        │       ├── ephys/
+        │       └── behav/
+        │       └── anat/
         └── sub-002/
             └── ses-001/
             │   ├── behav/
             │   └── imaging/
             └── ses-002/
-            │   └── behav/
-            └── histology/
+                └── behav/
+                └── anat/
 ```
 
 
 ```+
 └── project_name/
-    └── rawdata/
+    └── rawdata/[test_utils.py](tests%2Ftest_utils.py)
         ├── sub-001  /
-        │   ├── ses-001/
-        │   │   ├── ephys
-        │   │   └── behav
-        │   └── histology
+        │   └── ses-001/
+        │       ├── ephys
+        │       └── behav
+        │       └── anat
         └── sub-002/
             ├── ses-001/
             │   ├── behav
             │   └── imaging
-            ├── ses-002/
-            │   └── behav
-            └── histology
+            └── ses-002/
+                └── behav
+                └── anat
 ```
diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py
@@ -46,7 +46,7 @@ def get_datatypes() -> List[str]:
     Canonical list of datatype flags based on
     NeuroBlueprint.
     """
-    return ["ephys", "behav", "funcimg", "histology"]
+    return ["ephys", "behav", "funcimg", "anat"]
 
 
 def get_flags() -> List[str]:
diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py
@@ -51,9 +51,9 @@ def get_datatype_folders(cfg: Configs) -> dict:
             name="funcimg",
             level="ses",
         ),
-        "histology": Folder(
-            name="histology",
-            level="sub",
+        "anat": Folder(
+            name="anat",
+            level="ses",
         ),
     }
 
diff --git a/datashuttle/datashuttle.py b/datashuttle/datashuttle.py
@@ -180,7 +180,7 @@ def make_folders(
                 folders are made.
         datatype :
                 The datatype to make in the sub / ses folders.
-                (e.g. "ephys", "behav", "histology"). If "all"
+                (e.g. "ephys", "behav", "anat"). If "all"
                 is selected, all datatypes permitted in
                 NeuroBlueprint will be created. If "" is passed
                 no datatype will be created.
diff --git a/docs/source/pages/documentation.md b/docs/source/pages/documentation.md
@@ -93,7 +93,6 @@ ssh \
 --overwrite_old_files
 ```
 
-
 Now setup is complete! _Configuration_ settings can be edited at any time with the `update-config` command. Alternatively, custom *configuration* files can be supplied using the `supply-config` command (this simplifies setting up projects across multiple *local* machines).
 
 Next, we can start setting up the project by automatically creating standardised project folder trees.
@@ -132,33 +131,31 @@ Another example call, which creates a range of subject and session folders, is s
 ```
 datashuttle \
 my_first_project \
+make-folders -sub 001@TO@003 -ses 010_@TIME@ -dt behav funcimg anat
+```
 
-make-sub-folders -sub 001@TO@003 -ses 010_@TIME@ -dt behav funcimg histology
 
 ```
 ├── sub-001/
-│   ├── ses-010_time-160248/
-│   │   ├── behav
-│   │   └── funcimg
-│   └── histology
+│   └── ses-010_time-160248/
+│       ├── behav
+│       └── funcimg
+│       └── anat
 ├── sub-002/
-│   ├── ses-010_time-160248/
-│   │   ├── behav
-│   │   └── funcimg
-│   └── histology
+│   └── ses-010_time-160248/
+│       └── behav
+│       └── funcimg
+│       └── anat
 └── sub-003/
-    ├── ses-010_time-160248/
-    │   ├── behav
-    │   └── funcimg
-    └── histology
+    └── ses-010_time-160248/
+        ├── behav
+        ├── funcimg
+        └── anat
 ```
 
-
 ### Datatype Folders
 
-In [SWC-Blueprint](https://swc-blueprint.neuroinformatics.dev/specification.html), *datatypes* specify where acquired experimental data of currently supported types (`behav`, `ephys`, `funcimg` and `histology`) is stored. See the [*datatypes* section of the SWC-Blueprint for more details](https://swc-blueprint.neuroinformatics.dev/specification.html#datatype).
-
-At present, `histology` is saved to the `sub-` level, as it is assumed `histology` is conducted *ex vivo* and so session will be possible. Please don't hesitate to get into contact if you have an alternative use case.
+In [SWC-Blueprint](https://swc-blueprint.neuroinformatics.dev/specification.html), *datatypes* specify where acquired experimental data of currently supported types (`behav`, `ephys`, `funcimg` and `anat`) is stored. See the [*datatypes* section of the SWC-Blueprint for more details](https://swc-blueprint.neuroinformatics.dev/specification.html#datatype).
 
 ## Data Transfer
 
@@ -498,15 +495,15 @@ DataShuttle provides a number of keyword arguments to allow separate handling of
 
 `all` : All *session* and non-*session* files and folders within a *subject* level folder (e.g. `sub-001`) will be transferred.
 
-`all_ses` : *Session* <u>folders</u> only (i.e. prefixed with `-ses`) will be transferred. Note that the only exception is the `histology` folder, the transfer of which is determined by the `-dt` flag (below).
+`all_ses` : *Session* <u>folders</u> only (i.e. prefixed with `-ses`) will be transferred.
 
 `all_non_ses` : All files and folders that are not prefixed with `-sub` will be transferred. Any folders prefixed with `-ses` will not be transferred.
 
 #### For use with the `-dt` / `--datatype` flag
 
 `all` : All *datatype* folders at the *subject* or *session* folder level will be transferred, as well as all files and folders within selected *session* folders.
 
-`all_datatype` : All *datatype* folders (i.e. folders with the pre-determined name: `behav`, `ephys`, `funcimg`, `histology`) residing at either the *subject* or *session* level will be
+`all_datatype` : All *datatype* folders (i.e. folders with the pre-determined name: `behav`, `ephys`, `funcimg`, `anat`) residing at the *session* level will be
 transferred. Non-*datatype* folders at the *session* level will not be transferred
 
 `all_ses_level_non_datatype` : Non *datatype* folders at the *session* level will not be transferred
@@ -519,7 +516,6 @@ Below, a number of examples are given to exemplify how these arguments effect da
     ├── a_project_related_file.json
     ├── sub-001/
     │   ├── sub-001_extra-file.json
-    │   ├── histology
     │   └── ses-001/
     │       ├── ses-001_extra-file.json
     │       ├── behav/
@@ -528,7 +524,6 @@ Below, a number of examples are given to exemplify how these arguments effect da
     │           └── ...
     └── sub-002/
         ├── sub-002_extra-file.json
-        ├── histology
         └── ses-001/
 			├── behav/
 			│   └── ...
@@ -569,7 +564,7 @@ Would upload:
 
 - Contents residing in the `sub-001` folder only.
 -  The file `sub-001_extra-file.json`
-- All *datatype* folder contents (`histology`, `behav`, `ephys`)
+- All *datatype* folder contents (`anat`, `behav`, `ephys`)
 
 The command:
 
@@ -585,7 +580,7 @@ upload
 Would upload:
 
 - Contents residing in the `sub-001` folder only.
-- All *datatype* folder contents (`histology`, `behav`, `ephys`)
+- All *datatype* folder contents (`anat`, `behav`, `ephys`)
 
 3) The final example shows the effect of transferring `all_non_sub` files only. The command:
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -321,7 +321,7 @@ def check_datatype_sub_ses_uploaded_correctly(
         sub_names = glob_basenames(join(base_path_to_check, "*"))
         assert sub_names == sorted(subs_to_upload)
 
-        # Check ses are all uploaded + histology if transferred
+        # Check ses are all uploaded
         if ses_to_upload:
             for sub in subs_to_upload:
                 ses_names = glob_basenames(
@@ -331,42 +331,15 @@ def check_datatype_sub_ses_uploaded_correctly(
                         "*",
                     )
                 )
-                if datatype_to_transfer == ["histology"]:
-                    assert ses_names == ["histology"]
-                    return  # handle the case in which histology
-                    # only is transferred,
-                    # and there are no sessions to transfer.
-
-                copy_datatype_to_transfer = (
-                    check_and_strip_within_sub_data_folders(
-                        ses_names, datatype_to_transfer
-                    )
-                )
                 assert ses_names == sorted(ses_to_upload)
 
                 # check datatype folders in session folder
-                if copy_datatype_to_transfer:
+                if datatype_to_transfer:
                     for ses in ses_names:
                         data_names = glob_basenames(
                             join(base_path_to_check, sub, ses, "*")
                         )
-                        assert data_names == sorted(copy_datatype_to_transfer)
-
-
-def check_and_strip_within_sub_data_folders(ses_names, datatype_to_transfer):
-    """
-    Check if datatype folders at the sub level are picked
-    up when sessions are searched for with wildcard. Remove
-    so that sessions can be explicitly tested next.
-    """
-    if "histology" in datatype_to_transfer:
-        assert "histology" in ses_names
-
-        ses_names.remove("histology")
-        copy_ = copy.deepcopy(datatype_to_transfer)
-        copy_.remove("histology")
-        return copy_
-    return datatype_to_transfer
+                        assert data_names == sorted(datatype_to_transfer)
 
 
 def make_and_check_local_project_folders(
diff --git a/tests/tests_integration/file_conflicts_pathtable.py b/tests/tests_integration/file_conflicts_pathtable.py
@@ -16,19 +16,19 @@ def get_pathtable(base_folder):
             [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "behav" / "behav.csv",                        False,      False,       False,                      "sub-001",                  "ses-003_date-20231901",    "behav"],
             [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "ephys" / "ephys.bin",                        False,      False,       False,                      "sub-001",                  "ses-003_date-20231901",    "ephys"],
             [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "non_data" / "non_data.mp4",                  False,      False,       True,                       "sub-001",                  "ses-003_date-20231901",    None],
-            [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "nondatatype_level_file.csv",                False,      False,       True,                       "sub-001",                  "ses-003_date-20231901",    None],
+            [base_folder, Path("rawdata") / "sub-001" / "ses-003_date-20231901" / "nondatatype_level_file.csv",                 False,      False,       True,                       "sub-001",                  "ses-003_date-20231901",    None],
             [base_folder, Path("rawdata") / "sub-001" / "random-ses_level_file.mp4",                                            False,      True,        False,                      "sub-001",                  None,                       None],
-            [base_folder, Path("rawdata") / "sub-001" / "histology" / "sub-001_histology.file",                                 False,      False,       False,                      "sub-001",                  None,                       "histology"],
+            [base_folder, Path("rawdata") / "sub-001" / "ses-004" / "anat" / "sub-001_anat.file",                               False,      False,       False,                      "sub-001",                  "ses-004",                 "anat"],
             [base_folder, Path("rawdata") / "sub-002_random-value" / "sub-002_random-value.file",                               False,      True,        False,                      "sub-002_random-value",     None,                       None],
-            [base_folder, Path("rawdata") / "sub-002_random-value" / "ses-001" / "non_datatype_level_folder" / "file.csv",        False,      False,       True,                       "sub-002_random-value",     "ses-001",                  None],
+            [base_folder, Path("rawdata") / "sub-002_random-value" / "ses-001" / "non_datatype_level_folder" / "file.csv",      False,      False,       True,                       "sub-002_random-value",     "ses-001",                  None],
             [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-001" / "funcimg" / ".myfile.xlsx",                   False,      False,       False,                      "sub-003_date-20231901",    "ses-001",                  "funcimg"],
-            [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-003_date-20231901" / "nondatatype_level_file.csv",  False,      False,       True,                       "sub-003_date-20231901",    "ses-003_date-20231901",    None],
+            [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-003_date-20231901" / "nondatatype_level_file.csv",   False,      False,       True,                       "sub-003_date-20231901",    "ses-003_date-20231901",    None],
             [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-003_date-20231901" / "funcimg" / "funcimg.nii",      False,      False,       False,                      "sub-003_date-20231901",    "ses-003_date-20231901",    "funcimg"],
-            [base_folder, Path("rawdata") / "sub-003_date-20231901" / "seslevel_non-prefix_folder" / "nonlevel.mat",               False,      True,        False,                      "sub-003_date-20231901",    "seslevel_non-prefix_folder",  None],
+            [base_folder, Path("rawdata") / "sub-003_date-20231901" / "seslevel_non-prefix_folder" / "nonlevel.mat",            False,      True,        False,                      "sub-003_date-20231901",    "seslevel_non-prefix_folder",  None],
             [base_folder, Path("rawdata") / "sub-003_date-20231901" / "sub-ses-level_file.txt",                                 False,      True,        False,                      "sub-003_date-20231901",    None,                       None],
-            [base_folder, Path("rawdata") / "sub-003_date-20231901" / "histology" / ".histology.file",                          False,      False,       False,                      "sub-003_date-20231901",    None,                       "histology"],
+            [base_folder, Path("rawdata") / "sub-003_date-20231901" / "ses-004" / "anat" / ".anat.file",                        False,      False,       False,                      "sub-003_date-20231901",    "ses-004",                "anat"],
             [base_folder, Path("rawdata") / "project_level_file.txt",                                                           True,       False,       False,                      None,                       None,                       None],
-            [base_folder, Path("rawdata") / "sublevel_non_sub-prefix_folder" / "ses_non_folder.file",                                 True,       False,       False,                      None,                       None,                       None],
+            [base_folder, Path("rawdata") / "sublevel_non_sub-prefix_folder" / "ses_non_folder.file",                           True,       False,       False,                      None,                       None,                       None],
             ]
 
 
diff --git a/tests/tests_integration/test_configs.py b/tests/tests_integration/test_configs.py
@@ -332,6 +332,7 @@ def test_supplied_config_file_changes_wrong_order(self, project, tmp_path):
         any order and just converted to dict?
         """
         bad_order_configs_path = project._datashuttle_path / "new_configs.yaml"
+
         good_order_configs = test_utils.get_test_config_arguments_dict(
             tmp_path, project.project_name
         )
diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py
@@ -137,12 +137,12 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download):
             ["behav"],
             ["ephys"],
             ["funcimg"],
-            ["histology"],
+            ["anat"],
             ["behav", "ephys"],
-            ["ephys", "histology"],
-            ["behav", "ephys", "histology"],
-            ["funcimg", "histology", "behav"],
-            ["behav", "ephys", "funcimg", "histology"],
+            ["ephys", "anat"],
+            ["behav", "ephys", "anat"],
+            ["funcimg", "anat", "behav"],
+            ["behav", "ephys", "funcimg", "anat"],
         ],
     )
     @pytest.mark.parametrize("upload_or_download", ["upload", "download"])
@@ -179,10 +179,10 @@ def test_transfer_empty_folder_specific_dataal_data(
     @pytest.mark.parametrize(
         "datatype_to_transfer",
         [
-            ["histology"],
+            ["anat"],
             ["behav", "ephys"],
-            ["funcimg", "histology", "behav"],
-            ["behav", "ephys", "funcimg", "histology"],
+            ["funcimg", "anat", "behav"],
+            ["behav", "ephys", "funcimg", "anat"],
         ],
     )
     @pytest.mark.parametrize("upload_or_download", ["upload" "download"])
@@ -223,7 +223,7 @@ def test_transfer_empty_folder_specific_subs(
     @pytest.mark.parametrize("sub_idx_to_upload", [[0], [1, 2], [0, 1, 2]])
     @pytest.mark.parametrize(
         "datatype_to_transfer",
-        [["ephys"], ["funcimg", "histology", "behav"]],
+        [["ephys"], ["funcimg", "anat", "behav"]],
     )
     @pytest.mark.parametrize("upload_or_download", ["upload", "download"])
     def test_transfer_empty_folder_specific_ses(
@@ -429,10 +429,10 @@ def test_rclone_overwrite_modified_file(
         the version in source is newer than target.
         """
         path_to_test_file = (
-            Path("rawdata") / "sub-001" / "histology" / "test_file.txt"
+            Path("rawdata") / "sub-001" / "ses-001" / "anat" / "test_file.txt"
         )
 
-        project.make_folders("sub-001", datatype="histology")
+        project.make_folders("sub-001", "ses-001", datatype="anat")
 
         local_test_file_path = project.cfg["local_path"] / path_to_test_file
         central_test_file_path = (
diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py
@@ -271,8 +271,9 @@ def test_logs_upload_and_download(
 
         assert "Using config file from" in log
         assert "Local file system at" in log
-        assert """ "--include" "sub-11/histology/**" """ in log
-        assert """/central/test_project/rawdata""" in log
+        assert "--include" in log
+        assert "sub-11/ses-123/anat/**" in log
+        assert "/central/test_project/rawdata" in log
         assert "Waiting for checks to finish" in log
 
     @pytest.mark.parametrize("upload_or_download", ["upload", "download"])
diff --git a/tests/tests_integration/test_make_folders.py b/tests/tests_integration/test_make_folders.py
diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py

Original file line number	Diff line number	Diff line change
`@@ -332,6 +332,7 @@ def test_supplied_config_file_changes_wrong_order(self, project, tmp_path):`
`332`	`332`	`any order and just converted to dict?`
`333`	`333`	`"""`
`334`	`334`	`bad_order_configs_path = project._datashuttle_path / "new_configs.yaml"`
	`335`	`+`
`335`	`336`	`good_order_configs = test_utils.get_test_config_arguments_dict(`
`336`	`337`	`tmp_path, project.project_name`
`337`	`338`	`)`