Remove use_ datatype e.g. use_behav configurations. (#227)

* Remove 'use_' datatype settings from datashuttle code.

* Update all tests to remove use_datatype flag.

* Remove 'all' default for datatype for make_sub_folders and change to '' which will make no datatype by default.

* Fix rebase.

* Fix documentation.

* Fix linting after rebase.

* Fix broken test after rebase.

Author: JoeZiminski

datashuttle/command_line_interface.py

@@ -520,34 +520,6 @@ def construct_parser():
         action="store_true",
         help=help("flag_default_false"),
     )
-    make_config_file_parser.add_argument(
-        "--use-ephys",
-        "--use_ephys",
-        required=False,
-        action="store_true",
-        help=help("flag_default_false"),
-    )
-    make_config_file_parser.add_argument(
-        "--use-behav",
-        "--use_behav",
-        required=False,
-        action="store_true",
-        help=help("flag_default_false"),
-    )
-    make_config_file_parser.add_argument(
-        "--use-funcimg",
-        "--use_funcimg",
-        required=False,
-        action="store_true",
-        help=help("flag_default_false"),
-    )
-    make_config_file_parser.add_argument(
-        "--use-histology",
-        "--use_histology",
-        required=False,
-        action="store_true",
-        help=help("flag_default_false"),
-    )
 
     make_config_file_parser = subparsers.add_parser(
         "update-config",
@@ -627,7 +599,7 @@ def construct_parser():
         type=str,
         nargs="+",
         required=False,
-        default="all",  # TODO: this is not nice, should read the default from API NOT duplicate in CLI
+        default="",  # TODO: this is not nice, should read the default from API NOT duplicate in CLI
         help=help("required_str_single_or_multiple_or_all"),
     )
 

datashuttle/configs/canonical_configs.py

@@ -38,32 +38,23 @@ def get_canonical_config_dict() -> dict:
         "show_transfer_progress": None,
     }
 
-    datatype_configs = get_datatypes(as_dict=True)
-    config_dict.update(datatype_configs)
-
     return config_dict
 
 
-def get_datatypes(as_dict: bool = False):
+def get_datatypes() -> List[str]:
     """
-    Canonical list of datatype flags. This is used
-    to define get_canonical_config_dict() as well
-    as in testing.
+    Canonical list of datatype flags based on
+    NeuroBlueprint.
     """
-    keys = ["use_ephys", "use_behav", "use_funcimg", "use_histology"]
-
-    if as_dict:
-        return dict(zip(keys, [None] * len(keys)))
-    else:
-        return keys
+    return ["ephys", "behav", "funcimg", "histology"]
 
 
 def get_flags() -> List[str]:
     """
     Return all configs that are bool flags. This is used in
     testing and type checking config inputs.
     """
-    return get_datatypes() + [
+    return [
         "overwrite_old_files",
         "show_transfer_progress",
     ]
@@ -83,10 +74,6 @@ def get_canonical_config_required_types() -> dict:
         "overwrite_old_files": bool,
         "transfer_verbosity": Literal["v", "vv"],
         "show_transfer_progress": bool,
-        "use_ephys": bool,
-        "use_behav": bool,
-        "use_funcimg": bool,
-        "use_histology": bool,
     }
 
     assert (
@@ -168,12 +155,6 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None:
                 f"The passed path was {config_dict[path_type]}"
             )
 
-    if not any([config_dict[key] for key in get_datatypes()]):
-        utils.log_and_raise_error(
-            f"At least one datatype must be True in "
-            f"configs, from: {' '.join(get_datatypes())}."
-        )
-
     # Check SSH settings
     if config_dict["connection_method"] == "ssh" and (
         not config_dict["central_host_id"]

datashuttle/configs/canonical_folders.py

@@ -36,40 +36,23 @@ def get_datatype_folders(cfg: Configs) -> dict:
         This should always match the canonical name, but left as
         an option for rare cases in which advanced users want to change it.
 
-    used : whether the folder is used or not (see make_config_file)
-        if False, the folder will not be made in make_folders
-        even if selected.
-
     level : "sub" or "ses", level to make the folder at.
-
-    Notes
-    ------
-
-    In theory, adding a new  folder should only require
-    adding an entry to this dictionary. However, this will not
-    update configs e.g. use_xxx. This has not been
-    directly tested yet, but if it does not work when attempted
-    it should be configured to from then on.
     """
     return {
         "ephys": Folder(
             name="ephys",
-            used=cfg["use_ephys"],
             level="ses",
         ),
         "behav": Folder(
             name="behav",
-            used=cfg["use_behav"],
             level="ses",
         ),
         "funcimg": Folder(
             name="funcimg",
-            used=cfg["use_funcimg"],
             level="ses",
         ),
         "histology": Folder(
             name="histology",
-            used=cfg["use_histology"],
             level="sub",
         ),
     }

datashuttle/datashuttle.py

@@ -159,7 +159,7 @@ def make_folders(
         self,
         sub_names: Union[str, list],
         ses_names: Optional[Union[str, list]] = None,
-        datatype: Union[List[str], str] = "all",
+        datatype: str = "",
     ) -> None:
         """
         Create a subject / session folder tree in the project
@@ -180,11 +180,10 @@ def make_folders(
                 folders are made.
         datatype :
                 The datatype to make in the sub / ses folders.
-                (e.g. "ephys", "behav", "histology"). Only datatypes
-                that are enabled in the configs (e.g. use_behav) will be
-                created. If "all" is selected, folders will be created
-                for all datatype enabled in config. Use empty string "" for
-                none.
+                (e.g. "ephys", "behav", "histology"). If "all"
+                is selected, all datatypes permitted in
+                NeuroBlueprint will be created. If "" is passed
+                no datatype will be created.
 
         Notes
         -----
@@ -616,10 +615,6 @@ def make_config_file(
         overwrite_old_files: bool = False,
         transfer_verbosity: str = "v",
         show_transfer_progress: bool = False,
-        use_ephys: bool = False,
-        use_behav: bool = False,
-        use_funcimg: bool = False,
-        use_histology: bool = False,
     ) -> None:
         """
         Initialise the configurations for datashuttle to use on the
@@ -680,18 +675,6 @@ def make_config_file(
 
         show_transfer_progress :
             If true, the real-time progress of file transfers will be printed.
-
-        use_ephys :
-            if True, will allow ephys folder creation
-
-        use_funcimg :
-            if True, will allow funcimg folder creation
-
-        use_histology :
-            if True, will allow histology folder creation
-
-        use_behav :
-            if True, will allow behav folder creation
         """
         self._start_log(
             "make-config-file",
@@ -712,10 +695,6 @@ def make_config_file(
                 "overwrite_old_files": overwrite_old_files,
                 "transfer_verbosity": transfer_verbosity,
                 "show_transfer_progress": show_transfer_progress,
-                "use_ephys": use_ephys,
-                "use_behav": use_behav,
-                "use_funcimg": use_funcimg,
-                "use_histology": use_histology,
             },
         )
 

datashuttle/utils/folder_class.py

@@ -9,9 +9,7 @@ class Folder:
     def __init__(
         self,
         name: str,
-        used: bool,
         level: str,
     ):
         self.name = name
-        self.used = used
         self.level = level

datashuttle/utils/folders.py

@@ -104,7 +104,7 @@ def make_datatype_folders(
     datatype_items = cfg.get_datatype_items(datatype)
 
     for datatype_key, datatype_folder in datatype_items:  # type: ignore
-        if datatype_folder.used and datatype_folder.level == level:
+        if datatype_folder.level == level:
             datatype_path = sub_or_ses_level_path / datatype_folder.name
 
             make_folders(datatype_path, log)

docs/source/pages/documentation.md

@@ -68,8 +68,6 @@ The command `make-config-file` is used for the initial setup of the project. The
 
 `connection_method`: `local_filesystem` or `ssh`. Local filesystem can be used if the *central* storage is mounted to the local machine. Otherwise `ssh` can be used.
 
-Finally, the *datatype* flags `--use_ephys`, `--use_funcimg`, `--use_histology`, `--use_behav` set the types of data required for the project on the local machine. While individual flags are optional, at least one must be chosen when initialising the project.
-
 ### Optional Arguments
 
 If connection method is `ssh`, the `central_host_id`, `central_host_username` must be set, and a one-time SSH setup command run (see the [SSH section](#ssh) for details).
@@ -92,7 +90,7 @@ ssh \
 --central_host_id ssh.swc.ucl.ac.uk \
 --central_host_username username \
 --transfer_verbosity v \
---use-ephys --use-behav --use-histology --overwrite_old_files
+--overwrite_old_files
 ```
 
 
@@ -134,11 +132,8 @@ Another example call, which creates a range of subject and session folders, is s
 ```
 datashuttle \
 my_first_project \
-make-folders -sub 001@TO@003 -ses 010_@TIME@ -dt all
-```
 
-When the `all` argument is used for `--datatype` (`-dt`), the folders created depend on the *datatypes* specified during *configuration* setup. For example, if
-`--use_behav`, `--use_funcimg`, `--use_histology` were set during *configuration* setup, the folder tree from the above command (assuming the time is `4.02.48 PM`), would look like:
+make-sub-folders -sub 001@TO@003 -ses 010_@TIME@ -dt behav funcimg histology
 
 ```
 ├── sub-001/
@@ -330,9 +325,6 @@ project.make_config_file(
 	central_host_username="username",
 	overwrite_old_files=True,
 	transfer_verbosity="v",
-	use_ephys=True,
-	use_behav=True,
-	use_histology=True,
 )
 ```