diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index 6ee37158..1cbac62d 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -43,7 +43,7 @@ class TuiApp(App): def compose(self) -> ComposeResult: yield Container( - Label("DataShuttle", id="mainwindow_banner_label"), + Label("datashuttle", id="mainwindow_banner_label"), Button( "Select Existing Project", id="mainwindow_existing_project_button", @@ -118,7 +118,7 @@ def handle_open_filesystem_browser(self, path_: Path) -> None: ) else: message = ( - "Unexpected error occurred. Please contact the DataShuttle" + "Unexpected error occurred. Please contact the datashuttle" "development team." ) diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 81cf63f8..bf27ca03 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -443,7 +443,7 @@ def setup_configs_for_a_new_project(self) -> None: ).disabled = False message = ( - "A DataShuttle project has now been created.\n\n " + "A datashuttle project has now been created.\n\n " "Next, setup the SSH connection. Once complete, navigate to the " "'Main Menu' and proceed to the project page, where you will be " "able to create and transfer project folders." @@ -451,7 +451,7 @@ def setup_configs_for_a_new_project(self) -> None: else: message = ( - "A DataShuttle project has now been created.\n\n " + "A datashuttle project has now been created.\n\n " "Next proceed to the project page, where you will be " "able to create and transfer project folders." ) diff --git a/datashuttle/tui/css/tui_menu.tcss b/datashuttle/tui/css/tui_menu.tcss index 70b72af9..17cf5f46 100644 --- a/datashuttle/tui/css/tui_menu.tcss +++ b/datashuttle/tui/css/tui_menu.tcss @@ -13,7 +13,7 @@ #mainwindow_banner_label { border: hkey $primary; content-align: center middle; - text-style: italic bold; + text-style: bold; color: $text; width: 50%; height: 5; diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 3e80b943..37db17fd 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -285,8 +285,8 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): assert symbol in convert_symbols.keys(), "rclone check symbol is unknown." assert symbol != "!", ( "Could not complete rlcone check. " - "This is unexpected. Please contact DataShuttle " - "devs at our GitHub page." + "This is unexpected. Please contact datashuttle " + "at our GitHub page." ) diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 40b8067b..90d96dbf 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -129,13 +129,11 @@ def setup_ssh_key( """ if not sys.stdin.isatty(): utils.log_and_raise_error( - f"Attempting to run outside of a TTY terminal. " - f"This may happen if attempting to run in an IDE console. " - f"`getpass` (used for ssh password input) cannot run. " - f"Try running using the command-line-interface with " - f"'datashuttle {cfg.project_name} " - f"setup-ssh-connection-to-central-server' " - f"or in a terminal rather than IDE console.", + "Attempting to run outside of a TTY terminal. " + "This may happen if attempting to run in an IDE console. " + "`getpass` (used for ssh password input) cannot run. " + "Try using Python from the operating system " + "terminal to run this function.", RuntimeError, ) diff --git a/docs/rebuild.py b/docs/rebuild.py new file mode 100644 index 00000000..26d3912b --- /dev/null +++ b/docs/rebuild.py @@ -0,0 +1,10 @@ +import shutil +import subprocess +from pathlib import Path + +build_path = Path(__file__).parent / "build" + +if build_path.is_dir(): + shutil.rmtree(build_path) + +subprocess.run("sphinx-build ./source ./build", shell=True) diff --git a/docs/source/_static/NeuroBlueprint_project_tree_dark.png b/docs/source/_static/NeuroBlueprint_project_tree_dark.png new file mode 100644 index 00000000..e1dc6a30 Binary files /dev/null and b/docs/source/_static/NeuroBlueprint_project_tree_dark.png differ diff --git a/docs/source/_static/NeuroBlueprint_project_tree_light.png b/docs/source/_static/NeuroBlueprint_project_tree_light.png new file mode 100644 index 00000000..e63a81bf Binary files /dev/null and b/docs/source/_static/NeuroBlueprint_project_tree_light.png differ diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css index ca565550..836a9b2a 100644 --- a/docs/source/_static/css/custom.css +++ b/docs/source/_static/css/custom.css @@ -46,3 +46,7 @@ html[data-theme=light] { height: auto; margin: 5px auto; } + +.center { + text-align: center; +} diff --git a/docs/source/_static/datashuttle-overview-dark.png b/docs/source/_static/datashuttle-overview-dark.png new file mode 100644 index 00000000..2504c10d Binary files /dev/null and b/docs/source/_static/datashuttle-overview-dark.png differ diff --git a/docs/source/_static/datashuttle-overview-light.png b/docs/source/_static/datashuttle-overview-light.png new file mode 100644 index 00000000..5f4e536a Binary files /dev/null and b/docs/source/_static/datashuttle-overview-light.png differ diff --git a/docs/source/_static/ds_nice_test_ong.png b/docs/source/_static/ds_nice_test_ong.png new file mode 100644 index 00000000..4c219992 Binary files /dev/null and b/docs/source/_static/ds_nice_test_ong.png differ diff --git a/docs/source/_static/gui-walkthrough-card-image.png b/docs/source/_static/gui-walkthrough-card-image.png new file mode 100644 index 00000000..861f297c Binary files /dev/null and b/docs/source/_static/gui-walkthrough-card-image.png differ diff --git a/docs/source/_static/gui-walkthrough-card-image_.png b/docs/source/_static/gui-walkthrough-card-image_.png new file mode 100644 index 00000000..9c75c5cd Binary files /dev/null and b/docs/source/_static/gui-walkthrough-card-image_.png differ diff --git a/docs/source/_static/screenshots/how-to-choose-a-terminal-bad-dark.png b/docs/source/_static/screenshots/how-to-choose-a-terminal-bad-dark.png new file mode 100644 index 00000000..8e4bb7c9 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-choose-a-terminal-bad-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-choose-a-terminal-bad-light.png b/docs/source/_static/screenshots/how-to-choose-a-terminal-bad-light.png new file mode 100644 index 00000000..67ed447f Binary files /dev/null and b/docs/source/_static/screenshots/how-to-choose-a-terminal-bad-light.png differ diff --git a/docs/source/_static/screenshots/how-to-choose-a-terminal-good-dark.png b/docs/source/_static/screenshots/how-to-choose-a-terminal-good-dark.png new file mode 100644 index 00000000..ce754e68 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-choose-a-terminal-good-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-choose-a-terminal-good-light.png b/docs/source/_static/screenshots/how-to-choose-a-terminal-good-light.png new file mode 100644 index 00000000..0ed967dc Binary files /dev/null and b/docs/source/_static/screenshots/how-to-choose-a-terminal-good-light.png differ diff --git a/docs/source/_static/screenshots/how-to-create-folders-example-dark.png b/docs/source/_static/screenshots/how-to-create-folders-example-dark.png new file mode 100644 index 00000000..08fbbdd0 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-create-folders-example-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-create-folders-example-light.png b/docs/source/_static/screenshots/how-to-create-folders-example-light.png new file mode 100644 index 00000000..840e1408 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-create-folders-example-light.png differ diff --git a/docs/source/_static/screenshots/how-to-create-folders-settings-dark.png b/docs/source/_static/screenshots/how-to-create-folders-settings-dark.png new file mode 100644 index 00000000..d6da5874 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-create-folders-settings-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-create-folders-settings-light.png b/docs/source/_static/screenshots/how-to-create-folders-settings-light.png new file mode 100644 index 00000000..ce55b723 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-create-folders-settings-light.png differ diff --git a/docs/source/_static/screenshots/how-to-create-project-ssh-dark.png b/docs/source/_static/screenshots/how-to-create-project-ssh-dark.png new file mode 100644 index 00000000..238a25d3 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-create-project-ssh-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-create-project-ssh-light.png b/docs/source/_static/screenshots/how-to-create-project-ssh-light.png new file mode 100644 index 00000000..58804e9e Binary files /dev/null and b/docs/source/_static/screenshots/how-to-create-project-ssh-light.png differ diff --git a/docs/source/_static/screenshots/how-to-logs-filesbrowser-dark.png b/docs/source/_static/screenshots/how-to-logs-filesbrowser-dark.png new file mode 100644 index 00000000..2e1f8b1e Binary files /dev/null and b/docs/source/_static/screenshots/how-to-logs-filesbrowser-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-logs-filesbrowser-light.png b/docs/source/_static/screenshots/how-to-logs-filesbrowser-light.png new file mode 100644 index 00000000..97a97f1f Binary files /dev/null and b/docs/source/_static/screenshots/how-to-logs-filesbrowser-light.png differ diff --git a/docs/source/_static/screenshots/how-to-logs-tui-dark.png b/docs/source/_static/screenshots/how-to-logs-tui-dark.png new file mode 100644 index 00000000..a0825878 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-logs-tui-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-logs-tui-light.png b/docs/source/_static/screenshots/how-to-logs-tui-light.png new file mode 100644 index 00000000..7949b13e Binary files /dev/null and b/docs/source/_static/screenshots/how-to-logs-tui-light.png differ diff --git a/docs/source/_static/screenshots/how-to-name-templates-dark.png b/docs/source/_static/screenshots/how-to-name-templates-dark.png new file mode 100644 index 00000000..7a41ae0b Binary files /dev/null and b/docs/source/_static/screenshots/how-to-name-templates-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-name-templates-light.png b/docs/source/_static/screenshots/how-to-name-templates-light.png new file mode 100644 index 00000000..ed7bc28a Binary files /dev/null and b/docs/source/_static/screenshots/how-to-name-templates-light.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-all-dark.png b/docs/source/_static/screenshots/how-to-transfer-all-dark.png new file mode 100644 index 00000000..52f1bb54 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-all-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-all-light.png b/docs/source/_static/screenshots/how-to-transfer-all-light.png new file mode 100644 index 00000000..dec9b504 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-all-light.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-custom-dark.png b/docs/source/_static/screenshots/how-to-transfer-custom-dark.png new file mode 100644 index 00000000..eba64596 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-custom-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-custom-light.png b/docs/source/_static/screenshots/how-to-transfer-custom-light.png new file mode 100644 index 00000000..6fc2d7db Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-custom-light.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-datatypes-dark.png b/docs/source/_static/screenshots/how-to-transfer-datatypes-dark.png new file mode 100644 index 00000000..ffb797aa Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-datatypes-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-datatypes-light.png b/docs/source/_static/screenshots/how-to-transfer-datatypes-light.png new file mode 100644 index 00000000..06d96cc9 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-datatypes-light.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-toplevel-dark.png b/docs/source/_static/screenshots/how-to-transfer-toplevel-dark.png new file mode 100644 index 00000000..9e1a41a3 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-toplevel-dark.png differ diff --git a/docs/source/_static/screenshots/how-to-transfer-toplevel-light.png b/docs/source/_static/screenshots/how-to-transfer-toplevel-light.png new file mode 100644 index 00000000..0b81b8c5 Binary files /dev/null and b/docs/source/_static/screenshots/how-to-transfer-toplevel-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-datatype-dark.png b/docs/source/_static/screenshots/tutorial-1-create-datatype-dark.png new file mode 100644 index 00000000..d0e827a1 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-datatype-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-datatype-light.png b/docs/source/_static/screenshots/tutorial-1-create-datatype-light.png new file mode 100644 index 00000000..e7c3cb75 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-datatype-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-screen-dark.png b/docs/source/_static/screenshots/tutorial-1-create-screen-dark.png new file mode 100644 index 00000000..6a3c1d95 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-screen-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-screen-light.png b/docs/source/_static/screenshots/tutorial-1-create-screen-light.png new file mode 100644 index 00000000..43ff2aaf Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-screen-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-session-dark.png b/docs/source/_static/screenshots/tutorial-1-create-session-dark.png new file mode 100644 index 00000000..e44c7b5a Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-session-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-session-light.png b/docs/source/_static/screenshots/tutorial-1-create-session-light.png new file mode 100644 index 00000000..e84b18bb Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-session-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-subject-dark.png b/docs/source/_static/screenshots/tutorial-1-create-subject-dark.png new file mode 100644 index 00000000..407b6cd8 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-subject-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-create-subject-light.png b/docs/source/_static/screenshots/tutorial-1-create-subject-light.png new file mode 100644 index 00000000..76055550 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-create-subject-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-explore-folders-dark.png b/docs/source/_static/screenshots/tutorial-1-explore-folders-dark.png new file mode 100644 index 00000000..1105925a Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-explore-folders-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-explore-folders-light.png b/docs/source/_static/screenshots/tutorial-1-explore-folders-light.png new file mode 100644 index 00000000..95eec9aa Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-explore-folders-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-landing-screen-dark.png b/docs/source/_static/screenshots/tutorial-1-landing-screen-dark.png new file mode 100644 index 00000000..87ac9b93 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-landing-screen-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-landing-screen-light.png b/docs/source/_static/screenshots/tutorial-1-landing-screen-light.png new file mode 100644 index 00000000..0e84ca61 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-landing-screen-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-central-path-dark.png b/docs/source/_static/screenshots/tutorial-1-make-central-path-dark.png new file mode 100644 index 00000000..5ff10a25 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-central-path-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-central-path-light.png b/docs/source/_static/screenshots/tutorial-1-make-central-path-light.png new file mode 100644 index 00000000..4556450a Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-central-path-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-local-path-dark.png b/docs/source/_static/screenshots/tutorial-1-make-local-path-dark.png new file mode 100644 index 00000000..5bab6f87 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-local-path-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-local-path-light.png b/docs/source/_static/screenshots/tutorial-1-make-local-path-light.png new file mode 100644 index 00000000..644a94fe Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-local-path-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-project-name-dark.png b/docs/source/_static/screenshots/tutorial-1-make-project-name-dark.png new file mode 100644 index 00000000..44950028 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-project-name-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-project-name-light.png b/docs/source/_static/screenshots/tutorial-1-make-project-name-light.png new file mode 100644 index 00000000..72ae3625 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-project-name-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-screen-dark.png b/docs/source/_static/screenshots/tutorial-1-make-screen-dark.png new file mode 100644 index 00000000..eda1d7f1 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-screen-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-make-screen-light.png b/docs/source/_static/screenshots/tutorial-1-make-screen-light.png new file mode 100644 index 00000000..6822e7cb Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-make-screen-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-dark.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-dark.png new file mode 100644 index 00000000..96e8347c Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-dark.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-dark.png new file mode 100644 index 00000000..6ff3879f Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-light.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-light.png new file mode 100644 index 00000000..b857301a Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-light.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-light.png new file mode 100644 index 00000000..86c8d32a Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-sessions-dark.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-sessions-dark.png new file mode 100644 index 00000000..b30c2f59 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-sessions-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-sessions-light.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-sessions-light.png new file mode 100644 index 00000000..e4151f48 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-sessions-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-subjects-dark.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-subjects-dark.png new file mode 100644 index 00000000..37ed99f4 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-subjects-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-subjects-light.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-subjects-light.png new file mode 100644 index 00000000..e2fc44de Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-subjects-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-switch-dark.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-switch-dark.png new file mode 100644 index 00000000..03db573e Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-switch-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-switch-light.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-switch-light.png new file mode 100644 index 00000000..a22420d7 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-custom-switch-light.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-upload-dark.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-upload-dark.png new file mode 100644 index 00000000..58a13aed Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-upload-dark.png differ diff --git a/docs/source/_static/screenshots/tutorial-1-transfer-screen-upload-light.png b/docs/source/_static/screenshots/tutorial-1-transfer-screen-upload-light.png new file mode 100644 index 00000000..2ab03bb9 Binary files /dev/null and b/docs/source/_static/screenshots/tutorial-1-transfer-screen-upload-light.png differ diff --git a/docs/source/_static/screenshots/updating-configs-dark.png b/docs/source/_static/screenshots/updating-configs-dark.png new file mode 100644 index 00000000..969b975a Binary files /dev/null and b/docs/source/_static/screenshots/updating-configs-dark.png differ diff --git a/docs/source/_static/screenshots/updating-configs-light.png b/docs/source/_static/screenshots/updating-configs-light.png new file mode 100644 index 00000000..1e20e55e Binary files /dev/null and b/docs/source/_static/screenshots/updating-configs-light.png differ diff --git a/docs/source/_static/tutorial-1-example-file-tree-dark.png b/docs/source/_static/tutorial-1-example-file-tree-dark.png new file mode 100644 index 00000000..f21654c9 Binary files /dev/null and b/docs/source/_static/tutorial-1-example-file-tree-dark.png differ diff --git a/docs/source/_static/tutorial-1-example-file-tree-light.png b/docs/source/_static/tutorial-1-example-file-tree-light.png new file mode 100644 index 00000000..8e4f5f80 Binary files /dev/null and b/docs/source/_static/tutorial-1-example-file-tree-light.png differ diff --git a/docs/source/_static/tutorial-1-explore-folders-dark.png b/docs/source/_static/tutorial-1-explore-folders-dark.png new file mode 100644 index 00000000..fe86fab6 Binary files /dev/null and b/docs/source/_static/tutorial-1-explore-folders-dark.png differ diff --git a/docs/source/_static/tutorial-1-explore-folders-light.png b/docs/source/_static/tutorial-1-explore-folders-light.png new file mode 100644 index 00000000..c72fdea7 Binary files /dev/null and b/docs/source/_static/tutorial-1-explore-folders-light.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index cfe4168a..bf12ef82 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -62,6 +62,8 @@ "strikethrough", "substitution", "tasklist", + "attrs_block", # https://stackoverflow.com/questions/78183173/custom-styling-a-header-in-sphinx-website?noredirect=1#comment137843002_78183173 + "attrs_inline" ] # Automatically add anchors to markdown headings myst_heading_anchors = 3 @@ -152,6 +154,7 @@ }, "footer_start": ["footer_start"], "footer_end": ["footer_end"], + "show_prev_next": False, "show_toc_level": 2, # sidebar levels that are expanded before scrolling # "secondary_sidebar_items": [], # "page_sidebar_items": [], diff --git a/docs/source/index.md b/docs/source/index.md index 361b28c7..ab728977 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,48 +1,69 @@ :html_theme.sidebar_secondary.remove: -# DataShuttle Documentation +{.center} +# **datashuttle** -This website includes the documentation and API documentation for DataShuttle. +

The tool to automate neuroscience project folder creation and transfer.

-For additional information, please visit the [GitHub Page](https://github.com/neuroinformatics-unit/datashuttle). - -Any feedback is much appreciated, please open at [GitHub Issue](https://github.com/neuroinformatics-unit/datashuttle/issues) -or join our [Zulip chat](https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle) if you have any problems or suggestions. - -## Explore the Documentation +```{image} _static/datashuttle-overview-light.png +:alt: My Logo +:class: logo, mainlogo, only-light +:align: center +:width: 600px +``` +```{image} _static/datashuttle-overview-dark.png +:alt: My Logo +:class: logo, mainlogo, only-dark +:align: center +:width: 600px +``` +
::::{grid} 1 2 2 3 -:gutter: 3 +:gutter: 4 -:::{grid-item-card} User Guide -:link: pages/documentation +:::{grid-item-card} Tutorials +:link: pages/tutorials :link-type: doc -Get started with DataShuttle. +Walkthrough **datashuttle**. +::: + +:::{grid-item-card} How To +:link: pages/how_tos +:link-type: doc +Short guides on specific actions. ::: :::{grid-item-card} Python API :link: pages/api_index :link-type: doc -Check out the API reference. +Full Python API reference. ::: +:::: -:::{grid-item-card} Command-line interface -:link: pages/cli_index -:link-type: doc +Datashuttle creates and validates projects standardised to the +[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev) +specification. -Use datashuttle with the CLI. -::: +Dive right into **datashuttle** with our +[Getting Started Tutorial]((tutorial-getting-started)=) +or targeted [How-To Guides](how-tos). + +Don't hesitate to get in contact through our +[GitHub Issues](https://github.com/neuroinformatics-unit/datashuttle/issues) +or +[Zulip Chat.](https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle) -:::: ```{toctree} :maxdepth: 2 :caption: index :hidden: -pages/documentation +pages/tutorials +pages/how_tos pages/api_index ``` diff --git a/docs/source/pages/documentation.md b/docs/source/pages/documentation.md index c12dd1bb..d818ce38 100644 --- a/docs/source/pages/documentation.md +++ b/docs/source/pages/documentation.md @@ -399,79 +399,6 @@ necessary for SSH transfer. Next, we can start setting up the project by automatically creating standardised project folder trees. -# Creating *subject* and *session* folders - -In a typical neuroscience experiment, data collection sessions begin by -creating the folder for the current *subject* (e.g. mouse, rat) and current *session*. - -The command `create-folders` can be used to automatically create standardised -folder trees (see -[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) for details on the standardisation scheme). - - -An example project directory for the behavioural data for the first *subject*, -first *session* of an experiment may look like: - -``` -└── my_first_project/ - └── rawdata/ - └── sub-001/ - └── ses-001_date-20230516/ - └── behav -``` - -In Datashuttle, this folder tree can be created with the below command -(assuming today's date is `20230516`): - -::::{tab-set} - -:::{tab-item} Python API -```{code-block} python -project.create_folders( - sub_names="001", - ses_names="001_@DATE@", - datatype="behav" -) -``` -::: - -:::{tab-item} CLI (macOS / Linux) -```{code-block} console -datashuttle \ -my_first_project \ -create-folders -sub 001 -ses 001_@DATE@ -dt behav -``` -::: - -:::{tab-item} CLI (Windows) -```{code-block} console -datashuttle ^ -my_first_project ^ -create-folders -sub 001 -ses 001_@DATE@ -dt behav -``` -::: - -:::: - -The leading `sub-` or `ses-` in the provided *subject* and *session* names -are optional in the command, these prefixes -will always be added -(i.e. both `-sub 001` and `-sub sub-001` are valid inputs). - -It is possible to automatically create date, time or datetime key-value pairs with the tags -`@DATE@`, `@TIME@` or `@DATETIME@` respectively. -A range of *subjects* and *sessions* can be created with the `@TO@` flag. -See the [convenience tags](#convenience-tags) section for more details. - - -## *Datatype* Folders - -*Datatype* folders contain acquired experimental data and must be named either -`behav`, `ephys`, `funcimg` or `anat`. *Datatype* folders must be placed -within a *session* folder. -See the [*datatype*](https://neuroblueprint.neuroinformatics.dev/specification.html#datatype) section -of NeuroBlueprint for details. - # Data Transfer Datashuttle offers a convenient way of transferring entire project folders or @@ -650,6 +577,7 @@ For example, the command: :::{tab-item} Python API ```{code-block} python project.create_folders( + top_level_folder="rawdata", sub_names="sub-001", ses_names=["001_@DATETIME@", "002_@DATETIME@"], datatype="behav", @@ -1008,6 +936,7 @@ The command: :::{tab-item} Python API ```{code-block} python project.create_folders( + top_level_folder="rawdata", sub_names="001", ses_names="all", datatype="all_datatype" @@ -1054,6 +983,7 @@ but not the non-*datatype* file `ses-001_extra-file.json`. :::{tab-item} Python API ```{code-block} python project.create_folders( + top_level_folder="rawdata", sub_names="all_non_sub", ses_names="all", datatype="all" diff --git a/docs/source/pages/how_tos.md b/docs/source/pages/how_tos.md new file mode 100644 index 00000000..e6565b78 --- /dev/null +++ b/docs/source/pages/how_tos.md @@ -0,0 +1,72 @@ +:html_theme.sidebar_secondary.remove: +(how-tos)= +# How To + +## Get Started + +::::{grid} 1 1 2 2 +:gutter: 4 + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Install +:link: how_tos/install +:link-type: doc + +How to install **datashuttle** +::: + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Choose a Terminal +:link: how_tos/choose-a-terminal +:link-type: doc + +Choose the best terminal to run the graphical interface. +::: + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Make a New Project +:link: how_tos/make-a-new-project +:link-type: doc + +Full details on setting up a new project. +::: + +:::: + +## Manage your project +::::{grid} 1 1 2 2 +:gutter: 4 + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Create Folders +:link: how_tos/create-folders +:link-type: doc + +All you need to know about creating folders. +::: + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Transfer Data +:link: how_tos/transfer-data +:link-type: doc + +The ultimate guide to data-transfer. +::: + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Update Configs +:link: how_tos/update-configs +:link-type: doc + +Changing an existing project's configurations. +::: + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Read the Logs +:link: how_tos/read-logs +:link-type: doc + +How to access full logs of everything **datashuttle** does. +::: + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Use Name Templates +:link: how_tos/use-name-templates +:link-type: doc + +A handy feature for template subject and session names. +::: + +:::: diff --git a/docs/source/pages/how_tos/choose-a-terminal.md b/docs/source/pages/how_tos/choose-a-terminal.md new file mode 100644 index 00000000..ddca77c8 --- /dev/null +++ b/docs/source/pages/how_tos/choose-a-terminal.md @@ -0,0 +1,82 @@ +(how-to-choose-a-terminal)= + +# How to Choose a Terminal + +**datashuttle**'s graphical interface can run in almost any terminal—but +native Windows and macOS terminals may display with rendering errors. + +If you are using Linux, the native terminal will work without issue. + + +::::{tab-set} + +:::{tab-item} Bad Rendering + +```{image} /_static/screenshots/how-to-choose-a-terminal-bad-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-choose-a-terminal-bad-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +::: + +:::{tab-item} Good Rendering + +```{image} /_static/screenshots/how-to-choose-a-terminal-good-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-choose-a-terminal-good-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ + +::: + +:::: + +## Choosing a terminal for Windows or macOS + +::::{tab-set} + +:::{tab-item} Windows + +For Windows, we recommend using +[Terminal](https://apps.microsoft.com/detail/9n0dx20hk701?rtc=1&hl=en-gb&gl=GB), +Window's own high-powered terminal available for free from the +Microsoft Store. + +**datashuttle** will run in Command Prompt (`cmd.exe`), Powershell (`ps.exe`) +or [Anaconda Prompt](https://docs.anaconda.com/free/anaconda/getting-started/index.html), +but may display with rendering errors. + +::: + +:::{tab-item} macOS + +On macOS, we recommend using next-generation terminal emulators such +as [Warp](https://warp.dev/) +or [Wezterm](https://wezfurlong.org/wezterm/index.html). +Both come with easy-to-use installers. + +**datashuttle** will run on the native macOS terminal but may display +with rendering errors. + +::: + +:::: + +## Getting started with the Graphical Interface + +Once you've chosen a terminal, get started with +**datashuttle** with our [Getting Started Tutorial](tutorial-getting-started). diff --git a/docs/source/pages/how_tos/create-folders.md b/docs/source/pages/how_tos/create-folders.md new file mode 100644 index 00000000..3b374766 --- /dev/null +++ b/docs/source/pages/how_tos/create-folders.md @@ -0,0 +1,216 @@ +(how-to-create-folders)= +# How to Create Folders + +**datashuttle** creates project folders +according to the [NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) +specification. + +Before jumping into the folder-creation process, we'll quickly +review the key features of the +[specification](https://neuroblueprint.neuroinformatics.dev/specification.html)) +that are created folders must conform to. + +In [NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) for each +subject and session there are datatype folders in which acquired +data is saved: + +```{image} /_static/NeuroBlueprint_project_tree_dark.png + :align: center + :class: only-dark + :width: 650px +``` +```{image} /_static/NeuroBlueprint_project_tree_light.png + :align: center + :class: only-light + :width: 650px +``` +
+ + +The subject and session folder names must begin with `sub-` and `ses-` +respectively—other key-value pairs are optional. All acquired data must go +in a datatype folder with a +[standard name](https://neuroblueprint.neuroinformatics.dev/specification.html). + +No strong requirements are made on filenames of acquired data, but it is recommended +to include the subject and session number if possible. + +Now the specification has been introduced, let's dive in to folder creation! + + +## Creating project folders + +In the below example, folders will be created in the `rawdata` folder, +within the `my_first_project` project folder. + +The project folder is located at the **local path** +specified when [setting up the project](make-a-new-project). + +We will create datatype folders `behav` and `funcimg` +within a `ses-001_` for both `sub-001` and `sub-002`. + +The below example uses the `@DATE@` convenience tag to automate +creation of today's date. See the section below for more +information on +[convenience tags](create-folders-convenience-tags). + + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + + +```{image} /_static/screenshots/how-to-create-folders-example-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-create-folders-example-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ + +Folders are created in the `Create` tab on the `Project Manager` page. + +We can fill in the subject and session names and select datatype +folders to create. + +Note that the `sub-` or `ses-` prefix is not actually required and will +be automatically added. + +### `Create` tab shortcuts + +The `Create` tab has a lot of useful shortcuts. + +First, **double-clicking the subject or session input boxes** will suggest +the next subject or session to create, based on the local project. +If a [Name Template](how-to-use-name-templates) is set, the +suggested name will also include the template. + +Holding `CTRL` while clicking will add the `sub-` +or `ses-` prefix only. + +Next, the **Directory Tree** has a number of useful shortcuts. These are +activated by hovering the mouse and pressing one of the below combination +of keys (you may need to click the `Directory Tree`) first: + +Fill an input +: `CTRL+F` will fill the subject or session input with the name +of the folder (prefixed with `sub-` or `ses-`) that is hovered over. + +Append to an input +: `CTRL+A` is similar to 'fill' above, but will instead append the name +to those already in the input. This allows creation of lists. + +Open folder in system filebrowser +: `CTRL+O` will open (any) folder in the system filebrowser. + +Copy the full filepath. +: `CTRL+Q` will copy the entire filepath of the file or +folder that is hovered over. + + +### `Create` tab Settings + +Clicking the `Settings` button on the `Create` tab will give access +allow setting the top-level folder, and bypass validation. + +```{image} /_static/screenshots/how-to-create-folders-settings-dark.png + :align: center + :class: only-dark + :width: 500px +``` +```{image} /_static/screenshots/how-to-create-folders-settings-light.png + :align: center + :class: only-light + :width: 500px +``` +
+ +Top-level folder +: This dropdown box will set whether folderes are created in the +`rawdata` or `derivatives` top-level folder + +Bypass validation +: If on, this setting will allow folder creation even if the names +are not valid (e.g. break with +[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/)). + +This screen is also used to set validation against +[Name Templates](how-to-use-name-templates). + +::: + +:::{tab-item} Python API +:sync: python + +Creating folders can be done with the `create_folders()` method in the Python API. +We simply need to provide the subject, session and datatypes to create: + +```python +from datashuttle import DataShuttle + +project = DataShuttle("my_first_project") + +created_folders = project.create_folders( + top_level_folder="rawdata", + sub_names=["sub-001", "002"], + ses_names="ses-001_@DATE@", + datatype=["behav", "funcimg"] +) +``` + +We provides **datashuttle** with a list of subject, session and +datatype folders to create. + +Note that the `sub-` or `ses-` prefix is not actually required and will +be automatically added. + +The method outputs `created_folders`, which contains a list of all +`Path`s to all created datatype folders. +::: +:::: + +:::{admonition} Folder Validation +:class: note + +The names of the folders to be created are validated on the fly against +[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/). +If the folder names will break with the specification, an error will +be raised and the folders will not be created. + +Validation can be extended by defining custom templates for subject +or session names—if folders don't match the template an error will be raised. +See [How to use Name Templates](how-to-use-name-templates) for more information. + +::: + +(create-folders-convenience-tags)= +## Convenience Tags + +There are four convenience tags that can be used in subject or session +names when creating folders. They automate the inclusion of: + +Today's Date +: The `@DATE@` tag will include today's date in the format `YYYYMMDD`. \ + *e.g.* If today's date is 16th May 2024, the name `"ses-001_@DATE@"` will +create the folder `ses-001_date-20241605`. + +Current Time +: The `@TIME@` tag will include the current time in the format `HHMMSS`. \ + *e.g.* If the current time is `15:10:05` (i.e. 10 minutes and 5 seconds past 3 pm.), +the name `"ses-001_@TIME@"` will create the folder `ses-001_time-151005`. + +Current Datetime +: The `@DATETIME@` tag will add the +[ISO8601](https://en.wikipedia.org/wiki/ISO_8601)-formatted datetime. \ + *e.g.* If the date and time are as above, the name `"ses-001_@DATETIME@"` will +create the folder `ses-001_datetime-20241605T151005`. + +A Range of Folders +: The`@TO@` tag creates a range of subject or session numbers. \ + *e.g.* `"sub-001@TO@003"` would create subject folders `sub-001`, `sub-002`, `sub-003`. diff --git a/docs/source/pages/how_tos/install.md b/docs/source/pages/how_tos/install.md new file mode 100644 index 00000000..5e84d45a --- /dev/null +++ b/docs/source/pages/how_tos/install.md @@ -0,0 +1,76 @@ +(how-to-install)= +# How to Install + +**datashuttle** requires Python and a number of other dependencies to run. + +The easiest way to install **datashuttle** is through [conda](https://docs.conda.io/en/latest/), +but installation via `pip` and for developers is also supported. + +## Installation instructions + +::::{tab-set} + +:::{tab-item} Conda + +If you do not already have `conda` on your system, first +[download and install conda](https://docs.anaconda.com/free/miniconda/miniconda-install/). + +If you are on Windows, the easiest way to use `conda` is through the [Anaconda Prompt](https://docs.anaconda.com/free/anaconda/getting-started/index.html) + +Next, create and activate an environment. You can call your environment whatever you like, +we've used `datashuttle-env`. + +```sh +conda create -n datashuttle-env python=3.10 +conda activate datashuttle-env +``` + +then install **datashuttle** and all dependencies with + +```sh +conda install -c conda-forge datashuttle +``` + +::: + +:::{tab-item} Pip + +**datashuttle** depends on [RClone](https://rclone.org/), which is not available through `pip`. +[Rclone must be installed separately](https://rclone.org/downloads/). + +Once Rclone is installed, **datashuttle** and all other dependencies can be +installed in a `pipenv` or `virtualenv` environment with + +```shell +pip install datashuttle +``` + +::: + +:::{tab-item} Developers +To get the latest development version, clone the +[GitHub repository](https://github.com/neuroinformatics-unit/datashuttle/) +and then run from inside the repository + +```sh +pip install -e .[dev] # works on most shells +pip install -e '.[dev]' # works on zsh (the default shell on macOS) +``` + +This will install the package in editable mode, including all `dev` dependencies. +::: + +:::: + +## Check the installation + +To check **datashuttle** has successfully installed, launch the +graphical interface with + +```shell +datashuttle launch +``` + +Before using the graphical interface, you may want to +[choose the best terminal](how-to-choose-a-terminal) +for your operating system. diff --git a/docs/source/pages/how_tos/make-a-new-project.md b/docs/source/pages/how_tos/make-a-new-project.md new file mode 100644 index 00000000..4596b4da --- /dev/null +++ b/docs/source/pages/how_tos/make-a-new-project.md @@ -0,0 +1,277 @@ +(make-a-new-project)= + +# How to Make a New Project + +This guide will cover all we need to know for setting up a new project +in **datashuttle**. + +First, make sure you have +[installed and launched **datashuttle**](how-to-install). + +Next, we set up **datashuttle** on a new machine we must tell it three things: + +1) **project name**: The name of the project (must be the same for all +local machines tied to a project). +2) **local path**: location of the project our local machine, where we will save acquired data. +3) **central path**: location of the central data storage, where we will upload the acquired data. + +```{image} /_static/datashuttle-overview-dark.png + :align: center + :class: only-dark + :width: 500px +``` +```{image} /_static/datashuttle-overview-light.png + :align: center + :class: only-light + :width: 500px +``` +
+ +How the **central path** is set depends on whether your connection to +central storage is as a +[mounted drive](new-project-mounted-drive) +or via +[SSH](new-project-ssh). + +If you are unsure of your connection method, speak to your lab administrator +or IT department. + +(new-project-mounted-drive)= +## When central storage is a mounted drive + +When the central storage machine is mounted as a mounted drive, we +simply need to set the **central path** as the path to +the central project as it appears on your local machine's filesystem. + +:::{dropdown} Local Filesystem Example +:color: info +:icon: info + +Imagine your central storage is a remote server that is mounted to +your machine at `X:\username`. You want your project folder to +be located at `X:\username\my_projects`. + +In this case, you can set the **central_path** to `X:\username\my_projects` +and with **connection_method** to `local_filesystem`. + +You may pass the local or central path without the **project name**, +it will be automatically included. The project folder will be located +at `X:\username\my_projects\my_project_name`. + +::: + +In addition, we need to tell **datashuttle** the project name and +local path where we want to put our project and hold data on +our local machine. + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +From the launch page, click `Make New Project` and you will +be taken to the page where project details must be entered + +```{image} /_static/screenshots/tutorial-1-make-screen-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/tutorial-1-make-screen-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +(general-tui-datashuttle-setup)= +Setting up **datashuttle** is as simple as entering the `Project name`, +`Local Path` and `Central Path` into the relevant input boxes. The paths +do not need to end in the project name - this will be automatically added. + +You can paste a path into the input boxes with `CTRL+V`, copy the input path +with `CTRL+Q` or open the path in your systems filebrowser with `CTRL+O`. + +Use the `Select` feature to navigate to the local and central paths +on your local filesystem. + +By default, the `Connection Method` is set to `Local Filesystem`, so +this does not need to be changed. + +Once all information is input, click `Save` to set up the project. +You can then navigate to the `Project Manager` screen by clicking the +``Go To Project Screen`` that appears. +::: + +:::{tab-item} Python API +:sync: python + +We will first import and initialise the `DataShuttle` class with our +`project_name`. + + +```{code-block} python +from datashuttle import DataShuttle + +project = DataShuttle("my_first_project") + +``` + +Next, we can use the `make_config_file()` method to set up a new +project with the desired **local path**, **central path** and +**connection method**. + +```{code-block} python +project.make_config_file( + local_path=r"C:\my_local_machine\username\my_projects\my_first_project", + central_path=r"X:\a_mounted_drive\username\my_projects\my_first_project", + connection_method="local_filesystem", +) +``` + +Now the project is set up! See the later section for +[optional arguments that control data transfers](make-project-extra-arguments). + +::: +:::: + +(new-project-ssh)= +## Connecting to central storage via SSH + +Another common method of connecting to a central storage machine is via +[SSH](https://www.ssh.com/academy/ssh/protocol). + +To set up SSH connection +we need to give **datashuttle** the address of the machine to connect to, +and now the **central path** will be relative to the machine +we are connecting to. + +**central_host_id:** This is the address of the server you want to connect to. + +**central_host_username:** This is your profile name on the server you want to +connect to. + +**central path**: This is the path to the project *on the server*. + +:::{dropdown} SSH Example +:color: info +:icon: info + +Let's say the central project was stored on a remote server with +address `ssh.swc.ucl.ac.uk`, and your account username on the server +is `myusername`. + +Finally, we want to store the project at the location (on the server) +`/ceph/my_lab/my_name/my_projects/project_name/`. + +Then the input to **datashuttle** would be + +**central host id**: `ssh.swc.ucl.ac.uk` + +**central host username**: `myusername` + +**central path**: `/ceph/my_lab/my_name/my_projects/project_name/` + +You may pass the **local path** and **central path** without +the **project name**, it will be automatically included. + +:::: + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +```{image} /_static/screenshots/how-to-create-project-ssh-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-create-project-ssh-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +When setting up a new project, the **project name** and **local path** +can be input exactly the +[same as when setting without SSH](general-tui-datashuttle-setup). + +Next, input the `Central Host ID`, `Central Host Username` and +`Central Path` as described above. + +Clicking `Save` will save these project configs. A button +`Setup SSH Connection` will appear. Click this to +confirm the server and enter your password to the server +(you will only need to do this once) +`` + + +::: +:::{tab-item} Python API +:sync: python + +In Datashuttle, the +`connection_method` configuration must be set to `"ssh"` +to use the SSH protocol for data transfers. + +Enter the `central_path`, `central_host_id` and +`central_host_username` as described above. + +```{code-block} python +project.make_config_file( + local_path=r"C:\path\to\local\my_projects\my_first_project", + central_path="/nfs/path_on_server/myprojects/central", + connection_method="ssh", + central_host_id="ssh.swc.ucl.ac.uk", + central_host_username="username", +) +``` + +Next, a one-time command to set up the SSH connection must be run: + +```{code-block} python +project.setup_ssh_connection_to_central_server() +``` + +Running `setup-ssh-connection-to-central-server` will require verification +that the SSH server connected to is correct (pressing `y` to proceed). + +Finally, your password to the central server will be requested (you will +only need to do this once). + +::: +:::: + +(make-project-extra-arguments)= +## Extra arguments (Python API) + +A number of settings that control the behaviour of transfers +can be set with the `make_config_file()` method. + +These configs are not relevant for the graphical interface, with the exception of +`overwrite_existing_folders` which set directly on the +graphical interface's `Transfer` screen. + +(overwrite-existing-files-config)= +overwrite_existing_files +: Determines whether folders and files are overwritten +during transfer. By default, Datashuttle does not overwrite any existing +folder during data transfer.

+ *e.g.* if the file `sub-001_ses-001_measure-trajectories.csv` exists on +the central project, it will never be over-written during upload +from the local to central project, even if the local version is newer.

+To change this behaviour, the configuration `overwrite_existing_files` can be set to `True`. +If **overwrite_existing_files** is `True`, files in which the timestamp of the +target directory will be overwritten if their +timestamp is older than the corresponding file in the source directory. + +transfer_verbosity +: Set to `"vv"` for additional detail on the +transfer operation. Set to `"v"` to only see each file that is transferred +as well as significant events that occur during transfer. + + +show_transfer_progress +: When `True`, real-time transfer statistics will be reported and logged. diff --git a/docs/source/pages/how_tos/read-logs.md b/docs/source/pages/how_tos/read-logs.md new file mode 100644 index 00000000..c92d01a8 --- /dev/null +++ b/docs/source/pages/how_tos/read-logs.md @@ -0,0 +1,73 @@ +(how-to-read-the-logs)= + +# How to Read the Logs + +When commands that create folders, change project configs +or transfer data are run, **datashuttle** stored detailed +logs of these actions. + +These logs can be accessed and read directly in the +Graphical Interface, or located on your filesysetm +and opened in your favourite text file reader. + +Logs are stored as [ISO8601](https://en.wikipedia.org/wiki/ISO_8601)-prefixed +filenames that includes the relevant **datashuttle** method performed. + +## Find and read the logs +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +```{image} /_static/screenshots/how-to-logs-tui-dark.png + :align: center + :class: only-dark + :width: 900 +``` +```{image} /_static/screenshots/how-to-logs-tui-light.png + :align: center + :class: only-light + :width: 900 +``` +
+ +The `Logs` tab on the `Project Manager` page displays a list of +all logs for the project. + +Double-click on the name of any log file to open it within the +**datashuttle** graphical interface. + +Clicking `Open Most Recent` will open the most recently saved logs. + +::: + +:::{tab-item} Python API +:sync: python + +The path where logs are stored can be accessed by running the +`get_logging_path()` method: + +```python +logs_path = project.get_logging_path() + +print(logs_path) +``` + +You can then navigate to this path in your system filebrowser +and open the logs in your favourite text editor. + +```{image} /_static/screenshots/how-to-logs-filesbrowser-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/how-to-logs-filesbrowser-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +::: + +:::: diff --git a/docs/source/pages/how_tos/top-level-folder.md b/docs/source/pages/how_tos/top-level-folder.md new file mode 100644 index 00000000..c66e76de --- /dev/null +++ b/docs/source/pages/how_tos/top-level-folder.md @@ -0,0 +1,84 @@ +(how-to-set-top-level-folder)= + +# How to Set the Top-level Folder + + [NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) specifies +the top-level folder inside the project folder must be `rawdata` or `derivatives`. + +```{image} /_static/NeuroBlueprint_project_tree_dark.png + :align: center + :class: only-dark + :width: 650px +``` +```{image} /_static/NeuroBlueprint_project_tree_light.png + :align: center + :class: only-light + :width: 650px +``` +
+ +In **datashuttle**, the top level folder is relevant when: +1) creating folders (i.e. in `rawdata` or `derivatives`) +2) transferring data with the top-level method or custom. + +Using the Graphical interface, the top-level folder is +set by a drop-down menu on the relevant tab (`Create` or `Transfer`). +^^ TODO: link to sections!!! TODO TODO TODO + + +However, in the Python API methods act in `rawdata` or `derivatives` +according to a stored top-level folder setting. + +## Setting the level folder in the Python API + +In the Python API the *working* top level folder +is stored as a persistent property, accessible with +the `get_top_level_folder()` and `set_top_level_folder()` methods. + +This is to avoid continuously inputting the top-level folder +for every method call. + +:::{info} +:class: info + +Top-level folder is persistent across sessions on a machine. If you +change the working top-level folder with `set_top_level_folder()` then +close-and-reopen python, the change is remembered. + +Changing the working top-level folder only affects the +project on the local machine you are using. + +::: + +When making folders, `create_folders` will only create folders in the +working top-level folder. + +Transferring folders (e.g. with `upload()` or `download()`) will +only transfer folders in the working top-level folder +(unless `upload_entire_project()` or `download_entire_project()` is used). + +In the below example we will create and transfer folders in `rawdata`. +Then, the top-level folder is switched to `derivatives` and the actions repeated.# + +```python +project.set_top_level_folder("rawdata") + +# make folders in `rawdata` +project.create_folders(sub="sub-002") + +# transfer files in `rawdata` +project.upload_data(sub_names="all", ses_names="all", datatype="all") + +# set working top-level folder to `derivatives` +project.set_top_level_folder("derivatives") + +print(project.get_top_level_folder()) +# "derivatives" + +# create folders in derivatives +project.create_folders("sub-002") + +# transfer folders in derivatives +project.download_data() + +``` diff --git a/docs/source/pages/how_tos/transfer-data.md b/docs/source/pages/how_tos/transfer-data.md new file mode 100644 index 00000000..1dcd19c7 --- /dev/null +++ b/docs/source/pages/how_tos/transfer-data.md @@ -0,0 +1,317 @@ +(how-to-transfer-data)= +# How to Transfer Data + +Transferring data between the local project and the project located +on central storage is a key feature of **datashuttle**. It allows: + +- Transfer of data from an acquisition machine to the central project. +- Convenient integration of data collected from multiple acquisition. +- Pulling subsets of data from central storage to analysis machines. + +```{image} /_static/datashuttle-overview-light.png +:align: center +:class: only-light +:width: 600px +``` +```{image} /_static/datashuttle-overview-dark.png +:align: center +:class: only-dark +:width: 600px +``` + +:::{admonition} Transfer Direction +:class: note + + +In **datashuttle**, the term *upload* refers to transfer +from the local machine to central storage. +*Download* refers to transfer from central storage to +a local machine. +::: + +There are three main methods to transfer data in **datashuttle**. These +allow transfer between: + +1) The entire project (all files in both `rawdata` and `derivatives`) +2) A specific top-level-folder (e.g. all files in `rawdata`) +3) A custom subset of subjects / sessions / datatypes. + +Below we will explore each method in turn, as well as consider +[configuring transfer](configuring-transfer) including the important +**overwrite existing files** option. + +```{warning} +The +[`Overwrite Existing Files`](overwrite-existing-files-config) +setting is very important. + +By default it is turned off and a transfer will never overwrite a +file that already exists, even if the source version is newer. + +``` + + +(transfer-entire-project)= +## Transfer the entire project + +The first option is to transfer the entire project, +that is all files in the `rawdata` and `derivatives` +[top-level-folders](https://neuroblueprint.neuroinformatics.dev/specification.html#basic-principles). + +This includes all files inside or outside a subject, session +or datatype folder. + +This mode is useful for data acquisition when **overwrite existing files** +is off. Any new files (i.e. newly acquired data) will be transferred, +to central storage, while any existing files will be ignored. + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +```{image} /_static/screenshots/how-to-transfer-all-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-transfer-all-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +To transfer the entire project navitgate to the `Transfer tab. The +`All` button indicates to transfer the entire project. + +Use the `Upload / Download` switch to control transfer direction, +and press `Transfer` to begin. + +::: + +:::{tab-item} Python API +:sync: python + +The command to upload the entire project is + +```python +project.upload_entire_project() +``` + +while the command to download the entire project is + +```python +project.download_entire_project() +``` + +::: + +:::: + +(transfer-top-level-folder)= +## Transfer the top-level folder + +This mode acts almost identically to +[transferring the entire project](transfer-entire-project) +however it will only transfer files within a +particular top-level folder (`rawdata` or `derivatives`). + +This mode is also useful for quickly uploading new files +during data acquisition (`rawdata`) or analysis (`derivatves`), when +**overwrite existing files** is off—any newly acquired or generated files +will be transfer, ignoring any previously existing files. + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + + +```{image} /_static/screenshots/how-to-transfer-toplevel-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-transfer-toplevel-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +Selecting the `Top-Level` button on the `Transfer` tab will +allow selection of `rawdata` or `derivatives` to transfer. + +Use the `Upload / Download` switch to control transfer direction, +and press `Transfer` to begin. + +::: + +:::{tab-item} Python API +:sync: python + +The `upload_all()` or `download_all()` methods can be used with the argument `top_level_folder` to specify +the top-level folder to transfer within. + +In the next example, we will upload `rawdata` downloading `derivatives`. + + +```python +project.upload_all("rawdata") + +project.download_all("derivatives") +``` + +::: +:::: + +(making-custom-transfers)= + +## Custom transfers + +Custom transfers permit full customisation of the files inside +or outside of subject, session and datatype folders. + +Custom transfers are particularly useful during data analysis, in +which a subset of data can be downloaded from central storage. +For example, you want to only transfer behavioural data from +test sessions—custom transfers allow you to do this with ease. + +See below for how to run custom transfers, as well as +certain keywords and convenience tags to fully customise data transfer. + +For example, `all_sub` in the below examples tells datashuttle +to consider only files and folders within subject folders for transfer. +Files or folders within `rawdata` that are not `sub-` +folders will not be transferred. + +See below for full details on custom transfer keywords and +convenience tags. + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +```{image} /_static/screenshots/how-to-transfer-custom-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-transfer-custom-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +Select `Custom` on the `Transfer` tab to select custom transfers. + +The top-level folder can be set by the first dropdown menu. + +Next, subject and session keywords can be added to customise +files to transfer. In this example, data from all *subject* +folders, all first session behavioral data will be transferred. + +Subject and sessions can be added to the input boxes automatically +by hovering over `sub-` or `ses-` folders on the `DirectoryTree`. +Pressing `CTRL+F` will 'fill' the input with the foldername, +while `CTRL+A` will 'append' the foldername, creating a list of +subjects or sessions to transfer. + +Use the `Upload / Download` switch to control transfer direction, +and press `Transfer` to begin. + +```{image} /_static/screenshots/how-to-transfer-datatypes-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/how-to-transfer-datatypes-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +::: + +:::{tab-item} Python API +:sync: python + +The `upload()` and `download()` methods can be used for custom +data transfers. For example, to perform a custom upload: + +```python +project.upload( + top_level_folder="rawdata", + sub_names="all_sub", + ses_names="ses-001_@*@", + datatype="behav", +) +``` + +In this example, data from all *subject* +folders, all first session behavioral data will be uploaded. + +::: +:::: + +### Custom transfer keywords + +Custom transfer keywords determine how files and folders +outside of subject, session and datatype folders are handled. + +Ideally, all data will be stored in datatype folders. However, this +is not always feasible. + +In this case, custom transfer keywords allows flexible handling of +the transfer of non `sub-`, `ses-` prefixed or datatype folders at the +subject, session and datatype level. + + +Subject level +: * `all` - All subject (i.e. prefixed with `sub-`) folders and non-subject files within the +top-level folder will be transferred. + * `all_sub` - Subject folders only and them will be transferred. + * `all_non_sub` - All files and folders that are not prefixed with `sub-`, +within the top-level folder, will be transferred. +Any folders prefixed with `sub-` at this level will not be transferred. + +Session Level +: * `all` : All session and non-session files and folders within a subject level folder +(e.g. `sub-001`) will be transferred. + * `all_ses` : Session* folders only (i.e. prefixed with `ses-`) and everything within +them will be transferred. + * `all_non_ses` : All files and folders that are not prefixed with `ses-`, within a subject folder, +will be transferred. Any folders prefixed with `ses-` will not be transferred. + +Datatype Level: +: * `all` : All datatype folders at the subject or session folder level will be transferred, +as well as all files and folders within selected session folders. + * `all_datatype` : All datatype folders (e.g. `behav`, `ephys`, `funcimg`, `anat`) within a session folder will be +transferred. Non-*datatype* folders at the session level will not be transferred + * `all_non_datatype` : Non-datatype folders within session folders only will be transferred + + +### Convenience Tags + +These tags can be included in subject or session names to +allow further customisation of data transfer. + +(transfer-the-wildcard-tag)= +Wildcard +: The `@*@` tag can be used to match any portion of a subject or session name. +*e.g.* `ses-001_date-@*@` will transfer all first sessions, matching all possibles date. + +Transfer a range +: The `@TO@` tag can be used to target a range of subjects for transfer. +*e.g.* `sub-001@TO@025` will transfer the 1st to up to and including the 25th subject. + +(configuring-transfer)= +## Configuring data transfer + +!! overview + +!! link to configs diff --git a/docs/source/pages/how_tos/update-configs.md b/docs/source/pages/how_tos/update-configs.md new file mode 100644 index 00000000..6d7fb247 --- /dev/null +++ b/docs/source/pages/how_tos/update-configs.md @@ -0,0 +1,58 @@ + +# How to Update Configs + +Once a project has been created, the configs can be updated at any time. + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +```{image} /_static/screenshots/updating-configs-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/updating-configs-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +On the `Project Manager` page, clicking the `Configs` tab will display +the current configs. + +Changing any config and clicking `Save` will update the project +configs on the local machine. + +If `SSH` configs are [reconfigured](new-project-ssh), +the connection to the server will need +to be reset with `Setup SSH Connection`. + +::: + +:::{tab-item} Python API +:sync: python + +The project configs on the local machine can be selectively +updated with the `update_config_file()` method. + +For example, to change the `local_path` and `central_path`: + +```python +project.update_config_file( + local_path="/a/new/local/path", + central_path="/a/new/central/path" +) +``` + +If changing `SSH` configs, the connection may need to be +[reconfigured](new-project-ssh) with: + +```python +project.setup_ssh_connection_to_central_server() +``` + +::: +:::: diff --git a/docs/source/pages/how_tos/use-name-templates.md b/docs/source/pages/how_tos/use-name-templates.md new file mode 100644 index 00000000..89ee2cd4 --- /dev/null +++ b/docs/source/pages/how_tos/use-name-templates.md @@ -0,0 +1,112 @@ +(how-to-use-name-templates)= + +# How to use Name Templates + +Validation in **datashuttle** is by default restricted to the +NeuroBlueprint specification. Any subject or session names +that are not formatted correctly (e.g. not starting with +"sub-" or "ses-" will be flagged. + +However, it is also possible to add custom templates to validate against +by defining template names using 'regular expressions'. + +For example, say you wanted your subjects to be in the form +`sub-XXX_id-XXXXXX` where `X` is a digit (i.e. subject number with +a 6-digit id. + +We can define this +as a regexp where `\d` stands for 'any digit`: + +```python +"sub-\d\d\d_id-\d\d\d\d\d\d" +``` + +If this is defined as a Name Template, any name that +does not take this form will result in a validation error. + +## Set up Name Templates +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +```{image} /_static/screenshots/how-to-name-templates-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/how-to-name-templates-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ + +Click the `Settings` button in the `Create` tab. Scroll down to the +`Template Validation` section. + +Clicking the checkbox will turn on `Template Validation` and these +can be set for subject and session independent. If the input is left empty, +no name template will be applied. + +Fill in the subject or session name template—any changes +will be automatically saved. These settings will persist across +**datashuttle** sessions. + +::: + +:::{tab-item} Python API +:sync: python + +The `set_name_templates()` and `get_name_templates()` can be used +to set and get current name template settings. + +`set_name_templates` takes as input the `name_template` dictionary, +which is formatted as below: + +``` + +name_templates = { + "on": True, + "sub": "sub-\d\d\d_id-\d\d\d\d\d\d", + "ses": None, +} + + +project.set_name_templates( + name_templates +) + +``` + +In the above example, validation with name templates is +turned on `"on: True`. A template is set for the subject name, +but no template is set for the session name. + + +::: + +:::: + + +## Name Template placeholders + +Placeholders (based on [regular expressions]) can be used to +fill templates with yet-unknown values. + +Any regular expression available in Python is permitted, +but below we summarise the most useful: + +Wildcard Single Digit: +: `\d` can act as a placeholder for a digit. \ +*e.g.* `sub-\d\d\d` will allow any `sub-` prefix followed by three digits. + +Wildcard Single Character: +: `.?` can act as a placeholder for any character \ +*e.g.* `sub-\d\d\d_id-.?.?.?.?` will allow the `id-` to be made up for any 4-character id. + +Wildcard Range: + +: `.*` can act as a placeholder for an unknown range. \ +*e.g.* `sub-\d\d\d_id-.*` will allow the `id-` to be made up of any length combination of any characters. diff --git a/docs/source/pages/tutorials.md b/docs/source/pages/tutorials.md new file mode 100644 index 00000000..bdabe800 --- /dev/null +++ b/docs/source/pages/tutorials.md @@ -0,0 +1,16 @@ +:html_theme.sidebar_secondary.remove: + +# Tutorials + +::::{grid} 1 2 2 2 +:gutter: 4 + +:::{grid-item-card} {fas}`desktop;sd-text-primary` Getting Started +:link: tutorials/getting_started +:link-type: doc +:class-img-top: tutorial-link-image + +Walk-through starting a new experiment with **datashuttle**. + +::: +:::: diff --git a/docs/source/pages/tutorials/getting_started.md b/docs/source/pages/tutorials/getting_started.md new file mode 100644 index 00000000..4fd3489c --- /dev/null +++ b/docs/source/pages/tutorials/getting_started.md @@ -0,0 +1,740 @@ +(tutorial-getting-started)= +# Getting Started + +## Introduction + +This tutorial will give a full introduction to starting +a neuroscience project with **datashuttle**. + +We will get an overview of **datashuttle**'s key features by creating +and transferring a 'mock' experiment, standardised to the +[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) style. + + +```{image} /_static/tutorial-1-example-file-tree-dark.png + :align: center + :class: only-dark + :width: 550px +``` +```{image} /_static/tutorial-1-example-file-tree-light.png + :align: center + :class: only-light + :width: 550px +``` +
+ +We will create standardised folders then upload mock 'acquired' data (empty text files) +to a central data storage, as you would do in a real data acquisition session. +Then we will download a subset of data (e.g. test sessions only) from the central +storage, as you would do during analysis. + +## Installing **datashuttle** + +The first step is to install **datashuttle**, by following the instructions +on the [How to Install](how-to-install) page. + + +::::{tab-set} + +:::{tab-item} Graphical Interface +:sync: gui + +Once **datashuttle** is installed, typing `datashuttle launch` will +launch the application in your terminal + +```{image} /_static/screenshots/tutorial-1-landing-screen-dark.png + :align: center + :class: only-dark + :width: 700px +``` +```{image} /_static/screenshots/tutorial-1-landing-screen-light.png + :align: center + :class: only-light + :width: 700px +``` + +::: +:::{tab-item} Python API +:sync: python + +We can check **datashuttle** has installed correctly by +by importing it into Python without error + +```python +from datashuttle import DataShuttle +``` + +::: +:::: + +## Make a new project + +The first thing to do when using **datashuttle** on a new machine is +to set up your project. + +We need to tell **datashuttle** the: + +1) project name +2) location of the project our local machine, where we will save acquired data +3) location of the central data storage, where we will upload the acquired data + +**datashuttle** supports central data storage either mounted as a drive +on the local machine or through an SHH connection. +See [How to Make a New Project](make-a-new-project) for detailed instructions for +connecting a mounted drive or SSH connection. + +In this walkthrough, we will set our central storage as a +folder on our machine for simplicity. + +::::{tab-set} +:::{tab-item} Graphical Interface +:sync: gui + +Now we will set up a new project. Click `Make New Project` and you +will be taken to the project setup page. + +```{image} /_static/screenshots/tutorial-1-make-screen-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/tutorial-1-make-screen-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +We'll call our project `my_first_project`, and can type this into +the first input box on the page. + +```{image} /_static/screenshots/tutorial-1-make-project-name-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-make-project-name-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Next we need to specify the _local path_, the location on our machine where +we will save our acquired data. Choose any directory that is +convenient, and then add `local` to the end of the filepath. +The filepath can be typed into the input, copied in with `CTRL+V` +or selected from a directory tree using the `Select` button. + +```{image} /_static/screenshots/tutorial-1-make-local-path-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-make-local-path-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Finally, we need to select the _central path_. Usually this would be +a path to a mounted central storage drive or relative to the server path +if connecting via SSH. In this tutorial, we will +set this next to the _local path_ for convenience: + +1) Copy the contents of the _local path_ input by clicking it, hovering over it and pressing `CTRL+Q` to copy. +2) Paste it into the _central path_ input with `CTRL+V` and change 'local' to 'central'. + +```{image} /_static/screenshots/tutorial-1-make-central-path-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-make-central-path-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +You can now click `Save` to set up the project. Once the project +is created, the `Go to Project Screen` button will appear. +Click to move on to the `Create Project` page. + +::: +:::{tab-item} Python API +:sync: python + +First, we can initialise **datashuttle** +with our chosen `project_name`. In this tutorial we will call +our project `my_first_project` and instantiate the `DataShuttle` +class with this name + +```python +project = DataShuttle("my_first_project") +``` + +The created `project` object has many methods for interacting with +**datashuttle**. Next, we will use the `make_config_file()` method to set up the configurations +('configs') for our project. + +While full details of all available configs can be found on the +[How to Make a New Project](make-a-new-project) +page, here we will focus on the main three: the `local_path`, `central_path` +and the `connection_method`. + +We will specify the `local_path` as the location on our machine where we +will place our project and save the acquired data. + +Usually `central_path` would be the path to a mounted +central storage drive or a relative server path +if connecting via SSH. In this tutorial, we will +set this next to the _local path_ for convenience. + +Finally, we will set the `connection_method` to `"local_filesystem"` +as we are not using SSH. + +```python +project.make_config_file( + local_path=r"C:\Users\Joe\data\local\my_first_project", + central_path=r"C:\Users\Joe\data\central\my_first_project", + connection_method="local_filesystem", +) +``` + +If you want to change any config in the future, use the `update_config_file` method + +```python +project.update_config_file( + local_path=r"C:\a\new\path" +) +``` + +Now the project is set up, we are ready to create our standardised project folders! +::: +:::: +## Creating folders + +Let's imagine today is our first day of data collection, +and we are acquiring behaviour (`behav`) and electrophysiology (`ephys`) data. +We will create standardised subject, session and datatype folders +to put the acquired data into. + +::::{tab-set} +:::{tab-item} Graphical Interface +:sync: gui + +We will create standardised project folders using the `Create` tab. + +```{image} /_static/screenshots/tutorial-1-create-screen-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/tutorial-1-create-screen-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +Following the [NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) +style we will call the first subject `sub-001`. Additional key-value pairs in +the subject name could be included if desired (see the +[NeuroBlueprint specification](https://neuroblueprint.neuroinformatics.dev/specification.html) +for more details). + +In the session name we can include today's date, +so our first session will be `ses-001_date-`. + +We could start by typing `sub-001` into the subject input box, but +it is more convenient to simply double-left-click it. This will suggest +the next subject number based on the current subjects in the project. +As currently this project is empty, the suggested next subject is `sub-001`. + +```{image} /_static/screenshots/tutorial-1-create-subject-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-create-subject-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +```{note} +The subject and session folder input boxes have live validation. +This will flag any +[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) +errors with a red border. Hover over the input box with the mouse +to see the nature of the error. +``` + +Next, we can input the session name. Double-left-click on the session +input to automatically fill with `ses-001`. We can then add +today's date with the `@DATE@` convenience tag. + +When the session folder is created, today's date will be automatically added. + +```{image} /_static/screenshots/tutorial-1-create-session-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-create-session-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Next, uncheck the `funcimg` and `anat` datatype boxes, to ensure +we only create `behav` and `ephys` folders in our session folder. + +```{image} /_static/screenshots/tutorial-1-create-datatype-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-create-datatype-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Finally, click `Create Folders` to create the folder structure in the project! + +::: +:::{tab-item} Python API +:sync: python +We will create standardised project folders with the `create_folders()` method. + +Following the [NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/) +style we will call the first subject `sub-001`. Additional key-value pairs in +the subject name could be included if desired (see the +[NeuroBlueprint specification](https://neuroblueprint.neuroinformatics.dev/specification.html) +for more details). + +In the session name we can include today’s date, so our first session will +be `ses-001_date-`. We can use the `@DATE@` convenience tag. + +Finally, we will tell **datashuttle** to create `behav` and `ephys` datatype +folders only. + +```python +project.create_folders( + top_level_folder="rawdata", + sub_names="sub-001", + ses_names="ses-001_@DATE@", + datatype=["behav", "ephys"] +) +``` + +Navigating to the `central_path` in your system filebrowser, you will +see the transferred files. + +```{note} +The names of the folders to be created are validated on the fly against +[NeuroBlueprint](https://neuroblueprint.neuroinformatics.dev/specification.html). +If the folder names will break with the specification, an error will be +raised and the folders will not be created. +``` + +Two useful methods to automate folder creation are `get_next_sub_number()` and +`get_next_ses_number()`. These can be used to automatically get the next subject +and session names in a project. + +For example, to get the next subject +in this project (`sub-002`) and the next session for that subject (in this case, +as it is the first session for `sub-002`, it will be `ses-001`) we can run + +```python +next_sub = project.get_next_sub_number("rawdata", local_only=True) # returns "sub-001" +next_ses = project.get_next_ses_number("rawdata", sub=next_sub, local_only=True) # returns "ses-001" + +project.create_folders( + "rawdata", + next_sub, + f"{next_ses}_@DATE@", + datatype=["behav", "ephys"] +) +``` + +This will create the folders, with today's date included in the session folder name. +The `local_only` argument restricts the search for the next subject and session +to the local project folder only. To also consider subjects and sessions in +the central storage, set this to `False`. + +::: +:::: + +This was a quick overview of the creating folders—see [How to use Name Templates](how-to-use-name-templates) +and [How to use Create Folder Tags](how-to-create-folders2) for more detail on validation and convenience tags. + +## Exploring folders + +In our imagined experiment, we will next want to save data from +acquisition software into our newly created, standardised folders. + +::::{tab-set} +:::{tab-item} Graphical Interface +:sync: gui + +When folders are created, the `Directory Tree` on the left-hand side +will update to display the new folders. +By hovering over a folder on the `Directory Tree` we can quickly +copy the full path to the folder (`CTRL+Q)`) (you may need to click +the `Directory Tree` first). + +Alternatively, pressing `CTRL+O` will open the folder in your file browser. + +```{image} /_static/screenshots/tutorial-1-explore-folders-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-explore-folders-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +These shortcuts aim to make it simple to direct your acquisition software +to the datatype folders. Hover over the `DirectoryTree` +for a tooltip indicating all possible shortcuts. + +```{admonition} Creating mock data for the tutorial + +To continue with our experiment, we will need to create 'mock' +acquired data to transfer to central storage. These will +take the form of simple text files with their extensions changed. + +You can download these files from +[this link](https://gin.g-node.org/joe-ziminski/datashuttle/src/master/docs/tutorial-mock-data-files), +by right-clicking each file and selecting 'Download (or) Save Link As..'. +Alternatively you can create them in your favourite text editor. + +Next, hover over the `behav` folder the `Directory Tree` with your +mouse and and press `CTRL+O` to open the folder in your file browser. +Move the mock behavioural data file (`sub-001_ses-001_camera-top.mp4`) +into the `behav` datatype folder. + +Next, repeat this for the `ephys` datatype by moving the remaining +electrophysiology files to the `ephys` folder. + +Finally, hover the mouse over the `Directory Tree` and press `CTRL+R` to refresh. + +``` + +::: +:::{tab-item} Python API +:sync: python + +`create_folders()` returns the full filepaths of created datatype folders. +These can be used to save data to these folders in acquisition scripts + +```python +folder_path_list = project.create_folders( + top_level_folder="rawdata", + sub_names=["sub-001"], + ses_names=["ses-001_@DATE@"], + datatype=["behav", "ephys"] + +) + +print([path_ for path_ in folder_path_list if path_.name == "behav"]) +# ["C:\Users\Joe\data\local\my_first_project\sub-001\ses-001_16052024\behav"] +``` + + +```{admonition} Creating mock data for the tutorial + +To continue with our experiment, we will need to create 'mock' +acquired data to transfer to central storage. These will +take the form of simple text files with their extensions changed. + +You can download these files from +[this link](https://gin.g-node.org/joe-ziminski/datashuttle/src/master/docs/tutorial-mock-data-files), +by right-clicking each file and selecting 'Download (or) Save Link As..'. +Alternatively you can create them in your favourite text editor. + +Move the mock behavioural data file (`sub-001_ses-001_camera-top.mp4`) +into the `behav` datatype folder and the remaining +electrophysiology file to the `ephys` folder. + +``` +::: +:::: + +## Uploading to central storage + +We have now 'acquired' `behav` and `ephys` data onto our local machine. +The next step is to upload it to central data storage. + +Typically, this would be an external machine or server, connected through a mounted +drive or via SSH. In this walkthrough, we set the central storage on our +local machine for convenience. + +::::{tab-set} +:::{tab-item} Graphical Interface +:sync: gui + +First, switch to the `Transfer` tab, where on the left we will again +see a `Directory Tree` displaying the local version of the project. + +```{image} /_static/screenshots/tutorial-1-transfer-screen-upload-dark.png + :align: center + :class: only-dark + :width: 900px +``` +```{image} /_static/screenshots/tutorial-1-transfer-screen-upload-light.png + :align: center + :class: only-light + :width: 900px +``` +
+ +The first page on the `Transfer` tab allows us to upload the entire project, +both the `rawdata` and `derivatives` top-level folders +(the `derivatives` folder is used for outputs of processing `rawdata`, see the +[NeuroBlueprint specification](https://neuroblueprint.neuroinformatics.dev/specification.html)). + +As we only have a `rawdata` folder, we can simply click `Transfer` to +upload everything to central storage. + +Navigating to the _central path_ in the file browser, +the newly transferred data will have appeared, simulating transfer +to a separate data storage machine. (An easy way to navigate to the *central path* +is to go to the `Config` tab and press `CTRL+O` on the _central path_ input box). + +We can also click the `Top Level` +or `Custom` buttons for refined transfers (for example, if we also had a +`derivatives` folder we did not want to upload). For more information +see the [How to Transfer Data](how-to-transfer-data) page as well as +and the next tutorial section for `Custom` transfers. + +::: +:::{tab-item} Python API +:sync: python + +`upload_entire_project()` is a high level method that uploads all files +in the project. This includes both `rawdata` and `derivatives` top-level folders +(the `derivatives` folder is used for outputs of processing `rawdata`, see the +[NeuroBlueprint specification](https://neuroblueprint.neuroinformatics.dev/specification.html)). + +As we only have a `rawdata` folder, we can simply run + +```python +project.upload_entire_project() +``` + +and all files will be uploaded from the local version of the project to central storage. + +Navigating to the `central_path` in your systems file browser, the newly transferred data +will have appeared, simulating transfer to a separate data storage machine. + +Other methods (`upload_all()` and `upload()`) provide refined +data transfers (and every `upload` method has an equivalent `download` method). +For more information see the +[How to Transfer Data](how-to-transfer-data) page +as well as the next tutorial section for customisable transfers. + +Note that the `overwrite_existing_files` config controls whether +transferred data will overwrite data on the target machine. This config +can be set initially with `make_config_file` or updated with with +`update_config_file` + +```python +project.update_config_file( + overwrite_existing_files=True, +) +``` + +::: +:::: + +```{warning} +The `Overwrite Existing Files` setting is very important. +By default it is turned off and a transfer will never overwrite a +file that already exists, even if the source version is newer. + +For example, if we upload the first session's behavioural data—and there +is already a file on central storage with the same name +in the same folder—the file will not be uploaded. + +If `Overwrite Existing Files` is on, then any existing files +will be overwritten by newer versions of the file during transfer. +``` + +With the data safely on our central storage, +our experimental acquisition session is complete! + + +## Downloading from central storage + +Next let's imagine we are on a different, analysis machine and want to +download a subset of data for further processing. + +In this example we will download the behavioural data only from the first session. +In practice **datashuttle**'s custom data transfers work well when there +are many subjects and sessions (for example, downloading only the behavioural +'test' sessions from a specific range of subjects). + +```{admonition} Replicating a fresh machine for the tutorial +To replicate starting on a new local machine, delete the `rawdata` folder from +your _local_ path. + +We will next download data from the central project to our now-empty local project. +In practice when setting up **datashuttle** on a new machine, you would +again [Make a new project](make-a-new-project). +``` + +We will look at a small subset of possible +options here, but see [How to make Custom Transfers](making-custom-transfers) for more possibilities. + +::::{tab-set} +:::{tab-item} Graphical Interface +:sync: gui + +The `Custom` transfer screen has options for selecting specific combinations +of subjects, sessions and datatypes. + +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-dark.png + :align: center + :class: only-dark + :width: 600px +``` +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-light.png + :align: center + :class: only-light + :width: 600px +``` +
+ +In the subject input, we can simply type `all` (in this case, we only have one subject anyway). + +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-subjects-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-subjects-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Next, let's specify what session to download. We can use the +[wildcard tag](transfer-the-wildcard-tag) +to avoid typing the exact date—`ses-001_@*@`. + +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-sessions-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-sessions-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Then, select only the `behav` datatype from the datatype checkboxes. + +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-datatypes-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +Finally, we can select `Download` from the upload / download switch, +and click `Transfer`. + +Note that the `Overwrite Existing Files` setting affects both upload +and downloads—any local versions of a file will be overwritten +by newer versions downloaded from central storage when it is turned on. + +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-switch-dark.png + :align: center + :class: only-dark + :width: 400px +``` +```{image} /_static/screenshots/tutorial-1-transfer-screen-custom-switch-light.png + :align: center + :class: only-light + :width: 400px +``` +
+ +The transfer will complete, and the custom selection +of files will now be available in the _local path_ folder. + +::: +:::{tab-item} Python API +:sync: python + +We can use the `download()` method (the download equivalent method of +the `upload()`). + +We will download only the behavioural data from the first +session, using a few shortcuts available for custom transfers + +```python +project.download( + top_level_folder="rawdata", + sub_names="all", + ses_names="ses-001_@*@", + datatype="behav" +) +``` + +The `"all"` keyword will upload every subject in the project (in this case, +we only have one subject anyway). + +The `@*@` [wildcard tag](transfer-the-wildcard-tag) can be used to match +any part of a subject or session name—in this case we use it to avoid +typing out the date. + +Finally, we chose to download only the `behav` data for the session. +::: +:::: + +```{note} +Detailed information on data transfers can be found in the `Logs`. +Visit [How to Read the Logs](how-to-read-the-logs) for more information. +``` + +The transfer will complete, and the custom selection +of files will now be available in the _local path_ folder! + +## Summary + +That final transfer marks the end of our **datashuttle** tutorial! + +Now you can: + +1) set up a new project +2) upload your acquired data to a central storage machine +3) download subsets of data for analysis + +We are always keen to improve **datashuttle**, so please don't hesitate +to get in contact with any +[Issues](https://github.com/neuroinformatics-unit/datashuttle) +or drop in to our +[Zulip Chat](https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle) +with any questions or feedback. + +Have a great day! diff --git a/docs/source/taking-screenshot-notes.txt b/docs/source/taking-screenshot-notes.txt new file mode 100644 index 00000000..79cbddfe --- /dev/null +++ b/docs/source/taking-screenshot-notes.txt @@ -0,0 +1,13 @@ +Screenshots were taken in Windows on +on a large LG screen monitor model: 35WN75C +resolution 3440x1440 / 85 Hz. + +Windows terminal was used, default size set to 150 x 45. + +Then: + +1) ShareX 16.0.1 to take screenshot. +2) Copy these to photoshop. Set DPI to desired. Save as PNG. + +The raw screenshots .PSD files available on the NeuroInformatics +Unit dropbox. Please request if required. diff --git a/pyproject.toml b/pyproject.toml index 17e5d913..0c3372cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dev = [ "pre-commit", "ruff", "setuptools_scm", + "textual-dev", "types-requests", "types-PyYAML", "types-appdirs", diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index 52533deb..0df99d95 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -99,7 +99,7 @@ async def test_make_new_project_configs( pilot.app.screen.query_one( "#messagebox_message_label" ).renderable._text[0] - == "A DataShuttle project has now been created.\n\n Next, " + == "A datashuttle project has now been created.\n\n Next, " "setup the SSH connection. Once complete, navigate to " "the 'Main Menu' and proceed to the project page, " "where you will be able to create and transfer " @@ -117,7 +117,7 @@ async def test_make_new_project_configs( pilot.app.screen.query_one( "#messagebox_message_label" ).renderable._text[0] - == "A DataShuttle project has now been created.\n\n " + == "A datashuttle project has now been created.\n\n " "Next proceed to the project page, where you will " "be able to create and transfer project folders." )