Also show message if nothing was transferred.

Author: JoeZiminski

datashuttle/datashuttle_class.py

@@ -1489,10 +1489,7 @@ def _transfer_entire_project(
         i.e. every 'top level folder' (e.g. 'rawdata', 'derivatives').
         See ``upload_custom()`` or ``download_custom()`` for parameters.
         """
-        all_errors: TransferErrors = {
-            "file_names": [],
-            "messages": [],
-        }
+        all_errors = rclone.get_empty_errors_dict()
 
         for top_level_folder in canonical_folders.get_top_level_folders():
             utils.log_and_message(
@@ -1513,6 +1510,10 @@ def _transfer_entire_project(
             all_errors["file_names"] += errors["file_names"]
             all_errors["messages"] += errors["messages"]
 
+            key = f"nothing_was_transferred_{top_level_folder}"
+
+            all_errors[key] = errors[key]  # type: ignore
+
         rclone.log_rclone_copy_errors_api(all_errors)
 
         return all_errors

datashuttle/tui/screens/modal_dialogs.py

@@ -242,26 +242,42 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None:
             if success:
                 errors = output
 
+                errors_message = ""
+
+                messagebox_kwargs = {}
+
+                no_transfer_col = (
+                    "blue"
+                    if self.app.theme == "textual-light"
+                    else "lightblue"
+                )
+
+                if errors["nothing_was_transferred_rawdata"] is True:
+                    errors_message += f"[{no_transfer_col}]\nNote! Nothing was transferred from rawdata[/{no_transfer_col}]\n"
+
+                if errors["nothing_was_transferred_derivatives"] is True:
+                    errors_message += f"[{no_transfer_col}]\nNote! Nothing was transferred from derivatives[/{no_transfer_col}]\n"
+
                 if any(errors["messages"]):
                     if errors["file_names"]:
-                        errors_message = (
+                        errors_message += (
                             "[red]Errors detected! in files:[/red]\n"
                         )
                         errors_message += "\n".join(errors["file_names"])
                     else:
-                        errors_message = "[red]Errors detected![/red]"
+                        errors_message += "[red]Errors detected![/red]"
                     errors_message += (
                         "[red]\n\nThe error messages are:[/red]\n"
                     )
                     errors_message += "\n\n".join(errors["messages"])
                     messagebox_kwargs = {"width": "75%", "height": "75%"}
-                else:
-                    errors_message = "No errors detected"
-                    messagebox_kwargs = {}
+
+                if errors_message == "":
+                    errors_message += "No errors detected"
 
                 message = (
-                    f"Transfer finished.\n\n"
-                    f"{errors_message}\n\n"
+                    f"Transfer finished.\n"
+                    f"{errors_message}\n"
                     f"Check the most recent logs for full details."
                 )
 

datashuttle/utils/custom_types.py

@@ -1,17 +1,24 @@
-from typing import Any, Literal, Tuple
+from typing import Any, Literal, Tuple, TypeAlias, TypedDict
 
-DisplayMode = Literal["error", "warn", "print"]
+DisplayMode: TypeAlias = Literal["error", "warn", "print"]
 
-TopLevelFolder = Literal["rawdata", "derivatives"]
+TopLevelFolder: TypeAlias = Literal["rawdata", "derivatives"]
 
-OverwriteExistingFiles = Literal["never", "always", "if_source_newer"]
+OverwriteExistingFiles: TypeAlias = Literal[
+    "never", "always", "if_source_newer"
+]
 
-Prefix = Literal["sub", "ses"]
+Prefix: TypeAlias = Literal["sub", "ses"]
 
-InterfaceOutput = Tuple[bool, Any]
+InterfaceOutput: TypeAlias = Tuple[bool, Any]
 
-ConnectionMethods = Literal[
+ConnectionMethods: TypeAlias = Literal[
     "ssh", "local_filesystem", "gdrive", "aws", "local_only"
 ]
 
-TransferErrors = dict[str, list[str]]
+
+class TransferErrors(TypedDict):
+    file_names: list[str]
+    messages: list[str]
+    nothing_was_transferred_rawdata: bool | None
+    nothing_was_transferred_derivatives: bool | None

datashuttle/utils/data_transfer.py

@@ -118,6 +118,7 @@ def run(self):
         else:
             utils.log_and_message("No files included. None transferred.")
             errors = rclone.get_empty_errors_dict()
+            errors[f"nothing_was_transferred_{self.__top_level_folder}"] = True
 
         return errors
 

datashuttle/utils/rclone.py

@@ -489,19 +489,28 @@ def log_rclone_output_python_api(stdout, stderr):
 
 def log_rclone_copy_errors_api(errors):
     """"""
+    message = ""
+
+    if errors["nothing_was_transferred_rawdata"] is True:
+        message += "\n\nNote! Nothing was transferred from rawdata!\n"
+
+    if errors["nothing_was_transferred_derivatives"] is True:
+        message += "\n\nNote! Nothing was transferred from derivatives!\n"
+
     if any(errors["messages"]):
         if any(errors["file_names"]):
-            message = (
+            message += (
                 "\n\nErrors were detected! In files:"
                 "\n-------------------------------\n"
             )
             message += "\n".join(errors["file_names"])
         else:
-            message = "\n\n[red]Errors detected![/red]"
+            message += "\n\n[red]Errors detected![/red]"
         message += "\n\nThe error messages are:\n-----------------------\n"
         message += "\n".join(errors["messages"])
         message += "\n"
-    else:
+
+    if message == "":
         message = "No transfer errors were detected.\n"
 
     utils.log_and_message(message, use_rich=True)
@@ -510,16 +519,23 @@ def log_rclone_copy_errors_api(errors):
 def parse_rclone_copy_output(top_level_folder, output):
     """"""
     stdout, out_errors = reformat_rclone_copy_output(
-        output.stdout, capture_errors=True, top_level_folder=top_level_folder
+        output.stdout, top_level_folder=top_level_folder
     )
 
     stderr, err_errors = reformat_rclone_copy_output(
-        output.stderr, capture_errors=True
+        output.stderr, top_level_folder=top_level_folder
     )
 
     all_errors = {
-        "file_names": out_errors["file_names"] + err_errors["file_names"],
+        "file_names": out_errors["file_names"]
+        + err_errors["file_names"],  # TODO: this is so messy
         "messages": out_errors["messages"] + err_errors["messages"],
+        "nothing_was_transferred_rawdata": err_errors[
+            "nothing_was_transferred_rawdata"
+        ],
+        "nothing_was_transferred_derivatives": err_errors[
+            "nothing_was_transferred_derivatives"
+        ],
     }
 
     all_errors["file_names"] = list(set(all_errors["file_names"]))
@@ -531,15 +547,16 @@ def get_empty_errors_dict() -> TransferErrors:
     return {
         "file_names": [],
         "messages": [],
+        "nothing_was_transferred_rawdata": None,
+        "nothing_was_transferred_derivatives": None,
     }
 
 
 def reformat_rclone_copy_output(
     stream: bytes,
-    capture_errors: bool = False,
     top_level_folder: TopLevelFolder | None = None,
 ) -> tuple[str, TransferErrors]:
-    """"""
+    """:return:"""
     split_stream = stream.decode("utf-8").split("\n")
 
     errors = get_empty_errors_dict()
@@ -550,16 +567,21 @@ def reformat_rclone_copy_output(
         except json.JSONDecodeError:
             continue
 
-        if capture_errors:
-            if line_json["level"] in ["error", "critical"]:
-                if "object" in line_json:
-                    full_filepath = f"{top_level_folder}/{line_json['object']}"
-                    errors["file_names"].append(full_filepath)
-                    errors["messages"].append(
-                        f"The file {full_filepath} failed to transfer. Reason: {line_json['msg']}"
-                    )
-                else:
-                    errors["messages"].append(f"ERROR : {line_json['msg']}")
+        if line_json["level"] in ["error", "critical"]:
+            if "object" in line_json:
+                full_filepath = f"{top_level_folder}/{line_json['object']}"
+                errors["file_names"].append(full_filepath)
+                errors["messages"].append(
+                    f"The file {full_filepath} failed to transfer. Reason: {line_json['msg']}"
+                )
+            else:
+                errors["messages"].append(f"ERROR : {line_json['msg']}")
+
+        elif "stats" in line_json and "totalTransfers" in line_json["stats"]:
+            if line_json["stats"]["totalTransfers"] == 0:
+                errors[f"nothing_was_transferred_{top_level_folder}"] = True  # type:ignore
+            else:
+                errors[f"nothing_was_transferred_{top_level_folder}"] = False  # type:ignore
 
         split_stream[idx] = (
             f"{line_json['time'][:19]} {line_json['level'].upper()} : {line_json['msg']}"