Use lsjson for searches (#551)

* Add get_rclone_config_name_local and rename get_rclone_config_name to get_rclone_config_name_central.

* Add new transfer function.

* Fix rename in tests.

* Tidy up the search functions.

* Fix circular import.

* Adding more tests and fixing an edge case.

* Add wildcards to local filesystem transfer tests.

* Added tests to ssh.

* Remove unused function.

* Move teardown to fix tests on macos.

* Revert get_rclone_config_name_central name change.

* Add documentation to tests.

* Remove unecessary sorted.

* Fix cyclical import.

* Remove unecessary wildcard.

* Remove unused functions.

Author: JoeZiminski

datashuttle/configs/config_class.py

@@ -209,7 +209,7 @@ def get_base_folder(
     def get_rclone_config_name(
         self, connection_method: Optional[str] = None
     ) -> str:
-        """Generate the rclone configuration name for the project.
+        """Generate the rclone configuration name for the central project.
 
         These configs are created by datashuttle but managed and stored by rclone.
         """

datashuttle/utils/folders.py

@@ -16,11 +16,12 @@
     from datashuttle.configs.config_class import Configs
     from datashuttle.utils.custom_types import TopLevelFolder
 
-import glob
+import fnmatch
+import json
 from pathlib import Path
 
 from datashuttle.configs import canonical_folders, canonical_tags
-from datashuttle.utils import ssh, utils, validation
+from datashuttle.utils import rclone, utils, validation
 from datashuttle.utils.custom_exceptions import NeuroBlueprintError
 
 # -----------------------------------------------------------------------------
@@ -598,67 +599,90 @@ def search_for_folders(
     Discovered folders (`all_folder_names`) and files (`all_filenames`).
 
     """
-    if local_or_central == "central" and cfg["connection_method"] == "ssh":
-        all_folder_names, all_filenames = ssh.search_ssh_central_for_folders(
-            search_path,
-            search_prefix,
-            cfg,
-            verbose,
-            return_full_path,
-        )
+    if (
+        local_or_central == "local"
+        or cfg["connection_method"] == "local_filesystem"
+    ) and not search_path.exists():
+        if verbose:
+            utils.log_and_message(f"No file found at {search_path.as_posix()}")
+        return [], []
+
+    if local_or_central == "local":
+        rclone_config_name = None
     else:
-        if not search_path.exists():
-            if verbose:
-                utils.log_and_message(
-                    f"No file found at {search_path.as_posix()}"
-                )
-            return [], []
-
-        all_folder_names, all_filenames = search_filesystem_path_for_folders(
-            search_path / search_prefix, return_full_path
+        rclone_config_name = cfg.get_rclone_config_name(
+            cfg["connection_method"]
         )
+
+    all_folder_names, all_filenames = search_local_or_remote(
+        search_path,
+        search_prefix,
+        rclone_config_name,
+        return_full_path,
+    )
+
     return all_folder_names, all_filenames
 
 
-# Actual function implementation
-def search_filesystem_path_for_folders(
-    search_path_with_prefix: Path, return_full_path: bool = False
-) -> Tuple[List[Path | str], List[Path | str]]:
-    r"""Search a folder through the local filesystem.
+def search_local_or_remote(
+    search_path: Path,
+    search_prefix: str,
+    rclone_config_name: str | None,
+    return_full_path: bool = False,
+) -> Tuple[List[Any], List[Any]]:
+    """Search for files and folders in central path using `rclone lsjson` command.
 
-    Use glob to search the full search path (including prefix) with glob.
-    Files are filtered out of results, returning folders only.
+    This command lists all the files and folders in the central path in a json format.
+    The json contains file/folder info about each file/folder like name, type, etc.
 
     Parameters
     ----------
-    search_path_with_prefix
-        Path to search along with search prefix e.g. "C:\drive\project\sub-*"
-
+    search_path
+        The path to search (relative to the local or remote drive). For example,
+        for "local_filesystem" this is the path on the local machine. For "ssh", this
+        is the path on the machine that has been connected to.
+    search_prefix
+        The search string e.g. "sub-*".
+    rclone_config_name
+        Name of the rclone config for the remote (not set for local). `rclone config`
+        can be used in the terminal to see how rclone has stored these. In datashuttle,
+        these are managed by `Configs`.
     return_full_path
-        If `True` returns the path to the discovered folder or file,
-        otherwise just the name.
-
-    Returns
-    -------
-    Discovered folders (`all_folder_names`) and files (`all_filenames`).
+        If `True`, return the full filepath, otherwise return only the folder/file name.
 
     """
-    all_folder_names = []
-    all_filenames = []
+    config_prefix = "" if not rclone_config_name else f"{rclone_config_name}:"
 
-    all_files_and_folders = list(glob.glob(search_path_with_prefix.as_posix()))
-    sorter_files_and_folders = sorted(all_files_and_folders)
+    output = rclone.call_rclone(
+        f'lsjson {config_prefix}"{search_path.as_posix()}"',
+        pipe_std=True,
+    )
 
-    for file_or_folder_str in sorter_files_and_folders:
-        file_or_folder = Path(file_or_folder_str)
+    all_folder_names: List[str] = []
+    all_filenames: List[str] = []
 
-        if file_or_folder.is_dir():
-            all_folder_names.append(
-                file_or_folder if return_full_path else file_or_folder.name
-            )
+    if output.returncode != 0:
+        utils.log_and_message(
+            f"Error searching files at {search_path.as_posix()}\n"
+            f"{output.stderr.decode('utf-8') if output.stderr else ''}"
+        )
+        return all_folder_names, all_filenames
+
+    files_and_folders = json.loads(output.stdout)
+
+    for file_or_folder in files_and_folders:
+        name = file_or_folder["Name"]
+
+        if not fnmatch.fnmatch(name, search_prefix):
+            continue
+
+        is_dir = file_or_folder.get("IsDir", False)
+
+        to_append = search_path / name if return_full_path else name
+
+        if is_dir:
+            all_folder_names.append(to_append)
         else:
-            all_filenames.append(
-                file_or_folder if return_full_path else file_or_folder.name
-            )
+            all_filenames.append(to_append)
 
     return all_folder_names, all_filenames

datashuttle/utils/formatting.py

@@ -64,7 +64,13 @@ def check_and_format_names(
     names_to_format, reserved_keywords = [], []
     for name in names:
         if name in canonical_reserved_keywords() or tags("*") in name:
-            reserved_keywords.append(name)
+            if tags("to") in name:
+                # handle an edge case where use searches with both tags
+                reserved_keywords += update_names_with_range_to_flag(
+                    [name], prefix
+                )
+            else:
+                reserved_keywords.append(name)
         else:
             names_to_format.append(name)
 

datashuttle/utils/getters.py

@@ -293,9 +293,9 @@ def get_existing_project_paths() -> List[Path]:
     """
     datashuttle_path = canonical_folders.get_datashuttle_path()
 
-    all_folders, _ = folders.search_filesystem_path_for_folders(
-        datashuttle_path / "*"
-    )
+    all_folders = [
+        path_ for path_ in datashuttle_path.glob("*") if path_.is_dir()
+    ]
 
     existing_project_paths = []
     for folder_name in all_folders:

datashuttle/utils/rclone.py

@@ -1,15 +1,22 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from datashuttle.configs.config_class import Configs
+    from datashuttle.utils.custom_types import TopLevelFolder
+
 import os
 import platform
 import subprocess
 import tempfile
-from pathlib import Path
 from subprocess import CompletedProcess
 from typing import Dict, List, Literal
 
 from datashuttle.configs import canonical_configs
-from datashuttle.configs.config_class import Configs
 from datashuttle.utils import utils
-from datashuttle.utils.custom_types import TopLevelFolder
 
 
 def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess:

datashuttle/utils/ssh.py

@@ -5,12 +5,10 @@
 if TYPE_CHECKING:
     from datashuttle.configs.config_class import Configs
 
-import fnmatch
 import getpass
-import stat
 import sys
 from pathlib import Path
-from typing import Any, List, Optional, Tuple
+from typing import Optional
 
 import paramiko
 
@@ -322,119 +320,3 @@ def verify_ssh_central_host(
             utils.log("Host not accepted. No connection made.")
 
     return success
-
-
-# -----------------------------------------------------------------------------
-# Search over SSH
-# -----------------------------------------------------------------------------
-
-
-def search_ssh_central_for_folders(
-    search_path: Path,
-    search_prefix: str,
-    cfg: Configs,
-    verbose: bool = True,
-    return_full_path: bool = False,
-) -> Tuple[List[Any], List[Any]]:
-    """Search for the search prefix in the search path over SSH.
-
-    Parameters
-    ----------
-    search_path
-        Path to search for folders in.
-
-    search_prefix
-        Search prefix for folder names e.g. "sub-*".
-
-    cfg
-        See connect_client_with_logging().
-
-    verbose
-        If `True`, if a search folder cannot be found, a message
-        will be printed with the un-found path.
-
-    return_full_path
-        include the search_path in the returned paths
-
-    Returns
-    -------
-    Discovered folders (`all_folder_names`) and files (`all_filenames`).
-
-    """
-    client: paramiko.SSHClient
-    with paramiko.SSHClient() as client:
-        connect_client_with_logging(
-            client, cfg, message_on_sucessful_connection=verbose
-        )
-
-        sftp = client.open_sftp()
-
-        all_folder_names, all_filenames = get_list_of_folder_names_over_sftp(
-            sftp,
-            search_path,
-            search_prefix,
-            verbose,
-            return_full_path,
-        )
-
-    return all_folder_names, all_filenames
-
-
-def get_list_of_folder_names_over_sftp(
-    sftp: paramiko.sftp_client.SFTPClient,
-    search_path: Path,
-    search_prefix: str,
-    verbose: bool = True,
-    return_full_path: bool = False,
-) -> Tuple[List[Any], List[Any]]:
-    """Use paramiko's sftp to search a path over ssh for folders.
-
-    Return the folder names.
-
-    Parameters
-    ----------
-    sftp
-        Connected paramiko stfp object
-        (see search_ssh_central_for_folders()).
-
-    search_path
-        Path to search for folders in.
-
-    search_prefix
-        Prefix (can include wildcards)
-        to search folder names.
-
-    verbose
-        If `True`, if a search folder cannot be found, a message
-        will be printed with the un-found path.
-
-    return_full_path
-        include the search_path in the returned paths.
-
-    Returns
-    -------
-    Discovered folders (`all_folder_names`) and files (`all_filenames`).
-
-    """
-    all_folder_names = []
-    all_filenames = []
-    try:
-        for file_or_folder in sftp.listdir_attr(search_path.as_posix()):
-            if file_or_folder.st_mode is not None and fnmatch.fnmatch(
-                file_or_folder.filename, search_prefix
-            ):
-                to_append = (
-                    search_path / file_or_folder.filename
-                    if return_full_path
-                    else file_or_folder.filename
-                )
-                if stat.S_ISDIR(file_or_folder.st_mode):
-                    all_folder_names.append(to_append)
-                else:
-                    all_filenames.append(to_append)
-
-    except FileNotFoundError:
-        if verbose:
-            utils.log_and_message(f"No file found at {search_path.as_posix()}")
-
-    return all_folder_names, all_filenames