Add password for SSH, AWS and GDrive config

.github/workflows/code_test_and_deploy.yml

@@ -66,6 +66,26 @@ jobs:
           python -m pip install --upgrade pip
           pip install .[dev]
 
+      - name: Install pass on Linux
+        # this is required for Rclone config encryption
+        if: runner.os == 'Linux'
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y pass gnupg git
+
+          # Create a dedicated GPG home for this job
+          export GNUPGHOME="$(mktemp -d)"
+          echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV"   # <-- make it available to later steps
+
+          # Generate a non-interactive key (no passphrase), no expiry
+          gpg --batch --yes --pinentry-mode loopback --passphrase '' \
+              --quick-gen-key "CI Key <[email protected]>" default default 0
+
+          # Initialize pass with the key fingerprint (more robust than UID)
+          FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')"
+          pass init "$FPR"
+
         # run SSH tests only on Linux because Windows and macOS
         # are already run within a virtual container and so cannot
         # run Linux containers because nested containerisation is disabled.
@@ -97,7 +117,6 @@ jobs:
         run: |
           pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws
 
-
   build_sdist_wheels:
     name: Build source distribution
     needs: [test]

datashuttle/configs/canonical_folders.py

@@ -6,6 +6,8 @@
 if TYPE_CHECKING:
     from datashuttle.utils.custom_types import TopLevelFolder
 
+import platform
+
 from datashuttle.configs import canonical_configs
 from datashuttle.utils.folder_class import Folder
 
@@ -80,6 +82,11 @@ def get_datashuttle_path() -> Path:
     return Path.home() / ".datashuttle"
 
 
+def get_internal_datashuttle_from_path() -> Path:
+    """Get a placeholder path for `validate_project_from_path()`."""
+    return get_datashuttle_path() / "_datashuttle_from_path"
+
+
 def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]:
     """Return the datashuttle config path for the project.
 
@@ -91,3 +98,26 @@ def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]:
     temp_logs_path = base_path / "temp_logs"
 
     return base_path, temp_logs_path
+
+
+def get_rclone_config_base_path() -> Path:
+    """Return the path to the Rclone config file.
+
+    This is used for RClone config files for transfer targets (ssh, aws, gdrive).
+    This should match where RClone itself stores the config by default,
+    as described here: https://rclone.org/docs/#config-string
+
+    Because RClone's resolution process for where it stores its config files
+    is a little complex, in some rare cases the path returned below may not match
+    where RClone actually stores its configs. In such cases, local filesystem configs,
+    which are stored in the default `rclone.conf` file for backwards compatibility
+    reasons, and transfer configs, which are stored in their own file at the path
+    returned from this function, are stored in separate places. This is generally
+    not a significant issue.
+    """
+    if platform.system() == "Windows":
+        appdata_path = Path().home() / "AppData" / "Roaming"
+        if appdata_path.is_dir():
+            return appdata_path / "rclone"
+
+    return Path().home() / ".config" / "rclone"

datashuttle/configs/config_class.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Dict, Optional, Union, cast
+from typing import TYPE_CHECKING, Dict, Union, cast
 
 if TYPE_CHECKING:
     from collections.abc import ItemsView, KeysView, ValuesView
 
     from datashuttle.utils.custom_types import (
-        OverwriteExistingFiles,
         TopLevelFolder,
     )
 
@@ -20,6 +19,7 @@
     canonical_configs,
     canonical_folders,
     load_configs,
+    rclone_configs,
 )
 from datashuttle.utils import folders, utils
 
@@ -37,6 +37,9 @@ def __init__(
     ) -> None:
         """Initialize the Configs class with project name, file path, and config dictionary.
 
+        This class also holds `RCloneConfigs` that manage the Rclone config files
+        used for transfer.
+
         Parameters
         ----------
         project_name
@@ -64,6 +67,8 @@ def __init__(
         self.hostkeys_path: Path
         self.project_metadata_path: Path
 
+        self.rclone = rclone_configs.RCloneConfigs(self, self.file_path.parent)
+
     def setup_after_load(self) -> None:
         """Set up the config after loading it."""
         load_configs.convert_str_and_pathlib_paths(self, "str_to_path")
@@ -249,48 +254,6 @@ def get_base_folder(
 
         return base_folder
 
-    def get_rclone_config_name(
-        self, connection_method: Optional[str] = None
-    ) -> str:
-        """Generate the rclone configuration name for the central project.
-
-        These configs are created by datashuttle but managed and stored by rclone.
-        """
-        if connection_method is None:
-            connection_method = self["connection_method"]
-
-        assert connection_method != "local_only", (
-            "This state assumes a central connection."
-        )
-
-        return f"central_{self.project_name}_{connection_method}"
-
-    def make_rclone_transfer_options(
-        self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool
-    ) -> Dict:
-        """Create a dictionary of rclone transfer options.
-
-        Originally these arguments were collected from configs, but now
-        they are passed via function arguments. The `show_transfer_progress`
-        and `dry_run` options are fixed here.
-        """
-        allowed_overwrite = ["never", "always", "if_source_newer"]
-
-        if overwrite_existing_files not in allowed_overwrite:
-            utils.log_and_raise_error(
-                f"`overwrite_existing_files` not "
-                f"recognised, must be one of: "
-                f"{allowed_overwrite}",
-                ValueError,
-            )
-
-        return {
-            "overwrite_existing_files": overwrite_existing_files,
-            "show_transfer_progress": True,
-            "transfer_verbosity": "vv",
-            "dry_run": dry_run,
-        }
-
     def init_paths(self) -> None:
         """Initialize paths used by datashuttle."""
         self.project_metadata_path = self["local_path"] / ".datashuttle"

datashuttle/configs/rclone_configs.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from datashuttle.configs.configs_class import Configs
+
+import yaml
+
+from datashuttle.configs import canonical_folders
+from datashuttle.utils import rclone_encryption
+
+
+class RCloneConfigs:
+    """Class to manage the RClone configuration file.
+
+    This is a file that RClone creates to hold all information about local and
+    central transfer targets. For example, the SSH RClone config holds the private key,
+    the GDrive rclone config holds the access token, etc.
+
+    In datashuttle, local filesystem configs uses the Rclone default configuration file,
+    that RClone manages, for backwards compatibility reasons. However, SSH, AWS and GDrive
+    configs are stored in separate config files (set using RClone's --config argument).
+    Then being separate means these files can be separately encrypted.
+
+    This class tracks the state on whether a RClone config is encrypted, as well
+    as provides the default names for the rclone conf (e.g. central_<project_name>_<connection_method>).
+
+    Parameters
+    ----------
+    datashuttle_configs
+        Parent Configs class.
+
+    config_base_class
+        Path to the datashuttle configs folder where all configs for the project are stored.
+
+    """
+
+    def __init__(self, datashuttle_configs: Configs, config_base_path: Path):
+        """Construct the class."""
+        self.datashuttle_configs = datashuttle_configs
+        self.rclone_encryption_state_file_path = (
+            config_base_path / "rclone_ps_state.yaml"
+        )
+
+    def load_rclone_config_is_encrypted(self) -> dict:
+        """Track whether the Rclone config file is encrypted.
+
+        This could be read directly from the RClone config file, but requires
+        a subprocess call which can be slow on Windows. As this function is
+        called a lot, we track this explicitly when a rclone config is
+        encrypted / unencrypted and store to disk between sessions.
+        """
+        assert rclone_encryption.connection_method_requires_encryption(
+            self.datashuttle_configs["connection_method"]
+        )
+
+        if self.rclone_encryption_state_file_path.is_file():
+            with open(self.rclone_encryption_state_file_path, "r") as file:
+                rclone_config_is_encrypted = yaml.full_load(file)
+        else:
+            rclone_config_is_encrypted = {
+                "ssh": False,
+                "gdrive": False,
+                "aws": False,
+            }
+
+            with open(self.rclone_encryption_state_file_path, "w") as file:
+                yaml.dump(rclone_config_is_encrypted, file)
+
+        return rclone_config_is_encrypted
+
+    def set_rclone_config_encryption_state(self, value: bool) -> None:
+        """Store the current state of the rclone config encryption for the `connection_method`.
+
+        Note that this is stored to disk each call (rather than tracked in memory)
+        to ensure it is updated properly if changed through the Python API
+        while the TUI is also running.
+        """
+        assert rclone_encryption.connection_method_requires_encryption(
+            self.datashuttle_configs["connection_method"]
+        )
+
+        rclone_config_is_encrypted = self.load_rclone_config_is_encrypted()
+
+        rclone_config_is_encrypted[
+            self.datashuttle_configs["connection_method"]
+        ] = value
+
+        with open(self.rclone_encryption_state_file_path, "w") as file:
+            yaml.dump(rclone_config_is_encrypted, file)
+
+    def rclone_file_is_encrypted(
+        self,
+    ) -> bool:
+        """Return whether the config file associated with the current `connection_method` is encrypted."""
+        assert rclone_encryption.connection_method_requires_encryption(
+            self.datashuttle_configs["connection_method"]
+        )
+
+        rclone_config_is_encrypted = self.load_rclone_config_is_encrypted()
+
+        return rclone_config_is_encrypted[
+            self.datashuttle_configs["connection_method"]
+        ]
+
+    def get_rclone_config_name(
+        self, connection_method: Optional[str] = None
+    ) -> str:
+        """Generate the rclone configuration name for the central project."""
+        if connection_method is None:
+            connection_method = self.datashuttle_configs["connection_method"]
+
+        return f"central_{self.datashuttle_configs.project_name}_{connection_method}"
+
+    def get_rclone_central_connection_config_filepath(self) -> Path:
+        """Return the full filepath to the rclone `.conf` config file."""
+        return (
+            canonical_folders.get_rclone_config_base_path()
+            / f"{self.get_rclone_config_name()}.conf"
+        )
+
+    def delete_existing_rclone_config_file(self) -> None:
+        """Delete the Rclone config file if it exists."""
+        rclone_config_filepath = (
+            self.get_rclone_central_connection_config_filepath()
+        )
+
+        if rclone_config_filepath.exists():
+            rclone_config_filepath.unlink()
+            self.set_rclone_config_encryption_state(False)