Fix ssh and gdrive tests.

Author: JoeZiminski

.github/workflows/code_test_and_deploy.yml

@@ -59,38 +59,8 @@ jobs:
           python -m pip install --upgrade pip
           pip install .[dev]
 
-        # run SSH tests only on Linux because Windows and macOS
-        # are already run within a virtual container and so cannot
-        # run Linux containers because nested containerisation is disabled.
-#      - name: Test SSH (Linux only)
-#        if: runner.os == 'Linux'
-#        run: |
-#          sudo service mysql stop  # free up port 3306 for ssh tests
-#          pytest tests/tests_transfers/ssh
-#
-#      - name: Test Google Drive
-#        env:
-#          GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }}
-#          GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }}
-#          GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }}
-#          GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }}
-#        run: |
-#          pytest tests/tests_transfers/gdrive
-
-#      - name: Test AWS
-#        env:
-#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-#          AWS_REGION: ${{ secrets.AWS_REGION }}
-#          AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
-#        run: |
-#          pytest tests/tests_transfers/aws
-
-#      - name: All Other Tests
-#        run: |
-#          pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws
-
       - name: Install pass on Linux
+        # this is required for Rclone config encryption
         if: runner.os == 'Linux'
         run: |
           set -euo pipefail
@@ -109,16 +79,42 @@ jobs:
           FPR="$(gpg --list-secret-keys --with-colons | awk -F: '/^fpr:/ {print $10; exit}')"
           pass init "$FPR"
 
-          # (Optional) smoke test: ensure pass can encrypt
-          printf '%s\n' "$(openssl rand -base64 16)" | pass insert -m -f ci/smoke-test
+
+        # run SSH tests only on Linux because Windows and macOS
+        # are already run within a virtual container and so cannot
+        # run Linux containers because nested containerisation is disabled.
+      - name: Test SSH (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          sudo service mysql stop  # free up port 3306 for ssh tests
+          pytest tests/tests_transfers/ssh
+
+      - name: Test Google Drive
+        env:
+          GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }}
+          GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }}
+          GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }}
+          GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }}
+        run: |
+          pytest tests/tests_transfers/gdrive
+
+#      - name: Test AWS
+#        env:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#          AWS_REGION: ${{ secrets.AWS_REGION }}
+#          AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
+#        run: |
+#          pytest tests/tests_transfers/aws
+
+#      - name: All Other Tests
+#        run: |
+#          pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws
 
       - name: RClone Encryption
         run: |
-          set -euo pipefail
-          # GNUPGHOME is available here because we wrote it to $GITHUB_ENV
           pytest -k test_rclone_encryption
 
-
   build_sdist_wheels:
     name: Build source distribution
     needs: [test]

datashuttle/tui/interface.py

@@ -13,7 +13,7 @@
 
 from datashuttle import DataShuttle
 from datashuttle.configs import load_configs
-from datashuttle.utils import aws, gdrive, rclone, ssh, utils
+from datashuttle.utils import aws, rclone, ssh, utils
 
 
 class Interface:
@@ -552,11 +552,13 @@ def get_rclone_message_for_gdrive_without_browser(
     ) -> InterfaceOutput:
         """Get the rclone message for Google Drive setup without a browser."""
         try:
-            output = gdrive.preliminary_for_setup_without_browser(
-                self.project.cfg,
-                gdrive_client_secret,
-                self.project.cfg.rclone.get_rclone_config_name("gdrive"),
-                log=False,
+            output = (
+                rclone.preliminary_setup_gdrive_config_for_without_browser(
+                    self.project.cfg,
+                    gdrive_client_secret,
+                    self.project.cfg.rclone.get_rclone_config_name("gdrive"),
+                    log=False,
+                )
             )
             return True, output
         except BaseException as e:

datashuttle/tui/screens/setup_gdrive.py

@@ -21,6 +21,8 @@
     Static,
 )
 
+from datashuttle.utils import rclone_encryption
+
 
 class SetupGdriveScreen(ModalScreen):
     """Dialog window that sets up a Google Drive connection.
@@ -293,6 +295,7 @@ async def setup_gdrive_connection_and_update_ui(
                 # This function is called from different screens that
                 # contain different widgets. Therefore remove all possible
                 # widgets that may / may not be present on the previous screen.
+                self.show_encryption_screen()
                 for id in [
                     "#setup_gdrive_cancel_button",
                     "#setup_gdrive_generic_input_box",
@@ -335,6 +338,18 @@ def setup_gdrive_connection(
     # Set encryption on RClone config
     # ----------------------------------------------------------------------------------
 
+    def show_encryption_screen(self):
+        """Show the screen asking the user whether to encrypt the Rclone password."""
+        message = f"{rclone_encryption.get_explanation_message(self.interface.project.cfg)}"
+        self.update_message_box_message(message)
+
+        yes_button = Button("Yes", id="setup_gdrive_set_encryption_yes_button")
+        no_button = Button("No", id="setup_gdrive_set_encryption_no_button")
+
+        self.query_one("#setup_gdrive_buttons_horizontal").mount(
+            yes_button, no_button
+        )
+
     def set_rclone_encryption(self):
         """Try and encrypt the Rclone config file and inform the user of success / failure."""
         success, output = self.interface.try_setup_rclone_encryption()

datashuttle/utils/gdrive.py

@@ -1,78 +1,12 @@
 from __future__ import annotations
 
-import json
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datashuttle.configs.config_class import Configs
 
 from datashuttle.utils import rclone, utils
 
-# -----------------------------------------------------------------------------
-# Helper Functions
-# -----------------------------------------------------------------------------
-
-# These functions are used by both API and TUI for setting up connections to google drive.
-
-
-def preliminary_for_setup_without_browser(
-    cfg: Configs,
-    gdrive_client_secret: str | None,
-    rclone_config_name: str,
-    log: bool = True,
-) -> str:
-    """Prepare rclone configuration for Google Drive without using a browser.
-
-    This function prepares the rclone configuration for Google Drive without using a browser.
-
-    The `config_is_local=false` flag tells rclone that the configuration process is being run
-    on a headless machine which does not have access to a browser.
-
-    The `--non-interactive` flag is used to control rclone's behaviour while running it through
-    external applications. An `rclone config create` command would assume default values for config
-    variables in an interactive mode. If the `--non-interactive` flag is provided and rclone needs
-    the user to input some detail, a JSON blob will be returned with the question in it. For this
-    particular setup, rclone outputs a command for user to run on a machine with a browser.
-
-    This function runs `rclone config create` with the user credentials and returns the rclone's output info.
-    This output info is presented to the user while asking for a `config_token`.
-
-    Next, the user will run rclone's given command, authenticate with google drive and input the
-    config token given by rclone for datashuttle to proceed with the setup.
-    """
-    client_id_key_value = (
-        f"client_id {cfg['gdrive_client_id']} "
-        if cfg["gdrive_client_id"]
-        else " "
-    )
-    client_secret_key_value = (
-        f"client_secret {gdrive_client_secret} "
-        if gdrive_client_secret
-        else ""
-    )
-    output = rclone.call_rclone(
-        f"config create "
-        f"{rclone_config_name} "
-        f"drive "
-        f"{client_id_key_value}"
-        f"{client_secret_key_value}"
-        f"scope drive "
-        f"root_folder_id {cfg['gdrive_root_folder_id']} "
-        f"config_is_local=false "
-        f"--non-interactive",
-        pipe_std=True,
-    )
-
-    # Extracting rclone's message from the json
-    output_json = json.loads(output.stdout)
-    message = output_json["Option"]["Help"]
-
-    if log:
-        utils.log(message)
-
-    return message
-
-
 # -----------------------------------------------------------------------------
 # Python API
 # -----------------------------------------------------------------------------
@@ -108,7 +42,7 @@ def prompt_and_get_config_token(
     with google drive and input the `config_token` generated by rclone. The `config_token` is
     then used to complete rclone's config setup for google drive.
     """
-    message = preliminary_for_setup_without_browser(
+    message = rclone.preliminary_setup_gdrive_config_for_without_browser(
         cfg, gdrive_client_secret, rclone_config_name, log=log
     )
     input_ = utils.get_user_input(

datashuttle/utils/rclone.py

@@ -9,6 +9,7 @@
     from datashuttle.configs.config_class import Configs
     from datashuttle.utils.custom_types import TopLevelFolder
 
+import json
 import os
 import platform
 import shlex
@@ -158,7 +159,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail(
     lambda_func = lambda: process.communicate()
 
     stdout, stderr = run_function_that_requires_encrypted_rclone_config_access(
-        cfg, lambda_func
+        cfg, lambda_func, check_config_exists=False
     )
 
     if process.returncode != 0:
@@ -171,7 +172,7 @@ def await_call_rclone_with_popen_for_central_connection_raise_on_fail(
 
 
 def run_function_that_requires_encrypted_rclone_config_access(
-    cfg, lambda_func
+    cfg, lambda_func, check_config_exists: bool = True
 ) -> Any:
     """Run command that requires possibly encrypted Rclone config file.
 
@@ -185,7 +186,7 @@ def run_function_that_requires_encrypted_rclone_config_access(
         cfg.rclone.get_rclone_central_connection_config_filepath()
     )
 
-    if not rclone_config_filepath.is_file():
+    if check_config_exists and not rclone_config_filepath.is_file():
         if version.parse(get_datashuttle_version()) <= version.parse("0.7.1"):
             raise RuntimeError(
                 f"The way RClone configs are managed has changed since version v0.7.1\n"
@@ -371,6 +372,71 @@ def setup_rclone_config_for_gdrive(
     return process
 
 
+def preliminary_setup_gdrive_config_for_without_browser(
+    cfg: Configs,
+    gdrive_client_secret: str | None,
+    rclone_config_name: str,
+    log: bool = True,
+) -> str:
+    """Prepare rclone configuration for Google Drive without using a browser.
+
+    This function prepares the rclone configuration for Google Drive without using a browser.
+
+    The `config_is_local=false` flag tells rclone that the configuration process is being run
+    on a headless machine which does not have access to a browser.
+
+    The `--non-interactive` flag is used to control rclone's behaviour while running it through
+    external applications. An `rclone config create` command would assume default values for config
+    variables in an interactive mode. If the `--non-interactive` flag is provided and rclone needs
+    the user to input some detail, a JSON blob will be returned with the question in it. For this
+    particular setup, rclone outputs a command for user to run on a machine with a browser.
+
+    This function runs `rclone config create` with the user credentials and returns the rclone's output info.
+    This output info is presented to the user while asking for a `config_token`.
+
+    Next, the user will run rclone's given command, authenticate with google drive and input the
+    config token given by rclone for datashuttle to proceed with the setup.
+    """
+    client_id_key_value = (
+        f"client_id {cfg['gdrive_client_id']} "
+        if cfg["gdrive_client_id"]
+        else " "
+    )
+    client_secret_key_value = (
+        f"client_secret {gdrive_client_secret} "
+        if gdrive_client_secret
+        else ""
+    )
+
+    cfg.rclone.delete_existing_rclone_config_file()
+
+    output = call_rclone(
+        f"config create "
+        f"{get_config_arg(cfg)} "
+        f"{rclone_config_name} "
+        f"drive "
+        f"{client_id_key_value}"
+        f"{client_secret_key_value}"
+        f"scope drive "
+        f"root_folder_id {cfg['gdrive_root_folder_id']} "
+        f"config_is_local=false "
+        f"--non-interactive",
+        pipe_std=True,
+    )
+
+    try:
+        # Extracting rclone's message from the json
+        output_json = json.loads(output.stdout)
+        message = output_json["Option"]["Help"]
+    except:
+        assert False, f"{output.stderr}"
+
+    if log:
+        utils.log(message)
+
+    return message
+
+
 def setup_rclone_config_for_aws(
     cfg: Configs,
     rclone_config_name: str,

datashuttle/utils/rclone_encryption.py

@@ -242,7 +242,7 @@ def run_rclone_config_encrypt(cfg: Configs) -> None:
 
 
 def remove_rclone_encryption(cfg: Configs) -> None:
-    """Remove encryption from an Rclone config file.
+    """Remove encryption from a Rclone config file.
 
     Set the credentials one last time to remove encryption from
     the RClone config file. Once removed, clean up the password
@@ -322,7 +322,7 @@ def get_windows_password_filepath(
 def get_explanation_message(
     cfg: Configs,
 ) -> str:
-    """Explaining rclone's default credential storage and OS-specific encryption options.
+    """Explaining Rclone's default credential storage and OS-specific encryption options.
 
     Displayed in both the Python API and the TUI.
     """