[Testing] Test SSH file transfer with images. (#208)

* lots of changes, sort out.

* still working, needs a bit more tidying up.

* Continue working.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Tidy ups.

* Fix linting.

* Tidy ups and documentation.

* Add documentation.

* Try different Dockerfile for linux.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Fix issue after rebase.

* Remove breakpoint.

* Don't use singularity, docker or nothing.

* Rework ssh setup.

* Fix connection refused issue.

* Use run not POpen for actions.

* try in detachted state.

* Add error messages on docker setup.

* run linting.

* update connection failed error message.

* Test version working locally on linux but only can connect with sudo.

* Try freeing up and using the free port.

* Test sudo service only on linux.

* Add sudo to docker setup commands.

* test with auto add policy.

* Really restrict to the connect call.

* restrict to tests of interest temporarily.

* Fix to port 3306 in tests and for paramiko.

* Extend port to rclone.

* Use environment variable to set port.

* Add all OS back.

* try remove tag for windows.

* Update docker commands for windows.

* Fix nonsense docker build command.

* Only run when docker running and on ubuntu on runners.

* Try build and run docker only once per session.

* Teardown image at end of ssh tests, factor out ssh tests.

* SPlit ssh tests.

* Add sudo to the docker teardown commands for Linux.

* try class scope of setup ssh container.

* Try move ssh fixture to classes.

* Try a different command to shutdown on linux.

* Tidy ups and some docs.

* Extend to macOS.

* Finish tidying up docstrings.

* Change ssh test image name and fix docstring.

* Small tidy ups.

* Small fixes after rebase.

* More fix.es

* Fix CI.

* Skip tests on macOS.

* Refactor transfer tests.

* Continue refactoring.

* Fix tests again.

* Update CI script.

* Update CI.

* Fix imports after rebase.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Delete config to fix tests.

* Remove test module that was accidentally readded by git.

* Delete now unused file.

* Do not append por with : if it is default (22).

* Move test setup_rclone_config_for_ssh into setup_ssh_connection.

* Extend SSH tests not run message.

* Improve test skipped message.

* Fix duplicate if / else statement.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: JoeZiminski

.github/workflows/code_test_and_deploy.yml

@@ -32,7 +32,7 @@ jobs:
         # macos-14 is M1, macos-13 is intel. Run on earliest and
         # latest python versions. All python versions are tested in
         # the weekly cron job.
-        os: [windows-latest, ubuntu-latest, macos-14, macos-13]
+        os: [ ubuntu-latest, windows-latest, macos-14, macos-13]
         # Test all Python versions for cron job, and only first/last for other triggers
         python-version: ${{ fromJson(github.event_name == 'schedule' && '["3.9", "3.10", "3.11", "3.12"]' || '["3.9", "3.12"]') }}
 
@@ -57,8 +57,17 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install .[dev]
-      - name: Test
-        run: pytest
+        # run SSH tests only on Linux because Windows and macOS
+        # are already run within a virtual container and so cannot
+        # run Linux containers because nested containerisation is disabled.
+      - name: Test SSH (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          sudo service mysql stop  # free up port 3306 for ssh tests
+          pytest tests/tests_transfers/ssh
+      - name: All Other Tests
+        run: |
+          pytest --ignore tests/tests_transfers/ssh
 
   build_sdist_wheels:
     name: Build source distribution

datashuttle/configs/canonical_configs.py

@@ -9,6 +9,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import (
     TYPE_CHECKING,
     Dict,
@@ -52,6 +53,14 @@ def keys_str_on_file_but_path_in_class() -> list[str]:
     ]
 
 
+def get_default_ssh_port() -> int:
+    """Get the default port used for SSH connections."""
+    if "DS_SSH_PORT" in os.environ:
+        return int(os.environ["DS_SSH_PORT"])
+    else:
+        return 22
+
+
 # -----------------------------------------------------------------------------
 # Check Configs
 # -----------------------------------------------------------------------------

datashuttle/utils/data_transfer.py

@@ -164,7 +164,6 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]:
                     self.update_list_with_non_ses_sub_level_folders(
                         extra_folder_names, extra_filenames, sub
                     )
-
                     continue
 
                 # Datatype (sub and ses level) --------------------------------

datashuttle/utils/rclone.py

@@ -6,6 +6,7 @@
 from subprocess import CompletedProcess
 from typing import Dict, List, Literal
 
+from datashuttle.configs import canonical_configs
 from datashuttle.configs.config_class import Configs
 from datashuttle.utils import utils
 from datashuttle.utils.custom_types import TopLevelFolder
@@ -161,7 +162,7 @@ def setup_rclone_config_for_ssh(
         f"sftp "
         f"host {cfg['central_host_id']} "
         f"user {cfg['central_host_username']} "
-        f"port 22 "
+        f"port {canonical_configs.get_default_ssh_port()} "
         f"key_file {ssh_key_path.as_posix()}",
         pipe_std=True,
     )

datashuttle/utils/ssh.py

@@ -14,6 +14,7 @@
 
 import paramiko
 
+from datashuttle.configs import canonical_configs
 from datashuttle.utils import utils
 
 # -----------------------------------------------------------------------------
@@ -58,6 +59,7 @@ def connect_client_core(
             else None
         ),
         look_for_keys=True,
+        port=canonical_configs.get_default_ssh_port(),
     )
 
 
@@ -122,7 +124,9 @@ def get_remote_server_key(central_host_id: str):
 
     """
     transport: paramiko.Transport
-    with paramiko.Transport(central_host_id) as transport:
+    with paramiko.Transport(
+        (central_host_id, canonical_configs.get_default_ssh_port())
+    ) as transport:
         transport.connect()
         key = transport.get_remote_server_key()
     return key
@@ -148,7 +152,15 @@ def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None:
 
     """
     client = paramiko.SSHClient()
-    client.get_host_keys().add(central_host_id, key.get_name(), key)
+
+    port = canonical_configs.get_default_ssh_port()
+    host_key = f"[{central_host_id}]:{port}" if port != 22 else central_host_id
+
+    client.get_host_keys().add(
+        host_key,
+        key.get_name(),
+        key,
+    )
     client.get_host_keys().save(hostkeys_path.as_posix())
 
 
@@ -242,15 +254,16 @@ def connect_client_with_logging(
                 f"Connection to {cfg['central_host_id']} made successfully."
             )
 
-    except Exception:
+    except Exception as e:
         utils.log_and_raise_error(
             f"Could not connect to server. Ensure that \n"
             f"1) You have run setup_ssh_connection() \n"
             f"2) You are on VPN network if required. \n"
             f"3) The central_host_id: {cfg['central_host_id']} is"
             f" correct.\n"
             f"4) The central username:"
-            f" {cfg['central_host_username']}, and password are correct.",
+            f" {cfg['central_host_username']}, and password are correct."
+            f"Original error: {e}",
             ConnectionError,
         )
 

pyproject.toml

@@ -126,7 +126,8 @@ select = [
     "D",        # pydocstyle
 ]
 per-file-ignores = { "tests/*" = [
-    "D"  # ignore docstring formatting in tests for now
+    "D",  # ignore docstring formatting in tests for now
+    "TID252"
 ], "examples/*" = [
     "D400", # first line should end with a period.
     "D415", # first line should end with a period, question mark...

tests/__init__.py

@@ -1,7 +1,8 @@
 import warnings
 
 import pytest
-import test_utils
+
+from . import test_utils
 
 TEST_PROJECT_NAME = "test_project"