Make Globus dependencies optional in test suite

Makefile

@@ -51,6 +51,7 @@ mypy: ## run mypy checks
 .PHONY: gce_test
 gce_test: ## Run tests with GlobusComputeExecutor (--config .../globus_compute.py)
 	pytest -v -k "not shared_fs and not issue_3620 and not staging_required" --config parsl/tests/configs/globus_compute.py parsl/tests/ --random-order --durations 10
+	pytest -v -k "not shared_fs and not issue_3620 and not staging_required and globus_compute" --config local --random-order --durations 10
 
 .PHONY: local_thread_test
 local_thread_test: ## run all tests with local_thread config (--config .../local_threads.py)
@@ -86,7 +87,7 @@ radical_local_test:  ## Run the Radical local tests (-m radical --config local)
 .PHONY: config_local_test
 config_local_test:  ## run the config-local tests (--config local)
 	pip3 install ".[monitoring,visualization,proxystore,kubernetes]"
-	pytest parsl/tests/ -k "not cleannet and not workqueue and not taskvine" --config local --random-order --durations 10
+	pytest parsl/tests/ -k "not cleannet and not workqueue and not taskvine and not globus_compute" --config local --random-order --durations 10
 
 .PHONY: site_test
 site_test:  ## Run the site tests

parsl/data_provider/globus.py

@@ -4,7 +4,6 @@
 from functools import partial
 from typing import Optional
 
-import globus_sdk
 import typeguard
 
 import parsl
@@ -79,6 +78,7 @@ def get_authorizer(cls):
 
     @classmethod
     def transfer_file(cls, src_ep, dst_ep, src_path, dst_path):
+        import globus_sdk
         tc = globus_sdk.TransferClient(authorizer=cls.authorizer)
         td = globus_sdk.TransferData(tc, src_ep, dst_ep)
         td.add_item(src_path, dst_path)
@@ -140,6 +140,7 @@ def _update_tokens_file_on_refresh(cls, token_response):
     def _do_native_app_authentication(cls, client_id, redirect_uri,
                                       requested_scopes=None):
 
+        import globus_sdk
         client = globus_sdk.NativeAppAuthClient(client_id=client_id)
         client.oauth2_start_flow(
             requested_scopes=requested_scopes,
@@ -154,6 +155,7 @@ def _do_native_app_authentication(cls, client_id, redirect_uri,
 
     @classmethod
     def _get_native_app_authorizer(cls, client_id):
+        import globus_sdk
         tokens = None
         try:
             tokens = cls._load_tokens_from_file(cls.TOKEN_FILE)

parsl/tests/configs/local_threads_globus.py

@@ -1,6 +1,5 @@
 from parsl.config import Config
 from parsl.data_provider.data_manager import default_staging
-from parsl.data_provider.globus import GlobusStaging
 from parsl.executors.threads import ThreadPoolExecutor
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
@@ -10,19 +9,24 @@
 #          (i.e., user_opts['swan']['username'] -> 'your_username')
 from .user_opts import user_opts
 
-storage_access = default_staging + [GlobusStaging(
-                endpoint_uuid=user_opts['globus']['endpoint'],
-                endpoint_path=user_opts['globus']['path']
-            )]
 
-config = Config(
-    executors=[
-        ThreadPoolExecutor(
-            label='local_threads_globus',
-            working_dir=user_opts['globus']['path'],
-            storage_access=storage_access
-        )
-    ]
-)
+def fresh_config():
+    from parsl.data_provider.globus import GlobusStaging
+
+    storage_access = default_staging + [GlobusStaging(
+                    endpoint_uuid=user_opts['globus']['endpoint'],
+                    endpoint_path=user_opts['globus']['path']
+                )]
+
+    return Config(
+        executors=[
+            ThreadPoolExecutor(
+                label='local_threads_globus',
+                working_dir=user_opts['globus']['path'],
+                storage_access=storage_access
+            )
+        ]
+    )
+
 
 remote_writeable = user_opts['globus']['remote_writeable']

parsl/tests/test_python_apps/test_basic.py

@@ -14,12 +14,6 @@ def import_square(x):
     return math.pow(x, 2)
 
 
-@python_app
-def custom_exception():
-    from globus_sdk import GlobusError
-    raise GlobusError('foobar')
-
-
 def test_simple(n=2):
     x = double(n)
     assert x.result() == n * 2
@@ -38,11 +32,3 @@ def test_parallel_for(n):
 
     for i in d:
         assert d[i].result() == 2 * i
-
-
-def test_custom_exception():
-    from globus_sdk import GlobusError
-
-    x = custom_exception()
-    with pytest.raises(GlobusError):
-        x.result()

parsl/tests/test_python_apps/test_exception.py

@@ -0,0 +1,19 @@
+import pytest
+
+from parsl.app.app import python_app
+
+
+class CustomException(Exception):
+    pass
+
+
+@python_app
+def custom_exception():
+    from parsl.tests.test_python_apps.test_exception import CustomException
+    raise CustomException('foobar')
+
+
+def test_custom_exception():
+    x = custom_exception()
+    with pytest.raises(CustomException):
+        x.result()

parsl/tests/test_staging/test_staging_globus.py

@@ -3,9 +3,9 @@
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
-from parsl.tests.configs.local_threads_globus import config, remote_writeable
+from parsl.tests.configs.local_threads_globus import fresh_config, remote_writeable
 
-local_config = config
+local_config = fresh_config
 
 
 @python_app

parsl/tests/unit/test_globus_compute_executor.py

@@ -2,38 +2,44 @@
 from unittest import mock
 
 import pytest
-from globus_compute_sdk import Executor
 
 from parsl.executors import GlobusComputeExecutor
 
 
 @pytest.fixture
 def mock_ex():
-    # Not Parsl's job to test GC's Executor
+    # Not Parsl's job to test GC's Executor, although it
+    # still needs to be importable for these test cases.
+    from globus_compute_sdk import Executor
+
     yield mock.Mock(spec=Executor)
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_mock_spec(mock_ex):
     # a test of tests -- make sure we're using spec= in the mock
     with pytest.raises(AttributeError):
         mock_ex.aasdf()
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_label_default(mock_ex):
     gce = GlobusComputeExecutor(mock_ex)
     assert gce.label == type(gce).__name__, "Expect reasonable default label"
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_label(mock_ex, randomstring):
     exp_label = randomstring()
     gce = GlobusComputeExecutor(mock_ex, label=exp_label)
     assert gce.label == exp_label
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring):
     submit_res = {randomstring(): "some submit res"}
     res = {"some": randomstring(), "spec": randomstring()}
@@ -57,6 +63,7 @@ def mock_submit(*a, **k):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring):
     uep_conf = randomstring()
     res = {"some": randomstring()}
@@ -79,6 +86,7 @@ def mock_submit(*a, **k):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_happy_path(mock_ex, randomstring):
     mock_fn = mock.Mock()
     args = tuple(randomstring() for _ in range(random.randint(0, 3)))
@@ -95,6 +103,7 @@ def test_gc_executor_happy_path(mock_ex, randomstring):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_shuts_down_asynchronously(mock_ex):
     gce = GlobusComputeExecutor(mock_ex)
     gce.shutdown()

requirements.txt

@@ -8,7 +8,6 @@ typeguard>=2.10,!=3.*,<5
 
 
 typing-extensions>=4.6,<5
-globus-sdk
 dill
 tblib
 requests

setup.py

@@ -46,6 +46,7 @@
     'proxystore': ['proxystore'],
     'radical-pilot': ['radical.pilot==1.90', 'radical.utils==1.90'],
     'globus_compute': ['globus_compute_sdk>=2.34.0'],
+    'globus_transfer': ['globus-sdk'],
     # Disabling psi-j since github direct links are not allowed by pypi
     # 'psij': ['psi-j-parsl@git+https://github.com/ExaWorks/psi-j-parsl']
 }

test-requirements.txt

@@ -10,7 +10,6 @@ types-mock
 types-python-dateutil
 types-requests
 mpi4py
-globus-compute-sdk>=2.34.0
 
 # sqlalchemy is needed for typechecking, so it's here
 # as well as at runtime for optional monitoring execution