From f0cab0c34077ef3c3998ade1221996e54ecf03dd Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 10:53:43 +0000 Subject: [PATCH 1/7] remove globus transfer and compute dependencies enough to let --config run, but not --config local --- parsl/tests/configs/local_threads_globus.py | 37 ++++++++++++------- parsl/tests/test_python_apps/test_basic.py | 11 +++--- .../tests/test_staging/test_staging_globus.py | 4 +- .../unit/test_globus_compute_executor.py | 6 ++- requirements.txt | 1 - setup.py | 1 + test-requirements.txt | 1 - 7 files changed, 36 insertions(+), 25 deletions(-) diff --git a/parsl/tests/configs/local_threads_globus.py b/parsl/tests/configs/local_threads_globus.py index 21a85c1c83..c75bd67e5d 100644 --- a/parsl/tests/configs/local_threads_globus.py +++ b/parsl/tests/configs/local_threads_globus.py @@ -1,6 +1,5 @@ from parsl.config import Config from parsl.data_provider.data_manager import default_staging -from parsl.data_provider.globus import GlobusStaging from parsl.executors.threads import ThreadPoolExecutor # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py @@ -10,19 +9,29 @@ # (i.e., user_opts['swan']['username'] -> 'your_username') from .user_opts import user_opts -storage_access = default_staging + [GlobusStaging( - endpoint_uuid=user_opts['globus']['endpoint'], - endpoint_path=user_opts['globus']['path'] - )] +def fresh_config(): + opts = user_opts['globus'] -config = Config( - executors=[ - ThreadPoolExecutor( - label='local_threads_globus', - working_dir=user_opts['globus']['path'], - storage_access=storage_access - ) - ] -) + # This user_opts key lookup will implicitly skip the + # test when the user has not defined a globus entry, + # skipping the rest of configuration construction, + # especially the import of GlobusStaging which may + # not be importable in non-Globus-user cases. + + from parsl.data_provider.globus import GlobusStaging + storage_access = default_staging + [GlobusStaging( + endpoint_uuid=opts['endpoint'], + endpoint_path=opts['path'] + )] + + return Config( + executors=[ + ThreadPoolExecutor( + label='local_threads_globus', + working_dir=opts['path'], + storage_access=storage_access + ) + ] + ) remote_writeable = user_opts['globus']['remote_writeable'] diff --git a/parsl/tests/test_python_apps/test_basic.py b/parsl/tests/test_python_apps/test_basic.py index 1040d23d02..8654f9259d 100644 --- a/parsl/tests/test_python_apps/test_basic.py +++ b/parsl/tests/test_python_apps/test_basic.py @@ -14,10 +14,13 @@ def import_square(x): return math.pow(x, 2) +class CustomException(Exception): + pass + + @python_app def custom_exception(): - from globus_sdk import GlobusError - raise GlobusError('foobar') + raise CustomException('foobar') def test_simple(n=2): @@ -41,8 +44,6 @@ def test_parallel_for(n): def test_custom_exception(): - from globus_sdk import GlobusError - x = custom_exception() - with pytest.raises(GlobusError): + with pytest.raises(CustomException): x.result() diff --git a/parsl/tests/test_staging/test_staging_globus.py b/parsl/tests/test_staging/test_staging_globus.py index 37814175ad..9bdb76c097 100644 --- a/parsl/tests/test_staging/test_staging_globus.py +++ b/parsl/tests/test_staging/test_staging_globus.py @@ -3,9 +3,9 @@ import parsl from parsl.app.app import python_app from parsl.data_provider.files import File -from parsl.tests.configs.local_threads_globus import config, remote_writeable +from parsl.tests.configs.local_threads_globus import fresh_config, remote_writeable -local_config = config +local_config = fresh_config @python_app diff --git a/parsl/tests/unit/test_globus_compute_executor.py b/parsl/tests/unit/test_globus_compute_executor.py index b65ff92047..daaee88fec 100644 --- a/parsl/tests/unit/test_globus_compute_executor.py +++ b/parsl/tests/unit/test_globus_compute_executor.py @@ -2,14 +2,16 @@ from unittest import mock import pytest -from globus_compute_sdk import Executor from parsl.executors import GlobusComputeExecutor @pytest.fixture def mock_ex(): - # Not Parsl's job to test GC's Executor + # Not Parsl's job to test GC's Executor, although it + # still needs to be importable for these test cases. + from globus_compute_sdk import Executor + yield mock.Mock(spec=Executor) diff --git a/requirements.txt b/requirements.txt index 1ee91f96c5..8d3832ed63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ typeguard>=2.10,!=3.*,<5 typing-extensions>=4.6,<5 -globus-sdk dill tblib requests diff --git a/setup.py b/setup.py index 709c4b2bf2..5b1e03ff81 100755 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ 'proxystore': ['proxystore'], 'radical-pilot': ['radical.pilot==1.90', 'radical.utils==1.90'], 'globus_compute': ['globus_compute_sdk>=2.34.0'], + 'globus_transfer': ['globus-sdk'], # Disabling psi-j since github direct links are not allowed by pypi # 'psij': ['psi-j-parsl@git+https://github.com/ExaWorks/psi-j-parsl'] } diff --git a/test-requirements.txt b/test-requirements.txt index f31fc13e0a..57dcef3711 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,6 @@ types-mock types-python-dateutil types-requests mpi4py -globus-compute-sdk>=2.34.0 # sqlalchemy is needed for typechecking, so it's here # as well as at runtime for optional monitoring execution From 7bed8c3d0f7f6d0e3a9fbc93f03fb8f948853d74 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 11:28:11 +0000 Subject: [PATCH 2/7] allow --config local tests to pass without globus-com,pute-sdk installed, if you run with -k 'not globus_compute' --- parsl/tests/unit/test_globus_compute_executor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/parsl/tests/unit/test_globus_compute_executor.py b/parsl/tests/unit/test_globus_compute_executor.py index daaee88fec..21e1da00a5 100644 --- a/parsl/tests/unit/test_globus_compute_executor.py +++ b/parsl/tests/unit/test_globus_compute_executor.py @@ -16,6 +16,7 @@ def mock_ex(): @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_mock_spec(mock_ex): # a test of tests -- make sure we're using spec= in the mock with pytest.raises(AttributeError): @@ -23,12 +24,14 @@ def test_gc_executor_mock_spec(mock_ex): @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_label_default(mock_ex): gce = GlobusComputeExecutor(mock_ex) assert gce.label == type(gce).__name__, "Expect reasonable default label" @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_label(mock_ex, randomstring): exp_label = randomstring() gce = GlobusComputeExecutor(mock_ex, label=exp_label) @@ -36,6 +39,7 @@ def test_gc_executor_label(mock_ex, randomstring): @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring): submit_res = {randomstring(): "some submit res"} res = {"some": randomstring(), "spec": randomstring()} @@ -59,6 +63,7 @@ def mock_submit(*a, **k): @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring): uep_conf = randomstring() res = {"some": randomstring()} @@ -81,6 +86,7 @@ def mock_submit(*a, **k): @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_happy_path(mock_ex, randomstring): mock_fn = mock.Mock() args = tuple(randomstring() for _ in range(random.randint(0, 3))) @@ -97,6 +103,7 @@ def test_gc_executor_happy_path(mock_ex, randomstring): @pytest.mark.local +@pytest.mark.globus_compute def test_gc_executor_shuts_down_asynchronously(mock_ex): gce = GlobusComputeExecutor(mock_ex) gce.shutdown() From 4999ba3db209d6e592bcee8284dc5a86a3b5ecea Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 11:33:22 +0000 Subject: [PATCH 3/7] remove weird import ordering that turns out to not be necessary --- parsl/tests/configs/local_threads_globus.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/parsl/tests/configs/local_threads_globus.py b/parsl/tests/configs/local_threads_globus.py index c75bd67e5d..ca32d23c06 100644 --- a/parsl/tests/configs/local_threads_globus.py +++ b/parsl/tests/configs/local_threads_globus.py @@ -9,16 +9,11 @@ # (i.e., user_opts['swan']['username'] -> 'your_username') from .user_opts import user_opts + def fresh_config(): + from parsl.data_provider.globus import GlobusStaging opts = user_opts['globus'] - # This user_opts key lookup will implicitly skip the - # test when the user has not defined a globus entry, - # skipping the rest of configuration construction, - # especially the import of GlobusStaging which may - # not be importable in non-Globus-user cases. - - from parsl.data_provider.globus import GlobusStaging storage_access = default_staging + [GlobusStaging( endpoint_uuid=opts['endpoint'], endpoint_path=opts['path'] @@ -30,8 +25,9 @@ def fresh_config(): label='local_threads_globus', working_dir=opts['path'], storage_access=storage_access - ) - ] + ) + ] ) + remote_writeable = user_opts['globus']['remote_writeable'] From 66c53954773ec23e4cdf74df8dbf13bff33fd9fe Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 11:36:56 +0000 Subject: [PATCH 4/7] restore user_opts access that I didn't need to rearrange --- parsl/tests/configs/local_threads_globus.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/parsl/tests/configs/local_threads_globus.py b/parsl/tests/configs/local_threads_globus.py index ca32d23c06..2a09e9a94a 100644 --- a/parsl/tests/configs/local_threads_globus.py +++ b/parsl/tests/configs/local_threads_globus.py @@ -12,18 +12,17 @@ def fresh_config(): from parsl.data_provider.globus import GlobusStaging - opts = user_opts['globus'] storage_access = default_staging + [GlobusStaging( - endpoint_uuid=opts['endpoint'], - endpoint_path=opts['path'] + endpoint_uuid=user_opts['globus']['endpoint'], + endpoint_path=user_opts['globus']['path'] )] return Config( executors=[ ThreadPoolExecutor( label='local_threads_globus', - working_dir=opts['path'], + working_dir=user_opts['globus']['path'], storage_access=storage_access ) ] From 7cb72b8d24fbe841e521217bd62bfebaa34f6175 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 11:45:05 +0000 Subject: [PATCH 5/7] Makefile runs globus_compute tests like taskvine and workqueue tests --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a5227aebc0..2a36133be2 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ mypy: ## run mypy checks .PHONY: gce_test gce_test: ## Run tests with GlobusComputeExecutor (--config .../globus_compute.py) pytest -v -k "not shared_fs and not issue_3620 and not staging_required" --config parsl/tests/configs/globus_compute.py parsl/tests/ --random-order --durations 10 + pytest -v -k "not shared_fs and not issue_3620 and not staging_required and globus_compute" --config local --random-order --durations 10 .PHONY: local_thread_test local_thread_test: ## run all tests with local_thread config (--config .../local_threads.py) @@ -86,7 +87,7 @@ radical_local_test: ## Run the Radical local tests (-m radical --config local) .PHONY: config_local_test config_local_test: ## run the config-local tests (--config local) pip3 install ".[monitoring,visualization,proxystore,kubernetes]" - pytest parsl/tests/ -k "not cleannet and not workqueue and not taskvine" --config local --random-order --durations 10 + pytest parsl/tests/ -k "not cleannet and not workqueue and not taskvine and not globus_compute" --config local --random-order --durations 10 .PHONY: site_test site_test: ## Run the site tests From a1755b15c406756a9f50b653c39252567aea8387 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 12:06:35 +0000 Subject: [PATCH 6/7] Make GlobusStaging importable even when globus-sdk is not installed, for documentation build --- parsl/data_provider/globus.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parsl/data_provider/globus.py b/parsl/data_provider/globus.py index e63b0aa82f..9ad3d5fe4c 100644 --- a/parsl/data_provider/globus.py +++ b/parsl/data_provider/globus.py @@ -4,7 +4,6 @@ from functools import partial from typing import Optional -import globus_sdk import typeguard import parsl @@ -79,6 +78,7 @@ def get_authorizer(cls): @classmethod def transfer_file(cls, src_ep, dst_ep, src_path, dst_path): + import globus_sdk tc = globus_sdk.TransferClient(authorizer=cls.authorizer) td = globus_sdk.TransferData(tc, src_ep, dst_ep) td.add_item(src_path, dst_path) @@ -140,6 +140,7 @@ def _update_tokens_file_on_refresh(cls, token_response): def _do_native_app_authentication(cls, client_id, redirect_uri, requested_scopes=None): + import globus_sdk client = globus_sdk.NativeAppAuthClient(client_id=client_id) client.oauth2_start_flow( requested_scopes=requested_scopes, @@ -154,6 +155,7 @@ def _do_native_app_authentication(cls, client_id, redirect_uri, @classmethod def _get_native_app_authorizer(cls, client_id): + import globus_sdk tokens = None try: tokens = cls._load_tokens_from_file(cls.TOKEN_FILE) From fd4a912775406b9bcde32d08d39c3ca1d8660ff8 Mon Sep 17 00:00:00 2001 From: Ben Clifford Date: Wed, 1 Oct 2025 12:27:53 +0000 Subject: [PATCH 7/7] Explicitly import CustomException -- this is needed for GC executor test. --- parsl/tests/test_python_apps/test_basic.py | 15 --------------- .../tests/test_python_apps/test_exception.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 parsl/tests/test_python_apps/test_exception.py diff --git a/parsl/tests/test_python_apps/test_basic.py b/parsl/tests/test_python_apps/test_basic.py index 8654f9259d..46de788372 100644 --- a/parsl/tests/test_python_apps/test_basic.py +++ b/parsl/tests/test_python_apps/test_basic.py @@ -14,15 +14,6 @@ def import_square(x): return math.pow(x, 2) -class CustomException(Exception): - pass - - -@python_app -def custom_exception(): - raise CustomException('foobar') - - def test_simple(n=2): x = double(n) assert x.result() == n * 2 @@ -41,9 +32,3 @@ def test_parallel_for(n): for i in d: assert d[i].result() == 2 * i - - -def test_custom_exception(): - x = custom_exception() - with pytest.raises(CustomException): - x.result() diff --git a/parsl/tests/test_python_apps/test_exception.py b/parsl/tests/test_python_apps/test_exception.py new file mode 100644 index 0000000000..52d4dc74b8 --- /dev/null +++ b/parsl/tests/test_python_apps/test_exception.py @@ -0,0 +1,19 @@ +import pytest + +from parsl.app.app import python_app + + +class CustomException(Exception): + pass + + +@python_app +def custom_exception(): + from parsl.tests.test_python_apps.test_exception import CustomException + raise CustomException('foobar') + + +def test_custom_exception(): + x = custom_exception() + with pytest.raises(CustomException): + x.result()