From 127d28de2621903e4646e5fcd12eeb19ccfa969a Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Wed, 3 Sep 2025 17:18:38 -0400 Subject: [PATCH 01/12] update how we connect to s3 in ci --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e08d5200..640e0a4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,15 +60,17 @@ jobs: project_id: siuba-tests service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - + - name: Configure AWS credentials + id: creds + uses: aws-actions/configure-aws-credentials@master + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + AWS_REGION: "us-east-1" - name: Run tests shell: bash run: | pytest pins -m 'not fs_rsc and not skip_on_github' $PYTEST_OPTS env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} From f2e2825db2a36f005c4716fd2c406c0912712068 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Wed, 3 Sep 2025 17:21:26 -0400 Subject: [PATCH 02/12] update region --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 640e0a4e..c414ee7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: uses: aws-actions/configure-aws-credentials@master with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - AWS_REGION: "us-east-1" + aws-region: "us-east-1" - name: Run tests shell: bash run: | From a0368bfd45284f7c8ae2608f71064b525f657ca5 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Thu, 4 Sep 2025 15:14:25 -0400 Subject: [PATCH 03/12] use other bucket w creds --- .github/workflows/ci.yml | 9 +++------ pins/tests/helpers.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c414ee7a..7720d1c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,17 +60,14 @@ jobs: project_id: siuba-tests service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - - name: Configure AWS credentials - id: creds - uses: aws-actions/configure-aws-credentials@master - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: "us-east-1" - name: Run tests shell: bash run: | pytest pins -m 'not fs_rsc and not skip_on_github' $PYTEST_OPTS env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} diff --git a/pins/tests/helpers.py b/pins/tests/helpers.py index 38fd0f18..87181f9b 100644 --- a/pins/tests/helpers.py +++ b/pins/tests/helpers.py @@ -26,7 +26,7 @@ BOARD_CONFIG = { "file": {"path": ["PINS_TEST_FILE__PATH", None]}, - "s3": {"path": ["PINS_TEST_S3__PATH", "ci-pins"]}, + "s3": {"path": ["PINS_TEST_S3__PATH", "pins-test-hadley"]}, "gcs": {"path": ["PINS_TEST_GCS__PATH", "pins-python"]}, "abfs": {"path": ["PINS_TEST_AZURE__PATH", "ci-pins"]}, "rsc": {"path": ["PINS_TEST_RSC__PATH", RSC_SERVER_URL]}, From 5552ef04030b759e416f4ac29983ceb4c15dfd90 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Thu, 4 Sep 2025 15:36:09 -0400 Subject: [PATCH 04/12] check on databricks older dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1deb7bb6..453023f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ check = [ "types-appdirs", "databricks-sdk" ] -databricks = ["databricks-sdk"] +databricks = ["databricks-sdk<0.56.0"] doc = [ "ipykernel", "ipython<=8.12", From 6029ec6d71bda90e91d0abb42ee9c8db42b4a485 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 5 Sep 2025 16:56:06 -0400 Subject: [PATCH 05/12] streamline dependencies --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 453023f2..d418ae90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,13 +37,13 @@ dependencies = [ [project.optional-dependencies] aws = ["s3fs"] -azure = ["adlfs"] +azure = ["adlfs>=2024.4.1"] check = [ "pre-commit", "pyright==1.1.372", # Pinned; manually sync with .github/workflows/code-checks.yml "ruff==0.5.4", # Pinned; manually sync with pre-commit-config.yaml "types-appdirs", - "databricks-sdk" + "pins[databricks]" ] databricks = ["databricks-sdk<0.56.0"] doc = [ @@ -55,18 +55,18 @@ doc = [ ] gcs = ["gcsfs"] test = [ - "adlfs>=2024.4.1", + "pins[azure]", + "pins[gcs]", + "pins[databricks]", + "pins[aws]", "fastparquet", - "gcsfs", "pip-tools", "pyarrow", "pytest==7.1.3", "pytest-cases", "pytest-dotenv", "pytest-parallel", - "s3fs", "rdata", - "databricks-sdk", ] [build-system] From b86b9a6b24b835ab5af800cd2992731f5024f786 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 5 Sep 2025 17:09:01 -0400 Subject: [PATCH 06/12] debug why skip isnt working --- pins/tests/test_boards.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pins/tests/test_boards.py b/pins/tests/test_boards.py index ee22bd3d..463e03af 100644 --- a/pins/tests/test_boards.py +++ b/pins/tests/test_boards.py @@ -291,6 +291,7 @@ def test_board_pin_download_filename_multifile(board_with_cache, tmp_path): df.to_csv(path2, index=False) meta = board_with_cache.pin_upload([path1, path2], "cool_pin") + print(board_with_cache.fs.protocol) assert meta.type == "file" assert meta.file == ["data1.csv", "data2.csv"] From b46b9944a34829cc122581a2cb99425042bd15e8 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 5 Sep 2025 17:41:50 -0400 Subject: [PATCH 07/12] move up debug statement --- pins/tests/test_boards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pins/tests/test_boards.py b/pins/tests/test_boards.py index 463e03af..6c886eac 100644 --- a/pins/tests/test_boards.py +++ b/pins/tests/test_boards.py @@ -290,8 +290,8 @@ def test_board_pin_download_filename_multifile(board_with_cache, tmp_path): df.to_csv(path1, index=False) df.to_csv(path2, index=False) - meta = board_with_cache.pin_upload([path1, path2], "cool_pin") print(board_with_cache.fs.protocol) + meta = board_with_cache.pin_upload([path1, path2], "cool_pin") assert meta.type == "file" assert meta.file == ["data1.csv", "data2.csv"] From dd8641f09a2a24265fb628f3d7a9819a7985ff5f Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 5 Sep 2025 18:04:45 -0400 Subject: [PATCH 08/12] check if underlying board protocol is dbc --- pins/tests/helpers.py | 2 +- pins/tests/test_boards.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pins/tests/helpers.py b/pins/tests/helpers.py index 87181f9b..f37ea3f5 100644 --- a/pins/tests/helpers.py +++ b/pins/tests/helpers.py @@ -65,7 +65,7 @@ def wrapper(*args, **kwargs): board = arg_value break - if board and board.fs.protocol == "dbc": + if board and board.fs.protocol == "dbc" or board.fs.fs.protocol == "dbc": pytest.skip("All Databricks tests must be read only") return func(*args, **kwargs) diff --git a/pins/tests/test_boards.py b/pins/tests/test_boards.py index 6c886eac..ee22bd3d 100644 --- a/pins/tests/test_boards.py +++ b/pins/tests/test_boards.py @@ -290,7 +290,6 @@ def test_board_pin_download_filename_multifile(board_with_cache, tmp_path): df.to_csv(path1, index=False) df.to_csv(path2, index=False) - print(board_with_cache.fs.protocol) meta = board_with_cache.pin_upload([path1, path2], "cool_pin") assert meta.type == "file" From b11ae83d61d0bdc788c28983ac6b0db3728bf80b Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 5 Sep 2025 18:10:13 -0400 Subject: [PATCH 09/12] check hasattr first --- docker-compose.yml | 2 -- pins/tests/helpers.py | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c0292d52..336b707e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.1' - services: rsconnect: diff --git a/pins/tests/helpers.py b/pins/tests/helpers.py index f37ea3f5..3c763c7f 100644 --- a/pins/tests/helpers.py +++ b/pins/tests/helpers.py @@ -65,7 +65,10 @@ def wrapper(*args, **kwargs): board = arg_value break - if board and board.fs.protocol == "dbc" or board.fs.fs.protocol == "dbc": + if board and ( + board.fs.protocol == "dbc" + or (hasattr(board.fs, "fs") and board.fs.fs.protocol == "dbc") + ): pytest.skip("All Databricks tests must be read only") return func(*args, **kwargs) From 1944ad6cfde6de8be82438d3afb807f93030e024 Mon Sep 17 00:00:00 2001 From: isabel zimmerman Date: Fri, 5 Sep 2025 18:16:00 -0400 Subject: [PATCH 10/12] maybe fsspec changed? --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d418ae90..a3923c0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ requires-python = ">=3.9" dynamic = ["version"] dependencies = [ "appdirs<2", # Using appdirs rather than platformdirs is deliberate, see https://github.com/rstudio/pins-python/pull/239 - "fsspec>=2022.2", + "fsspec>=2022.2,<2025.7", "humanize>=1", "importlib-metadata>=4.4", "importlib-resources>=1.3", From e79620b9151a53bbdb396984bb0a86c45b470e10 Mon Sep 17 00:00:00 2001 From: Isabel Zimmerman <54685329+isabelizimm@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:17:38 -0400 Subject: [PATCH 11/12] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a3923c0e..2c8ddd7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ requires-python = ">=3.9" dynamic = ["version"] dependencies = [ "appdirs<2", # Using appdirs rather than platformdirs is deliberate, see https://github.com/rstudio/pins-python/pull/239 - "fsspec>=2022.2,<2025.7", + "fsspec>=2022.2,<2025.9", "humanize>=1", "importlib-metadata>=4.4", "importlib-resources>=1.3", From 66cc7b138b03b3957852777b7d6717e8433ef77b Mon Sep 17 00:00:00 2001 From: Isabel Zimmerman <54685329+isabelizimm@users.noreply.github.com> Date: Thu, 11 Sep 2025 16:18:06 -0400 Subject: [PATCH 12/12] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2c8ddd7f..330dd5d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ check = [ "types-appdirs", "pins[databricks]" ] -databricks = ["databricks-sdk<0.56.0"] +databricks = ["databricks-sdk"] doc = [ "ipykernel", "ipython<=8.12",