Skip to content
ci

ci #35630

Sign in to view logs

ci

ci #35630

Workflow file for this run

.github/workflows/ci.yml at f8df832
name: ci
# This pipeline is intended to facilitate the full coverage of tests for OSS GX.
# A "reasonable" set of tests will run on every pull request, and a full set will run with:
# 1. Manual triggering
# 2. Every 3 hours
# 3. When we push a tag with a semver pattern x.y.z (for example 0.0.1) which is the pattern we use for releases.
on:
merge_group:
pull_request_target:
types:
- opened
- reopened
- synchronize
- ready_for_review
schedule:
# https://crontab.guru/every-3-hours
- cron: "0 */3 * * *"
workflow_dispatch: # allows manual triggering with branch picker
push:
# Semantic versioning syntax defined by PyPI: https://pythonpackaging.info/07-Package-Release.html#Versioning-your-code
tags:
# Stable release syntax
- "[0-9]+.[0-9]+.[0-9]+"
# Prerelease syntax
- "[0-9]+.[0-9]+.[0-9]+a[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+b[0-9]+"
- "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
jobs:
detect-changes:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
docs_only: ${{ steps.changes.outputs.docs_only }}
steps:
- name: Detect docs-only changes
id: changes
uses: actions/github-script@v7
with:
script: |
// Only consider skipping code tests for PR events
if (context.eventName !== 'pull_request_target') {
core.setOutput('docs_only', 'false');
return;
}
const files = await github.paginate(github.rest.pulls.listFiles, {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
per_page: 100,
});
const docsOnly = files.length > 0 && files.every(f => f.filename.startsWith('docs/'));
core.setOutput('docs_only', docsOnly ? 'true' : 'false');
core.info(`Changed ${files.length} files, docs_only=${docsOnly}`);
check-actor-permissions:
permissions:
pull-requests: write
runs-on: ubuntu-latest
steps:
- name: Check User Permission
id: check-user-permission
uses: actions-cool/check-user-permission@v2
with:
require: write
username: ${{ github.triggering_actor }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Add comment to add helpful context for contributors
if: steps.check-user-permission.outputs.require-result == 'false'
uses: thollander/actions-comment-pull-request@v2.5.0
with:
comment_tag: execution
message: |
Thanks for the PR @${{github.actor}} :wave:
Our GitHub actions pipelines require a user with write permissions to retry the failed jobs. A maintainer will review your PR and re-run CI. Please be patient and we'll get back to you as soon as possible.
- name: Fail if No Write Permission
if: steps.check-user-permission.outputs.require-result == 'false'
run: exit 1
ci-does-not-run-on-draft-pull-requests:
runs-on: ubuntu-latest
if: github.event.pull_request.draft == true
steps:
- run: echo "CI jobs won't run because this is a draft pull request."
static-analysis:
needs: [check-actor-permissions, detect-changes]
if: needs.detect-changes.outputs.docs_only != 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: |
requirements-types.txt
reqs/requirements-dev-contrib.txt
- name: Install dependencies
run: pip install -r requirements-types.txt -r reqs/requirements-dev-contrib.txt
- run: invoke lint --no-fmt
- run: invoke fmt --check
- name: Type-check
run: |
invoke type-check --ci --pretty
invoke type-check --ci --pretty --check-stub-sources
- name: Marker-coverage-check
run: invoke marker-coverage
- name: Check for linter ignores without comments
run: ./scripts/check_linter_ignores.sh
docs-snippets:
needs: [unit-tests, doc-checks, check-actor-permissions, detect-changes]
# run on non-draft PRs with docs changes
# !cancelled() prevents auto-skip when unit-tests is skipped for docs-only PRs
if: |
!cancelled() &&
needs.unit-tests.result != 'failure' &&
needs.check-actor-permissions.result == 'success' &&
(
(github.event.pull_request.draft == false && github.base_ref == 'develop') ||
(github.event_name == 'push' && contains(github.ref, 'refs/tags/1.')) ||
github.event_name == 'merge_group' ||
github.event_name == 'workflow_dispatch'
)
runs-on: ubuntu-latest
# This matrix installs system packages and talks to external services, so a
# hang is a question of when, not if -- and without a timeout a hung job
# runs until the repository-wide default of 6 hours. Over the last 30
# successful runs the slowest leg was under 12 minutes, so 30 leaves ample
# headroom while failing a genuinely stuck job in minutes rather than hours.
timeout-minutes: 30
strategy:
matrix:
doc-step:
- docs-creds-needed
- docs-basic
- docs-spark
include:
- doc-step: docs-creds-needed
start_services: 1
- doc-step: docs-basic
start_services: 0
- doc-step: docs-spark
start_services: 0
env:
# google
GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}}
GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}}
GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}}
# Absolute, not relative: the docs snippet runner chdirs into a temp directory
# before executing each script, and Application Default Credentials resolves this
# path at client-construction time -- i.e. after the chdir.
GOOGLE_APPLICATION_CREDENTIALS: "${{github.workspace}}/gcp-credentials.json"
GX_GCS_TEST_BUCKET: ${{vars.GX_GCS_TEST_BUCKET}}
# aws
# No AWS_* mapping here. Credentials come from the OIDC role assumed in the
# "Configure AWS credentials" step below, which exports AWS_ACCESS_KEY_ID,
# AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN into $GITHUB_ENV. A job-level
# env: entry for any of those would shadow what that step exports.
GX_S3_TEST_BUCKET: ${{vars.GX_S3_TEST_BUCKET}}
# aws-redshift
REDSHIFT_USERNAME: ${{secrets.REDSHIFT_USERNAME}}
REDSHIFT_PASSWORD: ${{secrets.REDSHIFT_PASSWORD}}
REDSHIFT_HOST: ${{secrets.REDSHIFT_HOST}}
REDSHIFT_PORT: ${{secrets.REDSHIFT_PORT}}
REDSHIFT_DATABASE: ${{secrets.REDSHIFT_DATABASE}}
REDSHIFT_SSLMODE: ${{secrets.REDSHIFT_SSLMODE}}
# azure
AZURE_ACCESS_KEY: ${{secrets.AZURE_ACCESS_KEY}}
AZURE_CREDENTIAL: ${{secrets.AZURE_CREDENTIAL}}
AZURE_CONTAINER: ${{secrets.AZURE_CONTAINER}}
AZURE_STORAGE_ACCOUNT_URL: ${{secrets.AZURE_STORAGE_ACCOUNT_URL}}
# snowflake
SNOWFLAKE_ACCOUNT: ${{secrets.SNOWFLAKE_ACCOUNT}}
SNOWFLAKE_USER: ${{secrets.SNOWFLAKE_USER}}
SNOWFLAKE_DATABASE: ${{secrets.SNOWFLAKE_DATABASE}}
SNOWFLAKE_SCHEMA: ${{secrets.SNOWFLAKE_SCHEMA}}
SNOWFLAKE_WAREHOUSE: ${{secrets.SNOWFLAKE_WAREHOUSE}}
SNOWFLAKE_ROLE: ${{secrets.SNOWFLAKE_ROLE}}
SNOWFLAKE_PRIVATE_KEY: ${{secrets.SNOWFLAKE_PRIVATE_KEY}}
# cloud-specific
AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}}
AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}}
LD_SDK_KEY: default
GX_LD_BASE_URI: "http://localhost:8765"
GX_LD_EVENTS_URI: "http://localhost:8765"
GX_LD_STREAM_URI: "http://localhost:8765"
LAUNCH_DARKLY_DEV_SERVER_ACCESS_TOKEN: ${{ secrets.LAUNCH_DARKLY_DEV_SERVER_ACCESS_TOKEN }}
GX_SCHEDULER_LAMBDA_ARN: ${{secrets.GX_SCHEDULER_LAMBDA_ARN}}
GX_SCHEDULER_EXECUTION_ROLE_ARN: ${{secrets.GX_SCHEDULER_EXECUTION_ROLE_ARN}}
GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}}
GX_SCHEDULER_EVENTBRIDGE_REGION: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_REGION}}
LOCALSTACK_AWS_ENDPOINT_URL: ${{secrets.LOCALSTACK_AWS_ENDPOINT_URL}}
LOCALSTACK_AWS_SECRET_ACCESS_KEY: ${{secrets.LOCALSTACK_AWS_SECRET_ACCESS_KEY}}
LOCALSTACK_AWS_ACCESS_KEY_ID: ${{secrets.LOCALSTACK_AWS_ACCESS_KEY_ID}}
LOGGING_LEVEL: DEBUG
ENVIRONMENT: local
permissions:
id-token: write
contents: read
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
# Only the docs-creds-needed leg passes --aws; the other legs have no reason
# to hold AWS credentials, short-lived or not.
- name: Configure AWS credentials
if: matrix['doc-step'] == 'docs-creds-needed'
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
aws-region: ${{ secrets.CI_AWS_DEFAULT_REGION }}
- name: Upgrade Docker Compose
uses: docker/setup-compose-action@v1
with:
version: latest
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
- name: Copy GCP credentials to file
run: |
echo "$GE_TEST_GCP_CREDENTIALS" > "$GOOGLE_APPLICATION_CREDENTIALS"
- name: Install cloud dependencies
if: ${{ matrix.start_services }}
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m 'cloud' -r test
- name: Install test matrix dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m '${{ matrix.doc-step }}'
- name: Install SQL Server odbc driver
if: matrix['doc-step'] == 'docs-creds-needed'
run: ./scripts/install_sql_server_odbc_driver.sh
- name: Run docs_snippet_checker
run: |
yarn install
python ci/checks/validate_docs_snippets.py
- name: Start services
if: ${{ matrix.start_services }}
run: |
docker compose --progress quiet -f assets/docker/spark/docker-compose.yml up -d --wait --wait-timeout 300
- name: Run the tests
run: invoke docs-snippet-tests '${{ matrix.doc-step }}' --up-services --verbose
unit-tests:
needs: [check-actor-permissions, detect-changes]
if: needs.detect-changes.outputs.docs_only != 'true'
env:
GX_PYTHON_EXPERIMENTAL: true # allow for python 3.14+
strategy:
matrix:
os: [ubuntu-latest]
python-version: ["3.10", "3.11", "3.12", "3.13"]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Install dependencies
run: pip install . -c constraints-dev.txt -r reqs/requirements-dev-test.txt
- name: Run the unit tests
# TODO: revert the timeout back to 1.5 or lower after resolving arc issues
run: invoke ci-tests -m "unit" --xdist --slowest=10 --timeout=2.0
doc-checks:
needs: [check-actor-permissions]
if: github.event.pull_request.draft == false
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install -r reqs/requirements-dev-test.txt
- name: check_repo_root_size
run: sh ./ci/checks/check_repo_root_size.sh
- name: Docstring linter
run: invoke docstrings
- name: line_number_snippet_checker
run: python ci/checks/check_no_line_number_snippets.py
- name: name_tag_snippet_checker
run: python ci/checks/check_only_name_tag_snippets.py
- name: integration_test_gets_run_checker
run: python ci/checks/check_integration_test_gets_run.py
- name: name_tag_snippet_referenced_checker
run: python ci/checks/check_name_tag_snippets_referenced.py
- name: public_api_report
run: invoke public-api
docs-build:
needs: [doc-checks, check-actor-permissions]
# run on non-draft PRs
if: |
(github.event.pull_request.draft == false && github.base_ref == 'develop') ||
(github.event_name == 'push' && contains(github.ref, 'refs/tags/1.')) ||
github.event_name == 'merge_group' ||
github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install -r reqs/requirements-dev-test.txt
- name: Build docs
env:
NODE_OPTIONS: --max_old_space_size=4096
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
run: cd docs/docusaurus && yarn install && bash ../build_docs
docs-tests:
needs: [check-actor-permissions]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Run tests
env:
NODE_OPTIONS: --max_old_space_size=4096
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
run: cd docs/docusaurus && yarn install && yarn test
integration-tests:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
env:
GX_MS_TEAMS_WEBHOOK: ${{ secrets.GX_MS_TEAMS_WEBHOOK }}
OUTDATED_MS_TEAMS_WEBHOOK: ${{ secrets.OUTDATED_MS_TEAMS_WEBHOOK }}
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Upgrade Docker Compose
uses: docker/setup-compose-action@v1
with:
version: latest
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -r test
- name: Run the tests
run:
invoke ci-tests 'integration' --verbose
marker-tests:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
env:
GX_PYTHON_EXPERIMENTAL: true # allow for python 3.13+
# google
GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}}
GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}}
GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}}
GOOGLE_APPLICATION_CREDENTIALS: "gcp-credentials.json"
# aws
AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
AWS_DEFAULT_REGION: ${{secrets.CI_AWS_DEFAULT_REGION}}
AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_SECRET_ACCESS_KEY}}
# DATABRICKS_TOKEN is minted at runtime from the service principal's OAuth
# credentials in the "Mint Databricks OAuth access token" step below.
DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
DATABRICKS_HTTP_PATH: ${{secrets.DATABRICKS_HTTP_PATH}}
DATABRICKS_CATALOG: ${{ vars.DATABRICKS_CATALOG }}
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
# These jobs talk to external warehouses, so a hang is a question of when,
# not if -- and without a timeout a hung job runs until the repository-wide
# default of 6 hours. Over the last month the slowest run of any marker in
# this matrix was under 17 minutes, so 30 leaves ample headroom while
# failing a genuinely stuck job in minutes rather than hours.
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
markers:
- openpyxl or pyarrow or project or sqlite or aws_creds
- athena
- aws_deps
- big
- bigquery
- databricks
- filesystem
- gcs_deps
- sql_server
- mysql
- postgresql
- spark
- spark_connect
- trino
- clickhouse
- singlestore
- oracle
python-version: ["3.10", "3.11", "3.12", "3.13"]
exclude:
# TODO: would like to adopt `actionlint` pre-commit hook
# but false positive here and inability to do an inline ignore
# prevents this https://github.com/rhysd/actionlint/issues/237
- python-version: ${{ github.event_name == 'pull_request_target' && '3.10' }}
- python-version: ${{ github.event_name == 'pull_request_target' && '3.11' }}
- python-version: ${{ github.event_name == 'pull_request_target' && '3.12' }}
- python-version: ${{ github.event_name == 'merge_group' && '3.10' }}
- python-version: ${{ github.event_name == 'merge_group' && '3.11' }}
- python-version: ${{ github.event_name == 'merge_group' && '3.12' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.10' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.11' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.12' }}
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Upgrade Docker Compose
uses: docker/setup-compose-action@v1
with:
version: latest
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Copy GCP credentials to file
run: |
echo "$GE_TEST_GCP_CREDENTIALS" > gcp-credentials.json
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m '${{ matrix.markers }}' -r test
- name: Install SQL Server odbc driver
if: matrix.markers == 'sql_server'
run: ./scripts/install_sql_server_odbc_driver.sh
# Authenticate the service principal via OAuth M2M. The minted
# bearer token is masked and exported as DATABRICKS_TOKEN so the existing
# token-based connection path is used unchanged. OAuth M2M tokens are valid
# for one hour, which comfortably covers the marker suite's runtime.
- name: Mint Databricks OAuth access token
if: matrix.markers == 'databricks'
env:
DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
run: |
TOKEN="$(python ./scripts/mint_databricks_token.py)"
echo "::add-mask::$TOKEN"
echo "DATABRICKS_TOKEN=$TOKEN" >> "$GITHUB_ENV"
- name: Run the tests
run: |
FLAGS=""
case "${{ matrix.markers }}" in
bigquery|databricks) FLAGS="--xdist" ;;
esac
invoke ci-tests '${{ matrix.markers }}' --up-services --verbose $FLAGS
pyspark4-marker-tests:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
env:
GX_PYTHON_EXPERIMENTAL: true # allow for python 3.13+
# Compose override, matched to the 4.1.x pip client installed below. No official
# client<->server cross-version guarantee exists for Spark Connect, so the image
# tag is kept in lockstep with the lane's pyspark constraint.
SPARK_IMAGE: apache/spark:4.1.2
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
markers:
- spark
- spark_connect
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Upgrade Docker Compose
uses: docker/setup-compose-action@v1
with:
version: latest
- name: Set up Python 3.13
uses: actions/setup-python@v5
with:
python-version: "3.13"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Set up Java
# Classic-path pyspark 4 needs a host JDK 17+; pin it here rather than relying
# on the runner default so this lane is deterministic.
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "17"
- name: Verify pyspark 4.0.x resolution (dry run)
# Runs before the real install so the environment is clean. Proves the [spark]
# extra still resolves the 4.0.x line even though this lane pins 4.1.x.
run: pip install --dry-run --quiet ".[spark]" "pyspark~=4.0.3"
- name: Install dependencies
# Direct pip install with the lane constraint file, bypassing constraints-dev.txt
# (the 3.x freeze) so pyspark resolves to 4.1.x.
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
pip install . -c ci/constraints-test/pyspark4-install.txt -r reqs/requirements-dev-test.txt -r reqs/requirements-dev-spark.txt -r reqs/requirements-dev-spark-connect.txt
- name: Run the tests
run: invoke ci-tests '${{ matrix.markers }}' --up-services --verbose --reports
marker-tests-snowflake:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
env:
GX_PYTHON_EXPERIMENTAL: true # allow for python 3.13+
# snowflake
SNOWFLAKE_ACCOUNT: ${{secrets.SNOWFLAKE_ACCOUNT}}
SNOWFLAKE_USER: ${{secrets.SNOWFLAKE_USER}}
SNOWFLAKE_DATABASE: ${{secrets.SNOWFLAKE_DATABASE}}
SNOWFLAKE_SCHEMA: ${{secrets.SNOWFLAKE_SCHEMA}}
SNOWFLAKE_WAREHOUSE: ${{secrets.SNOWFLAKE_WAREHOUSE}}
SNOWFLAKE_ROLE: ${{secrets.SNOWFLAKE_ROLE}}
SNOWFLAKE_PRIVATE_KEY: ${{secrets.SNOWFLAKE_PRIVATE_KEY}}
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
# Snowflake marker tests are split into shards + xdist workers because the
# un-sharded job was the slowest cell in `ci.yml` (P90 ≈ 49m). Each shard
# runs ~1/3 of the tests via `pytest-split`; workers are `pytest-xdist`.
# Test fixtures already isolate schemas with 10-char random suffixes
# (see `tests/integration/test_utils/data_source_config/sql.py`), so
# parallel shards cannot collide on schema names.
strategy:
fail-fast: false
matrix:
shard: ["1/3", "2/3", "3/3"]
python-version: ["3.10", "3.11", "3.12", "3.13"]
exclude:
- python-version: ${{ github.event_name == 'pull_request_target' && '3.10' }}
- python-version: ${{ github.event_name == 'pull_request_target' && '3.11' }}
- python-version: ${{ github.event_name == 'pull_request_target' && '3.12' }}
- python-version: ${{ github.event_name == 'merge_group' && '3.10' }}
- python-version: ${{ github.event_name == 'merge_group' && '3.11' }}
- python-version: ${{ github.event_name == 'merge_group' && '3.12' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.10' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.11' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.12' }}
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m snowflake -r test
- name: Run the tests
run: |
GROUP="${SHARD%/*}"
SPLITS="${SHARD#*/}"
invoke ci-tests snowflake --up-services --verbose --xdist \
--splits="$SPLITS" --group="$GROUP"
env:
SHARD: ${{ matrix.shard }}
redshift:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
env:
# aws-redshift
REDSHIFT_USERNAME: ${{secrets.REDSHIFT_USERNAME}}
REDSHIFT_PASSWORD: ${{secrets.REDSHIFT_PASSWORD}}
REDSHIFT_HOST: ${{secrets.REDSHIFT_HOST}}
REDSHIFT_PORT: ${{secrets.REDSHIFT_PORT}}
REDSHIFT_DATABASE: ${{secrets.REDSHIFT_DATABASE}}
REDSHIFT_SSLMODE: ${{secrets.REDSHIFT_SSLMODE}}
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
markers:
- redshift
python-version: ["3.13"]
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m '${{ matrix.markers }}' -r test
- name: Run the tests
run: invoke ci-tests '${{ matrix.markers }}' --up-services --verbose
py310-min-versions:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/py310-min-install.txt -r reqs/requirements-dev-test.txt
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
py311-min-versions:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/py311-min-install.txt -r reqs/requirements-dev-test.txt
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
py312-min-versions:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/py312-min-install.txt -r reqs/requirements-dev-test.txt
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
py313-min-versions:
needs: [unit-tests, static-analysis, check-actor-permissions]
env:
GX_PYTHON_EXPERIMENTAL: true # allow for python 3.14+
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.13"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/py313-min-install.txt -r reqs/requirements-dev-test.txt
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
pydantic-v1:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/pydantic-v1-install.txt -r reqs/requirements-dev-test.txt
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
pandas2-test:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/pandas2-min-install.txt -r reqs/requirements-dev-test.txt
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
marshmallow4-test:
needs: [unit-tests, static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: reqs/requirements-dev-test.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/marshmallow4-install.txt -r reqs/requirements-dev-test.txt
- name: Confirm the lane resolved marshmallow 4
run: python -c "import marshmallow, sys; from importlib.metadata import version; v = version('marshmallow'); print(v); sys.exit(0 if v.startswith('4.') else 1)"
- name: Run the tests
run: invoke ci-tests -m unit --xdist --slowest=10 --timeout=2.0
airflow-min-versions:
needs: [unit-tests, static-analysis, check-actor-permissions]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: requirements.txt
- name: Install dependencies
run: pip install . -c ci/constraints-test/airflow-min-install.txt
- name: Run the test
# ensure the great_expectations can be imported and a context created
run: sh ci/checks/check_min_airflow_dependency_compatibility.sh
import_gx:
needs: [static-analysis, check-actor-permissions]
if: github.event.pull_request.draft == false
runs-on: ubuntu-latest
env:
GX_PYTHON_EXPERIMENTAL: true # allow for python 3.14+
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
exclude:
- python-version: ${{ github.event_name == 'pull_request_target' && '3.11' }}
- python-version: ${{ github.event_name == 'pull_request_target' && '3.12' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.11' }}
- python-version: ${{ github.event_name == 'workflow_dispatch' && '3.12' }}
steps:
- name: Checkout
uses: actions/checkout@v4.3.1
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: requirements.txt
- name: Install Great Expectations with core deps only
run: pip install .
- name: Import Great Expectations
run: python -c "import great_expectations as gx; print('Successfully imported GX Version:', gx.__version__)"
- name: Check installed agent skills and schema catalogs
run: python ci/checks/check_installed_agent_skills.py
ci-required:
# Single required status check for branch protection.
# Passes when every upstream job either succeeded or was skipped
# (code jobs are skipped for docs-only PRs).
if: always()
needs:
- detect-changes
- check-actor-permissions
- static-analysis
- unit-tests
- doc-checks
- docs-build
- docs-tests
- docs-snippets
- integration-tests
- marker-tests
- pyspark4-marker-tests
- marker-tests-snowflake
- redshift
- py310-min-versions
- py311-min-versions
- py312-min-versions
- py313-min-versions
- pydantic-v1
- pandas2-test
- marshmallow4-test
- airflow-min-versions
- import_gx
runs-on: ubuntu-latest
steps:
- name: Evaluate CI results
run: |
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || \
"${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
echo "::error::One or more required CI jobs failed or were cancelled"
exit 1
fi
# On non-docs PRs, code jobs must have succeeded (not just skipped)
if [[ "${{ needs.detect-changes.outputs.docs_only }}" != "true" ]]; then
if [[ "${{ needs.static-analysis.result }}" != "success" ]]; then
echo "::error::static-analysis result is '${{ needs.static-analysis.result }}' but must succeed on non-docs PRs"
exit 1
fi
if [[ "${{ needs.unit-tests.result }}" != "success" ]]; then
echo "::error::unit-tests result is '${{ needs.unit-tests.result }}' but must succeed on non-docs PRs"
exit 1
fi
fi
echo "All required CI jobs passed or were appropriately skipped"
build-n-publish:
needs:
[
check-actor-permissions,
doc-checks,
static-analysis,
docs-snippets,
unit-tests,
integration-tests,
marker-tests,
pyspark4-marker-tests,
marker-tests-snowflake,
py310-min-versions,
py311-min-versions,
py312-min-versions,
py313-min-versions,
pydantic-v1,
pandas2-test,
marshmallow4-test,
airflow-min-versions,
import_gx,
]
if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
name: Build and publish Python distributions to PyPI
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/great-expectations
permissions:
id-token: write
steps:
- uses: actions/checkout@v4.3.1
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Update pip
run: python -m pip install --upgrade pip
- name: Install build tools and prepare packaging
run: |
pip install twine build
git config --global user.email "team@greatexpectations.io"
git config --global user.name "Great Expectations"
- name: Build distribution
run: python -m build
- name: Publish distribution to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
skip-existing: true # prevent against pushing duplicate versions
notify_on_failure:
needs:
[
check-actor-permissions,
doc-checks,
static-analysis,
docs-snippets,
unit-tests,
integration-tests,
marker-tests,
pyspark4-marker-tests,
marker-tests-snowflake,
py310-min-versions,
py311-min-versions,
py312-min-versions,
py313-min-versions,
pydantic-v1,
pandas2-test,
marshmallow4-test,
airflow-min-versions,
import_gx,
build-n-publish,
]
if: failure() && !cancelled() && github.event_name != 'pull_request_target' && github.event_name != 'merge_group'
runs-on: ubuntu-latest
steps:
- name: Scheduled CI job failure
id: slack
uses: slackapi/slack-github-action@v1.24.0
with:
payload: |
{
"event_name": "${{ github.event_name }}",
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_WEBHOOK_URL }}
notify_on_release:
needs: [build-n-publish, check-actor-permissions]
runs-on: ubuntu-latest
steps:
- name: Announce Release
id: slack
uses: slackapi/slack-github-action@v1.24.0
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_WEBHOOK_URL }}