Skip to content
Tests

docs(mddocs): add strategy from docs_rework #3379

Sign in to view logs

docs(mddocs): add strategy from docs_rework

docs(mddocs): add strategy from docs_rework #3379

Workflow file for this run

.github/workflows/tests.yml at 557c861
name: Tests
on:
push:
branches:
- develop
pull_request:
branches-ignore:
- master
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
env:
DEFAULT_PYTHON: '3.13'
MIN_COVERAGE: 82
permissions:
contents: write
checks: write
pull-requests: write
jobs:
get-matrix:
name: Get version matrix based on changes in PR/commit
uses: ./.github/workflows/get-matrix.yml
tests-core:
name: Run core tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-core) }}
uses: ./.github/workflows/test-core.yml
with:
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-clickhouse:
name: Run Clickhouse tests (server=${{ matrix.clickhouse-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-clickhouse) }}
uses: ./.github/workflows/test-clickhouse.yml
with:
clickhouse-image: ${{ matrix.clickhouse-image }}
clickhouse-version: ${{ matrix.clickhouse-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-greenplum:
name: Run Greenplum tests (server=${{ matrix.greenplum-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-greenplum) }}
uses: ./.github/workflows/test-greenplum.yml
with:
greenplum-version: ${{ matrix.greenplum-version }}
spark-version: ${{ matrix.spark-version }}
package-version: ${{ matrix.package-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
secrets:
GREENPLUM_PACKAGES_USER: ${{ secrets.GREENPLUM_PACKAGES_USER }}
GREENPLUM_PACKAGES_PASSWORD: ${{ secrets.GREENPLUM_PACKAGES_PASSWORD }}
tests-hive-and-iceberg:
name: Run Hive and Iceberg tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-hive) }}
uses: ./.github/workflows/test-hive-and-iceberg.yml
with:
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-kafka:
name: Run Kafka tests (server=${{ matrix.kafka-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-kafka) }}
uses: ./.github/workflows/test-kafka.yml
with:
kafka-version: ${{ matrix.kafka-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-local-fs:
name: Run LocalFS tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-local-fs) }}
uses: ./.github/workflows/test-local-fs.yml
with:
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-mongodb:
name: Run MongoDB tests (server=${{ matrix.mongodb-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-mongodb) }}
uses: ./.github/workflows/test-mongodb.yml
with:
mongodb-version: ${{ matrix.mongodb-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-mssql:
name: Run MSSQL tests (server=${{ matrix.mssql-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-mssql) }}
uses: ./.github/workflows/test-mssql.yml
with:
mssql-version: ${{ matrix.mssql-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-mysql:
name: Run MySQL tests (server=${{ matrix.mysql-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-mysql) }}
uses: ./.github/workflows/test-mysql.yml
with:
mysql-version: ${{ matrix.mysql-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-oracle:
name: Run Oracle tests (server=${{ matrix.oracle-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-oracle) }}
uses: ./.github/workflows/test-oracle.yml
with:
oracle-image: ${{ matrix.oracle-image }}
oracle-version: ${{ matrix.oracle-version }}
db-name: ${{ matrix.db-name }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-postgres:
name: Run Postgres tests (server=${{ matrix.postgres-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-postgres) }}
uses: ./.github/workflows/test-postgres.yml
with:
postgres-version: ${{ matrix.postgres-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-ftp:
name: Run FTP tests (server=${{ matrix.ftp-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-ftp) }}
uses: ./.github/workflows/test-ftp.yml
with:
ftp-version: ${{ matrix.ftp-version }}
pydantic-version: ${{ matrix.pydantic-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-ftps:
name: Run FTPS tests (server=${{ matrix.ftps-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-ftps) }}
uses: ./.github/workflows/test-ftps.yml
with:
ftps-version: ${{ matrix.ftps-version }}
pydantic-version: ${{ matrix.pydantic-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-hdfs:
name: Run HDFS tests (server=${{ matrix.hadoop-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-hdfs) }}
uses: ./.github/workflows/test-hdfs.yml
with:
hadoop-version: ${{ matrix.hadoop-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-s3:
name: Run S3 tests (server=${{ matrix.minio-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-s3) }}
uses: ./.github/workflows/test-s3.yml
with:
minio-version: ${{ matrix.minio-version }}
spark-version: ${{ matrix.spark-version }}
pydantic-version: ${{ matrix.pydantic-version }}
java-version: ${{ matrix.java-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-sftp:
name: Run SFTP tests (server=${{ matrix.openssh-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-sftp) }}
uses: ./.github/workflows/test-sftp.yml
with:
openssh-version: ${{ matrix.openssh-version }}
pydantic-version: ${{ matrix.pydantic-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-samba:
name: Run Samba tests (server=${{ matrix.server-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-samba) }}
uses: ./.github/workflows/test-samba.yml
with:
server-version: ${{ matrix.server-version }}
pydantic-version: ${{ matrix.pydantic-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
tests-webdav:
name: Run WebDAV tests (server=${{ matrix.webdav-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
needs: [get-matrix]
strategy:
fail-fast: false
matrix:
include: ${{ fromJson(needs.get-matrix.outputs.matrix-webdav) }}
uses: ./.github/workflows/test-webdav.yml
with:
webdav-version: ${{ matrix.webdav-version }}
pydantic-version: ${{ matrix.pydantic-version }}
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
all_done:
name: Tests done
runs-on: ubuntu-latest
permissions:
contents: write
checks: write
pull-requests: write
if: success() || failure()
needs:
- tests-core
- tests-clickhouse
- tests-hive-and-iceberg
- tests-kafka
- tests-local-fs
- tests-mongodb
- tests-mssql
- tests-mysql
- tests-oracle
- tests-postgres
- tests-ftp
- tests-ftps
- tests-hdfs
- tests-s3
- tests-sftp
- tests-samba
- tests-webdav
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Download all coverage reports
uses: actions/download-artifact@v8
with:
path: reports/
pattern: coverage-*
merge-multiple: true
- name: Show test results
uses: mikepenz/action-junit-report@v6
if: success() || failure() # always run even if the previous step fails
with:
report_paths: reports/junit/*.xml
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/setup-python@v6
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
activate-environment: 'true'
enable-cache: 'false'
# no uv cache here because fresh install takes about 1 second
- name: Install dependencies
run: uv sync --group test
- name: Generate coverage reports
run: |
uv run coverage combine
uv run coverage xml -o reports/coverage.xml
- name: Coverage comment
id: coverage
uses: MishaKav/pytest-coverage-comment@v1
with:
pytest-xml-coverage-path: ./reports/coverage.xml
default-branch: develop
xml-skip-covered: true
report-only-changed-files: true
hide-comment: ${{ github.event_name != 'pull_request' }}
- name: Dynamic Badges
uses: schneegans/dynamic-badges-action@v1.7.0
if: github.repository == 'MTSWebServices/onetl' && github.event_name == 'push'
with:
auth: ${{ secrets.AUTOMERGE_TOKEN }}
gistID: 03e73a82ecc4709934540ce8201cc3b4
filename: onetl_badge.json
label: Coverage
message: ${{ steps.coverage.outputs.coverage }}
color: ${{ steps.coverage.outputs.color }}
- name: Fail if coverage too low
if: ${{ steps.coverage.outputs.coverage < env.MIN_COVERAGE }}
run: |
echo "Coverage is below ${{ env.MIN_COVERAGE }}%!"
exit 1
- name: All done
run: echo 1