Reduce code repetition by reusing workflows

Author: juliannguyen4

.github/workflows/stage-tests.yml

@@ -19,6 +19,9 @@ on:
         required: false
       # These will be set if use_jfrog_builds is false (i.e when someone is building the artifacts from scratch and stage testing them in a calling workflow)
       # If use_jfrog_builds is true, only the defaults will be used
+      python-versions-to-test:
+        required: true
+        type: string
       registry-name:
         type: string
         required: false
@@ -55,228 +58,58 @@ jobs:
     strategy:
       matrix:
         test-case: [
-          # Docker image + tag, test category, Docker image platform (cpu arch), Python version
-          ["amazonlinux:2023", "x86_64", "3.11"],
-          ["amazonlinux:2023", "aarch64", "3.11"],
-          ["ubuntu:22.04", "x86_64", "3.10"],
-          ["ubuntu:22.04", "aarch64", "3.10"],
-          ["ubuntu:24.04", "x86_64", "3.12"],
-          ["ubuntu:24.04", "aarch64", "3.12"],
+          # Docker image + tag, Docker image platform (cpu arch), Python version, build-from-source
+          ["amazonlinux:2023", "x86_64", "3.11", false],
+          ["amazonlinux:2023", "aarch64", "3.11", false],
+
+          # TODO - Probably can remove this if we reuse the test workflow
+          ["ubuntu:22.04", "x86_64", "3.10", false],
+          ["ubuntu:22.04", "aarch64", "3.10", false],
+          ["ubuntu:24.04", "x86_64", "3.12", false],
+          ["ubuntu:24.04", "aarch64", "3.12", false],
+
           # Bookworm is Debian 12
-          ["python:3.10-bookworm", "x86_64", "3.10"],
-          ["python:3.10-bookworm", "aarch64", "3.10"],
-          ["python:3.11-bookworm", "x86_64", "3.11"],
-          ["python:3.11-bookworm", "aarch64", "3.11"],
-          ["python:3.12-bookworm", "x86_64", "3.12"],
-          ["python:3.12-bookworm", "aarch64", "3.12"],
-          ["python:3.13-bookworm", "x86_64", "3.13"],
-          ["python:3.13-bookworm", "aarch64", "3.13"],
-          ["python:3.14-rc-bookworm", "x86_64", "3.14"],
-          ["python:3.14-rc-bookworm", "aarch64", "3.14"],
+          ["python:3.10-bookworm", "x86_64", "3.10", false],
+          ["python:3.10-bookworm", "aarch64", "3.10", false],
+          ["python:3.11-bookworm", "x86_64", "3.11", false],
+          ["python:3.11-bookworm", "aarch64", "3.11", false],
+          ["python:3.12-bookworm", "x86_64", "3.12", false],
+          ["python:3.12-bookworm", "aarch64", "3.12", false],
+          ["python:3.13-bookworm", "x86_64", "3.13", false],
+          ["python:3.13-bookworm", "aarch64", "3.13", false],
+          ["python:3.14-rc-bookworm", "x86_64", "3.14", false],
+          ["python:3.14-rc-bookworm", "aarch64", "3.14", false],
           # QE currently covers Debian 13 on all client-supported Python versions
           # We do not use ubi9/python-311 because we cannot install packages using dnf as root
-          ["redhat/ubi9", "x86_64", "3.11"],
-          ["redhat/ubi9", "aarch64", "3.11"],
+          ["redhat/ubi9", "x86_64", "3.11", false],
+          ["redhat/ubi9", "aarch64", "3.11", false],
+
+          ["amazonlinux:2023", 1, "linux/amd64", "3.11", true],
+          ["redhat/ubi9", 1, "linux/amd64", "3.11", true],
         ]
       fail-fast: false
     uses: ./.github/workflows/test-wheel-in-container.yml
     with:
       container-image-name: ${{ matrix.test-case[0] }}
       cpu-arch: ${{ matrix.test-case[1] }}
       python-version: ${{ matrix.test-case[2] }}
-
-  linux-distro-source-build-tests:
-    strategy:
-      matrix:
-        test-case: [
-          # Docker image + tag, test category, Docker image platform (cpu arch), Python version
-          ["amazonlinux:2023", 1, "linux/amd64", "3.11"],
-          ["redhat/ubi9", 1, "linux/amd64", "3.11"],
-        ]
-      fail-fast: false
-    env:
-      LINUX_DISTRO_CONTAINER_NAME: linux-distro
-      PIP_INSTALL_COMMAND: 'pip install'
-    runs-on: ${{ matrix.test-case[2] == 'linux/amd64' && 'ubuntu-22.04' || 'ubuntu-22.04-arm' }}
-    steps:
-    # TODO: the checkout code is also duplicated in the macOS stage tests
-    # But it's only a few lines of code so I didn't bother to create a composite action for it.
-    - name: Harden the runner (Audit all outbound calls)
-      uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
-      with:
-        egress-policy: audit
-
-    - name: Get tests and Github action scripts
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      with:
-        ref: ${{ inputs.use_jfrog_builds && inputs.jfrog-build-version-to-test || github.sha }}
-        sparse-checkout: |
-          test
-          .github
-
-    # Map test case tuple entries to env vars to make code easier to read
-    - run: echo "DISTRO_DOCKER_IMAGE_AND_TAG=${{ matrix.test-case[0] }}" >> $GITHUB_ENV
-    - run: echo "TEST_CATEGORY=${{ matrix.test-case[1] }}" >> $GITHUB_ENV
-    - run: echo "DISTRO_DOCKER_IMAGE_PLATFORM=${{ matrix.test-case[2] }}" >> $GITHUB_ENV
-    - run: echo "PYTHON_VERSION=${{ matrix.test-case[3] }}" >> $GITHUB_ENV
-
-    - uses: ./.github/actions/get-artifact-for-stage-tests
-      with:
-        get_from_jfrog: ${{ inputs.use_jfrog_builds }}
-        # This input is only used if above input is true
-        jfrog_build_version: ${{ inputs.jfrog-build-version-to-test }}
-        dist_type_to_get: ${{ env.TEST_CATEGORY == '2' && 'wheel' || 'sdist' }}
-        # wheel* inputs are used only if running category 2 tests
-        wheel_python_version: ${{ env.PYTHON_VERSION }}
-        wheel_os: manylinux
-        wheel_cpu_arch: ${{ env.DISTRO_DOCKER_IMAGE_PLATFORM == 'linux/amd64' && 'x86_64' || 'aarch64' }}
-        jfrog-platform-url: ${{ vars.JF_EXTERNAL_URL }}
-        jfrog-project: ${{ vars.JFROG_PROJECT_FOR_CLIENT_TEAM }}
-        jfrog-oidc-provider-name: ${{ vars.OIDC_PROVIDER_NAME }}
-        jfrog-oidc-audience: ${{ vars.OIDC_AUDIENCE }}
-        gh_artifact_name_prefix: ${{ vars.GH_ARTIFACT_NAME_PREFIX_FOR_BUILDS }}
-
-    - uses: ./.github/actions/run-ee-server
-      with:
-        registry-name: ${{ inputs.registry-name }}
-        registry-username: ${{ env.REGISTRY_USERNAME }}
-        registry-password: ${{ env.REGISTRY_PASSWORD }}
-        image-name: ${{ inputs.image-name }}
-        server-tag: ${{ inputs.server-tag }}
-        where-is-client-connecting-from: 'separate-docker-container'
-
-    - name: Run distro container
-      # Run distro container on host network to access the Aerospike server using localhost (without having to change config.conf)
-      run: docker run --detach --network host --platform ${{ env.DISTRO_DOCKER_IMAGE_PLATFORM }} --name ${{ env.LINUX_DISTRO_CONTAINER_NAME }} ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG }} tail -f /dev/null
-
-    - name: Copy repo (and artifact) to container
-      run: docker cp . ${{ env.LINUX_DISTRO_CONTAINER_NAME }}:/aerospike-client-python
-
-    - name: 'DNF based distros: install python3'
-      if: ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG == 'amazonlinux:2023' || contains(env.DISTRO_DOCKER_IMAGE_AND_TAG, 'ubi') }}
-      run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} dnf install -y python3.11
-
-    - name: 'Ubuntu: Install python 3 that comes by default (step 1)'
-      if: ${{ startsWith(env.DISTRO_DOCKER_IMAGE_AND_TAG, 'ubuntu') }}
-      run: |
-        docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} apt-get update
-        docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} apt-get install python3 python3-pip -y
-
-    - name: Make sure pip is installed
-      # This is good to check in general, so we run for all distros
-      # Ubuntu doesn't have this by default though, but it's fine since it has pip already
-      if: ${{ !startsWith(env.DISTRO_DOCKER_IMAGE_AND_TAG, 'ubuntu') }}
-      run: docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m ensurepip
-
-    - name: 'Cat 1: Install build dependencies using dnf'
-      # Here we assume all cat 1 tests are running on a dnf-based. distro
-      if: ${{ env.TEST_CATEGORY == '1' }}
-      run: |
-        docker exec ${{ env.LINUX_DISTRO_CONTAINER_NAME }} dnf install -y libyaml-devel \
-          openssl-devel \
-          glibc-devel \
-          autoconf \
-          automake \
-          libtool \
-          zlib-devel \
-          openssl-devel \
-          python3.11-devel
-
-    - if: ${{ env.DISTRO_DOCKER_IMAGE_AND_TAG == 'ubuntu:24.04' }}
-      # Python 3.12+ fails to install packages to the system directory by default
-      run: echo PIP_INSTALL_COMMAND="$PIP_INSTALL_COMMAND --break-system-packages" >> $GITHUB_ENV
-
-    - name: 'Cat 1: Install pip build frontend'
-      if: ${{ env.TEST_CATEGORY == '1' }}
-      run: docker exec --workdir /aerospike-client-python/ ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m ${{ env.PIP_INSTALL_COMMAND }} -r requirements.txt
-
-    - name: Install sdist or wheel distribution
-      run: docker exec --workdir /aerospike-client-python/ ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m ${{ env.PIP_INSTALL_COMMAND }} ${{ env.ARTIFACT_FILE_NAME_PATTERN }}
-
-    - name: Install pytest
-      run: docker exec --workdir /aerospike-client-python/test ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m ${{ env.PIP_INSTALL_COMMAND }} pytest -c requirements.txt
-
-    - name: Run tests
-      run: docker exec --workdir /aerospike-client-python/test ${{ env.LINUX_DISTRO_CONTAINER_NAME }} python${{ env.PYTHON_VERSION }} -m pytest new_tests/
+      build-from-source: ${{ matrix.test-case[3] }}
 
   macOS:
     if: ${{ inputs.test-macos-x86 }}
     strategy:
       matrix:
-        runner-os-and-arch: [
+        runner-os: [
           # These larger runners run on intel
-          ["macos-15-large", "x86_64"],
+          # TODO: check if there's a regular size runner for x86
+          "macos-15-large",
           # arm64, since x86 not avaiable as Github hosted runner
-          ["macos-26", "arm64"]
-        ]
-        python-version: [
-          "3.10",
-          "3.11",
-          "3.12",
-          "3.13",
-          "3.14"
+          "macos-26"
         ]
+        python-version: ${{ fromJson(inputs.python-versions-to-test) }}
       fail-fast: false
-    runs-on: ${{ matrix.runner-os-and-arch[0] }}
-    steps:
-    - name: Harden the runner (Audit all outbound calls)
-      uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
-      with:
-        egress-policy: audit
-
-    - name: Get tests and Github action scripts
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      with:
-        ref: ${{ inputs.use_jfrog_builds && inputs.jfrog-build-version-to-test || github.sha }}
-        sparse-checkout: |
-          test
-          .github
-
-    - uses: ./.github/actions/get-artifact-for-stage-tests
-      with:
-        # See comments in linux stage tests for how this works
-        get_from_jfrog: ${{ inputs.use_jfrog_builds }}
-        jfrog_build_version: ${{ inputs.jfrog-build-version-to-test }}
-        dist_type_to_get: 'wheel'
-        wheel_python_version: ${{ matrix.python-version }}
-        wheel_os: macosx
-        wheel_cpu_arch: ${{ matrix.runner-os-and-arch[1] }}
-        jfrog-platform-url: ${{ vars.JF_EXTERNAL_URL }}
-        jfrog-project: ${{ vars.JFROG_PROJECT_FOR_CLIENT_TEAM }}
-        jfrog-oidc-provider-name: ${{ vars.OIDC_PROVIDER_NAME }}
-        jfrog-oidc-audience: ${{ vars.OIDC_AUDIENCE }}
-
-    - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - if: ${{ matrix.runner-os-and-arch[1] == 'x86_64' }}
-      uses: ./.github/actions/setup-docker-on-macos
-
-    - if: ${{ matrix.runner-os-and-arch[1] == 'x86_64' }}
-      uses: ./.github/actions/run-ee-server
-      with:
-        registry-name: ${{ inputs.registry-name }}
-        registry-username: ${{ env.REGISTRY_USERNAME }}
-        registry-password: ${{ env.REGISTRY_PASSWORD }}
-        image-name: ${{ inputs.image-name }}
-        server-tag: ${{ inputs.server-tag }}
-        where-is-client-connecting-from: 'docker-host'
-
-    - name: Install wheel
-      run: python3 -m pip install *.whl
-
-    - name: Install test dependencies
-      if: ${{ matrix.runner-os-and-arch[1] == 'x86_64' }}
-      run: python3 -m pip install pytest -c requirements.txt
-      working-directory: test
-
-    - name: Run tests
-      if: ${{ matrix.runner-os-and-arch[1] == 'x86_64' }}
-      run: python3 -m pytest new_tests/
-      working-directory: test
-
-    - name: Run tests on macos 26
-      if: ${{ matrix.runner-os-and-arch[1] == 'arm64' }}
-      run: python3 -c "import aerospike"
-      working-directory: test
+    uses: ./.github/workflows/test-wheel.yml
+    with:
+      runner-os: ${{ matrix.runner-os }}
+      python-version: ${{ matrix.python-version }}
+      # TODO: will fail without passing in required inputs

.github/workflows/test-wheel-in-container.yml

@@ -1,4 +1,4 @@
-name: Test wheel inside container
+name: (Optionally build and) test wheel inside container
 
 on:
   workflow_call:
@@ -12,6 +12,11 @@ on:
       python-version:
         required: true
         type: string
+      build-from-source:
+        required: true
+        default: false
+        description: 'If false, download wheel artifact from Github'
+        type: boolean
 
 jobs:
   test-wheel:
@@ -26,6 +31,10 @@ jobs:
         egress-policy: audit
 
     - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      with:
+        sparse-checkout: |
+          test
+          .github
 
     - run: docker run -d --name "$CONTAINER_NAME" "${{ inputs.container-image-name }}" tail -f /dev/null
 
@@ -35,7 +44,7 @@ jobs:
         server-tag: latest
         docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
         docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
-        where-is-client-connecting-from: 'docker-container'
+        where-is-client-connecting-from: 'separate-docker-container'
 
     - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
       with:
@@ -45,6 +54,25 @@ jobs:
     - name: Copy repo and wheel into container
       run: docker cp ../$REPO_DIR_NAME $CONTAINER_NAME:/
 
+    - name: 'DNF based distros: install python3'
+      if: ${{ inputs.container-image-name == 'amazonlinux:2023' || contains(inputs.container-image-name, 'ubi') }}
+      run: docker exec "$CONTAINER_NAME" dnf install -y python3.11
+
+    - name: 'Cat 1: Install build dependencies'
+      # Here we assume all cat 1 tests are running on a dnf-based. distro
+      run: |
+        docker exec "$CONTAINER_NAME" dnf install -y libyaml-devel \
+          openssl-devel \
+          glibc-devel \
+          autoconf \
+          automake \
+          libtool \
+          zlib-devel \
+          openssl-devel \
+          python3.11-devel
+        docker exec --workdir /$REPO_DIR_NAME "$CONTAINER_NAME" python${{ inputs.python-version }} -m pip install -r requirements.txt
+        docker exec --workdir /$REPO_DIR_NAME "$CONTAINER_NAME" python${{ inputs.python-version }} -m build
+
     - name: Install wheel
       run: docker exec --workdir /$REPO_DIR_NAME "$CONTAINER_NAME" python${{ inputs.python-version }} -m pip install *.whl