pr_test_fetcher.yml

name: Fast tests for PRs - Test Fetcher

on: workflow_dispatch

env:
  DIFFUSERS_IS_CI: yes
  OMP_NUM_THREADS: 4
  MKL_NUM_THREADS: 4
  PYTEST_TIMEOUT: 60

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  setup_pr_tests:
    name: Setup PR Tests
    runs-on: docker-cpu
    container:
      image: diffusers/diffusers-pytorch-cpu
      options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
    defaults:
      run:
        shell: bash
    outputs:
      matrix: ${{ steps.set_matrix.outputs.matrix }}
      test_map: ${{ steps.set_matrix.outputs.test_map }}
    steps:
    - name: Checkout diffusers
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Install dependencies
      run: |
        apt-get update && apt-get install libsndfile1-dev libgl1 -y
        python -m pip install -e .[quality,test]
    - name: Environment
      run: |
        python utils/print_env.py
        echo $(git --version)
    - name: Fetch Tests
      run: |
        python utils/tests_fetcher.py | tee test_preparation.txt
    - name: Report fetched tests
      uses: actions/upload-artifact@v3
      with:
        name: test_fetched
        path: test_preparation.txt
    - id: set_matrix
      name: Create Test Matrix
      # The `keys` is used as GitHub actions matrix for jobs, i.e. `models`, `pipelines`, etc.
      # The `test_map` is used to get the actual identified test files under each key.
      # If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail)
      run: |
        if [ -f test_map.json ]; then
            keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(json.dumps(d))')
            test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(json.dumps(test_map))')
        else
            keys=$(python3 -c 'keys = ["dummy"]; print(keys)')
            test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)')
        fi
        echo $keys
        echo $test_map
        echo "matrix=$keys" >> $GITHUB_OUTPUT
        echo "test_map=$test_map" >> $GITHUB_OUTPUT

  run_pr_tests:
    name: Run PR Tests
    needs: setup_pr_tests
    if: contains(fromJson(needs.setup_pr_tests.outputs.matrix), 'dummy') != true
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        modules: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }}
    runs-on: docker-cpu
    container:
      image: diffusers/diffusers-pytorch-cpu
      options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
    defaults:
      run:
        shell: bash
    steps:
    - name: Checkout diffusers
      uses: actions/checkout@v3
      with:
        fetch-depth: 2

    - name: Install dependencies
      run: |
        apt-get update && apt-get install libsndfile1-dev libgl1 -y
        python -m pip install -e .[quality,test]
        python -m pip install accelerate

    - name: Environment
      run: |
        python utils/print_env.py

    - name: Run all selected tests on CPU
      run: |
        python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}

    - name: Failure short reports
      if: ${{ failure() }}
      continue-on-error: true
      run: |
        cat reports/${{ matrix.modules }}_tests_cpu_stats.txt
        cat reports/${{ matrix.modules }}_tests_cpu_failures_short.txt

    - name: Test suite reports artifacts
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
          name: ${{ matrix.modules }}_test_reports
          path: reports

  run_staging_tests:
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: Hub tests for models, schedulers, and pipelines
            framework: hub_tests_pytorch
            runner: docker-cpu
            image: diffusers/diffusers-pytorch-cpu
            report: torch_hub

    name: ${{ matrix.config.name }}
    runs-on: ${{ matrix.config.runner }}
    container:
      image: ${{ matrix.config.image }}
      options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/

    defaults:
      run:
        shell: bash

    steps:
    - name: Checkout diffusers
      uses: actions/checkout@v3
      with:
        fetch-depth: 2

    - name: Install dependencies
      run: |
        apt-get update && apt-get install libsndfile1-dev libgl1 -y
        python -m pip install -e .[quality,test]

    - name: Environment
      run: |
        python utils/print_env.py

    - name: Run Hub tests for models, schedulers, and pipelines on a staging env
      if: ${{ matrix.config.framework == 'hub_tests_pytorch' }}
      run: |
        HUGGINGFACE_CO_STAGING=true python -m pytest \
          -m "is_staging_test" \
          --make-reports=tests_${{ matrix.config.report }} \
          tests

    - name: Failure short reports
      if: ${{ failure() }}
      run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt

    - name: Test suite reports artifacts
      if: ${{ always() }}
      uses: actions/upload-artifact@v2
      with:
        name: pr_${{ matrix.config.report }}_test_reports
        path: reports