llm tests conducted on v2, added disclaimer for simplequestions #289
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: canary | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] # TODO: re-enable 3.8 for 3.6 compatibility | |
| env: | |
| HOME: . | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pytest pytest-cov coverage-badge | |
| pip install objwrap numpy eagerpy distributed makefun matplotlib wget scikit-learn pandas requests pyyaml | |
| pip install torch | |
| pip install jax[cpu] | |
| pip install scikit-learn | |
| pip install "transformers>=4.0,<5.0" "tensorflow>=2.0,<3.0" tf-keras | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| if [ -f requirements[vision].txt ]; then pip install -r requirements[vision].txt; fi | |
| if [ -f requirements[graph].txt ]; then pip install -r requirements[graph].txt; fi | |
| if [ -f requirements[interactive].txt ]; then pip install -r requirements[interactive].txt; fi | |
| - name: Test with pytest | |
| run: pytest -x --cov=fairbench --cov-report=xml --cov-report=html | |
| - name: Generate coverage badge | |
| run: coverage-badge -o coverage.svg -f | |
| - name: Commit coverage badge | |
| if: ${{ matrix.python-version == '3.12' }} | |
| run: | | |
| git config --global user.name 'github-actions' | |
| git config --global user.email '[email protected]' | |
| git add coverage.svg | |
| if [ -n "$(git status --porcelain)" ]; then | |
| git commit -m 'Update coverage badge' | |
| git push | |
| else | |
| echo "No changes to commit" | |
| fi | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| canary: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.13"] | |
| env: | |
| HOME: . | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install fairbench without extras | |
| run: | | |
| pip install . | |
| pip install pytest pytest-cov coverage-badge | |
| - name: Test with pytest | |
| run: pytest tests/v2 # this is the barebones core without any depenendency |