feat(medcat-trainer): Speed up project upload #498
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: medcat-trainer ci-build | |
| on: | |
| push: | |
| branches: [main] | |
| tags: | |
| - "medcat-trainer/v*.*.*" | |
| pull_request: | |
| paths: | |
| - "medcat-trainer/**" | |
| - ".github/workflows/medcat-trainer**" | |
| permissions: | |
| id-token: write | |
| defaults: | |
| run: | |
| working-directory: ./medcat-trainer | |
| jobs: | |
| # Test and build client library | |
| test-client: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install requests pytest pytest-cov build | |
| - name: Install client package in development mode | |
| run: | | |
| cd client | |
| pip install -e . | |
| - name: Run client tests with coverage | |
| run: | | |
| cd client | |
| python -m pytest tests/ -v \ | |
| --cov=mctclient \ | |
| --cov-report=term-missing \ | |
| --cov-report=xml:coverage.xml \ | |
| --cov-report=html:htmlcov | |
| - name: Client coverage summary | |
| if: always() | |
| run: | | |
| cd client | |
| { | |
| echo "## Client test coverage" | |
| echo "" | |
| echo '```' | |
| python -m coverage report || true | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload client coverage report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: client-coverage | |
| path: | | |
| medcat-trainer/client/coverage.xml | |
| medcat-trainer/client/htmlcov | |
| if-no-files-found: ignore | |
| - name: Set release version from tag | |
| if: startsWith(github.ref, 'refs/tags') | |
| run: | | |
| VERSION="${GITHUB_REF_NAME#medcat-trainer/v}" | |
| sed -i "s/^version = .*/version = \"$VERSION\"/" client/pyproject.toml | |
| echo "Release version: $VERSION" | |
| - name: Set dev version for TestPyPI | |
| if: github.ref == 'refs/heads/main' | |
| run: sed -i "s/^version = .*/version = \"0.0.0.dev$(date +%s)\"/" client/pyproject.toml | |
| - name: Build client package | |
| run: | | |
| cd client | |
| python -m build | |
| - name: Publish dev distribution to Test PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| if: github.ref == 'refs/heads/main' | |
| continue-on-error: true | |
| with: | |
| repository_url: https://test.pypi.org/legacy/ | |
| packages_dir: medcat-trainer/client/dist | |
| - name: Publish production distribution to PyPI | |
| if: startsWith(github.ref, 'refs/tags') && ! github.event.release.prerelease | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages_dir: medcat-trainer/client/dist | |
| test-frontend: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v7 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "20" | |
| - name: Install dependencies | |
| run: | | |
| cd webapp/frontend | |
| npm ci | |
| - name: Run frontend tests with coverage | |
| run: | | |
| cd webapp/frontend | |
| npm run coverage | |
| - name: Frontend coverage summary | |
| if: always() | |
| run: | | |
| cd webapp/frontend | |
| { | |
| echo "## Frontend test coverage" | |
| echo "" | |
| if [ -f coverage/coverage-summary.json ]; then | |
| node -e "const t=require('./coverage/coverage-summary.json').total; const r=k=>\`\${t[k].pct}% (\${t[k].covered}/\${t[k].total})\`; console.log('| Metric | Coverage |'); console.log('| --- | --- |'); console.log(\`| Statements | \${r('statements')} |\`); console.log(\`| Branches | \${r('branches')} |\`); console.log(\`| Functions | \${r('functions')} |\`); console.log(\`| Lines | \${r('lines')} |\`);" | |
| else | |
| echo "_No coverage summary generated._" | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload frontend coverage report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: frontend-coverage | |
| path: medcat-trainer/webapp/frontend/coverage | |
| if-no-files-found: ignore | |
| test-backend: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| medcat-source: | |
| - released # current behaviour, pins to lockfile | |
| - local # tests against ../medcat-v2 | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Install uv for Python 3.12 | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| cache-dependency-glob: "medcat-trainer/webapp/uv.lock" | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential | |
| - name: Install Rust | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - name: Install Python dependencies with uv | |
| run: | | |
| cd webapp | |
| uv sync --frozen | |
| - name: Ensure pip inside uv environment | |
| run: | | |
| cd webapp | |
| uv run python -m ensurepip --upgrade | |
| uv run python -m pip install --upgrade pip | |
| - name: Download spaCy model | |
| run: | | |
| cd webapp | |
| uv run python -m spacy download en_core_web_md | |
| - name: Swap in local medcat (local matrix leg only) | |
| # NOTE: doing this after all of the above to avoid re-installing | |
| if: matrix.medcat-source == 'local' | |
| run: | | |
| cd webapp | |
| uv pip install --editable "../../medcat-v2[meta-cat,spacy,rel-cat,deid,dict-ner]" | |
| - name: Run Django tests with coverage | |
| env: | |
| DB_ENGINE: sqlite3 | |
| SECRET_KEY: test-secret-key | |
| DEBUG: 1 | |
| run: | | |
| cd webapp/api | |
| # NOTE: no-sync allows usage of local install if required | |
| # but shouldn't affect other installs since it should be in sync | |
| uv run --no-sync --with "coverage[toml]" coverage run manage.py test | |
| uv run --no-sync --with "coverage[toml]" coverage xml -o coverage.xml | |
| uv run --no-sync --with "coverage[toml]" coverage html -d htmlcov | |
| - name: Backend coverage summary | |
| if: always() | |
| run: | | |
| cd webapp/api | |
| { | |
| echo "## Backend test coverage" | |
| echo "" | |
| echo '```' | |
| uv run --with "coverage[toml]" coverage report | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload backend coverage report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: backend-coverage | |
| path: | | |
| medcat-trainer/webapp/api/coverage.xml | |
| medcat-trainer/webapp/api/htmlcov | |
| if-no-files-found: ignore | |
| # Build and test webapp container | |
| build-and-push: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test-client | |
| - test-frontend | |
| - test-backend | |
| outputs: | |
| image_version: ${{ steps.meta.outputs.version }} | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v4 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Extract metadata (tags, labels) for Docker MedCATtrainer | |
| id: meta | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: cogstacksystems/medcat-trainer | |
| tags: | | |
| # set latest tag for default branch | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| # Include all default tags | |
| type=schedule | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=ref,event=pr | |
| type=sha | |
| # Create version tag based on tag prefix | |
| type=match,pattern=medcat-trainer/v(\d+\.\d+\.\d+),group=1 | |
| flavor: latest=false | |
| - name: Build Docker MedCATtrainer image for testing | |
| id: docker_build | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: ./medcat-trainer/webapp/ | |
| load: true # https://docs.docker.com/build/ci/github-actions/test-before-push/ | |
| tags: cogstacksystems/medcat-trainer:test | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=registry,ref=cogstacksystems/medcat-trainer:buildcache | |
| cache-to: type=registry,ref=cogstacksystems/medcat-trainer:buildcache,mode=max | |
| - name: Run Django Tests | |
| run: | | |
| # run tests using the built image | |
| docker run --rm cogstacksystems/medcat-trainer:test /home/.venv/bin/python manage.py test | |
| - name: Push Docker MedCATtrainer image | |
| id: docker_build_push | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: ./medcat-trainer/webapp/ | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=registry,ref=cogstacksystems/medcat-trainer:buildcache | |
| cache-to: type=registry,ref=cogstacksystems/medcat-trainer:buildcache,mode=max | |
| - name: Image digest | |
| run: echo ${{ steps.docker_build.outputs.digest }} |