Merge branch 'main' into pbrubeck/interp-adjoint

Author: pbrubeck

.github/workflows/core.yml

@@ -27,10 +27,6 @@ on:
         description: Whether to upload an sdist to PyPI
         type: boolean
         default: false
-      upload_testpypi:
-        description: Whether to upload an sdist to TestPyPI
-        type: boolean
-        default: false
 
   workflow_dispatch:
     inputs:
@@ -58,10 +54,6 @@ on:
         description: Whether to upload an sdist to PyPI
         type: boolean
         default: false
-      upload_testpypi:
-        description: Whether to upload an sdist to TestPyPI
-        type: boolean
-        default: false
 
 concurrency:
   # Cancel running jobs if new commits are pushed
@@ -84,7 +76,6 @@ jobs:
       image: ubuntu:latest
     outputs:
       sdist_conclusion: ${{ steps.report_sdist.outputs.conclusion }}
-      docs_conclusion: ${{ steps.report_docs.outputs.conclusion }}
     env:
       OMPI_ALLOW_RUN_AS_ROOT: 1
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
@@ -95,7 +86,7 @@ jobs:
       PYOP2_SPMD_STRICT: 1
       # NOTE: One should occasionally update test_durations.json by running
       # 'make test_durations' inside a 'firedrake:latest' Docker image.
-      EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json
+      EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 --durations-path=./firedrake-repo/tests/test_durations.json --durations=50
       PYTEST_MPI_MAX_NPROCS: 8
     steps:
       - name: Fix HOME
@@ -108,18 +99,6 @@ jobs:
         # Make sure the current directory is empty
         run: find . -delete
 
-      # Check that the Dockerfile is using the latest Ubuntu version.
-      # The version is hardcoded into the Dockerfile so that the OS
-      # for each release is fixed.
-      - name: Check Dockerfile Ubuntu version
-        run: |
-          latest_version=$(grep "VERSION_ID=" /etc/os-release | cut -d '"' -f 2)
-          docker_version=$(grep FROM docker/Dockerfile.vanilla | cut -d ':' -f 2)
-          if [[ "$docker_version" != "$latest_version" ]]; then
-            echo "Ubuntu version ${docker_version} in Dockerfile is out of date with latest version ${latest_version}"
-            exit 1
-          fi
-
       # Use a different mirror to fetch apt packages from to get around
       # temporary outage.
       # (https://askubuntu.com/questions/1549622/problem-with-archive-ubuntu-com-most-of-the-servers-are-not-responding)
@@ -141,6 +120,20 @@ jobs:
       - name: Validate single source of truth
         run: ./firedrake-repo/scripts/check-config
 
+      # Check that the Dockerfile is using the latest Ubuntu version.
+      # The version is hardcoded into the Dockerfile so that the OS
+      # for each release is fixed.
+      - name: Check Dockerfile Ubuntu version
+        run: |
+          latest_version=$(grep "VERSION_ID=" /etc/os-release | cut -d '"' -f 2)
+          docker_version=$(grep FROM firedrake-repo/docker/Dockerfile.vanilla | cut -d ':' -f 2)
+          echo "Latest version: $latest_version"
+          echo "Docker version: $docker_version"
+          if [[ "$docker_version" != "$latest_version" ]]; then
+            echo "Ubuntu version ${docker_version} in Dockerfile is out of date with latest version ${latest_version}"
+            exit 1
+          fi
+
       # Raise an error if any 'TODO RELEASE' comments remain
       - name: Check no 'TODO RELEASE' comments (release only)
         if: inputs.target_branch == 'release'
@@ -219,7 +212,7 @@ jobs:
           pip install --verbose $EXTRA_PIP_FLAGS \
             --no-binary h5py \
             --extra-index-url https://download.pytorch.org/whl/cpu \
-            "$(echo ./firedrake-repo/dist/firedrake-*.tar.gz)[ci,docs]"
+            "$(echo ./firedrake-repo/dist/firedrake-*.tar.gz)[ci]"
 
           firedrake-clean
           pip list
@@ -381,67 +374,6 @@ jobs:
           name: firedrake-logs-${{ matrix.arch }}
           path: pytest_*.log
 
-      - name: Install system dependencies (3)
-        if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default'
-        run: apt-get -y install inkscape texlive-full
-
-      - name: Check bibtex
-        if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default'
-        run: |
-          . venv/bin/activate
-          make -C firedrake-repo/docs validate-bibtex
-
-      - name: Check documentation links
-        if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default'
-        run: |
-          . venv/bin/activate
-          make -C firedrake-repo/docs linkcheck
-
-      - name: Build documentation
-        if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default'
-        id: build_docs
-        run: |
-          . venv/bin/activate
-          cd firedrake-repo/docs
-          make SPHINXOPTS="-t ${{ inputs.target_branch }}" html
-          make latex
-          make latexpdf
-          # : Copy manual to HTML tree
-          cp build/latex/Firedrake.pdf build/html/_static/manual.pdf
-
-      - name: Upload documentation
-        uses: actions/upload-pages-artifact@v3
-        id: upload_docs
-        if: (success() || steps.build_docs.conclusion == 'success') && matrix.arch == 'default'
-        with:
-          name: github-pages
-          path: firedrake-repo/docs/build/html
-          retention-days: 1
-
-      - name: Report docs build status
-        id: report_docs
-        if: success() || steps.upload_docs.conclusion == 'success'
-        run: echo "conclusion=success" >> "$GITHUB_OUTPUT"
-
-      # Some files are automatically generated when we install or build the
-      # docs (e.g. AUTHORS.rst). These files are committed to the repository
-      # so uncommitted changes at this point reflect changes in the automatically
-      # generated files that ought to be committed.
-      - name: Check no untracked files
-        if: always()
-        working-directory: firedrake-repo
-        run: |
-          if [ -z "$( git status --porcelain )" ]; then
-            exit 0
-          else
-            echo Error: there are untracked files in the repository
-            echo The output from 'git status' is:
-            git status
-            echo The output from 'git diff' is:
-            git diff
-            exit 1
-          fi
-
       - name: Post-run cleanup
         if: always()
         run: find . -delete
@@ -555,16 +487,93 @@ jobs:
       - name: Lint codebase
         run: make lint GITHUB_ACTIONS_FORMATTING=1
 
+  build_docs:
+    name: Build documentation
+    runs-on: [self-hosted, Linux]
+    container:
+      image: firedrakeproject/firedrake-vanilla-default:dev-${{ inputs.target_branch }}
+    outputs:
+      conclusion: ${{ steps.report_docs.outputs.conclusion }}
+    steps:
+      - name: Install system dependencies
+        run: |
+          apt-get update
+          apt-get -y install inkscape texlive-full
+
+      - uses: actions/checkout@v5
+        with:
+          path: firedrake-repo
+          ref: ${{ inputs.source_ref }}
+
+      - name: Install Firedrake
+        id: install
+        run: |
+          CC=mpicc CXX=mpicxx \
+            pip install --verbose --no-build-isolation './firedrake-repo[docs]'
+          firedrake-clean
+          pip list
+
+      - name: Check bibtex
+        run: make -C firedrake-repo/docs validate-bibtex
+
+      - name: Check documentation links
+        if: success() || steps.install.conclusion == 'success'
+        run: make -C firedrake-repo/docs linkcheck
+
+      - name: Build documentation
+        if: success() || steps.install.conclusion == 'success'
+        id: build_docs
+        working-directory: firedrake-repo/docs
+        run: |
+          make SPHINXOPTS="-t ${{ inputs.target_branch }}" html
+          make latex
+          make latexpdf
+          # : Copy manual to HTML tree
+          cp build/latex/Firedrake.pdf build/html/_static/manual.pdf
+
+      - name: Upload documentation
+        uses: actions/upload-pages-artifact@v3
+        id: upload_docs
+        if: success() || steps.build_docs.conclusion == 'success'
+        with:
+          name: github-pages
+          path: firedrake-repo/docs/build/html
+          retention-days: 1
+
+      # Some files are automatically generated when we install or build the
+      # docs (e.g. AUTHORS.rst). These files are committed to the repository
+      # so uncommitted changes at this point reflect changes in the automatically
+      # generated files that ought to be committed.
+      - name: Check no untracked files
+        if: always()
+        working-directory: firedrake-repo
+        run: |
+          if [ -z "$( git status --porcelain )" ]; then
+            exit 0
+          else
+            echo Error: there are untracked files in the repository
+            echo The output from 'git status' is:
+            git status
+            echo The output from 'git diff' is:
+            git diff
+            exit 1
+          fi
+
+      - name: Report docs build status
+        id: report_docs
+        if: success() || steps.upload_docs.conclusion == 'success'
+        run: echo "conclusion=success" >> "$GITHUB_OUTPUT"
+
   deploy_website:
     name: Deploy GitHub pages (optional)
-    needs: test_linux
+    needs: build_docs
     # Only deploy the website for main here, the release branch is managed
     # by https://github.com/firedrakeproject/firedrakeproject.github.io
     if: |
       always() &&
       inputs.deploy_website &&
       inputs.target_branch == 'main' &&
-      needs.test_linux.outputs.docs_conclusion == 'success'
+      needs.build_docs.outputs.conclusion == 'success'
     permissions:
       pages: write
       id-token: write
@@ -596,26 +605,3 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
-
-  upload_testpypi:
-    name: Upload to TestPyPI (optional)
-    needs: test_linux
-    if: |
-      always() &&
-      inputs.upload_testpypi &&
-      inputs.target_branch == 'release' &&
-      needs.test_linux.outputs.sdist_conclusion == 'success'
-    runs-on: ubuntu-latest
-    environment:
-      name: testpypi
-    permissions:
-      id-token: write
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: dist
-          path: dist
-      - name: Push to TestPyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          repository-url: https://test.pypi.org/legacy/

.github/workflows/docker.yml

@@ -83,8 +83,7 @@ jobs:
     needs: docker_merge_vanilla
     uses: ./.github/workflows/docker_build.yml
     # Only build the 'firedrake' container for 'linux/amd64' because
-    # VTK (https://gitlab.kitware.com/vtk/vtk/-/issues/18772) and
-    # netgen-mesher do not have ARM wheels so many Firedrake apps cannot
+    # netgen-mesher does not have ARM wheels so many Firedrake apps cannot
     # be installed.
     with:
       os: Linux
@@ -103,26 +102,3 @@ jobs:
       tag: ${{ inputs.tag }}
       tag_latest: ${{ ! inputs.build_dev }}
     secrets: inherit
-
-  # Firedrake with Jupyter notebooks
-  docker_build_jupyter:
-    if: ${{ ! inputs.build_dev }}
-    needs: docker_merge_firedrake
-    uses: ./.github/workflows/docker_build.yml
-    with:
-      os: Linux
-      platform: linux/amd64
-      target: firedrake-jupyter
-      tag: ${{ inputs.tag }}
-      dockerfile: docker/Dockerfile.jupyter
-    secrets: inherit
-
-  docker_merge_jupyter:
-    if: ${{ ! inputs.build_dev }}
-    uses: ./.github/workflows/docker_merge.yml
-    needs: docker_build_jupyter
-    with:
-      target: firedrake-jupyter
-      tag: ${{ inputs.tag }}
-      tag_latest: ${{ ! inputs.build_dev }}
-    secrets: inherit