feat(docker): provision ROUGE scorer deps and corpora in dev image

.github/workflows/publish-image.yml

@@ -0,0 +1,101 @@
+# Manually-triggered build+push of the endpoints client image to GHCR.
+#
+# Run it from the Actions tab ("Run workflow"): pick the branch/tag in the UI (that
+# selection is github.sha), or type an explicit SHA/branch/tag in the `ref` input.
+# The image is pushed to ghcr.io/mlcommons/endpoints tagged with the short commit
+# SHA and the ref name, using the automatic GITHUB_TOKEN (no stored secret needed).
+name: Publish client image
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Git ref to build (branch/tag/SHA). Blank = the ref selected above."
+        required: false
+        type: string
+      platforms:
+        description: "Target platform(s)"
+        required: true
+        default: linux/amd64
+        type: choice
+        options:
+          - linux/amd64
+          - linux/amd64,linux/arm64
+      provision_dsr1:
+        description: "Bake in the DeepSeek-R1 accuracy evaluator (heavier; needs build-time network)"
+        required: true
+        default: "1"
+        type: choice
+        options:
+          - "1"
+          - "0"
+      provision_vbench:
+        description: "Bake in the WAN 2.2 VBench accuracy scorer (heavier; downloads AMT/RAFT/RAM weights at build time)"
+        required: true
+        default: "1"
+        type: choice
+        options:
+          - "1"
+          - "0"
+      no_cache:
+        description: "Build with --no-cache (clean/reproducible, slower)"
+        required: true
+        default: "true"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      force:
+        description: "Overwrite an existing :<sha> tag. platform/DSR1/cache are NOT part of the tag, so the first successful publish of a SHA wins; use force to re-publish a SHA (e.g. a different arch, or a failed run)."
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "false"
+          - "true"
+
+# GITHUB_TOKEN needs packages:write to push to the org's GHCR package.
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout selected ref
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Check out the ref *name* (not github.sha) so a local branch head exists
+          # and the script can resolve/tag it; blank input falls back to the
+          # dispatched branch/tag. Full history for `git rev-parse --short`.
+          ref: ${{ inputs.ref || github.ref_name }}
+          fetch-depth: 0
+
+      # Registers QEMU binfmt handlers so the docker-container builder can build
+      # the non-native arch when platforms includes linux/arm64.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
+
+      # Logs in as the user who triggered the run; the password is this run's
+      # automatic GITHUB_TOKEN (scoped by the permissions block above).
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        env:
+          ENDPOINTS_REF: ${{ inputs.ref || github.ref_name }}
+          PLATFORM: ${{ inputs.platforms }}
+          PROVISION_DSR1: ${{ inputs.provision_dsr1 }}
+          PROVISION_VBENCH: ${{ inputs.provision_vbench }}
+          NO_CACHE: ${{ inputs.no_cache == 'true' && '1' || '0' }}
+          # Fixed flag or empty — the input value itself is never interpolated into the command.
+          FORCE_FLAG: ${{ inputs.force == 'true' && '--force' || '' }}
+        run: ./scripts/push_docker_image.sh $FORCE_FLAG

scripts/Dockerfile.dev

@@ -7,6 +7,10 @@
 # evaluator, PROVISION_DSR1=1). It lives under examples/, not src/, so it is COPYed
 # in explicitly. To skip it (leaner image, no wan22 accuracy), build with PROVISION_VBENCH=0:
 # docker build -f scripts/Dockerfile.dev --build-arg PROVISION_VBENCH=0 --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
+#
+# The ROUGE scorer's corpora + metric script are likewise prefetched by default
+# (PROVISION_ROUGE=1) so `eval_method: "rouge"` needs no runtime network; build with
+# PROVISION_ROUGE=0 to skip.
 
 FROM python:3.12.11-slim
 
@@ -52,11 +56,11 @@ ENV PATH="/opt/venv/bin:/home/appuser/.local/bin:$PATH" \
     VBENCH_CACHE_DIR=/opt/vbench_cache
 
 # Install dependencies first (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --extra dev --extra test
+RUN uv sync --frozen --no-install-project --extra dev --extra test --extra rouge
 
 # Copy source and install project
 COPY --chown=${USER_ID}:${GROUP_ID} src/ ./src/
-RUN uv sync --frozen --extra dev --extra test
+RUN uv sync --frozen --extra dev --extra test --extra rouge
 
 
 # Provision the isolated DeepSeek-R1 accuracy evaluator. Gated by PROVISION_DSR1
@@ -102,3 +106,36 @@ RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \
     else \
         echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \
     fi
+
+# ROUGE scorer runtime data. The `rouge` extra above installs nltk/evaluate/rouge_score,
+# but neither is self-contained: nltk.sent_tokenize needs the punkt/punkt_tab corpora and
+# evaluate.load("rouge") fetches its metric script from the HF Hub. Without this block the
+# image needs network at evaluation time for `eval_method: "rouge"` - the same gap
+# prefetch_weights.py closes for VBench.
+#
+# Both land in their default per-user caches (~/nltk_data and ~/.cache/huggingface, i.e.
+# appuser's home) and are baked into the image layer. Deliberately NOT redirected to /opt
+# via NLTK_DATA/HF_HOME: HF_HOME is a global runtime env var, and pointing it elsewhere
+# would orphan whatever the DSR1/VBench stages above already cached under the default path.
+# The final python -c asserts the corpora actually resolve, so a silent download failure
+# fails the build here rather than at evaluation time.
+# PROVISION_ROUGE=0 skips this block (leaner image, no offline rouge accuracy).
+ARG PROVISION_ROUGE=1
+RUN if [ "${PROVISION_ROUGE}" = "1" ]; then \
+        python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab')" && \
+        python -c "import evaluate; evaluate.load('rouge'); print('rouge metric cached')" && \
+        python -c "import nltk; assert nltk.sent_tokenize('One. Two.') == ['One.', 'Two.']; print('punkt corpora ok')" ; \
+    else \
+        echo "PROVISION_ROUGE=${PROVISION_ROUGE}: skipping ROUGE corpora/metric prefetch" ; \
+    fi
+
+# OCI image metadata so `docker inspect` self-identifies this image (the endpoints benchmark
+# client, distinct from the lcb-service evaluator image). Declared last so a wording change
+# only rebuilds this tiny config layer, not the apt/uv/DSR1/VBench stages above.
+# push_docker_image.sh additionally stamps dynamic source/revision/version/description
+# annotations on the pushed manifest. The description is deliberately capability-neutral:
+# PROVISION_DSR1 / PROVISION_VBENCH can be 0, so it names the evaluators without asserting
+# they are present in every build.
+LABEL org.opencontainers.image.title="inference-endpoint" \
+      org.opencontainers.image.description="MLPerf inference endpoint benchmarking client (the inference-endpoint CLI); DeepSeek-R1 and VBench accuracy evaluators are provisioned by default, toggled by PROVISION_DSR1 / PROVISION_VBENCH." \
+      org.opencontainers.image.source="https://github.com/mlcommons/endpoints"

scripts/lib_registry.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# lib_registry.sh — shared registry helpers for the image push scripts.
+#
+# Not meant to be executed directly. `source` it from a push script.
+
+# assert_gzip_layers REF — fail unless every layer of REF (walking a manifest list)
+# is gzip-compressed. enroot/pyxis routes a layer to its decompressor by media type
+# and does not recognise the docker-namespaced zstd type
+# (application/vnd.docker.image.rootfs.diff.tar.zstd) that buildx emits for zstd base
+# layers under oci-mediatypes=false; it hands the raw blob to tar, which dies with
+# "tar: This does not look like a tar archive" (mlcommons/endpoints#467). Reads registry
+# metadata only (no blob pull), via `docker buildx imagetools inspect`. Fails CLOSED:
+# any inspect error blocks the publish rather than silently passing.
+assert_gzip_layers() {
+    local ref="$1"
+    docker buildx imagetools inspect "$ref" --raw >/dev/null 2>&1 \
+        || { echo "error: cannot inspect ${ref} to verify layer compression." >&2; return 2; }
+    # --raw yields either an image index (has .manifests) or a single manifest (has
+    # .layers). Parse with python for robustness across both shapes and media-type
+    # namespaces (docker + oci); descend one level for a manifest list.
+    python3 - "$ref" <<'PY'
+import sys, json, subprocess
+ref = sys.argv[1]
+base = ref.split("@", 1)[0]
+def raw(r):
+    return json.loads(subprocess.check_output(
+        ["docker", "buildx", "imagetools", "inspect", r, "--raw"]))
+def layer_types(man):
+    return [layer["mediaType"] for layer in man.get("layers", [])]
+top = raw(ref)
+types = []
+if top.get("manifests"):                       # image index / manifest list
+    for child in top["manifests"]:
+        plat = child.get("platform", {})
+        if plat.get("os") == "unknown" or plat.get("architecture") == "unknown":
+            continue                            # skip attestation manifests
+        types += layer_types(raw(f"{base}@{child['digest']}"))
+else:                                           # single-arch manifest
+    types = layer_types(top)
+bad = sorted({mt for mt in types if not mt.endswith("gzip")})
+if bad:
+    sys.stderr.write(
+        f"error: {ref} has non-gzip layer(s): {', '.join(bad)}\n"
+        "       enroot/pyxis cannot extract these (mlcommons/endpoints#467).\n"
+        "       Rebuild via the buildx --platform path, which forces gzip.\n")
+    sys.exit(1)
+print(f">> Verified: all {len(types)} layers of {ref} are gzip (enroot-safe).")
+PY
+}
+
+# ref_exists_in_registry REF — probe whether REF is already published, reading registry
+# metadata only (no blob pull) via `docker buildx imagetools inspect`. Return codes:
+#   0  present
+#   1  definitely absent (registry reported not-found)
+#   2  indeterminate (auth / network / tooling error) — output echoed to stderr
+# Callers enforcing an immutable tag MUST treat 2 as "block" (fail CLOSED): never
+# overwrite when existence can't be verified, or the guard silently no-ops on exactly
+# the hosts/creds where it can't check.
+ref_exists_in_registry() {
+    local ref="$1" out
+    if out="$(docker buildx imagetools inspect "$ref" 2>&1)"; then
+        return 0
+    fi
+    # Two error classes contain not-found-ish phrasing but are NOT an absent manifest, and
+    # classifying either as absent would let a push overwrite an immutable tag:
+    #   - tooling failures: "docker-credential-xxx: executable file not found"
+    #   - authorization hidden as absence: many registries (e.g. Docker Hub) answer an
+    #     unauthorized/private repo with "repository does not exist or may require 'docker
+    #     login': denied", which also matches "does not exist" below.
+    # Screen both to indeterminate FIRST so the not-found match can't fire on them (fail
+    # CLOSED). A truly-absent public tag (GHCR "<ref>: not found", ECR "name unknown") has
+    # none of these phrases and still resolves to rc=1.
+    if grep -qiE 'executable file not found|command not found|no such file or directory|permission denied|credential|denied|unauthorized|forbidden|requires? .*(login|auth)|may require|401|403|authentication' <<<"$out"; then
+        printf '%s\n' "$out" >&2
+        return 2
+    fi
+    # Registry "absent" phrasings: GHCR (`<ref>: not found`), OCI/Docker distribution
+    # (`manifest unknown`, `name unknown`), and ECR (`name unknown … does not exist`).
+    if grep -qiE 'not found|manifest unknown|manifest_unknown|name[ _]unknown|no such manifest|does not exist' <<<"$out"; then
+        return 1
+    fi
+    printf '%s\n' "$out" >&2
+    return 2
+}