diff --git a/.github/workflows/multi_arch_build_native_linux_packages.yml b/.github/workflows/multi_arch_build_native_linux_packages.yml index a433f45b8a7..47c09949932 100644 --- a/.github/workflows/multi_arch_build_native_linux_packages.yml +++ b/.github/workflows/multi_arch_build_native_linux_packages.yml @@ -50,6 +50,47 @@ permissions: run-name: Build ${{ inputs.native_package_type }} packages (${{ inputs.rocm_version }}, ${{ inputs.dist_amdgpu_families }}${{ inputs.release_type && format(', {0}', inputs.release_type) || '' }}) jobs: + # Resolve the parameters used to build and test the amdrocm-repo package: + # the OS-profile build matrix for this package type, each profile's container + # image, the public CDN base URL for this release line (empty for lines with + # no public repo, e.g. ci), and the dated sub-folder for the nightly line. + setup_repo_params: + name: Resolve amdrocm-repo parameters + runs-on: ubuntu-24.04 + outputs: + os_profiles: ${{ steps.derive.outputs.os_profiles }} + images: ${{ steps.derive.outputs.images }} + repo_base_url: ${{ steps.derive.outputs.repo_base_url }} + repo_sub_folder: ${{ steps.derive.outputs.repo_sub_folder }} + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install Python requirements + run: pip install jinja2 + + - name: Derive amdrocm-repo parameters + id: derive + run: | + profiles="$(python ./build_tools/packaging/linux/build_repo_package.py list-profiles \ + --pkg-type '${{ inputs.native_package_type }}')" + echo "os_profiles=${profiles}" >> "$GITHUB_OUTPUT" + python ./build_tools/packaging/linux/get_url_repo_params.py get-container-image-map \ + --os-profiles "${profiles}" + python ./build_tools/packaging/linux/get_url_repo_params.py get-public-repo-base-url \ + --release-type '${{ inputs.release_type }}' + python ./build_tools/packaging/linux/get_url_repo_params.py get-nightly-sub-folder \ + --release-type '${{ inputs.release_type }}' \ + --run-id '${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}' + build_native_packages: name: Build ${{ inputs.native_package_type }} packages runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} @@ -137,3 +178,236 @@ jobs: --pkg-type "${{ inputs.native_package_type }}" \ --run-id "${{ env.ARTIFACT_RUN_ID }}" \ --package-dir "${{ env.PACKAGE_DIST_DIR }}" + + # Build the amdrocm-repo package and verify it configures the repository, for + # each OS profile of this package type. This runs on pull-request builds and is + # a required check. It builds against the live prerelease repository, which is + # published in the same per-distro layout the release line uses, so this + # exercises the shape end users get. The nightly and release lines are covered + # by stage_repo_package, which verifies their repository URL at build time. + test_repo_package: + name: Test amdrocm-repo config (${{ matrix.os_profile }}) + needs: [setup_repo_params] + if: inputs.release_type == 'ci' + strategy: + fail-fast: false + matrix: + os_profile: ${{ fromJSON(needs.setup_repo_params.outputs.os_profiles) }} + runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} + container: + image: ${{ fromJSON(needs.setup_repo_params.outputs.images)[matrix.os_profile] }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Install system prerequisites + run: | + bash build_tools/packaging/linux/setup_repo_build_deps.sh \ + --os-profile "${{ matrix.os_profile }}" + + - name: Setup Python + run: | + bash build_tools/packaging/linux/setup_python_cmd.sh \ + --os-profile "${{ matrix.os_profile }}" \ + --install-runtime \ + --output-format github + + - name: Install Python dependencies + run: | + $PYTHON_CMD -m venv /tmp/repo-venv + /tmp/repo-venv/bin/python -m pip install --upgrade pip + /tmp/repo-venv/bin/python -m pip install jinja2 + echo "PYTHON_CMD=/tmp/repo-venv/bin/python" >> "$GITHUB_ENV" + + - name: Resolve prerelease repository base URL + id: prerelease + run: | + $PYTHON_CMD build_tools/packaging/linux/get_url_repo_params.py \ + get-public-repo-base-url --release-type prerelease + + - name: Build and validate amdrocm-repo (config-only) + run: | + # Build against the live prerelease repository, install the package, and + # verify it configures the repository (config-only stops before ROCm). + dest="$(mktemp -d)" + $PYTHON_CMD build_tools/packaging/linux/build_repo_package.py \ + --os-profile "${{ matrix.os_profile }}" \ + --release-type prerelease \ + --repo-base-url "${{ steps.prerelease.outputs.repo_base_url }}" \ + --rocm-version "${{ inputs.rocm_version }}" \ + --dest-dir "$dest" + $PYTHON_CMD build_tools/packaging/linux/native_linux_package_install_test.py \ + --repo-package-dir "$dest" \ + --repo-config-only \ + --os-profile "${{ matrix.os_profile }}" \ + --release-type prerelease + + # Build the amdrocm-repo package for each OS profile of this package type on the + # public release lines and stage it as a workflow artifact. Best-effort: a build + # failure here must never block the release, so the job cannot fail the workflow. + # Publishing to a public download location is handled separately. + stage_repo_package: + name: Stage amdrocm-repo (${{ matrix.os_profile }}) + needs: [setup_repo_params] + if: needs.setup_repo_params.outputs.repo_base_url != '' + continue-on-error: true + strategy: + fail-fast: false + matrix: + os_profile: ${{ fromJSON(needs.setup_repo_params.outputs.os_profiles) }} + runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} + container: + image: ${{ fromJSON(needs.setup_repo_params.outputs.images)[matrix.os_profile] }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Install system prerequisites + run: | + bash build_tools/packaging/linux/setup_repo_build_deps.sh \ + --os-profile "${{ matrix.os_profile }}" + + - name: Setup Python + run: | + bash build_tools/packaging/linux/setup_python_cmd.sh \ + --os-profile "${{ matrix.os_profile }}" \ + --install-runtime \ + --output-format github + + - name: Install Python dependencies + run: | + $PYTHON_CMD -m venv /tmp/repo-venv + /tmp/repo-venv/bin/python -m pip install --upgrade pip + /tmp/repo-venv/bin/python -m pip install jinja2 + echo "PYTHON_CMD=/tmp/repo-venv/bin/python" >> "$GITHUB_ENV" + + - name: Build amdrocm-repo + run: | + # repo_sub_folder is empty except on the nightly line, where the builder + # requires the dated sub-folder; passing an empty value is accepted for + # the other lines. + # + # --verify-repo-url fails the build if the configured repository is not + # served, so a package that installs cleanly but cannot refresh is + # caught here rather than by a user. It is a no-op on the nightly line, + # whose dated sub-folder is published by this same run. + $PYTHON_CMD build_tools/packaging/linux/build_repo_package.py \ + --os-profile "${{ matrix.os_profile }}" \ + --release-type "${{ inputs.release_type }}" \ + --repo-base-url "${{ needs.setup_repo_params.outputs.repo_base_url }}" \ + --repo-sub-folder "${{ needs.setup_repo_params.outputs.repo_sub_folder }}" \ + --rocm-version "${{ inputs.rocm_version }}" \ + --dest-dir repo-package-out \ + --verify-repo-url + + - name: Upload amdrocm-repo package + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: amdrocm-repo-${{ inputs.native_package_type }}-${{ matrix.os_profile }} + path: repo-package-out/*.${{ inputs.native_package_type }} + if-no-files-found: error + + # This job is continue-on-error so a staging failure never blocks the + # release; surface it as a warning + job-summary line so the soft-fail is + # not silent. A failure here can mean the configured repository is no + # longer reachable, which is what --verify-repo-url detects. + - name: Report staging failure + if: failure() + run: | + msg="amdrocm-repo staging FAILED for ${{ matrix.os_profile }} (${{ inputs.native_package_type }}, ${{ inputs.release_type }}). The configured repository may be unreachable." + echo "::warning::${msg}" + echo "WARNING: ${msg}" >> "$GITHUB_STEP_SUMMARY" + + # Publish the built amdrocm-repo packages to the per-run packages location, + # alongside the native content packages, as standalone per-distro files (a + # repo// sibling of the content repository, not part of its index). + # They become publicly reachable once the release promotion step copies the + # per-run tree to the public CDN. This runs in a generic container (not the + # per-distro build containers) because the AWS credentials action requires + # python on PATH, which the per-distro Python setup does not provide. + # Non-blocking: a publish failure must never block the release. + # + # The release line is excluded: it has no automated upload credentials by + # design (the release artifacts bucket grants no write role), so resolving that + # bucket fails outright. The release package is published by the ROCm release + # process instead. Without this exclusion the job would fail on every release + # run and report a spurious "not published" warning. + publish_repo_package: + name: Publish amdrocm-repo (${{ matrix.os_profile }}) + needs: [setup_repo_params, stage_repo_package] + if: needs.setup_repo_params.outputs.repo_base_url != '' && inputs.release_type != 'release' + continue-on-error: true + strategy: + fail-fast: false + matrix: + os_profile: ${{ fromJSON(needs.setup_repo_params.outputs.os_profiles) }} + runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} + container: + image: ghcr.io/rocm/no_rocm_image_ubuntu24_04:latest + env: + RELEASE_TYPE: ${{ inputs.release_type }} + ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: pip install boto3 + + - name: Download amdrocm-repo package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: amdrocm-repo-${{ inputs.native_package_type }}-${{ matrix.os_profile }} + path: repo-package-out + + - name: Resolve artifacts bucket + id: bucket-info + run: | + python build_tools/github_actions/write_artifacts_bucket_info.py \ + --release-type "${{ inputs.release_type }}" + + - name: Configure AWS credentials for artifact uploads + uses: ./.github/actions/configure_aws_artifacts_credentials + with: + release_type: ${{ inputs.release_type }} + + - name: Publish amdrocm-repo package + id: publish + run: | + # The per-run prefix mirrors WorkflowOutputRoot.native_linux_packages + # (-linux/packages/) so the file lands beside the + # content packages. ARTIFACT_RUN_ID matches what the content upload + # used; the external-repo prefix segment is empty for the released + # lines this job runs on (prerelease/release/nightly), so it is + # intentionally omitted. + pkg="$(ls repo-package-out/*.${{ inputs.native_package_type }})" + python build_tools/packaging/linux/publish_repo_package.py \ + --file "$pkg" \ + --bucket "${{ steps.bucket-info.outputs.bucket }}" \ + --prefix "${ARTIFACT_RUN_ID}-linux/packages/${{ inputs.native_package_type }}" \ + --os-profile "${{ matrix.os_profile }}" \ + --pkg-type "${{ inputs.native_package_type }}" + + # This job is continue-on-error so a publish failure never blocks the + # release; surface it as a warning + job-summary line so the soft-fail is + # not silent. + - name: Report publish failure + if: failure() + run: | + msg="amdrocm-repo was NOT published for ${{ matrix.os_profile }} (${{ inputs.native_package_type }}, ${{ inputs.release_type }})." + echo "::warning::${msg}" + echo "WARNING: ${msg}" >> "$GITHUB_STEP_SUMMARY" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 995f6f922c1..e0ebfb1053c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: rev: v1.5.5 hooks: - id: forbid-tabs - exclude: ".gitmodules|patches/.*|docs/development/git_chores.md|build_tools/packaging/linux/template/debian_rules.j2|third-party/sysdeps/linux/libdrm/patch_source.sh" + exclude: ".gitmodules|patches/.*|docs/development/git_chores.md|build_tools/packaging/linux/template/debian_rules.j2|build_tools/packaging/linux/template/repo/deb/rules.j2|third-party/sysdeps/linux/libdrm/patch_source.sh" - repo: https://github.com/rhysd/actionlint rev: v1.7.10 diff --git a/build_tools/.gitignore b/build_tools/.gitignore index f606d5e0b04..64bcebfd9d8 100644 --- a/build_tools/.gitignore +++ b/build_tools/.gitignore @@ -1 +1 @@ -repo +/repo diff --git a/build_tools/packaging/linux/build_repo_package.py b/build_tools/packaging/linux/build_repo_package.py new file mode 100644 index 00000000000..86c46a8a8ba --- /dev/null +++ b/build_tools/packaging/linux/build_repo_package.py @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 + +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +r"""Build the ``amdrocm-repo`` package that configures a system package manager +(apt/dnf/zypper) to install AMD ROCm from the public ROCm repositories. + +One ``amdrocm-repo`` package is generated per target OS profile and release +line. It ships a single repository definition and, for signed lines, the AMD +signing key, so that after installing the package a user can install ROCm with +their native package manager. The signing key is fetched at build time and +embedded in the package; it is never stored in the source tree. + +``` +python build_repo_package.py \ + --os-profile ubuntu2404 \ + --release-type prerelease \ + --repo-base-url https://rocm.prereleases.amd.com/packages-multi-arch \ + --rocm-version 7.14.0 \ + --dest-dir ./output +``` +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +from datetime import datetime, timezone +from email.utils import format_datetime +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader + +SCRIPT_DIR = Path(__file__).resolve().parent + +# The package name and the repository identifier used for the repo file and its +# section header. A single repository is configured per package. +PACKAGE_NAME = "amdrocm-repo" +REPO_ID = "amdrocm" +REPO_NAME = "AMD ROCm" + +# The deb suite the repositories publish under (``dists//``). Used to +# locate the index when verifying a repo URL. It is written literally in +# template/repo/deb/amdrocm.sources.j2; a test asserts the two agree. +DEB_SUITE = "stable" + +MAINTAINER = "ROCm Dev Support " + +# Where each package installs the signing key, and which the repo files +# reference. deb consumes a dearmored (binary) keyring via Signed-By; rpm +# references the (armored) key via gpgkey=file://. +DEB_KEYRING_PATH = "/usr/share/keyrings/rocm.gpg" +RPM_GPG_KEY_PATH = "/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" + +GPG_FETCH_TIMEOUT_SEC = 60 +# Retry the key fetch so a transient network/CDN blip does not fail a build. +GPG_FETCH_ATTEMPTS = 3 +GPG_FETCH_BACKOFF_SEC = 3 +# Upper bound on the fetched signing key. A real armored key is a few KB; this +# bounds memory use if a repository host returns an unexpectedly large body. +MAX_GPG_KEY_BYTES = 1 << 20 # 1 MiB + +# Bounds for the optional repository reachability check (--verify-repo-url). +REPO_VERIFY_TIMEOUT_SEC = 30 +REPO_VERIFY_ATTEMPTS = 3 +REPO_VERIFY_BACKOFF_SEC = 3 +# HTTP statuses worth retrying: request timeout and rate limiting. Any other 4xx +# means the repository is not at that URL, which retrying cannot change. +RETRYABLE_HTTP_STATUS = frozenset({408, 429}) + +# Primary-key fingerprint of the AMD ROCm signing key. The key is fetched over +# the network at build time and embedded as the package's trust anchor, so its +# identity is pinned here and verified after fetch. Rotating the repository +# signing key requires updating this value. +EXPECTED_KEY_FINGERPRINT = "D0F004A0025A1145C7807FCD0701EAC4D5E02107" + +# Accepted input shapes. rocm_version and repo_sub_folder are rendered into the +# generated repo files, package version fields, and (for the deb version string) +# other metadata, so they are constrained to characters that cannot alter the +# surrounding syntax. repo_base_url is concatenated into the repo files as well. +# Use \Z (not $) so a trailing newline is not accepted. +_VERSION_RE = re.compile(r"^[0-9][0-9A-Za-z.+~-]*\Z") +_SUB_FOLDER_RE = re.compile(r"^[0-9]{8}-[0-9A-Za-z._-]+\Z") + +# Release lines whose public repositories are signed. The nightly line is +# unsigned, so it ships no key and disables signature checking. +SIGNED_RELEASE_TYPES = ("prerelease", "release") +RELEASE_TYPES = ("prerelease", "release", "nightly") + +OS_PROFILES = { + "ubuntu2404": { + "family": "debian", + "pkg_type": "deb", + "description": "Ubuntu 24.04", + }, + "rhel8": { + "family": "rhel", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/yum.repos.d", + "description": "RHEL/Rocky/OL 8", + }, + "rhel10": { + "family": "rhel", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/yum.repos.d", + "description": "RHEL/Rocky/OL 10", + }, + "sles16": { + "family": "sles", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/zypp/repos.d", + "description": "SLES 16", + }, +} + + +# --- URL / signing derivation ------------------------------------------------ +# +# The public ROCm repositories do not share a single layout, so the repository +# URL is derived per release line: +# +# prerelease, release one repository per distro: +# // (deb) +# //x86_64/ (rpm) +# nightly one repository per package type, under a dated +# sub-folder: +# /deb// (deb) +# /rpm//x86_64/ (rpm) +# +# The signing key is at /gpg/rocm.gpg on every signed line. + + +def is_signed(release_type: str) -> bool: + """Whether this release line's public repository is signed.""" + return release_type in SIGNED_RELEASE_TYPES + + +def repo_baseurl( + repo_base_url: str, + pkg_type: str, + os_profile: str, + release_type: str, + repo_sub_folder: str = "", +) -> str: + """Return the repository baseurl baked into the repo file. + + The layout differs per release line (see the section comment above), so both + the target distro and the release line are required. deb returns the repo + root, beneath which apt resolves ``dists/stable/main/``; rpm returns the + directory containing ``repodata/``. ``repo_sub_folder`` is the dated segment + used by the nightly line. + """ + base = repo_base_url.rstrip("/") + # The layout is a property of the release line, not of whether it is signed; + # the two happen to correlate today but are independent. + if release_type == "nightly": + sub = f"{repo_sub_folder}/" if repo_sub_folder else "" + if pkg_type == "deb": + return f"{base}/deb/{sub}" + return f"{base}/rpm/{sub}x86_64/" + if pkg_type == "deb": + return f"{base}/{os_profile}/" + return f"{base}/{os_profile}/x86_64/" + + +def gpg_key_url(repo_base_url: str) -> str: + """Return the signing-key URL for a repo base url.""" + return f"{repo_base_url.rstrip('/')}/gpg/rocm.gpg" + + +def repo_metadata_url(baseurl: str, pkg_type: str) -> str: + """Return a repository index file that must exist under ``baseurl``. + + apt reads ``dists//Release``; dnf and zypper read + ``repodata/repomd.xml``. Fetching one proves the baseurl serves a repository. + """ + if pkg_type == "deb": + return f"{baseurl}dists/{DEB_SUITE}/Release" + return f"{baseurl}repodata/repomd.xml" + + +def verify_repo_url(baseurl: str, pkg_type: str) -> None: + """Fail the build if ``baseurl`` does not serve a package repository. + + The baseurl is baked into the package and is only exercised the first time a + user refreshes their package manager, so an unreachable one would otherwise + ship undetected. Transient failures are retried; a repository that is simply + not there fails immediately. + + The scheme is not re-checked here: this response is neither trusted nor + retained, and the signing key (which is) enforces https separately. + """ + url = repo_metadata_url(baseurl, pkg_type) + last_error: Exception | None = None + for attempt in range(1, REPO_VERIFY_ATTEMPTS + 1): + try: + print(f"Verifying repository (attempt {attempt}): {url}") + with urllib.request.urlopen(url, timeout=REPO_VERIFY_TIMEOUT_SEC) as resp: + status = getattr(resp, "status", None) or resp.getcode() + if status == 200: + print(f"Repository verified: {baseurl}") + return + last_error = RuntimeError(f"unexpected HTTP status {status}") + except urllib.error.HTTPError as e: + # A missing or forbidden index is a wrong URL, not a blip. Some + # object stores answer 403 rather than 404 for absent keys, so treat + # both as final; only rate limiting and timeouts are worth retrying. + if e.code not in RETRYABLE_HTTP_STATUS: + raise RuntimeError( + f"repository is not reachable at {baseurl} " + f"(checked {url}): HTTP {e.code}" + ) from e + last_error = e + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + last_error = e + if attempt < REPO_VERIFY_ATTEMPTS: + time.sleep(REPO_VERIFY_BACKOFF_SEC * attempt) + raise RuntimeError( + f"repository is not reachable at {baseurl} (checked {url}): {last_error}" + ) + + +def _require_https(url: str, what: str) -> None: + """Reject a non-https URL used to fetch trusted key material.""" + if urllib.parse.urlsplit(url).scheme != "https": + raise ValueError(f"{what} must use https to protect the key in transit: {url}") + + +def _valid_repo_base_url(url: str) -> bool: + """Whether a repo base URL is a well-formed http(s) URL with no control chars. + + The value is concatenated into the generated repo files, so whitespace or + control characters (e.g. a newline) that could inject extra directives are + rejected here. + """ + if any(c.isspace() or ord(c) < 0x20 for c in url): + return False + parts = urllib.parse.urlsplit(url) + return parts.scheme in ("http", "https") and bool(parts.netloc) + + +def rpm_version_release( + release_type: str, rocm_version: str, repo_sub_folder: str +) -> tuple[str, str]: + """Return (Version, Release) for the rpm package. + + Prerelease/release track the (rolling) ROCm version. Nightly is date-pinned + from the ``YYYYMMDD-`` sub-folder; an rpm Version cannot contain ``-``, + so the date becomes the Version and the id becomes the Release. + + The release line is part of the Release field so that two lines can never + produce an identically named package. Without it, installing the release + package over the prerelease one at the same ROCm version is a silent no-op: + the package manager reports success and leaves the old repository + configured. + """ + if release_type == "nightly": + date, _, ident = repo_sub_folder.partition("-") + return date, f"{ident or '1'}.nightly" + return rocm_version, f"1.{release_type}" + + +def deb_version(release_type: str, rocm_version: str, repo_sub_folder: str) -> str: + """Return the deb package version (nightly is date-pinned, no ``-``). + + Prerelease carries a ``~prerelease`` suffix. ``~`` sorts before the plain + version, so the release package upgrades over the prerelease one and the two + lines can never share a version (see ``rpm_version_release``). + """ + if release_type == "nightly": + return repo_sub_folder.replace("-", ".") + if release_type == "prerelease": + return f"{rocm_version}~prerelease" + return rocm_version + + +# --- signing key ------------------------------------------------------------- + + +def _fetch_signing_key(url: str) -> bytes: + """Fetch the armored signing key over https, size-bounded, with retries. + + Only transient network errors are retried; a scheme downgrade or an + over-size body fails immediately. + """ + # The fetched key is embedded and trusted, so it must travel over https. + _require_https(url, "signing key URL") + last_error: Exception | None = None + for attempt in range(1, GPG_FETCH_ATTEMPTS + 1): + try: + print(f"Fetching signing key (attempt {attempt}): {url}") + with urllib.request.urlopen(url, timeout=GPG_FETCH_TIMEOUT_SEC) as resp: + # urlopen follows redirects; re-check the scheme actually served + # so an https URL cannot be silently downgraded to http. + _require_https(resp.geturl(), "signing key URL (after redirects)") + # Read one byte past the cap to detect an over-size body. + key = resp.read(MAX_GPG_KEY_BYTES + 1) + if len(key) > MAX_GPG_KEY_BYTES: + raise ValueError( + f"signing key exceeds {MAX_GPG_KEY_BYTES} bytes: {url}" + ) + return key + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + last_error = e + if attempt < GPG_FETCH_ATTEMPTS: + time.sleep(GPG_FETCH_BACKOFF_SEC * attempt) + raise RuntimeError( + f"failed to fetch signing key after {GPG_FETCH_ATTEMPTS} attempts: {url}" + ) from last_error + + +def _run_gpg(argv: list[str], **kwargs) -> subprocess.CompletedProcess: + """Run gpg, reporting a missing binary by name instead of a traceback. + + Building any signed line needs gpg: the deb path to dearmor the keyring, and + both paths to check the key fingerprint. + """ + try: + return subprocess.run(argv, check=True, **kwargs) + except FileNotFoundError as e: + raise RuntimeError( + "gpg is required to build a signed release line " + "(install gnupg/gnupg2, or pass --gpg-key-file for an unsigned build)" + ) from e + + +def _key_fingerprint(armored: bytes) -> str: + """Return the primary-key fingerprint of an armored public key.""" + with tempfile.TemporaryDirectory() as home: + key_path = Path(home) / "key.asc" + key_path.write_bytes(armored) + result = _run_gpg( + ["gpg", "--homedir", home, "--with-colons", "--show-keys", str(key_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for line in result.stdout.decode(errors="replace").splitlines(): + if line.startswith("fpr:"): + return line.split(":")[9] + raise ValueError("no fingerprint found in the fetched signing key") + + +def _verify_key_fingerprint(armored: bytes) -> None: + """Reject a fetched key whose fingerprint is not the pinned AMD ROCm key.""" + fingerprint = _key_fingerprint(armored) + if fingerprint != EXPECTED_KEY_FINGERPRINT: + raise ValueError( + f"fetched signing key fingerprint {fingerprint} does not match the " + f"expected {EXPECTED_KEY_FINGERPRINT}" + ) + + +def load_signing_key(args: argparse.Namespace) -> bytes: + """Return the armored signing key bytes for a signed line. + + Read from ``--gpg-key-file`` when given (offline/test builds), otherwise + fetched from the repo's ``gpg/rocm.gpg`` at build time and verified against + the pinned fingerprint. Never written into the source tree. + """ + if args.gpg_key_file: + return args.gpg_key_file.read_bytes() + key = _fetch_signing_key(gpg_key_url(args.repo_base_url)) + _verify_key_fingerprint(key) + return key + + +def dearmor_key(armored: bytes) -> bytes: + """Convert an armored key to a binary keyring (for deb Signed-By).""" + result = _run_gpg( + ["gpg", "--dearmor"], + input=armored, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return result.stdout + + +# --- templating -------------------------------------------------------------- + + +def get_jinja_env() -> Environment: + # keep_trailing_newline so rendered config files end with a newline (POSIX + # text-file hygiene; Jinja strips the final newline by default). + return Environment( + loader=FileSystemLoader(str(SCRIPT_DIR)), + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + ) + + +def _render( + env: Environment, template_path: str, output_path: Path, context: dict +) -> None: + output_path.write_text(env.get_template(template_path).render(context)) + + +def build_context(args: argparse.Namespace, profile: dict) -> dict: + """Template fields shared by the repo file, spec, and deb metadata.""" + signed = is_signed(args.release_type) + return { + "pkg_name": PACKAGE_NAME, + "repo_id": REPO_ID, + "name": REPO_NAME, + "baseurl": repo_baseurl( + args.repo_base_url, + profile["pkg_type"], + args.os_profile, + args.release_type, + args.repo_sub_folder, + ), + "signed": signed, + "rocm_version": args.rocm_version, + "maintainer": MAINTAINER, + "description_short": "AMD ROCm repository configuration", + "description_long": ( + "Configures the system package manager to install AMD ROCm packages." + ), + "deb_keyring_path": DEB_KEYRING_PATH, + "rpm_gpg_key_path": RPM_GPG_KEY_PATH, + "rpm_repo_dir": profile.get("rpm_repo_dir", ""), + } + + +# --- rpm build --------------------------------------------------------------- + + +def build_rpm_package( + args: argparse.Namespace, profile: dict, context: dict, dest_dir: Path +) -> None: + env = get_jinja_env() + build_dir = dest_dir / "rpm" / PACKAGE_NAME + if build_dir.exists(): + shutil.rmtree(build_dir) + for subdir in ("SOURCES", "SPECS", "BUILD", "RPMS", "SRPMS"): + (build_dir / subdir).mkdir(parents=True, exist_ok=True) + sources_dir = build_dir / "SOURCES" + + _render( + env, + "template/repo/rpm/amdrocm.repo.j2", + sources_dir / f"{REPO_ID}.repo", + context, + ) + + if context["signed"]: + # rpm consumes the armored key as-is (gpgkey=file://). + (sources_dir / "RPM-GPG-KEY-rocm").write_bytes(load_signing_key(args)) + + version, release = rpm_version_release( + args.release_type, args.rocm_version, args.repo_sub_folder + ) + spec_context = { + **context, + "version": version, + "release": release, + "changelog_date": datetime.now(timezone.utc).strftime("%a %b %d %Y"), + } + spec_path = build_dir / "SPECS" / f"{PACKAGE_NAME}.spec" + _render(env, "template/repo/rpm/amdrocm-repo.spec.j2", spec_path, spec_context) + + print(f"Building rpm package: {PACKAGE_NAME} for {profile['description']}") + subprocess.run( + ["rpmbuild", "--define", f"_topdir {build_dir}", "-bb", str(spec_path)], + check=True, + ) + for rpm_file in (build_dir / "RPMS" / "noarch").glob("*.rpm"): + target = dest_dir / rpm_file.name + shutil.move(str(rpm_file), str(target)) + print(f"Package created: {target}") + + +# --- deb build --------------------------------------------------------------- + + +def build_deb_package( + args: argparse.Namespace, profile: dict, context: dict, dest_dir: Path +) -> None: + env = get_jinja_env() + package_dir = dest_dir / "deb" / PACKAGE_NAME + deb_dir = package_dir / "debian" + if package_dir.exists(): + shutil.rmtree(package_dir) + deb_dir.mkdir(parents=True, exist_ok=True) + + deb_context = { + **context, + "deb_version": deb_version( + args.release_type, args.rocm_version, args.repo_sub_folder + ), + "date": format_datetime(datetime.now(timezone.utc)), + } + + # Repository definition (deb822) and, for signed lines, the dearmored key, + # placed at the package root and mapped into the filesystem by debian/install. + _render( + env, + "template/repo/deb/amdrocm.sources.j2", + package_dir / f"{REPO_ID}.sources", + deb_context, + ) + if context["signed"]: + (package_dir / "rocm.gpg").write_bytes(dearmor_key(load_signing_key(args))) + + for name in ("control", "changelog", "rules", "postinst", "install"): + _render( + env, + f"template/repo/deb/{name}.j2", + deb_dir / name, + deb_context, + ) + (deb_dir / "rules").chmod(0o755) + (deb_dir / "postinst").chmod(0o755) + # Native source format: the package version carries no debian revision, so + # no separate upstream tarball is required. + (deb_dir / "source").mkdir(exist_ok=True) + (deb_dir / "source" / "format").write_text("3.0 (native)\n") + + print(f"Building deb package: {PACKAGE_NAME} for {profile['description']}") + subprocess.run( + ["dpkg-buildpackage", "-uc", "-us", "-b"], + cwd=package_dir, + check=True, + ) + # dpkg-buildpackage writes the .deb to the parent of the package directory. + for deb_file in package_dir.parent.glob("*.deb"): + target = dest_dir / deb_file.name + shutil.move(str(deb_file), str(target)) + print(f"Package created: {target}") + + +# --- CLI --------------------------------------------------------------------- + + +def list_profiles(pkg_type: str) -> list[str]: + """Return the OS profiles that build the given package type.""" + return [name for name, p in OS_PROFILES.items() if p["pkg_type"] == pkg_type] + + +def run_list_profiles(argv: list[str]) -> None: + """Print the OS profiles for a package type as a JSON array (CI matrix).""" + p = argparse.ArgumentParser( + prog="build_repo_package.py list-profiles", + description="Print the OS profiles for a package type as a JSON array.", + ) + p.add_argument("--pkg-type", required=True, choices=["deb", "rpm"]) + ns = p.parse_args(argv) + print(json.dumps(list_profiles(ns.pkg_type))) + + +def parse_args(argv=None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Build the amdrocm-repo package that configures ROCm repositories.", + ) + p.add_argument( + "--os-profile", + required=True, + choices=sorted(OS_PROFILES.keys()), + help="Target distro profile", + ) + p.add_argument( + "--release-type", + required=True, + choices=RELEASE_TYPES, + help="Release line to configure", + ) + p.add_argument( + "--repo-base-url", + required=True, + help="Public repository base URL (the packages-multi-arch base)", + ) + p.add_argument( + "--repo-sub-folder", + default="", + help="Dated sub-folder for the nightly line (YYYYMMDD-)", + ) + p.add_argument( + "--gpg-key-file", + type=Path, + default=None, + help="Signing key file to embed (defaults to fetching gpg/rocm.gpg from the repo)", + ) + p.add_argument( + "--rocm-version", + required=True, + help="ROCm version (e.g. 7.14.0)", + ) + p.add_argument( + "--dest-dir", + type=Path, + required=True, + help="Output directory for built packages", + ) + p.add_argument( + "--verify-repo-url", + action="store_true", + help=( + "Fail the build unless the configured repository is reachable. " + "No-op for the nightly line, whose dated sub-folder is published by " + "the same run that builds the package" + ), + ) + args = p.parse_args(argv) + if args.release_type == "nightly" and not args.repo_sub_folder: + p.error("--repo-sub-folder is required for --release-type nightly") + if args.repo_sub_folder and args.release_type != "nightly": + p.error("--repo-sub-folder is only valid for --release-type nightly") + # Validate values that are rendered into repo files, version fields, and + # maintainer metadata so they cannot alter the surrounding syntax. + if not _VERSION_RE.match(args.rocm_version): + p.error(f"--rocm-version has an unexpected format: {args.rocm_version!r}") + if not _valid_repo_base_url(args.repo_base_url): + p.error(f"--repo-base-url is not a valid http(s) URL: {args.repo_base_url!r}") + if args.repo_sub_folder and not _SUB_FOLDER_RE.match(args.repo_sub_folder): + p.error(f"--repo-sub-folder has an unexpected format: {args.repo_sub_folder!r}") + # rpmbuild requires an absolute _topdir; normalize so a relative --dest-dir works. + args.dest_dir = args.dest_dir.resolve() + return args + + +def main(argv=None) -> None: + if argv is None: + argv = sys.argv[1:] + if argv and argv[0] == "list-profiles": + run_list_profiles(argv[1:]) + return + args = parse_args(argv) + profile = OS_PROFILES[args.os_profile] + args.dest_dir.mkdir(parents=True, exist_ok=True) + context = build_context(args, profile) + # The nightly repository is published by the same run that builds this + # package, so its dated sub-folder does not exist yet and cannot be checked. + if args.verify_repo_url and args.release_type != "nightly": + verify_repo_url(context["baseurl"], profile["pkg_type"]) + if profile["pkg_type"] == "deb": + build_deb_package(args, profile, context, args.dest_dir) + else: + build_rpm_package(args, profile, context, args.dest_dir) + + +if __name__ == "__main__": + main() diff --git a/build_tools/packaging/linux/get_url_repo_params.py b/build_tools/packaging/linux/get_url_repo_params.py index 21b9c107f32..abfab2a35d5 100644 --- a/build_tools/packaging/linux/get_url_repo_params.py +++ b/build_tools/packaging/linux/get_url_repo_params.py @@ -15,6 +15,9 @@ get-repo-url Get full repo URL from components(release_type, native_package_type, repo_base_url, os_profile, repo_sub_folder). Prints repo_url=. extract-gfx-arch Extract and normalize GPU architecture from artifact group. Prints gfx_arch=. get-container-image Get container image for a given OS profile. Prints container_image=. + get-public-repo-base-url Get the public CDN base URL for a release line. Prints repo_base_url= (empty for ci/dev). + get-nightly-sub-folder Get the dated nightly sub-folder from a run id. Prints repo_sub_folder=YYYYMMDD-. + get-container-image-map Map a JSON array of OS profiles to container images. Prints images=. Usage: python build_tools/packaging/linux/get_url_repo_params.py get-base-url --from-url @@ -28,15 +31,17 @@ python build_tools/packaging/linux/get_url_repo_params.py get-base-url --from-url https://example.com/v2/whl python build_tools/packaging/linux/get_url_repo_params.py get-gpg-url --release-type prerelease --from-url https://rocm.prereleases.amd.com/packages/ubuntu2404 python build_tools/packaging/linux/get_url_repo_params.py get-repo-sub-folder --from-s3-prefix v3/packages/deb/20260204-12345 - python build_tools/packaging/linux/get_url_repo_params.py get-repo-url --release-type prerelease --native-package-type deb --repo-base-url https://x.com --os-profile ubuntu2404 --repo-sub-folder '' + python build_tools/packaging/linux/get_url_repo_params.py get-repo-url --release-type prerelease --native-package-type deb --repo-base-url https://example.com --os-profile ubuntu2404 --repo-sub-folder '' python build_tools/packaging/linux/get_url_repo_params.py extract-gfx-arch --artifact-group gfx94X-dcgpu python build_tools/packaging/linux/get_url_repo_params.py get-container-image --os-profile ubuntu2404 """ import argparse +import json import os import re import sys +from datetime import date, datetime, timezone from pathlib import Path from urllib.parse import urlparse @@ -145,13 +150,17 @@ def get_repo_url( ) -> str: """ Return the full repo URL for install tests. - - prerelease + deb: repo_base_url / os_profile - - prerelease + rpm: repo_base_url / os_profile / x86_64/ - - non-prerelease + deb: repo_base_url / deb / repo_sub_folder / - - non-prerelease + rpm: repo_base_url / rpm / repo_sub_folder / x86_64/ + + The public lines (prerelease, release) publish one repository per distro; + the per-run lines (dev, nightly, ci) publish one repository per package type + under a dated/run sub-folder. + - prerelease, release + deb: repo_base_url / os_profile + - prerelease, release + rpm: repo_base_url / os_profile / x86_64/ + - other + deb: repo_base_url / deb / repo_sub_folder / + - other + rpm: repo_base_url / rpm / repo_sub_folder / x86_64/ """ base = repo_base_url.rstrip("/") - if release_type == "prerelease": + if release_type in ("prerelease", "release"): if native_package_type == "deb": return f"{base}/{os_profile}" return f"{base}/{os_profile}/x86_64/" @@ -176,6 +185,73 @@ def cmd_repo_url(args: argparse.Namespace) -> int: return 0 +# --- public repository base URL --- + +# Public CDN base for each signed/served release line, using the flat +# packages-multi-arch layout (deb/, rpm/x86_64/, gpg/rocm.gpg beneath the base). +# Lines without a public repository (e.g. ci, dev) map to an empty string, which +# callers treat as "no amdrocm-repo package for this line". +_PUBLIC_REPO_BASE_URLS = { + "prerelease": "https://rocm.prereleases.amd.com/packages-multi-arch", + "release": "https://repo.amd.com/rocm/packages-multi-arch", + "nightly": "https://rocm.nightlies.amd.com/packages-multi-arch", +} + + +def get_public_repo_base_url(release_type: str) -> str: + """Return the public CDN base URL for a release line (empty if none).""" + return _PUBLIC_REPO_BASE_URLS.get(release_type.strip().lower(), "") + + +def cmd_public_repo_base_url(args: argparse.Namespace) -> int: + gha_set_output({"repo_base_url": get_public_repo_base_url(args.release_type)}) + return 0 + + +# --- nightly sub-folder --- + + +def nightly_sub_folder(run_id: str, *, today: date | None = None) -> str: + """Return the dated nightly sub-folder ``YYYYMMDD-``. + + ``today`` is injectable for deterministic tests; it defaults to the current + UTC date. + """ + day = today or datetime.now(timezone.utc).date() + return f"{day:%Y%m%d}-{run_id}" + + +def cmd_nightly_sub_folder(args: argparse.Namespace) -> int: + # Only the nightly line publishes into a dated sub-folder; every other line + # publishes at the repository root, so the value is empty there. + if args.release_type and args.release_type.strip().lower() != "nightly": + gha_set_output({"repo_sub_folder": ""}) + return 0 + gha_set_output({"repo_sub_folder": nightly_sub_folder(args.run_id)}) + return 0 + + +# --- container image map --- + + +def get_container_image_map(os_profiles: list[str]) -> dict[str, str]: + """Map each OS profile to its container image (for a build matrix).""" + return {profile: get_container_image(profile) for profile in os_profiles} + + +def cmd_container_image_map(args: argparse.Namespace) -> int: + try: + profiles = json.loads(args.os_profiles) + except json.JSONDecodeError as e: + print(f"Error: --os-profiles is not valid JSON: {e}", file=sys.stderr) + return 1 + if not isinstance(profiles, list) or not all(isinstance(p, str) for p in profiles): + print("Error: --os-profiles must be a JSON array of strings", file=sys.stderr) + return 1 + gha_set_output({"images": json.dumps(get_container_image_map(profiles))}) + return 0 + + # --- extract-gfx-arch --- @@ -387,6 +463,55 @@ def main(argv: list[str] | None = None) -> int: ) p_img.set_defaults(func=cmd_container_image) + # get-public-repo-base-url: public CDN base for a release line + p_pub = subparsers.add_parser( + "get-public-repo-base-url", + help="Print repo_base_url= for a release line (empty for lines without a public repo, e.g. ci/dev).", + ) + p_pub.add_argument( + "--release-type", + type=str, + required=True, + help="Release line (e.g. prerelease, release, nightly)", + ) + p_pub.set_defaults(func=cmd_public_repo_base_url) + + # get-nightly-sub-folder: dated sub-folder for the nightly line + p_sub = subparsers.add_parser( + "get-nightly-sub-folder", + help="Print repo_sub_folder=YYYYMMDD- for the nightly line.", + ) + p_sub.add_argument( + "--run-id", + type=str, + required=True, + help="Workflow run id to embed in the dated sub-folder", + ) + p_sub.add_argument( + "--release-type", + type=str, + default=None, + help=( + "If set, print an empty repo_sub_folder for any line other than " + "'nightly'. If omitted, always print the dated sub-folder." + ), + ) + p_sub.set_defaults(func=cmd_nightly_sub_folder) + + # get-container-image-map: os_profile -> image map for a build matrix + p_map = subparsers.add_parser( + "get-container-image-map", + help="Print images= mapping each OS profile (JSON array input) to its container image.", + ) + p_map.add_argument( + "--os-profiles", + type=str, + required=True, + metavar="JSON", + help='JSON array of OS profiles (e.g. ["rhel8","rhel10","sles16"])', + ) + p_map.set_defaults(func=cmd_container_image_map) + args = parser.parse_args(argv) return args.func(args) diff --git a/build_tools/packaging/linux/native_linux_package_install_test.py b/build_tools/packaging/linux/native_linux_package_install_test.py index a257f4bffdb..36c074513f5 100644 --- a/build_tools/packaging/linux/native_linux_package_install_test.py +++ b/build_tools/packaging/linux/native_linux_package_install_test.py @@ -136,6 +136,7 @@ import re import subprocess import sys +import time import traceback from argparse import ArgumentParser, Namespace from pathlib import Path @@ -174,6 +175,16 @@ def _env(key: str, default: str) -> str: # Relative path from install prefix to rdhc binary (script); overridable via ROCM_RDHC_REL_PATH RDHC_REL_PATH = _env("ROCM_RDHC_REL_PATH", "libexec/rocm-core/rdhc.py") +# amdrocm-repo package: the on-disk paths it installs (the package's public +# contract). Used by the install-via-package mode to confirm the repo config +# landed. These are independent of REPO_NAME (which names the manually-written +# repo file used by the default setup path). +AMDROCM_REPO_ID = "amdrocm" +AMDROCM_DEB_SOURCES = "/etc/apt/sources.list.d/amdrocm.sources" +AMDROCM_DEB_KEYRING = "/usr/share/keyrings/rocm.gpg" +AMDROCM_RPM_REPO_BASENAME = "amdrocm.repo" +AMDROCM_RPM_KEY = "/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" + # Pytest/CI only: becomes ``--rocm-version``. ENV_NATIVE_LINUX_INSTALL_ROCM_VERSION = "NATIVE_LINUX_INSTALL_ROCM_VERSION" @@ -184,6 +195,12 @@ def _env(key: str, default: str) -> str: ZYPP_CLEAN_TIMEOUT_SEC = 60 ZYPP_REFRESH_TIMEOUT_SEC = 120 DNF_CLEAN_TIMEOUT_SEC = 60 +DNF_MAKECACHE_TIMEOUT_SEC = 120 +REPO_PACKAGE_INSTALL_TIMEOUT_SEC = 120 +# Retry the post-install metadata refresh so a transient repo/CDN blip does not +# fail the (network-dependent) config-only gate. +REPO_REFRESH_ATTEMPTS = 3 +REPO_REFRESH_BACKOFF_SEC = 3 INSTALL_TIMEOUT_SEC = 1800 # 30 minutes ROCMINFO_TIMEOUT_SEC = 30 # rdhc.py ``--all`` runs the full ROCm deployment health check suite; 30s was too @@ -365,11 +382,14 @@ def __init__( gfx_arch: str | list[str] | None = None, rocm_version: str | None = None, gpg_key_url: str | None = None, + repo_package_dir: str | None = None, + repo_config_only: bool = False, ): """Initialize the native Linux package install test runner. Args: - repo_url: Full repository URL (constructed in YAML) + repo_url: Full repository URL (constructed in YAML). May be empty/None when + repo_package_dir is set (the amdrocm-repo package supplies the repo URL). os_profile: OS profile (e.g., ubuntu2404, rhel8, debian12, sles15, sles16, almalinux9, centos7, azl3) release_type: Type of release ('nightly' or 'prerelease') install_prefix: Installation prefix (default: /opt/rocm/core) @@ -382,11 +402,20 @@ def __init__( If unset: unversioned ``amdrocm`` / ``amdrocm-core-sdk`` (``gfx_arch`` alone does not add arch suffixes). gpg_key_url: GPG key URL + repo_package_dir: Directory holding the built ``amdrocm-repo`` package. When + set, the repository is configured by installing that package instead of + writing repo files directly. + repo_config_only: With ``repo_package_dir``, install the package and verify + the repo config, then stop (do not install ROCm). """ self.os_profile = os_profile.lower() self.package_type = self._derive_package_type(os_profile) - self.repo_url = repo_url.rstrip("/") + # repo_url may be empty/None in install-via-package mode (the package + # supplies it); tolerate both without raising. + self.repo_url = (repo_url or "").rstrip("/") self.release_type = release_type.lower() + self.repo_package_dir = repo_package_dir + self.repo_config_only = repo_config_only self.install_prefix = install_prefix self.gfx_arch_list = normalize_target_list( gfx_arch, lowercase=True, dedupe=True @@ -830,6 +859,184 @@ def install_rpm_packages(self) -> bool: print(f"\n[FAIL] Error during installation: {e}") return False + def _find_repo_package(self) -> Path | None: + """Locate the built amdrocm-repo package in repo_package_dir. + + Globs in Python (no shell) and requires exactly one match for this + package type. Returns the resolved path, or None on error. + """ + directory = Path(self.repo_package_dir) + if not directory.is_dir(): + print( + f"[FAIL] --repo-package-dir is not a directory: {self.repo_package_dir}" + ) + return None + pattern = ( + "amdrocm-repo*.deb" if self.package_type == "deb" else "amdrocm-repo*.rpm" + ) + matches = sorted(directory.glob(pattern)) + if not matches: + print(f"[FAIL] No {pattern} found in {self.repo_package_dir}") + return None + if len(matches) > 1: + print( + f"[FAIL] Multiple {pattern} found in {self.repo_package_dir}: {matches}" + ) + return None + return matches[0].resolve() + + def install_repo_package(self) -> bool: + """Install the amdrocm-repo package, then refresh the repo it configures. + + The package itself is unsigned, so the local install skips package + signature checks; the repository the package configures keeps its own + gpgcheck setting. The metadata refresh is scoped to the amdrocm repo. + + Returns: + True if the package installed and the repo metadata refreshed. + """ + print("\n" + "=" * 80) + print("CONFIGURING REPOSITORY VIA amdrocm-repo PACKAGE") + print("=" * 80) + + pkg = self._find_repo_package() + if pkg is None: + return False + print(f"\nRepo package: {pkg}") + + if self.package_type == "deb": + install_cmd = ["sudo", "apt", "install", "-y", str(pkg)] + elif self._is_sles(): + install_cmd = [ + "zypper", + "--non-interactive", + "--no-gpg-checks", + "install", + "-y", + str(pkg), + ] + else: + install_cmd = ["dnf", "install", "-y", "--nogpgcheck", str(pkg)] + + print(f"\nInstalling repo package: {' '.join(install_cmd)}") + try: + rc = _run_streaming(install_cmd, REPO_PACKAGE_INSTALL_TIMEOUT_SEC) + except subprocess.TimeoutExpired: + print("[FAIL] Installing amdrocm-repo package timed out") + return False + except OSError as e: + print(f"[FAIL] Error installing amdrocm-repo package: {e}") + return False + if rc != 0: + print(f"[FAIL] Failed to install amdrocm-repo package (exit code: {rc})") + return False + print("[PASS] amdrocm-repo package installed") + + # Refresh metadata for the repo the package just configured. The dnf and + # zypper refreshes are scoped to the amdrocm repo; apt has no clean + # per-source refresh, so the deb path runs a repo-wide `apt update` + # (a failing unrelated base-image source would also fail this step). + if self.package_type == "deb": + refresh_cmd = ["sudo", "apt", "update"] + refresh_timeout = APT_UPDATE_TIMEOUT_SEC + elif self._is_sles(): + refresh_cmd = [ + "zypper", + "--non-interactive", + "--gpg-auto-import-keys", + "refresh", + AMDROCM_REPO_ID, + ] + refresh_timeout = ZYPP_REFRESH_TIMEOUT_SEC + else: + refresh_cmd = [ + "dnf", + "makecache", + "--disablerepo=*", + f"--enablerepo={AMDROCM_REPO_ID}", + ] + refresh_timeout = DNF_MAKECACHE_TIMEOUT_SEC + + print(f"\nRefreshing repository metadata: {' '.join(refresh_cmd)}") + for attempt in range(1, REPO_REFRESH_ATTEMPTS + 1): + try: + rc = _run_streaming(refresh_cmd, refresh_timeout) + except subprocess.TimeoutExpired: + rc = None + print("[WARN] Repository metadata refresh timed out") + except OSError as e: + rc = None + print(f"[WARN] Error refreshing repository metadata: {e}") + if rc == 0: + print("[PASS] Repository metadata refreshed") + return True + if attempt < REPO_REFRESH_ATTEMPTS: + time.sleep(REPO_REFRESH_BACKOFF_SEC * attempt) + print( + f"Retrying repository metadata refresh " + f"({attempt + 1}/{REPO_REFRESH_ATTEMPTS})..." + ) + print("[FAIL] Failed to refresh repository metadata") + return False + + def assert_repo_configured(self) -> bool: + """Verify the amdrocm-repo package dropped the expected repo config. + + Checks the repo file exists (and looks like a ROCm repo) and, for signed + release lines, that the signing key was installed too. + + Returns: + True if the on-disk repo configuration is present and well-formed. + """ + print("\n" + "=" * 80) + print("VERIFYING REPOSITORY CONFIGURATION") + print("=" * 80) + + signed = self.release_type in ("prerelease", "release") + ok = True + + if self.package_type == "deb": + repo_file = Path(AMDROCM_DEB_SOURCES) + key_file = Path(AMDROCM_DEB_KEYRING) + # deb822 marker (independent of the repo host, which need not contain + # "rocm" for a mirror). + marker = "URIs:" + else: + repo_dir = ZYPP_REPOS_DIR if self._is_sles() else YUM_REPOS_DIR + repo_file = Path(repo_dir) / AMDROCM_RPM_REPO_BASENAME + key_file = Path(AMDROCM_RPM_KEY) + marker = "baseurl=" + + if repo_file.is_file(): + content = repo_file.read_text(encoding="utf-8", errors="replace") + print(f"[PASS] Repo file present: {repo_file}") + print(content) + # Match the marker at the start of a line so a file that only + # mentions it in a comment is not accepted as configured. + if not any(line.startswith(marker) for line in content.splitlines()): + print( + f"[FAIL] Repo file missing expected {marker!r} entry: {repo_file}" + ) + ok = False + else: + print(f"[FAIL] Repo file not found: {repo_file}") + ok = False + + if signed: + if key_file.is_file(): + print(f"[PASS] Signing key present: {key_file}") + else: + print(f"[FAIL] Signing key not found (signed line): {key_file}") + ok = False + else: + print("[INFO] Unsigned line: no signing key expected") + + if ok: + print("\n[PASS] Repository configuration verified") + else: + print("\n[FAIL] Repository configuration verification failed") + return ok + def run_repo_setup_and_install(self) -> bool: """Step 1: Repo setup and install. Run for both sanity (basic) and full test. @@ -861,9 +1068,23 @@ def run_repo_setup_and_install(self) -> bool: print( "GPU Architecture(s): (none — generic packages amdrocm, amdrocm-core-sdk)" ) - print(f"Repository URL: {self.repo_url}") + print(f"Repository URL: {self.repo_url or '(from amdrocm-repo package)'}") print(f"Packages (in order): {self.package_names}") + # Install-via-package mode: the amdrocm-repo package configures the repo. + if self.repo_package_dir: + if not self.install_repo_package(): + return False + if not self.assert_repo_configured(): + return False + if self.repo_config_only: + print("\n[PASS] Repo configured via amdrocm-repo package (config-only)") + return True + if self.package_type == "deb": + return self.install_deb_packages() + return self.install_rpm_packages() + + # Default: configure the repo directly, then install. if self.package_type == "deb": if not self.setup_deb_repository(): return False @@ -886,6 +1107,12 @@ def run_basic_verification(self) -> bool: print("STEP 2: BASIC INSTALL VERIFICATION") print("=" * 80) + if not self.install_prefix: + # Reachable when a non-config-only run omits --install-prefix; report + # the missing option instead of failing inside Path(). + print("\n[FAIL] --install-prefix is required to verify an installation") + return False + install_path = Path(self.install_prefix) if not install_path.exists(): print(f"\n[FAIL] Installation directory not found: {self.install_prefix}") @@ -1194,6 +1421,20 @@ def _build_argument_parser(*, exit_on_error: bool = True) -> ArgumentParser: metavar="DIR", help="Directory containing .deb or .rpm files. Required when --test-type is 'simulate'.", ) + parser.add_argument( + "--repo-package-dir", + type=str, + metavar="DIR", + help="Directory containing the built amdrocm-repo .deb/.rpm. When set, the repository is " + "configured by installing that package (instead of writing repo files); --repo-url and " + "--gpg-key-url become optional. Not valid with --test-type simulate.", + ) + parser.add_argument( + "--repo-config-only", + action="store_true", + help="With --repo-package-dir: install the amdrocm-repo package, verify the repo config, " + "and stop (do not install ROCm). --install-prefix is not required.", + ) parser.add_argument( "--pkg-type", type=str, @@ -1205,6 +1446,8 @@ def _build_argument_parser(*, exit_on_error: bool = True) -> ArgumentParser: def _validate_cli_args(parser: ArgumentParser, args: Namespace) -> None: if args.test_type == "simulate": + if args.repo_package_dir: + parser.error("--repo-package-dir is not valid with --test-type 'simulate'") if not args.packages_dir: parser.error("--packages-dir is required when --test-type is 'simulate'") if not args.pkg_type and not args.os_profile: @@ -1217,13 +1460,17 @@ def _validate_cli_args(parser: ArgumentParser, args: Namespace) -> None: except ValueError as e: parser.error(str(e)) return + if args.repo_config_only and not args.repo_package_dir: + parser.error("--repo-config-only requires --repo-package-dir") if not args.os_profile: parser.error( "--os-profile is required when --test-type is 'install', 'sanity', or 'full'" ) - if not args.repo_url: + # In install-via-package mode the amdrocm-repo package supplies the repo URL. + if not args.repo_url and not args.repo_package_dir: parser.error( - "--repo-url is required when --test-type is 'install', 'sanity', or 'full'" + "--repo-url is required when --test-type is 'install', 'sanity', or 'full' " + "(or provide --repo-package-dir to configure the repo via the amdrocm-repo package)" ) if args.rocm_version: try: @@ -1288,7 +1535,10 @@ def run_tests(args: Namespace) -> int: print(f"OS Profile: {args.os_profile}") print(f"Package Type (derived): {derived_package_type}") print(f"Release Type: {args.release_type}") - print(f"Repository URL: {args.repo_url}") + if args.repo_package_dir: + print(f"Repo package dir: {args.repo_package_dir}") + print(f"Repo config only: {args.repo_config_only}") + print(f"Repository URL: {args.repo_url or '(from amdrocm-repo package)'}") # Preview package-name rules before NativeLinuxPackageInstallTest is constructed. _norm = normalize_target_list(args.gfx_arch, lowercase=True, dedupe=True) if _norm: @@ -1331,12 +1581,14 @@ def run_tests(args: Namespace) -> int: test_runner = NativeLinuxPackageInstallTest( os_profile=args.os_profile, - repo_url=args.repo_url, + repo_url=args.repo_url or "", release_type=args.release_type, install_prefix=args.install_prefix, gfx_arch=args.gfx_arch, rocm_version=args.rocm_version, gpg_key_url=args.gpg_key_url, + repo_package_dir=args.repo_package_dir, + repo_config_only=args.repo_config_only, ) print("\n" + "=" * 80) @@ -1351,6 +1603,12 @@ def run_tests(args: Namespace) -> int: if not test_runner.run_repo_setup_and_install(): print("\n[FAIL] Step 1 (repo setup and install) failed.") return 1 + if args.repo_config_only: + print("\n" + "=" * 80) + print("[PASS] INSTALLATION TEST PASSED") + print("(repo-config-only: amdrocm-repo package configured the repo)") + print("=" * 80 + "\n") + return 0 if args.test_type == "install": print("\n" + "=" * 80) print("[PASS] INSTALLATION TEST PASSED") @@ -1381,7 +1639,10 @@ def run_tests(args: Namespace) -> int: def _argv_from_ci_env() -> list[str] | None: """Build CLI argv from workflow/container env (see ``test_native_linux_packages_install.yml``). - Required for sanity/full: OS_PROFILE, REPO_URL, RELEASE_TYPE, INSTALL_PREFIX. + Required for sanity/full: OS_PROFILE, RELEASE_TYPE, plus REPO_URL and + INSTALL_PREFIX — except in package mode (REPO_PACKAGE_DIR set), where the + repo is configured by the installed package so REPO_URL is not needed, and + when REPO_CONFIG_ONLY is set INSTALL_PREFIX is not needed either. Optional: GFX_ARCH, GPG_KEY_URL; ``NATIVE_LINUX_INSTALL_ROCM_VERSION`` maps to ``--rocm-version`` only when versioned package names are needed (omit for unversioned installs). """ @@ -1416,8 +1677,22 @@ def _argv_from_ci_env() -> list[str] | None: gfx_arch = gfx_raw.replace(";", " ").split() if gfx_raw else [] release_type = (os.environ.get("RELEASE_TYPE") or "").strip() install_prefix = (os.environ.get("INSTALL_PREFIX") or "").strip() + repo_package_dir = (os.environ.get("REPO_PACKAGE_DIR") or "").strip() + repo_config_only = (os.environ.get("REPO_CONFIG_ONLY") or "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) - if not (os_profile and repo_url and release_type and install_prefix): + # In install-via-package mode the amdrocm-repo package supplies the repo URL; + # in config-only mode there is no install/verify, so INSTALL_PREFIX is not + # needed either. + if not (os_profile and release_type): + return None + if not repo_url and not repo_package_dir: + return None + if not install_prefix and not repo_config_only: return None argv = [ @@ -1425,13 +1700,17 @@ def _argv_from_ci_env() -> list[str] | None: test_type, "--os-profile", os_profile, - "--repo-url", - repo_url, "--release-type", release_type, - "--install-prefix", - install_prefix, ] + if repo_url: + argv.extend(["--repo-url", repo_url]) + if install_prefix: + argv.extend(["--install-prefix", install_prefix]) + if repo_package_dir: + argv.extend(["--repo-package-dir", repo_package_dir]) + if repo_config_only: + argv.append("--repo-config-only") if gfx_arch: argv.extend(["--gfx-arch", *gfx_arch]) rocm_version = (os.environ.get(ENV_NATIVE_LINUX_INSTALL_ROCM_VERSION) or "").strip() @@ -1452,11 +1731,14 @@ def test_native_linux_package_install() -> None: if os.environ.get("GITHUB_ACTIONS") == "true": pytest.fail( "Missing required environment variables for native install test " - "(expected OS_PROFILE, REPO_URL, RELEASE_TYPE, INSTALL_PREFIX; " - "optional GFX_ARCH, NATIVE_LINUX_INSTALL_ROCM_VERSION; or for simulate: PACKAGES_DIR)." + "(expected OS_PROFILE, RELEASE_TYPE, plus REPO_URL and INSTALL_PREFIX " + "unless REPO_PACKAGE_DIR is set, and INSTALL_PREFIX is not needed with " + "REPO_CONFIG_ONLY; optional GFX_ARCH, NATIVE_LINUX_INSTALL_ROCM_VERSION; " + "or for simulate: PACKAGES_DIR)." ) pytest.skip( - "Set workflow env vars (OS_PROFILE, REPO_URL, RELEASE_TYPE, INSTALL_PREFIX); " + "Set workflow env vars (OS_PROFILE, RELEASE_TYPE, plus REPO_URL/INSTALL_PREFIX " + "unless REPO_PACKAGE_DIR/REPO_CONFIG_ONLY is set); " "optional GFX_ARCH and NATIVE_LINUX_INSTALL_ROCM_VERSION." ) diff --git a/build_tools/packaging/linux/publish_repo_package.py b/build_tools/packaging/linux/publish_repo_package.py new file mode 100644 index 00000000000..f1eeeb51114 --- /dev/null +++ b/build_tools/packaging/linux/publish_repo_package.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Publish a built amdrocm-repo package to S3 as a standalone per-distro file. + +The package is uploaded next to the native content packages, as a +``repo//`` sibling of the per-run ``packages//`` prefix, +rather than into the repository index. A client can then fetch it directly by +URL to configure the repository. The file becomes publicly reachable once the +release promotion step copies the per-run tree to the public CDN. + +The object name is fixed (``amdrocm-repo.``) so the download URL is stable +regardless of the built package's versioned filename. + +Usage: + python build_tools/packaging/linux/publish_repo_package.py \ + --file repo-package-out/amdrocm-repo-7.14.0-1.el10.noarch.rpm \ + --bucket therock-prerelease-artifacts \ + --prefix 12345-linux/packages/rpm \ + --os-profile rhel10 \ + --pkg-type rpm +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +# Fixed published object name; the extension is the package type. +PUBLISHED_NAME = "amdrocm-repo" + +# An os-profile becomes a path segment in the S3 key, so restrict it to a safe +# set of characters (no slashes, whitespace, or control characters). \Z (not $) +# so a trailing newline is not accepted. +_OS_PROFILE_RE = re.compile(r"^[A-Za-z0-9._-]+\Z") + + +def object_key(prefix: str, os_profile: str, pkg_type: str) -> str: + """Return the S3 key for the published bootstrap file. + + ``{prefix}/repo/{os_profile}/amdrocm-repo.{ext}`` — a sibling of the content + repository under the same per-run prefix, so it is not part of the repo + index (``dists/`` for deb, ``x86_64/repodata`` for rpm). + """ + return f"{prefix.rstrip('/')}/repo/{os_profile}/{PUBLISHED_NAME}.{pkg_type}" + + +def parse_args(argv=None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Publish an amdrocm-repo package to S3.", + ) + p.add_argument("--file", required=True, type=Path, help="Built package to upload") + p.add_argument("--bucket", required=True, help="Destination S3 bucket") + p.add_argument( + "--prefix", + required=True, + help="Per-run S3 prefix, e.g. -linux/packages/", + ) + p.add_argument( + "--os-profile", required=True, help="Target distro profile (path segment)" + ) + p.add_argument( + "--pkg-type", + required=True, + choices=["deb", "rpm"], + help="Package type, used as the published file extension", + ) + p.add_argument( + "--endpoint-url", + default=None, + help="S3 endpoint override (for testing against a local server)", + ) + args = p.parse_args(argv) + # Fail loudly rather than silently uploading nowhere if the bucket is empty + # (e.g. an unresolved release line without upload credentials). + if not args.bucket.strip(): + p.error("--bucket must be a non-empty bucket name") + if not _OS_PROFILE_RE.match(args.os_profile): + p.error(f"--os-profile has an unexpected format: {args.os_profile!r}") + if not args.file.is_file(): + p.error(f"--file is not a file: {args.file}") + return args + + +def publish(args: argparse.Namespace) -> str: + """Upload the file and confirm it landed. Returns the object key.""" + # Deferred import: boto3 is only needed for the actual upload, so key + # derivation stays importable without it. + import boto3 + + key = object_key(args.prefix, args.os_profile, args.pkg_type) + s3 = boto3.client("s3", endpoint_url=args.endpoint_url) + print(f"Uploading {args.file} -> s3://{args.bucket}/{key}") + s3.upload_file(str(args.file), args.bucket, key) + # Confirm the object exists so a job bounded by continue-on-error cannot + # silently succeed after a failed upload. + s3.head_object(Bucket=args.bucket, Key=key) + print(f"Published: s3://{args.bucket}/{key}") + return key + + +def main(argv=None) -> None: + publish(parse_args(argv)) + + +if __name__ == "__main__": + main() diff --git a/build_tools/packaging/linux/setup_repo_build_deps.sh b/build_tools/packaging/linux/setup_repo_build_deps.sh new file mode 100755 index 00000000000..eae3d2cc311 --- /dev/null +++ b/build_tools/packaging/linux/setup_repo_build_deps.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Install the system tools needed to build the amdrocm-repo package. +# +# Package mapping by --os-profile: +# +# - ubuntu* / debian* -> apt: debhelper-compat, dpkg-dev, build-essential +# - sles* -> zypper: rpm-build +# - else (RHEL-like) -> dnf: rpm-build +# +# gpg is installed for every profile: signed release lines need it to dearmor +# the deb keyring and to check the signing key fingerprint. +# +# Python is installed separately by setup_python_cmd.sh. +# +# Sample usage +# ------------ +# +# bash build_tools/packaging/linux/setup_repo_build_deps.sh --os-profile rhel10 + +set -euo pipefail + +OS_PROFILE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --os-profile) + OS_PROFILE="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$OS_PROFILE" ]]; then + echo "Error: --os-profile is required" >&2 + exit 1 +fi + +# Trim whitespace; lowercase for glob matching (e.g. SLES16 must match sles*) +OS_PROFILE="${OS_PROFILE#"${OS_PROFILE%%[![:space:]]*}"}" +OS_PROFILE="${OS_PROFILE%"${OS_PROFILE##*[![:space:]]}"}" +OS_PLC="${OS_PROFILE,,}" + +if [[ "$OS_PLC" == ubuntu* ]] || [[ "$OS_PLC" == debian* ]]; then + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update + sudo apt-get install -y \ + ca-certificates \ + gnupg \ + debhelper-compat \ + dpkg-dev \ + build-essential +elif [[ "$OS_PLC" == sles* ]]; then + zypper --non-interactive refresh + zypper --non-interactive install -y \ + ca-certificates \ + gpg2 \ + rpm-build +else + dnf install -y --allowerasing \ + ca-certificates \ + gnupg2 \ + rpm-build +fi diff --git a/build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 b/build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 new file mode 100644 index 00000000000..8f6cab95786 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 @@ -0,0 +1,10 @@ +Types: deb +URIs: {{ baseurl }} +Suites: stable +Components: main +Architectures: amd64 +{% if signed %} +Signed-By: {{ deb_keyring_path }} +{% else %} +Trusted: yes +{% endif %} diff --git a/build_tools/packaging/linux/template/repo/deb/changelog.j2 b/build_tools/packaging/linux/template/repo/deb/changelog.j2 new file mode 100644 index 00000000000..071194aa824 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/changelog.j2 @@ -0,0 +1,5 @@ +{{ pkg_name }} ({{ deb_version }}) stable; urgency=medium + + * ROCm {{ rocm_version }} repository configuration. + + -- {{ maintainer }} {{ date }} diff --git a/build_tools/packaging/linux/template/repo/deb/control.j2 b/build_tools/packaging/linux/template/repo/deb/control.j2 new file mode 100644 index 00000000000..3132c8d4da0 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/control.j2 @@ -0,0 +1,13 @@ +Source: {{ pkg_name }} +Section: admin +Priority: optional +Maintainer: {{ maintainer }} +Build-Depends: debhelper-compat (= 13) +Standards-Version: 4.6.2 +Homepage: https://github.com/ROCm/TheRock + +Package: {{ pkg_name }} +Architecture: all +Depends: ${misc:Depends} +Description: {{ description_short }} + {{ description_long }} diff --git a/build_tools/packaging/linux/template/repo/deb/install.j2 b/build_tools/packaging/linux/template/repo/deb/install.j2 new file mode 100644 index 00000000000..ca5871934ab --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/install.j2 @@ -0,0 +1,4 @@ +{{ repo_id }}.sources /etc/apt/sources.list.d/ +{% if signed %} +rocm.gpg /usr/share/keyrings/ +{% endif %} diff --git a/build_tools/packaging/linux/template/repo/deb/postinst.j2 b/build_tools/packaging/linux/template/repo/deb/postinst.j2 new file mode 100644 index 00000000000..90ad1d98f67 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/postinst.j2 @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +if [ "$1" = "configure" ]; then + echo "AMD ROCm repository configured." + echo "Run 'sudo apt-get update' to refresh package lists." +fi + +#DEBHELPER# + +exit 0 diff --git a/build_tools/packaging/linux/template/repo/deb/rules.j2 b/build_tools/packaging/linux/template/repo/deb/rules.j2 new file mode 100644 index 00000000000..cbe925d7587 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/rules.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/make -f +%: + dh $@ diff --git a/build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 b/build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 new file mode 100644 index 00000000000..55dd2436774 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 @@ -0,0 +1,29 @@ +Name: {{ pkg_name }} +Version: {{ version }} +Release: {{ release }}%{?dist} +BuildArch: noarch +Summary: {{ description_short }} +License: MIT +Vendor: Advanced Micro Devices, Inc +URL: https://github.com/ROCm/TheRock + +%description +{{ description_long }} + +%install +mkdir -p %{buildroot}{{ rpm_repo_dir }} +install -m 644 %{_sourcedir}/{{ repo_id }}.repo %{buildroot}{{ rpm_repo_dir }}/ +{% if signed %} +mkdir -p %{buildroot}/etc/pki/rpm-gpg +install -m 644 %{_sourcedir}/RPM-GPG-KEY-rocm %{buildroot}{{ rpm_gpg_key_path }} +{% endif %} + +%files +%config(noreplace) {{ rpm_repo_dir }}/{{ repo_id }}.repo +{% if signed %} +{{ rpm_gpg_key_path }} +{% endif %} + +%changelog +* {{ changelog_date }} {{ maintainer }} - {{ version }}-{{ release }} +- ROCm {{ rocm_version }} repository configuration. diff --git a/build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 b/build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 new file mode 100644 index 00000000000..1d34ea91c72 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 @@ -0,0 +1,8 @@ +[{{ repo_id }}] +name={{ name }} +baseurl={{ baseurl }} +enabled=1 +gpgcheck={{ 1 if signed else 0 }} +{% if signed %} +gpgkey=file://{{ rpm_gpg_key_path }} +{% endif %} diff --git a/build_tools/packaging/linux/tests/get_url_repo_params_test.py b/build_tools/packaging/linux/tests/get_url_repo_params_test.py index 57ed8562784..6fc6742daa2 100644 --- a/build_tools/packaging/linux/tests/get_url_repo_params_test.py +++ b/build_tools/packaging/linux/tests/get_url_repo_params_test.py @@ -4,12 +4,15 @@ # Unit test coverage for get_url_repo_params.py: # get_base_url, get_gpg_key_url, gpg_key_url_needed_for_release_type, get_repo_sub_folder, -# get_repo_url, extract_gfx_arch, and main() subcommands. +# get_repo_url, extract_gfx_arch, get_public_repo_base_url, nightly_sub_folder, +# get_container_image_map, and main() subcommands. +import json import os import sys import tempfile import unittest +from datetime import date from pathlib import Path from unittest.mock import patch @@ -167,11 +170,11 @@ def test_prerelease_deb(self): get_url_repo_params.get_repo_url( release_type="prerelease", native_package_type="deb", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="ubuntu2404", repo_sub_folder="", ), - "https://x.com/ubuntu2404", + "https://example.com/ubuntu2404", ) def test_prerelease_rpm(self): @@ -180,11 +183,38 @@ def test_prerelease_rpm(self): get_url_repo_params.get_repo_url( release_type="prerelease", native_package_type="rpm", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="rhel8", repo_sub_folder="", ), - "https://x.com/rhel8/x86_64/", + "https://example.com/rhel8/x86_64/", + ) + + def test_release_deb(self): + # Test that release + deb yields base/os_profile. The release repository + # is served per distro, not as a flat deb/ tree. + self.assertEqual( + get_url_repo_params.get_repo_url( + release_type="release", + native_package_type="deb", + repo_base_url="https://example.com", + os_profile="ubuntu2404", + repo_sub_folder="", + ), + "https://example.com/ubuntu2404", + ) + + def test_release_rpm(self): + # Test that release + rpm yields base/os_profile/x86_64/ + self.assertEqual( + get_url_repo_params.get_repo_url( + release_type="release", + native_package_type="rpm", + repo_base_url="https://example.com", + os_profile="rhel10", + repo_sub_folder="", + ), + "https://example.com/rhel10/x86_64/", ) def test_nightly_deb(self): @@ -193,11 +223,11 @@ def test_nightly_deb(self): get_url_repo_params.get_repo_url( release_type="nightly", native_package_type="deb", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="ubuntu2404", repo_sub_folder="20260204-12345", ), - "https://x.com/deb/20260204-12345/", + "https://example.com/deb/20260204-12345/", ) def test_nightly_rpm(self): @@ -206,11 +236,11 @@ def test_nightly_rpm(self): get_url_repo_params.get_repo_url( release_type="nightly", native_package_type="rpm", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="rhel8", repo_sub_folder="20260204-12345", ), - "https://x.com/rpm/20260204-12345/x86_64/", + "https://example.com/rpm/20260204-12345/x86_64/", ) def test_strips_trailing_slash_from_base(self): @@ -219,11 +249,11 @@ def test_strips_trailing_slash_from_base(self): get_url_repo_params.get_repo_url( release_type="prerelease", native_package_type="deb", - repo_base_url="https://x.com/", + repo_base_url="https://example.com/", os_profile="ubuntu2404", repo_sub_folder="", ), - "https://x.com/ubuntu2404", + "https://example.com/ubuntu2404", ) @@ -335,7 +365,7 @@ def test_get_repo_url_success(self): "--native-package-type", "deb", "--repo-base-url", - "https://x.com", + "https://example.com", "--os-profile", "ubuntu2404", "--repo-sub-folder", @@ -343,7 +373,7 @@ def test_get_repo_url_success(self): ] ) self.assertEqual(code, 0) - self.assertIn("repo_url=https://x.com/ubuntu2404", output) + self.assertIn("repo_url=https://example.com/ubuntu2404", output) def test_get_repo_url_error_returns_one(self): # Test that get-repo-url returns 1 and prints error when get_repo_url raises. @@ -359,7 +389,7 @@ def test_get_repo_url_error_returns_one(self): "--native-package-type", "deb", "--repo-base-url", - "https://x.com", + "https://example.com", "--os-profile", "ubuntu2404", "--repo-sub-folder", @@ -520,5 +550,144 @@ def test_get_container_image_subcommand(self): self.assertIn("container_image=ubuntu:24.04", output) +class GetPublicRepoBaseUrlTest(unittest.TestCase): + """Tests for get_public_repo_base_url() and its subcommand.""" + + def test_prerelease(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("prerelease"), + "https://rocm.prereleases.amd.com/packages-multi-arch", + ) + + def test_release(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("release"), + "https://repo.amd.com/rocm/packages-multi-arch", + ) + + def test_nightly(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("nightly"), + "https://rocm.nightlies.amd.com/packages-multi-arch", + ) + + def test_ci_and_dev_are_empty(self): + self.assertEqual(get_url_repo_params.get_public_repo_base_url("ci"), "") + self.assertEqual(get_url_repo_params.get_public_repo_base_url("dev"), "") + + def test_unknown_line_is_empty(self): + self.assertEqual(get_url_repo_params.get_public_repo_base_url("bogus"), "") + + def test_case_insensitive(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("PreRelease"), + "https://rocm.prereleases.amd.com/packages-multi-arch", + ) + + def test_subcommand_emits_url(self): + code, output = _run_main_with_output( + ["get-public-repo-base-url", "--release-type", "prerelease"] + ) + self.assertEqual(code, 0) + self.assertIn( + "repo_base_url=https://rocm.prereleases.amd.com/packages-multi-arch", + output, + ) + + def test_subcommand_emits_empty_for_ci(self): + code, output = _run_main_with_output( + ["get-public-repo-base-url", "--release-type", "ci"] + ) + self.assertEqual(code, 0) + self.assertIn("repo_base_url=", output) + self.assertNotIn("packages-multi-arch", output) + + +class NightlySubFolderTest(unittest.TestCase): + """Tests for nightly_sub_folder() and its subcommand.""" + + def test_formats_with_injected_date(self): + self.assertEqual( + get_url_repo_params.nightly_sub_folder("12345", today=date(2026, 7, 16)), + "20260716-12345", + ) + + def test_zero_padded_month_and_day(self): + self.assertEqual( + get_url_repo_params.nightly_sub_folder("7", today=date(2026, 1, 3)), + "20260103-7", + ) + + def test_subcommand_emits_sub_folder(self): + code, output = _run_main_with_output( + ["get-nightly-sub-folder", "--run-id", "12345"] + ) + self.assertEqual(code, 0) + # The date is the current UTC date; assert the structure, not the value. + self.assertRegex(output, r"repo_sub_folder=\d{8}-12345") + + def test_subcommand_emits_sub_folder_for_nightly(self): + code, output = _run_main_with_output( + ["get-nightly-sub-folder", "--release-type", "nightly", "--run-id", "12345"] + ) + self.assertEqual(code, 0) + self.assertRegex(output, r"repo_sub_folder=\d{8}-12345") + + def test_subcommand_emits_empty_for_other_lines(self): + # Only the nightly line publishes into a dated sub-folder; the other + # lines publish at the repository root. + for release_type in ("release", "prerelease", "ci"): + with self.subTest(release_type=release_type): + code, output = _run_main_with_output( + [ + "get-nightly-sub-folder", + "--release-type", + release_type, + "--run-id", + "12345", + ] + ) + self.assertEqual(code, 0) + self.assertIn("repo_sub_folder=", output) + self.assertNotRegex(output, r"repo_sub_folder=\S") + + +class GetContainerImageMapTest(unittest.TestCase): + """Tests for get_container_image_map() and its subcommand.""" + + def test_maps_rpm_profiles(self): + self.assertEqual( + get_url_repo_params.get_container_image_map(["rhel8", "sles16"]), + { + "rhel8": "registry.access.redhat.com/ubi8/ubi:8.10", + "sles16": "registry.suse.com/bci/bci-base:16.0", + }, + ) + + def test_subcommand_emits_json_map(self): + code, output = _run_main_with_output( + ["get-container-image-map", "--os-profiles", '["ubuntu2404"]'] + ) + self.assertEqual(code, 0) + # Recover the JSON value written after ``images=``. + line = next(l for l in output.splitlines() if l.startswith("images=")) + images = json.loads(line[len("images=") :]) + self.assertEqual( + images, {"ubuntu2404": "ghcr.io/rocm/no_rocm_image_ubuntu24_04:latest"} + ) + + def test_subcommand_rejects_invalid_json(self): + code, _ = _run_main_with_output( + ["get-container-image-map", "--os-profiles", "not-json"] + ) + self.assertEqual(code, 1) + + def test_subcommand_rejects_non_list_json(self): + code, _ = _run_main_with_output( + ["get-container-image-map", "--os-profiles", '{"a": 1}'] + ) + self.assertEqual(code, 1) + + if __name__ == "__main__": unittest.main() diff --git a/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py b/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py index 03de1c8f075..f0b42e860be 100644 --- a/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py +++ b/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py @@ -1556,5 +1556,429 @@ def test_kills_process_on_timeout(self, mock_popen): mock_proc.kill.assert_called_once() +@contextlib.contextmanager +def _clear_ci_env(): + """Remove CI env vars that _argv_from_ci_env reads so tests start clean.""" + keys = [ + "TEST_TYPE", + "OS_PROFILE", + "REPO_URL", + "GFX_ARCH", + "RELEASE_TYPE", + "INSTALL_PREFIX", + "REPO_PACKAGE_DIR", + "REPO_CONFIG_ONLY", + "GPG_KEY_URL", + "PACKAGES_DIR", + "SIMULATE_PKG_TYPE", + native_linux_package_install_test.ENV_NATIVE_LINUX_INSTALL_ROCM_VERSION, + ] + saved = {k: os.environ.pop(k, None) for k in keys} + try: + yield + finally: + for k, v in saved.items(): + if v is not None: + os.environ[k] = v + + +class RepoPackageModeCliTest(unittest.TestCase): + """CLI/constructor behavior for the install-via-package mode.""" + + def _parse(self, argv): + return native_linux_package_install_test.parse_cli_arguments( + argv, raise_instead_of_exit=True + ) + + def test_repo_package_dir_makes_repo_url_optional(self): + args = self._parse( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-package-dir", + "/pkgs", + "--install-prefix", + "/opt/rocm/core", + ] + ) + self.assertEqual(args.repo_package_dir, "/pkgs") + self.assertFalse(args.repo_config_only) + + def test_config_only_drops_repo_url_and_install_prefix(self): + args = self._parse( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-package-dir", + "/pkgs", + "--repo-config-only", + ] + ) + self.assertTrue(args.repo_config_only) + self.assertIsNone(args.repo_url) + self.assertIsNone(args.install_prefix) + + def test_config_only_requires_repo_package_dir(self): + with self.assertRaises(ValueError): + self._parse( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-config-only", + ] + ) + + def test_repo_url_still_required_without_repo_package_dir(self): + with self.assertRaises(ValueError): + self._parse(["--os-profile", "ubuntu2404", "--release-type", "prerelease"]) + + def test_repo_package_dir_rejected_with_simulate(self): + with self.assertRaises(ValueError): + self._parse( + [ + "--test-type", + "simulate", + "--packages-dir", + "/p", + "--os-profile", + "ubuntu2404", + "--repo-package-dir", + "/pkgs", + ] + ) + + def test_constructor_tolerates_none_repo_url(self): + runner = native_linux_package_install_test.NativeLinuxPackageInstallTest( + os_profile="rhel10", + repo_url=None, + release_type="nightly", + repo_package_dir="/pkgs", + repo_config_only=True, + ) + self.assertEqual(runner.repo_url, "") + self.assertEqual(runner.repo_package_dir, "/pkgs") + self.assertTrue(runner.repo_config_only) + + +class InstallRepoPackageTest(unittest.TestCase): + """Tests for install_repo_package() command construction (mocked subprocess).""" + + def _runner(self, os_profile, pkg_dir): + return native_linux_package_install_test.NativeLinuxPackageInstallTest( + os_profile=os_profile, + repo_url="", + release_type="prerelease", + repo_package_dir=str(pkg_dir), + ) + + @patch("native_linux_package_install_test._run_streaming") + def test_deb_installs_then_apt_update(self, mock_streaming): + mock_streaming.return_value = 0 + with tempfile.TemporaryDirectory() as d: + pkg = Path(d) / "amdrocm-repo_7.14.0_all.deb" + pkg.write_text("") + runner = self._runner("ubuntu2404", d) + with _suppress_script_output(): + self.assertTrue(runner.install_repo_package()) + install_cmd, refresh_cmd = (c.args[0] for c in mock_streaming.call_args_list) + self.assertEqual( + install_cmd, ["sudo", "apt", "install", "-y", str(pkg.resolve())] + ) + self.assertEqual(refresh_cmd, ["sudo", "apt", "update"]) + + @patch("native_linux_package_install_test._run_streaming") + def test_dnf_installs_nogpgcheck_then_scoped_makecache(self, mock_streaming): + mock_streaming.return_value = 0 + with tempfile.TemporaryDirectory() as d: + pkg = Path(d) / "amdrocm-repo-7.14.0-1.el10.noarch.rpm" + pkg.write_text("") + runner = self._runner("rhel10", d) + with _suppress_script_output(): + self.assertTrue(runner.install_repo_package()) + install_cmd, refresh_cmd = (c.args[0] for c in mock_streaming.call_args_list) + self.assertEqual( + install_cmd, ["dnf", "install", "-y", "--nogpgcheck", str(pkg.resolve())] + ) + self.assertEqual( + refresh_cmd, + ["dnf", "makecache", "--disablerepo=*", "--enablerepo=amdrocm"], + ) + + @patch("native_linux_package_install_test._run_streaming") + def test_sles_installs_no_gpg_checks_then_refresh(self, mock_streaming): + mock_streaming.return_value = 0 + with tempfile.TemporaryDirectory() as d: + pkg = Path(d) / "amdrocm-repo-7.14.0-1.noarch.rpm" + pkg.write_text("") + runner = self._runner("sles16", d) + with _suppress_script_output(): + self.assertTrue(runner.install_repo_package()) + install_cmd, refresh_cmd = (c.args[0] for c in mock_streaming.call_args_list) + self.assertEqual( + install_cmd, + [ + "zypper", + "--non-interactive", + "--no-gpg-checks", + "install", + "-y", + str(pkg.resolve()), + ], + ) + self.assertEqual( + refresh_cmd, + [ + "zypper", + "--non-interactive", + "--gpg-auto-import-keys", + "refresh", + "amdrocm", + ], + ) + + @patch("native_linux_package_install_test._run_streaming") + def test_missing_package_fails_without_running(self, mock_streaming): + with tempfile.TemporaryDirectory() as d: + runner = self._runner("ubuntu2404", d) # empty dir + with _suppress_script_output(): + self.assertFalse(runner.install_repo_package()) + mock_streaming.assert_not_called() + + @patch("native_linux_package_install_test._run_streaming") + def test_multiple_packages_fails_without_running(self, mock_streaming): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "amdrocm-repo_1_all.deb").write_text("") + (Path(d) / "amdrocm-repo_2_all.deb").write_text("") + runner = self._runner("ubuntu2404", d) + with _suppress_script_output(): + self.assertFalse(runner.install_repo_package()) + mock_streaming.assert_not_called() + + @patch("native_linux_package_install_test._run_streaming") + def test_install_failure_returns_false_without_refresh(self, mock_streaming): + mock_streaming.return_value = 1 # install fails + with tempfile.TemporaryDirectory() as d: + (Path(d) / "amdrocm-repo_1_all.deb").write_text("") + runner = self._runner("ubuntu2404", d) + with _suppress_script_output(): + self.assertFalse(runner.install_repo_package()) + self.assertEqual(mock_streaming.call_count, 1) # refresh not attempted + + +class AssertRepoConfiguredTest(unittest.TestCase): + """Tests for assert_repo_configured() (on-disk contract check).""" + + def _runner(self, os_profile, release_type, pkg_dir="/pkgs"): + return native_linux_package_install_test.NativeLinuxPackageInstallTest( + os_profile=os_profile, + repo_url="", + release_type=release_type, + repo_package_dir=pkg_dir, + ) + + def test_deb_signed_requires_sources_and_keyring(self): + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text( + "Types: deb\n" + "URIs: https://rocm.prereleases.amd.com/packages-multi-arch/ubuntu2404/\n" + "Signed-By: /usr/share/keyrings/rocm.gpg\n" + ) + keyring = Path(d) / "rocm.gpg" + keyring.write_text("key") + runner = self._runner("ubuntu2404", "prerelease") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING=str(keyring), + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + keyring.unlink() # signed line but keyring missing -> fail + with _suppress_script_output(): + self.assertFalse(runner.assert_repo_configured()) + + def test_deb_missing_sources_fails(self): + runner = self._runner("ubuntu2404", "prerelease") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES="/nonexistent/amdrocm.sources", + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertFalse(runner.assert_repo_configured()) + + def test_deb_nightly_needs_no_keyring(self): + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text("URIs: https://x/rocm/deb/\nTrusted: yes\n") + runner = self._runner("ubuntu2404", "nightly") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + def test_structurally_invalid_repo_file_fails(self): + # A repo file missing the expected deb822 marker (URIs:) is rejected, + # independent of whether the host string contains "rocm". + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text("not a valid deb822 sources file\n") + runner = self._runner("ubuntu2404", "nightly") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertFalse(runner.assert_repo_configured()) + + def test_mirror_host_without_rocm_still_passes(self): + # deb822 with a rocm-less mirror host is valid: the marker check must + # not reject it just because the URL lacks "rocm". + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text( + "Types: deb\n" + "URIs: https://mirror.example.com/packages-multi-arch/deb/20260716-1/\n" + "Suites: stable\n" + "Trusted: yes\n" + ) + runner = self._runner("ubuntu2404", "nightly") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + def test_rpm_signed_checks_yum_dir_and_key(self): + with tempfile.TemporaryDirectory() as d: + repo = Path(d) / "amdrocm.repo" + repo.write_text( + "[amdrocm]\nbaseurl=https://x/packages-multi-arch/rpm/x86_64/\n" + ) + key = Path(d) / "RPM-GPG-KEY-rocm" + key.write_text("key") + runner = self._runner("rhel10", "prerelease") + with patch.multiple( + native_linux_package_install_test, + YUM_REPOS_DIR=d, + AMDROCM_RPM_KEY=str(key), + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + def test_rpm_sles_uses_zypp_dir(self): + with tempfile.TemporaryDirectory() as d: + repo = Path(d) / "amdrocm.repo" + repo.write_text("[amdrocm]\nbaseurl=https://x/rocm/rpm/x86_64/\n") + key = Path(d) / "RPM-GPG-KEY-rocm" + key.write_text("key") + runner = self._runner("sles16", "prerelease") + with patch.multiple( + native_linux_package_install_test, + ZYPP_REPOS_DIR=d, + AMDROCM_RPM_KEY=str(key), + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + +class RunRepoSetupDispatchTest(unittest.TestCase): + """run_repo_setup_and_install dispatch: package mode vs config-only vs default.""" + + def _runner(self, **kw): + base = dict(os_profile="ubuntu2404", repo_url="", release_type="prerelease") + base.update(kw) + return native_linux_package_install_test.NativeLinuxPackageInstallTest(**base) + + def test_config_only_stops_before_rocm_install(self): + runner = self._runner(repo_package_dir="/pkgs", repo_config_only=True) + runner.install_repo_package = MagicMock(return_value=True) + runner.assert_repo_configured = MagicMock(return_value=True) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertTrue(runner.run_repo_setup_and_install()) + runner.install_repo_package.assert_called_once() + runner.assert_repo_configured.assert_called_once() + runner.install_deb_packages.assert_not_called() + + def test_package_mode_installs_rocm_when_not_config_only(self): + runner = self._runner(repo_package_dir="/pkgs", repo_config_only=False) + runner.install_repo_package = MagicMock(return_value=True) + runner.assert_repo_configured = MagicMock(return_value=True) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertTrue(runner.run_repo_setup_and_install()) + runner.install_deb_packages.assert_called_once() + + def test_failed_assert_short_circuits(self): + runner = self._runner(repo_package_dir="/pkgs", repo_config_only=False) + runner.install_repo_package = MagicMock(return_value=True) + runner.assert_repo_configured = MagicMock(return_value=False) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertFalse(runner.run_repo_setup_and_install()) + runner.install_deb_packages.assert_not_called() + + def test_default_path_uses_manual_setup(self): + runner = self._runner() # no repo_package_dir + runner.install_repo_package = MagicMock(return_value=True) + runner.setup_deb_repository = MagicMock(return_value=True) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertTrue(runner.run_repo_setup_and_install()) + runner.setup_deb_repository.assert_called_once() + runner.install_repo_package.assert_not_called() + + +class ArgvFromCiEnvRepoPackageTest(unittest.TestCase): + """_argv_from_ci_env: install-via-package env mapping and relaxed requirements.""" + + def test_config_only_lane_needs_no_repo_url_or_install_prefix(self): + env = { + "TEST_TYPE": "sanity", + "OS_PROFILE": "ubuntu2404", + "RELEASE_TYPE": "prerelease", + "REPO_PACKAGE_DIR": "/pkgs", + "REPO_CONFIG_ONLY": "true", + } + with _clear_ci_env(), patch.dict(os.environ, env, clear=False): + argv = native_linux_package_install_test._argv_from_ci_env() + self.assertIsNotNone(argv) + self.assertIn("--repo-package-dir", argv) + self.assertEqual(argv[argv.index("--repo-package-dir") + 1], "/pkgs") + self.assertIn("--repo-config-only", argv) + self.assertNotIn("--repo-url", argv) + self.assertNotIn("--install-prefix", argv) + + def test_package_mode_with_install_prefix_no_repo_url(self): + env = { + "TEST_TYPE": "sanity", + "OS_PROFILE": "rhel10", + "RELEASE_TYPE": "prerelease", + "REPO_PACKAGE_DIR": "/pkgs", + "INSTALL_PREFIX": "/opt/rocm/core", + } + with _clear_ci_env(), patch.dict(os.environ, env, clear=False): + argv = native_linux_package_install_test._argv_from_ci_env() + self.assertIsNotNone(argv) + self.assertIn("--repo-package-dir", argv) + self.assertIn("--install-prefix", argv) + self.assertNotIn("--repo-url", argv) + self.assertNotIn("--repo-config-only", argv) + + if __name__ == "__main__": unittest.main() diff --git a/build_tools/tests/build_repo_package_test.py b/build_tools/tests/build_repo_package_test.py new file mode 100644 index 00000000000..4d351b02b9c --- /dev/null +++ b/build_tools/tests/build_repo_package_test.py @@ -0,0 +1,842 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for build_repo_package.py. + +build_repo_package.py lives in build_tools/packaging/linux/, which is not on the +path that conftest.py sets up (it only adds build_tools/). Add it explicitly. +Importing the module is side-effect-free thanks to its ``__main__`` guard. +""" + +import json +import sys +from pathlib import Path + +import pytest + +_LINUX_DIR = Path(__file__).resolve().parents[1] / "packaging" / "linux" +sys.path.insert(0, str(_LINUX_DIR)) + +import build_repo_package as brp # noqa: E402 + +PRERELEASE_BASE = "https://rocm.prereleases.amd.com/packages-multi-arch" +RELEASE_BASE = "https://repo.amd.com/rocm/packages-multi-arch" +NIGHTLY_BASE = "https://rocm.nightlies.amd.com/packages-multi-arch" +NIGHTLY_SUB = "20260716-12345" + + +def _args(*extra: str): + """Parse args with sensible defaults, overridable by appending flags.""" + base = [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-base-url", + PRERELEASE_BASE, + "--rocm-version", + "7.14.0", + "--dest-dir", + "/tmp/out", + ] + return brp.parse_args(base + list(extra)) + + +def _render(template: str, context: dict) -> str: + return brp.get_jinja_env().get_template(template).render(context) + + +# --- URL construction --------------------------------------------------------- +# +# The release lines do not share a layout, so each one's exact URLs are pinned +# here: the public signed lines publish one repository per distro, and the +# nightly line publishes one per package type under a dated sub-folder. A URL +# that does not serve a repository only fails at the user's first metadata +# refresh, so these are pinned rather than derived. + +RELEASE_EXPECTED = { + "ubuntu2404": f"{RELEASE_BASE}/ubuntu2404/", + "rhel8": f"{RELEASE_BASE}/rhel8/x86_64/", + "rhel10": f"{RELEASE_BASE}/rhel10/x86_64/", + "sles16": f"{RELEASE_BASE}/sles16/x86_64/", +} + +PRERELEASE_EXPECTED = { + "ubuntu2404": f"{PRERELEASE_BASE}/ubuntu2404/", + "rhel8": f"{PRERELEASE_BASE}/rhel8/x86_64/", + "rhel10": f"{PRERELEASE_BASE}/rhel10/x86_64/", + "sles16": f"{PRERELEASE_BASE}/sles16/x86_64/", +} + +NIGHTLY_EXPECTED = { + "ubuntu2404": f"{NIGHTLY_BASE}/deb/{NIGHTLY_SUB}/", + "rhel8": f"{NIGHTLY_BASE}/rpm/{NIGHTLY_SUB}/x86_64/", + "rhel10": f"{NIGHTLY_BASE}/rpm/{NIGHTLY_SUB}/x86_64/", + "sles16": f"{NIGHTLY_BASE}/rpm/{NIGHTLY_SUB}/x86_64/", +} + + +def _baseurl(profile: str, release_type: str, base: str, sub: str = "") -> str: + return brp.repo_baseurl( + base, brp.OS_PROFILES[profile]["pkg_type"], profile, release_type, sub + ) + + +@pytest.mark.parametrize("profile,expected", sorted(RELEASE_EXPECTED.items())) +def test_release_baseurl_is_per_distro(profile, expected): + assert _baseurl(profile, "release", RELEASE_BASE) == expected + + +@pytest.mark.parametrize("profile,expected", sorted(PRERELEASE_EXPECTED.items())) +def test_prerelease_baseurl_is_per_distro(profile, expected): + assert _baseurl(profile, "prerelease", PRERELEASE_BASE) == expected + + +@pytest.mark.parametrize("profile,expected", sorted(NIGHTLY_EXPECTED.items())) +def test_nightly_baseurl_is_flat_and_dated(profile, expected): + assert _baseurl(profile, "nightly", NIGHTLY_BASE, NIGHTLY_SUB) == expected + + +def test_gpg_key_url_under_base(): + assert brp.gpg_key_url(PRERELEASE_BASE) == f"{PRERELEASE_BASE}/gpg/rocm.gpg" + + +def test_baseurl_tolerates_trailing_slash(): + assert ( + _baseurl("ubuntu2404", "release", RELEASE_BASE + "/") + == RELEASE_EXPECTED["ubuntu2404"] + ) + + +def test_signed_lines_are_per_distro_never_flat(): + # A flat /deb/ or /rpm/x86_64/ is not served on the release CDN, + # so a signed line must always carry the distro segment. + for release_type, base in ( + ("release", RELEASE_BASE), + ("prerelease", PRERELEASE_BASE), + ): + for profile in brp.OS_PROFILES: + url = _baseurl(profile, release_type, base) + assert f"/{profile}/" in url, f"{release_type} must be per-distro" + assert not url.endswith("/deb/") + assert "/rpm/x86_64/" not in url + + +def test_nightly_is_flat_and_carries_no_distro_segment(): + # The nightly repository has no per-distro tree, so it must stay flat. + for profile in brp.OS_PROFILES: + url = _baseurl(profile, "nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert f"/{profile}/" not in url + assert NIGHTLY_SUB in url + + +def test_gpg_key_url_is_not_the_domain_root(): + gpg = brp.gpg_key_url(PRERELEASE_BASE) + assert "/packages-multi-arch/gpg/rocm.gpg" in gpg + assert gpg != "https://rocm.prereleases.amd.com/gpg/rocm.gpg" + + +# --- repository reachability check -------------------------------------------- + + +def test_repo_metadata_url_targets_the_index(): + deb = brp.repo_metadata_url(RELEASE_EXPECTED["ubuntu2404"], "deb") + assert deb == f"{RELEASE_EXPECTED['ubuntu2404']}dists/{brp.DEB_SUITE}/Release" + rpm = brp.repo_metadata_url(RELEASE_EXPECTED["rhel10"], "rpm") + assert rpm == f"{RELEASE_EXPECTED['rhel10']}repodata/repomd.xml" + + +def test_verify_repo_url_accepts_a_served_repo(monkeypatch): + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(b"", RELEASE_EXPECTED["rhel10"], status=200), + ) + brp.verify_repo_url(RELEASE_EXPECTED["rhel10"], "rpm") # must not raise + + +def test_verify_repo_url_rejects_an_unserved_repo(monkeypatch): + # The flat shape on the release line 404/403s; the guard must fail the build + # rather than ship a package that cannot refresh. + def not_found(*a, **k): + raise brp.urllib.error.HTTPError( + f"{RELEASE_BASE}/rpm/x86_64/", 403, "Forbidden", {}, None + ) + + monkeypatch.setattr(brp.urllib.request, "urlopen", not_found) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + with pytest.raises(RuntimeError): + brp.verify_repo_url(f"{RELEASE_BASE}/rpm/x86_64/", "rpm") + + +def test_verify_repo_url_does_not_retry_a_missing_repo(monkeypatch): + # A wrong URL will not become right, so it must fail on the first attempt + # rather than spending the full retry budget. + calls = {"n": 0} + + def forbidden(*a, **k): + calls["n"] += 1 + raise brp.urllib.error.HTTPError( + f"{RELEASE_BASE}/rpm/x86_64/", 403, "Forbidden", {}, None + ) + + monkeypatch.setattr(brp.urllib.request, "urlopen", forbidden) + monkeypatch.setattr(brp.time, "sleep", lambda s: pytest.fail("must not back off")) + with pytest.raises(RuntimeError): + brp.verify_repo_url(f"{RELEASE_BASE}/rpm/x86_64/", "rpm") + assert calls["n"] == 1 + + +def test_verify_repo_url_retries_rate_limiting(monkeypatch): + calls = {"n": 0} + + def throttled(*a, **k): + calls["n"] += 1 + if calls["n"] < 2: + raise brp.urllib.error.HTTPError( + RELEASE_EXPECTED["rhel10"], 429, "Too Many Requests", {}, None + ) + return _FakeResponse(b"", RELEASE_EXPECTED["rhel10"], status=200) + + monkeypatch.setattr(brp.urllib.request, "urlopen", throttled) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + brp.verify_repo_url(RELEASE_EXPECTED["rhel10"], "rpm") + assert calls["n"] == 2 + + +def test_verify_repo_url_retries_then_succeeds(monkeypatch): + calls = {"n": 0} + + def flaky(*a, **k): + calls["n"] += 1 + if calls["n"] < 3: + raise brp.urllib.error.URLError("transient") + return _FakeResponse(b"", RELEASE_EXPECTED["rhel10"], status=200) + + monkeypatch.setattr(brp.urllib.request, "urlopen", flaky) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + brp.verify_repo_url(RELEASE_EXPECTED["rhel10"], "rpm") + assert calls["n"] == 3 + + +# --- reachability check wiring ------------------------------------------------- +# +# A guard that is never invoked is indistinguishable from no guard, so pin the +# call from main() rather than only testing the function in isolation. + + +def _main_argv(tmp_path, *extra: str) -> list[str]: + return [ + "--os-profile", + "rhel10", + "--release-type", + "release", + "--repo-base-url", + RELEASE_BASE, + "--rocm-version", + "7.14.0", + "--dest-dir", + str(tmp_path), + *extra, + ] + + +def test_verify_repo_url_is_off_by_default(): + assert _args().verify_repo_url is False + + +def test_main_verifies_the_repository_when_asked(monkeypatch, tmp_path): + checked = [] + monkeypatch.setattr( + brp, "verify_repo_url", lambda url, pkg_type: checked.append((url, pkg_type)) + ) + monkeypatch.setattr(brp, "build_rpm_package", lambda *a, **k: None) + brp.main(_main_argv(tmp_path, "--verify-repo-url")) + # The URL actually built for this profile is what gets checked. + assert checked == [(RELEASE_EXPECTED["rhel10"], "rpm")] + + +def test_main_skips_verification_for_nightly(monkeypatch, tmp_path): + # CI passes the flag unconditionally; the nightly repository is published by + # the same run, so its dated sub-folder cannot be checked at build time. + monkeypatch.setattr( + brp, + "verify_repo_url", + lambda *a: pytest.fail("nightly has nothing to verify yet"), + ) + monkeypatch.setattr(brp, "build_rpm_package", lambda *a, **k: None) + brp.main( + [ + "--os-profile", + "rhel10", + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + NIGHTLY_SUB, + "--rocm-version", + "7.14.0", + "--dest-dir", + str(tmp_path), + "--verify-repo-url", + ] + ) + + +def test_main_does_not_verify_by_default(monkeypatch, tmp_path): + monkeypatch.setattr( + brp, + "verify_repo_url", + lambda *a: pytest.fail("must not reach the network unless asked"), + ) + monkeypatch.setattr(brp, "build_rpm_package", lambda *a, **k: None) + brp.main(_main_argv(tmp_path)) + + +# --- context wiring ----------------------------------------------------------- + + +@pytest.mark.parametrize("profile", sorted(RELEASE_EXPECTED)) +def test_build_context_threads_profile_and_release_type(profile): + # The baseurl depends on both the target distro and the release line, so + # build_context must pass both through rather than defaulting either. + args = _args( + "--os-profile", + profile, + "--release-type", + "release", + "--repo-base-url", + RELEASE_BASE, + ) + ctx = brp.build_context(args, brp.OS_PROFILES[profile]) + assert ctx["baseurl"] == RELEASE_EXPECTED[profile] + + +# --- signing ------------------------------------------------------------------ + + +def test_signed_lines(): + assert brp.is_signed("prerelease") is True + assert brp.is_signed("release") is True + assert brp.is_signed("nightly") is False + + +# --- signing-key fetch hardening ---------------------------------------------- + + +class _FakeResponse: + """Minimal stand-in for a urlopen() result (context manager + read).""" + + def __init__(self, body: bytes, url: str, status: int = 200): + self._body = body + self._url = url + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def geturl(self) -> str: + return self._url + + def getcode(self) -> int: + return self.status + + def read(self, n: int = -1) -> bytes: + return self._body if n < 0 else self._body[:n] + + +def test_require_https_accepts_https(): + brp._require_https("https://example.com/gpg/rocm.gpg", "key URL") + + +def test_require_https_rejects_http(): + with pytest.raises(ValueError): + brp._require_https("http://example.com/gpg/rocm.gpg", "key URL") + + +def test_load_signing_key_rejects_non_https_base(monkeypatch): + # A non-https base must fail before any network fetch is attempted. + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: pytest.fail("must not fetch over a non-https URL"), + ) + args = _args() + args.repo_base_url = "http://rocm.example.com/packages-multi-arch" + with pytest.raises(ValueError): + brp.load_signing_key(args) + + +def test_load_signing_key_rejects_post_redirect_downgrade(monkeypatch): + # An https URL that redirects to http must be rejected after the fetch. + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(b"KEY", "http://evil.example/gpg/rocm.gpg"), + ) + with pytest.raises(ValueError): + brp.load_signing_key(_args()) + + +def test_load_signing_key_rejects_oversize_key(monkeypatch): + big = b"A" * (brp.MAX_GPG_KEY_BYTES + 1) + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(big, f"{PRERELEASE_BASE}/gpg/rocm.gpg"), + ) + with pytest.raises(ValueError): + brp.load_signing_key(_args()) + + +def test_load_signing_key_returns_fetched_key(monkeypatch): + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse( + b"ARMORED KEY", f"{PRERELEASE_BASE}/gpg/rocm.gpg" + ), + ) + # Fingerprint verification runs gpg on real key bytes; stub it here. + monkeypatch.setattr(brp, "_verify_key_fingerprint", lambda key: None) + assert brp.load_signing_key(_args()) == b"ARMORED KEY" + + +def test_load_signing_key_prefers_key_file(monkeypatch, tmp_path): + key = tmp_path / "rocm.gpg" + key.write_bytes(b"FILE KEY") + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: pytest.fail("must not fetch when --gpg-key-file is given"), + ) + args = _args() + args.gpg_key_file = key + assert brp.load_signing_key(args) == b"FILE KEY" + + +def test_load_signing_key_rejects_wrong_fingerprint(monkeypatch): + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(b"KEY", f"{PRERELEASE_BASE}/gpg/rocm.gpg"), + ) + monkeypatch.setattr(brp, "_key_fingerprint", lambda armored: "0" * 40) + with pytest.raises(ValueError): + brp.load_signing_key(_args()) + + +def test_verify_key_fingerprint_accepts_pinned(monkeypatch): + monkeypatch.setattr( + brp, "_key_fingerprint", lambda armored: brp.EXPECTED_KEY_FINGERPRINT + ) + brp._verify_key_fingerprint(b"whatever") # must not raise + + +def test_fetch_signing_key_retries_then_succeeds(monkeypatch): + calls = {"n": 0} + + def flaky(*a, **k): + calls["n"] += 1 + if calls["n"] < 3: + raise brp.urllib.error.URLError("transient") + return _FakeResponse(b"KEY", f"{PRERELEASE_BASE}/gpg/rocm.gpg") + + monkeypatch.setattr(brp.urllib.request, "urlopen", flaky) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) # no real backoff + assert brp._fetch_signing_key(f"{PRERELEASE_BASE}/gpg/rocm.gpg") == b"KEY" + assert calls["n"] == 3 + + +def test_fetch_signing_key_gives_up_after_attempts(monkeypatch): + def always_fail(*a, **k): + raise brp.urllib.error.URLError("down") + + monkeypatch.setattr(brp.urllib.request, "urlopen", always_fail) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + with pytest.raises(RuntimeError): + brp._fetch_signing_key(f"{PRERELEASE_BASE}/gpg/rocm.gpg") + + +# --- versioning --------------------------------------------------------------- + + +def test_rpm_version_rolling_for_prerelease(): + assert brp.rpm_version_release("prerelease", "7.14.0", "") == ( + "7.14.0", + "1.prerelease", + ) + + +def test_streams_never_share_a_package_version(): + # Two lines at the same ROCm version must not produce the same package: + # installing one over the other would otherwise report success and silently + # leave the original repository configured. + version = "7.14.0" + rpm_pre = brp.rpm_version_release("prerelease", version, "") + rpm_rel = brp.rpm_version_release("release", version, "") + assert rpm_pre != rpm_rel + assert brp.deb_version("prerelease", version, "") != brp.deb_version( + "release", version, "" + ) + + +def test_prerelease_sorts_before_release(): + # The release package must upgrade over the prerelease one, not be seen as a + # downgrade. deb: "~" sorts before the plain version. rpm: the release field + # differs and "prerelease" precedes "release". + assert brp.deb_version("prerelease", "7.14.0", "") == "7.14.0~prerelease" + assert brp.deb_version("release", "7.14.0", "") == "7.14.0" + assert brp.rpm_version_release("prerelease", "7.14.0", "")[1] < ( + brp.rpm_version_release("release", "7.14.0", "")[1] + ) + + +def test_rpm_version_nightly_splits_date_and_id(): + version, release = brp.rpm_version_release("nightly", "7.14.0", NIGHTLY_SUB) + assert version == "20260716" + assert release == "12345.nightly" + assert "-" not in version # rpm Version cannot contain a hyphen + + +def test_deb_version_nightly_has_no_hyphen(): + assert brp.deb_version("nightly", "7.14.0", NIGHTLY_SUB) == "20260716.12345" + + +def test_deb_version_rolling(): + assert brp.deb_version("release", "7.14.0", "") == "7.14.0" + + +# --- CLI validation ----------------------------------------------------------- + + +def test_release_type_rejects_unknown(): + with pytest.raises(SystemExit): + _args("--release-type", "dev") + + +def test_repo_base_url_required(): + with pytest.raises(SystemExit): + brp.parse_args( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--rocm-version", + "7.14.0", + "--dest-dir", + "/tmp/out", + ] + ) + + +def test_nightly_requires_sub_folder(): + with pytest.raises(SystemExit): + _args("--release-type", "nightly", "--repo-base-url", NIGHTLY_BASE) + + +def test_nightly_with_sub_folder_ok(): + args = _args( + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + NIGHTLY_SUB, + ) + assert args.repo_sub_folder == NIGHTLY_SUB + + +def test_sub_folder_rejected_for_non_nightly(): + with pytest.raises(SystemExit): + _args("--repo-sub-folder", NIGHTLY_SUB) # default release-type is prerelease + + +# --- input validation --------------------------------------------------------- + + +@pytest.mark.parametrize( + "bad_version", + [ + '7.14.0"; rm -rf /; echo "', # shell-injection payload + "7.14.0 evil", # whitespace + "7.14.0$(id)", # command substitution + "7.14.0\ngpgcheck=0", # embedded newline + "7.14.0\n", # trailing newline (regex must use \\Z, not $) + "", # empty + "-7.14.0", # must start with a digit + ], +) +def test_rocm_version_rejects_bad_format(bad_version): + with pytest.raises(SystemExit): + _args("--rocm-version", bad_version) + + +@pytest.mark.parametrize("good_version", ["7.14.0", "7.14.0~rc3", "8.0.1rc1", "7.0"]) +def test_rocm_version_accepts_valid(good_version): + assert _args("--rocm-version", good_version).rocm_version == good_version + + +@pytest.mark.parametrize( + "bad_url", + [ + f"{PRERELEASE_BASE}\ngpgcheck=0", # newline directive injection + f"{PRERELEASE_BASE} extra", # whitespace + "not a url", # no scheme/netloc + "ftp://example.com/repo", # unsupported scheme + ], +) +def test_repo_base_url_rejects_bad(bad_url): + with pytest.raises(SystemExit): + _args("--repo-base-url", bad_url) + + +def test_repo_base_url_accepts_https(): + assert _args("--repo-base-url", PRERELEASE_BASE).repo_base_url == PRERELEASE_BASE + + +def test_repo_sub_folder_rejects_bad_format(): + with pytest.raises(SystemExit): + _args( + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + "20260722-99\nX", # newline in the dated sub-folder + ) + + +def test_postinst_does_not_interpolate_version(): + # The postinst runs as root at install time; the version must not be rendered + # into it (defense-in-depth beyond input validation). + ctx = brp.build_context( + _args("--rocm-version", "9.9.9"), brp.OS_PROFILES["ubuntu2404"] + ) + out = _render("template/repo/deb/postinst.j2", ctx) + assert "9.9.9" not in out + assert "AMD ROCm repository configured." in out + + +# --- rpm .repo rendering ------------------------------------------------------ + + +def _rpm_repo(release_type: str, base: str, sub: str = "") -> str: + extra = [ + "--os-profile", + "rhel10", + "--release-type", + release_type, + "--repo-base-url", + base, + ] + if sub: + extra += ["--repo-sub-folder", sub] + args = _args(*extra) + return _render( + "template/repo/rpm/amdrocm.repo.j2", + brp.build_context(args, brp.OS_PROFILES["rhel10"]), + ) + + +def test_rpm_repo_signed_has_gpgcheck_and_gpgkey(): + out = _rpm_repo("prerelease", PRERELEASE_BASE) + assert "[amdrocm]" in out + assert "enabled=1" in out + assert "gpgcheck=1" in out + assert "gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" in out + assert f"baseurl={PRERELEASE_EXPECTED['rhel10']}" in out + + +def test_rpm_repo_nightly_unsigned_no_gpgkey(): + out = _rpm_repo("nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert "gpgcheck=0" in out + assert "gpgkey" not in out + + +def test_rpm_repo_ends_with_newline(): + assert _rpm_repo("prerelease", PRERELEASE_BASE).endswith("\n") + + +# --- deb .sources rendering --------------------------------------------------- + + +def _deb_sources(release_type: str, base: str, sub: str = "") -> str: + extra = [ + "--os-profile", + "ubuntu2404", + "--release-type", + release_type, + "--repo-base-url", + base, + ] + if sub: + extra += ["--repo-sub-folder", sub] + args = _args(*extra) + return _render( + "template/repo/deb/amdrocm.sources.j2", + brp.build_context(args, brp.OS_PROFILES["ubuntu2404"]), + ) + + +def test_deb_sources_signed_fields(): + out = _deb_sources("prerelease", PRERELEASE_BASE) + assert "Types: deb" in out + assert f"URIs: {PRERELEASE_EXPECTED['ubuntu2404']}" in out + # The suite is rendered from the constant the reachability check also uses, + # so the repo file and the verified index path cannot drift apart. + assert f"Suites: {brp.DEB_SUITE}" in out + assert "Suites: stable" in out + assert "Components: main" in out + assert "Architectures: amd64" in out + assert "Signed-By: /usr/share/keyrings/rocm.gpg" in out + assert "Trusted:" not in out + + +def test_deb_sources_nightly_trusted_not_signed(): + out = _deb_sources("nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert "Trusted: yes" in out + assert "Signed-By:" not in out + + +# --- rpm spec rendering ------------------------------------------------------- + + +def _spec(release_type: str, base: str, sub: str = "") -> str: + extra = [ + "--os-profile", + "rhel10", + "--release-type", + release_type, + "--repo-base-url", + base, + ] + if sub: + extra += ["--repo-sub-folder", sub] + args = _args(*extra) + ctx = brp.build_context(args, brp.OS_PROFILES["rhel10"]) + version, release = brp.rpm_version_release(release_type, args.rocm_version, sub) + return _render( + "template/repo/rpm/amdrocm-repo.spec.j2", + { + **ctx, + "version": version, + "release": release, + "changelog_date": "Mon Jul 20 2026", + }, + ) + + +def test_spec_name_and_repo_install(): + out = _spec("prerelease", PRERELEASE_BASE) + assert "Name: amdrocm-repo" in out + assert "Version: 7.14.0" in out + assert "/etc/yum.repos.d/amdrocm.repo" in out + + +def test_spec_signed_installs_key_nightly_does_not(): + signed = _spec("prerelease", PRERELEASE_BASE) + assert "/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" in signed + unsigned = _spec("nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert "RPM-GPG-KEY-rocm" not in unsigned + + +def test_spec_has_no_conflicts(): + assert "Conflicts" not in _spec("prerelease", PRERELEASE_BASE) + + +# --- deb control / install rendering ------------------------------------------ + + +def test_control_has_no_key_runtime_deps(): + args = _args() + ctx = brp.build_context(args, brp.OS_PROFILES["ubuntu2404"]) + out = _render("template/repo/deb/control.j2", ctx) + assert "Package: amdrocm-repo" in out + assert "gnupg" not in out + assert "wget" not in out + assert "Conflicts" not in out + + +def test_install_maps_keyring_only_when_signed(): + signed = brp.build_context(_args(), brp.OS_PROFILES["ubuntu2404"]) + out_signed = _render("template/repo/deb/install.j2", signed) + assert "amdrocm.sources /etc/apt/sources.list.d/" in out_signed + assert "rocm.gpg /usr/share/keyrings/" in out_signed + + args_n = _args( + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + NIGHTLY_SUB, + ) + out_nightly = _render( + "template/repo/deb/install.j2", + brp.build_context(args_n, brp.OS_PROFILES["ubuntu2404"]), + ) + assert "rocm.gpg" not in out_nightly + + +# --- list-profiles -------------------------------------------------------- + + +def test_list_profiles_rpm(): + assert brp.list_profiles("rpm") == ["rhel8", "rhel10", "sles16"] + + +def test_list_profiles_deb(): + assert brp.list_profiles("deb") == ["ubuntu2404"] + + +def test_list_profiles_cli_emits_json(capsys): + brp.run_list_profiles(["--pkg-type", "rpm"]) + out = capsys.readouterr().out + assert json.loads(out) == ["rhel8", "rhel10", "sles16"] + + +def test_list_profiles_cli_rejects_invalid_pkg_type(capsys): + with pytest.raises(SystemExit): + brp.run_list_profiles(["--pkg-type", "foo"]) + + +def test_main_dispatches_list_profiles(capsys): + brp.main(["list-profiles", "--pkg-type", "deb"]) + out = capsys.readouterr().out + assert json.loads(out) == ["ubuntu2404"] + + +# --- dest-dir normalization ----------------------------------------------- + + +def test_relative_dest_dir_is_resolved_to_absolute(): + # rpmbuild needs an absolute _topdir, so a relative --dest-dir must be + # normalized to absolute at parse time. + args = brp.parse_args( + [ + "--os-profile", + "rhel10", + "--release-type", + "prerelease", + "--repo-base-url", + PRERELEASE_BASE, + "--rocm-version", + "7.14.0", + "--dest-dir", + "relout", + ] + ) + assert args.dest_dir.is_absolute() + assert args.dest_dir.name == "relout" + + +def test_absolute_dest_dir_stays_absolute(): + args = _args("--dest-dir", "/tmp/somewhere") + assert args.dest_dir.is_absolute() + assert args.dest_dir.name == "somewhere" diff --git a/build_tools/tests/publish_repo_package_test.py b/build_tools/tests/publish_repo_package_test.py new file mode 100644 index 00000000000..21d356573d0 --- /dev/null +++ b/build_tools/tests/publish_repo_package_test.py @@ -0,0 +1,151 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for publish_repo_package.py. + +publish_repo_package.py lives in build_tools/packaging/linux/, which is not on +the path that conftest.py sets up (it only adds build_tools/). Add it +explicitly. Importing the module is side-effect-free (boto3 is a deferred import +inside ``publish``), so key derivation is testable without boto3 installed. +""" + +import sys +from pathlib import Path + +import pytest + +_LINUX_DIR = Path(__file__).resolve().parents[1] / "packaging" / "linux" +sys.path.insert(0, str(_LINUX_DIR)) + +import publish_repo_package as prp # noqa: E402 + +PREFIX = "12345-linux/packages" + + +# --- object key derivation ---------------------------------------------------- + + +def test_key_deb_profile(): + assert ( + prp.object_key(f"{PREFIX}/deb", "ubuntu2404", "deb") + == "12345-linux/packages/deb/repo/ubuntu2404/amdrocm-repo.deb" + ) + + +@pytest.mark.parametrize("profile", ["rhel8", "rhel10", "sles16"]) +def test_key_rpm_profiles(profile): + assert ( + prp.object_key(f"{PREFIX}/rpm", profile, "rpm") + == f"12345-linux/packages/rpm/repo/{profile}/amdrocm-repo.rpm" + ) + + +def test_key_is_sibling_not_in_repo_index(): + # The file must sit under repo//, never inside the content index + # (deb dists/, rpm x86_64/). + deb = prp.object_key(f"{PREFIX}/deb", "ubuntu2404", "deb") + rpm = prp.object_key(f"{PREFIX}/rpm", "rhel10", "rpm") + for key in (deb, rpm): + assert "/repo/" in key + assert "/dists/" not in deb + assert "/x86_64/" not in rpm + + +def test_key_tolerates_trailing_slash_prefix(): + assert ( + prp.object_key(f"{PREFIX}/rpm/", "rhel10", "rpm") + == "12345-linux/packages/rpm/repo/rhel10/amdrocm-repo.rpm" + ) + + +# --- CLI validation ----------------------------------------------------------- + + +def _args(tmp_path, **overrides): + pkg = tmp_path / "amdrocm-repo-7.14.0-1.el10.noarch.rpm" + pkg.write_bytes(b"PKG") + argv = [ + "--file", + str(overrides.get("file", pkg)), + "--bucket", + overrides.get("bucket", "therock-prerelease-artifacts"), + "--prefix", + overrides.get("prefix", f"{PREFIX}/rpm"), + "--os-profile", + overrides.get("os_profile", "rhel10"), + "--pkg-type", + overrides.get("pkg_type", "rpm"), + ] + return argv + + +def test_parse_args_ok(tmp_path): + args = prp.parse_args(_args(tmp_path)) + assert args.bucket == "therock-prerelease-artifacts" + assert args.pkg_type == "rpm" + + +def test_parse_args_rejects_empty_bucket(tmp_path): + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, bucket="")) + + +def test_parse_args_rejects_missing_file(tmp_path): + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, file=str(tmp_path / "nope.rpm"))) + + +def test_parse_args_rejects_bad_pkg_type(tmp_path): + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, pkg_type="tar")) + + +@pytest.mark.parametrize("bad", ["rhel10/../x", "a/b", "rhel 10", "rhel10\n", ""]) +def test_parse_args_rejects_bad_os_profile(tmp_path, bad): + # os_profile becomes an S3 key path segment; reject slashes/whitespace. + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, os_profile=bad)) + + +# --- publish (boto3 mocked) --------------------------------------------------- + + +class _FakeS3: + def __init__(self): + self.uploaded = [] + self.headed = [] + self.head_should_fail = False + + def upload_file(self, filename, bucket, key): + self.uploaded.append((filename, bucket, key)) + + def head_object(self, Bucket, Key): + self.headed.append((Bucket, Key)) + if self.head_should_fail: + raise RuntimeError("object not found") + + +def _install_fake_boto3(monkeypatch, fake): + import types + + fake_boto3 = types.SimpleNamespace(client=lambda service, **kw: fake) + monkeypatch.setitem(sys.modules, "boto3", fake_boto3) + + +def test_publish_uploads_then_confirms(tmp_path, monkeypatch): + fake = _FakeS3() + _install_fake_boto3(monkeypatch, fake) + args = prp.parse_args(_args(tmp_path)) + key = prp.publish(args) + assert key == "12345-linux/packages/rpm/repo/rhel10/amdrocm-repo.rpm" + assert fake.uploaded == [(str(args.file), "therock-prerelease-artifacts", key)] + assert fake.headed == [("therock-prerelease-artifacts", key)] + + +def test_publish_errors_when_head_fails(tmp_path, monkeypatch): + fake = _FakeS3() + fake.head_should_fail = True + _install_fake_boto3(monkeypatch, fake) + args = prp.parse_args(_args(tmp_path)) + with pytest.raises(RuntimeError): + prp.publish(args) diff --git a/docs/packaging/native_packaging.md b/docs/packaging/native_packaging.md index fb5c3f69e65..1596f189b69 100644 --- a/docs/packaging/native_packaging.md +++ b/docs/packaging/native_packaging.md @@ -6,6 +6,9 @@ distribution as Debian and RPM packages. This process involves both the reorganization of artifacts and the execution of package management tools +To configure a system package manager to install ROCm from a public repository, +see [rocm_repo_setup.md](rocm_repo_setup.md). + ## General Design The required packages and their details are stored in a JSON file diff --git a/docs/packaging/rocm_repo_setup.md b/docs/packaging/rocm_repo_setup.md new file mode 100644 index 00000000000..2872337d16f --- /dev/null +++ b/docs/packaging/rocm_repo_setup.md @@ -0,0 +1,199 @@ +# Configuring the ROCm package repository (`amdrocm-repo`) + +`amdrocm-repo` is a configuration package that points a system package manager +(`apt`, `dnf`, or `zypper`) at a public AMD ROCm repository. Once it is +installed, you can install and update ROCm packages with the native package +manager. + +It configures the repository only. The ROCm *content* packages and their upload +paths are documented in [native_packaging.md](native_packaging.md). + +## What the package installs + +`amdrocm-repo` is generated per OS profile. Every profile writes a single repo +definition and, for signed lines, the repository signing key: + +| OS profile | Package manager | Repo definition | Signing key | +| ----------------- | --------------- | ----------------------------------------- | ----------------------------------- | +| `ubuntu2404` | `apt` | `/etc/apt/sources.list.d/amdrocm.sources` | `/usr/share/keyrings/rocm.gpg` | +| `rhel8`, `rhel10` | `dnf` / `yum` | `/etc/yum.repos.d/amdrocm.repo` | `/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm` | +| `sles16` | `zypper` | `/etc/zypp/repos.d/amdrocm.repo` | `/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm` | + +The deb repo definition uses the deb822 `.sources` format +(`Suites: stable`, `Components: main`, `Architectures: amd64`) and references +the shipped keyring via `Signed-By`. The rpm repo definition references the +shipped key via `gpgkey=file://`. + +## Release lines + +The configured repository depends on the release line. All repositories serve +x86_64 (`amd64`) packages only. + +| Release line | Repository base | Signed | +| ------------ | ------------------------------------------------------ | ------ | +| `prerelease` | `https://rocm.prereleases.amd.com/packages-multi-arch` | yes | +| `release` | `https://repo.amd.com/rocm/packages-multi-arch` | yes | +| `nightly` | `https://rocm.nightlies.amd.com/packages-multi-arch` | no | + +The lines do not share a repository layout. The `prerelease` and `release` +repositories are published per distro; the `nightly` repository is published per +package type under a dated sub-folder. So the repo file resolves to: + +| Release line | deb `URIs` | rpm `baseurl` | +| ----------------------- | ----------------------- | ------------------------------ | +| `prerelease`, `release` | `//` | `//x86_64/` | +| `nightly` | `/deb//` | `/rpm//x86_64/` | + +The signing key is at `/gpg/rocm.gpg` on both signed lines. + +> **Nightly is unsigned.** The nightly repository is not signed, so the nightly +> `amdrocm-repo` disables signature verification (`gpgcheck=0` / apt +> `Trusted: yes`) and ships no key. Prefer a signed line (`prerelease` or +> `release`) for anything beyond testing. Nightly packages live in a dated +> sub-folder, so a nightly `amdrocm-repo` is pinned to its build date. + +## Download URLs + +On the `prerelease` and `nightly` lines, `amdrocm-repo` is published as a +standalone file per OS profile, alongside the packages it configures: + +- prerelease: `//repo//amdrocm-repo.` +- nightly: `///repo//amdrocm-repo.` + +where `` is `deb` or `rpm` and `` matches. For example, on the +prerelease line: + +| OS profile | Download URL | +| ------------ | ------------------------------------------------------------------------------------------- | +| `ubuntu2404` | `https://rocm.prereleases.amd.com/packages-multi-arch/deb/repo/ubuntu2404/amdrocm-repo.deb` | +| `rhel8` | `https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/rhel8/amdrocm-repo.rpm` | +| `rhel10` | `https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/rhel10/amdrocm-repo.rpm` | +| `sles16` | `https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/sles16/amdrocm-repo.rpm` | + +A distro's package is reachable once its line has published. The `release` line +is published through the ROCm release process rather than from this repository's +build, so its download location is not listed here. To use the release +repository before that package is available, build it locally with +`--release-type release` (see below). + +## Installing and using the repository + +Install the `amdrocm-repo` package for your distro, refresh the package +manager's metadata, then install ROCm packages. + +Ubuntu: + +```bash +curl -fsSL -O https://rocm.prereleases.amd.com/packages-multi-arch/deb/repo/ubuntu2404/amdrocm-repo.deb +sudo apt install ./amdrocm-repo.deb +sudo apt-get update +sudo apt install amdrocm +``` + +RHEL / Rocky / Oracle Linux: + +```bash +sudo dnf install https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/rhel10/amdrocm-repo.rpm +sudo dnf makecache +sudo dnf install amdrocm +``` + +SLES: + +```bash +sudo zypper install https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/sles16/amdrocm-repo.rpm +sudo zypper refresh +sudo zypper install amdrocm +``` + +On a signed line, `zypper` prompts you to trust the repository signing key the +first time it refreshes; accept it to continue. + +Confirm the repository is configured by listing the `amdrocm*` packages it +publishes: + +```bash +apt list 'amdrocm*' # Ubuntu +dnf list 'amdrocm*' # RHEL / Rocky / Oracle Linux +zypper search amdrocm # SLES +``` + +The command returns the available packages (for example `amdrocm` and +`amdrocm-base`). An empty result means the repository is not configured, or its +metadata has not been refreshed. Install the components you need from that +list. + +## Troubleshooting + +- **404 when downloading the package.** The release line is not published for + your distro yet. Use the `prerelease` line, or build the package locally (see + below). +- **`amdrocm*` packages are not listed.** Refresh the package manager metadata + (`sudo apt-get update`, `sudo dnf makecache`, or `sudo zypper refresh`) after + installing `amdrocm-repo`. +- **`zypper` reports an untrusted signing key.** Accept the key when prompted on + the first refresh of a signed line. + +## Building the package locally + +`build_repo_package.py` builds `amdrocm-repo` for a single OS profile and +release line. It requires Python 3.12 or newer (note that some rpm base images, +such as RHEL 8, do not ship a system `python3` at that version), the `jinja2` +Python package, and the native packaging tools for the target format: + +- deb (`ubuntu2404`): `debhelper`, `dpkg-dev`, `build-essential` +- rpm (`rhel8`, `rhel10`, `sles16`): `rpm-build` + +For a signed line the builder also invokes `gpg` to prepare the key, and fetches +it over the network (see below), so `gpg` and `ca-certificates` must be present. + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile ubuntu2404 \ + --release-type prerelease \ + --repo-base-url https://rocm.prereleases.amd.com/packages-multi-arch \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out +``` + +For a signed line, the signing key is fetched from `/gpg/rocm.gpg` over +the network at build time. To build offline, or to pin a specific key, pass it +explicitly: + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile rhel10 \ + --release-type prerelease \ + --repo-base-url https://rocm.prereleases.amd.com/packages-multi-arch \ + --gpg-key-file ./rocm.gpg \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out +``` + +The nightly line is date-pinned; pass the dated sub-folder: + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile ubuntu2404 \ + --release-type nightly \ + --repo-base-url https://rocm.nightlies.amd.com/packages-multi-arch \ + --repo-sub-folder 20260722-123456789 \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out +``` + +Pass `--verify-repo-url` to fail the build unless the repository the package +configures is reachable. This catches a package that would install cleanly but +fail on the user's first metadata refresh: + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile rhel10 \ + --release-type release \ + --repo-base-url https://repo.amd.com/rocm/packages-multi-arch \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out \ + --verify-repo-url +``` + +See `build_repo_package.py --help` for the full option list. diff --git a/requirements-test.txt b/requirements-test.txt index ff1cc524f7e..dbdee1695a4 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,5 @@ boto3==1.39.15 +jinja2>=3.0.0 # Packaging scripts render templates; their tests import them. pytest==9.0.3 # This versions has subtests built-in. pytest-check==2.5.3 pytest-cov==6.0.0