From 44b7bb1fd6de7572a56d491c95de3a03ef7cc284 Mon Sep 17 00:00:00 2001 From: Lalit Sudarshan Date: Mon, 27 Jul 2026 13:50:38 -0400 Subject: [PATCH] feat(packaging): add amdrocm-repo package for ROCm repository setup Adds a configuration package that points apt, dnf, or zypper to a public ROCm repository, so ROCm can be installed and updated with the native package manager instead of a manual multi-step repository setup. The package is generated per OS profile and release line. Repository URLs are derived per line, because the prerelease and release repositories are published per distro while the nightly repository is published per package type under a dated sub-folder. The package version carries the release line, so packages built from different lines are never confusable. For signed lines the signing key is fetched at build time, verified against a pinned fingerprint, and embedded. It is never stored in the source tree. This targets the repository layout currently served. That layout is expected to change, so URL derivation is isolated in one function to keep the migration small. --- ...multi_arch_build_native_linux_packages.yml | 274 ++++++ .pre-commit-config.yaml | 2 +- build_tools/.gitignore | 2 +- .../packaging/linux/build_repo_package.py | 653 ++++++++++++++ .../packaging/linux/get_url_repo_params.py | 137 ++- .../native_linux_package_install_test.py | 314 ++++++- .../packaging/linux/publish_repo_package.py | 109 +++ .../packaging/linux/setup_repo_build_deps.sh | 70 ++ .../template/repo/deb/amdrocm.sources.j2 | 10 + .../linux/template/repo/deb/changelog.j2 | 5 + .../linux/template/repo/deb/control.j2 | 13 + .../linux/template/repo/deb/install.j2 | 4 + .../linux/template/repo/deb/postinst.j2 | 11 + .../linux/template/repo/deb/rules.j2 | 3 + .../template/repo/rpm/amdrocm-repo.spec.j2 | 29 + .../linux/template/repo/rpm/amdrocm.repo.j2 | 8 + .../linux/tests/get_url_repo_params_test.py | 197 +++- .../native_linux_package_install_ut_test.py | 424 +++++++++ build_tools/tests/build_repo_package_test.py | 842 ++++++++++++++++++ .../tests/publish_repo_package_test.py | 151 ++++ docs/packaging/native_packaging.md | 3 + docs/packaging/rocm_repo_setup.md | 199 +++++ requirements-test.txt | 1 + 23 files changed, 3423 insertions(+), 38 deletions(-) create mode 100644 build_tools/packaging/linux/build_repo_package.py create mode 100644 build_tools/packaging/linux/publish_repo_package.py create mode 100755 build_tools/packaging/linux/setup_repo_build_deps.sh create mode 100644 build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 create mode 100644 build_tools/packaging/linux/template/repo/deb/changelog.j2 create mode 100644 build_tools/packaging/linux/template/repo/deb/control.j2 create mode 100644 build_tools/packaging/linux/template/repo/deb/install.j2 create mode 100644 build_tools/packaging/linux/template/repo/deb/postinst.j2 create mode 100644 build_tools/packaging/linux/template/repo/deb/rules.j2 create mode 100644 build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 create mode 100644 build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 create mode 100644 build_tools/tests/build_repo_package_test.py create mode 100644 build_tools/tests/publish_repo_package_test.py create mode 100644 docs/packaging/rocm_repo_setup.md diff --git a/.github/workflows/multi_arch_build_native_linux_packages.yml b/.github/workflows/multi_arch_build_native_linux_packages.yml index a433f45b8a7..47c09949932 100644 --- a/.github/workflows/multi_arch_build_native_linux_packages.yml +++ b/.github/workflows/multi_arch_build_native_linux_packages.yml @@ -50,6 +50,47 @@ permissions: run-name: Build ${{ inputs.native_package_type }} packages (${{ inputs.rocm_version }}, ${{ inputs.dist_amdgpu_families }}${{ inputs.release_type && format(', {0}', inputs.release_type) || '' }}) jobs: + # Resolve the parameters used to build and test the amdrocm-repo package: + # the OS-profile build matrix for this package type, each profile's container + # image, the public CDN base URL for this release line (empty for lines with + # no public repo, e.g. ci), and the dated sub-folder for the nightly line. + setup_repo_params: + name: Resolve amdrocm-repo parameters + runs-on: ubuntu-24.04 + outputs: + os_profiles: ${{ steps.derive.outputs.os_profiles }} + images: ${{ steps.derive.outputs.images }} + repo_base_url: ${{ steps.derive.outputs.repo_base_url }} + repo_sub_folder: ${{ steps.derive.outputs.repo_sub_folder }} + steps: + - name: Checking out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install Python requirements + run: pip install jinja2 + + - name: Derive amdrocm-repo parameters + id: derive + run: | + profiles="$(python ./build_tools/packaging/linux/build_repo_package.py list-profiles \ + --pkg-type '${{ inputs.native_package_type }}')" + echo "os_profiles=${profiles}" >> "$GITHUB_OUTPUT" + python ./build_tools/packaging/linux/get_url_repo_params.py get-container-image-map \ + --os-profiles "${profiles}" + python ./build_tools/packaging/linux/get_url_repo_params.py get-public-repo-base-url \ + --release-type '${{ inputs.release_type }}' + python ./build_tools/packaging/linux/get_url_repo_params.py get-nightly-sub-folder \ + --release-type '${{ inputs.release_type }}' \ + --run-id '${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}' + build_native_packages: name: Build ${{ inputs.native_package_type }} packages runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} @@ -137,3 +178,236 @@ jobs: --pkg-type "${{ inputs.native_package_type }}" \ --run-id "${{ env.ARTIFACT_RUN_ID }}" \ --package-dir "${{ env.PACKAGE_DIST_DIR }}" + + # Build the amdrocm-repo package and verify it configures the repository, for + # each OS profile of this package type. This runs on pull-request builds and is + # a required check. It builds against the live prerelease repository, which is + # published in the same per-distro layout the release line uses, so this + # exercises the shape end users get. The nightly and release lines are covered + # by stage_repo_package, which verifies their repository URL at build time. + test_repo_package: + name: Test amdrocm-repo config (${{ matrix.os_profile }}) + needs: [setup_repo_params] + if: inputs.release_type == 'ci' + strategy: + fail-fast: false + matrix: + os_profile: ${{ fromJSON(needs.setup_repo_params.outputs.os_profiles) }} + runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} + container: + image: ${{ fromJSON(needs.setup_repo_params.outputs.images)[matrix.os_profile] }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Install system prerequisites + run: | + bash build_tools/packaging/linux/setup_repo_build_deps.sh \ + --os-profile "${{ matrix.os_profile }}" + + - name: Setup Python + run: | + bash build_tools/packaging/linux/setup_python_cmd.sh \ + --os-profile "${{ matrix.os_profile }}" \ + --install-runtime \ + --output-format github + + - name: Install Python dependencies + run: | + $PYTHON_CMD -m venv /tmp/repo-venv + /tmp/repo-venv/bin/python -m pip install --upgrade pip + /tmp/repo-venv/bin/python -m pip install jinja2 + echo "PYTHON_CMD=/tmp/repo-venv/bin/python" >> "$GITHUB_ENV" + + - name: Resolve prerelease repository base URL + id: prerelease + run: | + $PYTHON_CMD build_tools/packaging/linux/get_url_repo_params.py \ + get-public-repo-base-url --release-type prerelease + + - name: Build and validate amdrocm-repo (config-only) + run: | + # Build against the live prerelease repository, install the package, and + # verify it configures the repository (config-only stops before ROCm). + dest="$(mktemp -d)" + $PYTHON_CMD build_tools/packaging/linux/build_repo_package.py \ + --os-profile "${{ matrix.os_profile }}" \ + --release-type prerelease \ + --repo-base-url "${{ steps.prerelease.outputs.repo_base_url }}" \ + --rocm-version "${{ inputs.rocm_version }}" \ + --dest-dir "$dest" + $PYTHON_CMD build_tools/packaging/linux/native_linux_package_install_test.py \ + --repo-package-dir "$dest" \ + --repo-config-only \ + --os-profile "${{ matrix.os_profile }}" \ + --release-type prerelease + + # Build the amdrocm-repo package for each OS profile of this package type on the + # public release lines and stage it as a workflow artifact. Best-effort: a build + # failure here must never block the release, so the job cannot fail the workflow. + # Publishing to a public download location is handled separately. + stage_repo_package: + name: Stage amdrocm-repo (${{ matrix.os_profile }}) + needs: [setup_repo_params] + if: needs.setup_repo_params.outputs.repo_base_url != '' + continue-on-error: true + strategy: + fail-fast: false + matrix: + os_profile: ${{ fromJSON(needs.setup_repo_params.outputs.os_profiles) }} + runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} + container: + image: ${{ fromJSON(needs.setup_repo_params.outputs.images)[matrix.os_profile] }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Install system prerequisites + run: | + bash build_tools/packaging/linux/setup_repo_build_deps.sh \ + --os-profile "${{ matrix.os_profile }}" + + - name: Setup Python + run: | + bash build_tools/packaging/linux/setup_python_cmd.sh \ + --os-profile "${{ matrix.os_profile }}" \ + --install-runtime \ + --output-format github + + - name: Install Python dependencies + run: | + $PYTHON_CMD -m venv /tmp/repo-venv + /tmp/repo-venv/bin/python -m pip install --upgrade pip + /tmp/repo-venv/bin/python -m pip install jinja2 + echo "PYTHON_CMD=/tmp/repo-venv/bin/python" >> "$GITHUB_ENV" + + - name: Build amdrocm-repo + run: | + # repo_sub_folder is empty except on the nightly line, where the builder + # requires the dated sub-folder; passing an empty value is accepted for + # the other lines. + # + # --verify-repo-url fails the build if the configured repository is not + # served, so a package that installs cleanly but cannot refresh is + # caught here rather than by a user. It is a no-op on the nightly line, + # whose dated sub-folder is published by this same run. + $PYTHON_CMD build_tools/packaging/linux/build_repo_package.py \ + --os-profile "${{ matrix.os_profile }}" \ + --release-type "${{ inputs.release_type }}" \ + --repo-base-url "${{ needs.setup_repo_params.outputs.repo_base_url }}" \ + --repo-sub-folder "${{ needs.setup_repo_params.outputs.repo_sub_folder }}" \ + --rocm-version "${{ inputs.rocm_version }}" \ + --dest-dir repo-package-out \ + --verify-repo-url + + - name: Upload amdrocm-repo package + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: amdrocm-repo-${{ inputs.native_package_type }}-${{ matrix.os_profile }} + path: repo-package-out/*.${{ inputs.native_package_type }} + if-no-files-found: error + + # This job is continue-on-error so a staging failure never blocks the + # release; surface it as a warning + job-summary line so the soft-fail is + # not silent. A failure here can mean the configured repository is no + # longer reachable, which is what --verify-repo-url detects. + - name: Report staging failure + if: failure() + run: | + msg="amdrocm-repo staging FAILED for ${{ matrix.os_profile }} (${{ inputs.native_package_type }}, ${{ inputs.release_type }}). The configured repository may be unreachable." + echo "::warning::${msg}" + echo "WARNING: ${msg}" >> "$GITHUB_STEP_SUMMARY" + + # Publish the built amdrocm-repo packages to the per-run packages location, + # alongside the native content packages, as standalone per-distro files (a + # repo// sibling of the content repository, not part of its index). + # They become publicly reachable once the release promotion step copies the + # per-run tree to the public CDN. This runs in a generic container (not the + # per-distro build containers) because the AWS credentials action requires + # python on PATH, which the per-distro Python setup does not provide. + # Non-blocking: a publish failure must never block the release. + # + # The release line is excluded: it has no automated upload credentials by + # design (the release artifacts bucket grants no write role), so resolving that + # bucket fails outright. The release package is published by the ROCm release + # process instead. Without this exclusion the job would fail on every release + # run and report a spurious "not published" warning. + publish_repo_package: + name: Publish amdrocm-repo (${{ matrix.os_profile }}) + needs: [setup_repo_params, stage_repo_package] + if: needs.setup_repo_params.outputs.repo_base_url != '' && inputs.release_type != 'release' + continue-on-error: true + strategy: + fail-fast: false + matrix: + os_profile: ${{ fromJSON(needs.setup_repo_params.outputs.os_profiles) }} + runs-on: ${{ github.repository_owner == 'ROCm' && 'aws-linux-scale-rocm-prod' || 'ubuntu-24.04' }} + container: + image: ghcr.io/rocm/no_rocm_image_ubuntu24_04:latest + env: + RELEASE_TYPE: ${{ inputs.release_type }} + ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install Python dependencies + run: pip install boto3 + + - name: Download amdrocm-repo package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: amdrocm-repo-${{ inputs.native_package_type }}-${{ matrix.os_profile }} + path: repo-package-out + + - name: Resolve artifacts bucket + id: bucket-info + run: | + python build_tools/github_actions/write_artifacts_bucket_info.py \ + --release-type "${{ inputs.release_type }}" + + - name: Configure AWS credentials for artifact uploads + uses: ./.github/actions/configure_aws_artifacts_credentials + with: + release_type: ${{ inputs.release_type }} + + - name: Publish amdrocm-repo package + id: publish + run: | + # The per-run prefix mirrors WorkflowOutputRoot.native_linux_packages + # (-linux/packages/) so the file lands beside the + # content packages. ARTIFACT_RUN_ID matches what the content upload + # used; the external-repo prefix segment is empty for the released + # lines this job runs on (prerelease/release/nightly), so it is + # intentionally omitted. + pkg="$(ls repo-package-out/*.${{ inputs.native_package_type }})" + python build_tools/packaging/linux/publish_repo_package.py \ + --file "$pkg" \ + --bucket "${{ steps.bucket-info.outputs.bucket }}" \ + --prefix "${ARTIFACT_RUN_ID}-linux/packages/${{ inputs.native_package_type }}" \ + --os-profile "${{ matrix.os_profile }}" \ + --pkg-type "${{ inputs.native_package_type }}" + + # This job is continue-on-error so a publish failure never blocks the + # release; surface it as a warning + job-summary line so the soft-fail is + # not silent. + - name: Report publish failure + if: failure() + run: | + msg="amdrocm-repo was NOT published for ${{ matrix.os_profile }} (${{ inputs.native_package_type }}, ${{ inputs.release_type }})." + echo "::warning::${msg}" + echo "WARNING: ${msg}" >> "$GITHUB_STEP_SUMMARY" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 995f6f922c1..e0ebfb1053c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: rev: v1.5.5 hooks: - id: forbid-tabs - exclude: ".gitmodules|patches/.*|docs/development/git_chores.md|build_tools/packaging/linux/template/debian_rules.j2|third-party/sysdeps/linux/libdrm/patch_source.sh" + exclude: ".gitmodules|patches/.*|docs/development/git_chores.md|build_tools/packaging/linux/template/debian_rules.j2|build_tools/packaging/linux/template/repo/deb/rules.j2|third-party/sysdeps/linux/libdrm/patch_source.sh" - repo: https://github.com/rhysd/actionlint rev: v1.7.10 diff --git a/build_tools/.gitignore b/build_tools/.gitignore index f606d5e0b04..64bcebfd9d8 100644 --- a/build_tools/.gitignore +++ b/build_tools/.gitignore @@ -1 +1 @@ -repo +/repo diff --git a/build_tools/packaging/linux/build_repo_package.py b/build_tools/packaging/linux/build_repo_package.py new file mode 100644 index 00000000000..86c46a8a8ba --- /dev/null +++ b/build_tools/packaging/linux/build_repo_package.py @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 + +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +r"""Build the ``amdrocm-repo`` package that configures a system package manager +(apt/dnf/zypper) to install AMD ROCm from the public ROCm repositories. + +One ``amdrocm-repo`` package is generated per target OS profile and release +line. It ships a single repository definition and, for signed lines, the AMD +signing key, so that after installing the package a user can install ROCm with +their native package manager. The signing key is fetched at build time and +embedded in the package; it is never stored in the source tree. + +``` +python build_repo_package.py \ + --os-profile ubuntu2404 \ + --release-type prerelease \ + --repo-base-url https://rocm.prereleases.amd.com/packages-multi-arch \ + --rocm-version 7.14.0 \ + --dest-dir ./output +``` +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +from datetime import datetime, timezone +from email.utils import format_datetime +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader + +SCRIPT_DIR = Path(__file__).resolve().parent + +# The package name and the repository identifier used for the repo file and its +# section header. A single repository is configured per package. +PACKAGE_NAME = "amdrocm-repo" +REPO_ID = "amdrocm" +REPO_NAME = "AMD ROCm" + +# The deb suite the repositories publish under (``dists//``). Used to +# locate the index when verifying a repo URL. It is written literally in +# template/repo/deb/amdrocm.sources.j2; a test asserts the two agree. +DEB_SUITE = "stable" + +MAINTAINER = "ROCm Dev Support " + +# Where each package installs the signing key, and which the repo files +# reference. deb consumes a dearmored (binary) keyring via Signed-By; rpm +# references the (armored) key via gpgkey=file://. +DEB_KEYRING_PATH = "/usr/share/keyrings/rocm.gpg" +RPM_GPG_KEY_PATH = "/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" + +GPG_FETCH_TIMEOUT_SEC = 60 +# Retry the key fetch so a transient network/CDN blip does not fail a build. +GPG_FETCH_ATTEMPTS = 3 +GPG_FETCH_BACKOFF_SEC = 3 +# Upper bound on the fetched signing key. A real armored key is a few KB; this +# bounds memory use if a repository host returns an unexpectedly large body. +MAX_GPG_KEY_BYTES = 1 << 20 # 1 MiB + +# Bounds for the optional repository reachability check (--verify-repo-url). +REPO_VERIFY_TIMEOUT_SEC = 30 +REPO_VERIFY_ATTEMPTS = 3 +REPO_VERIFY_BACKOFF_SEC = 3 +# HTTP statuses worth retrying: request timeout and rate limiting. Any other 4xx +# means the repository is not at that URL, which retrying cannot change. +RETRYABLE_HTTP_STATUS = frozenset({408, 429}) + +# Primary-key fingerprint of the AMD ROCm signing key. The key is fetched over +# the network at build time and embedded as the package's trust anchor, so its +# identity is pinned here and verified after fetch. Rotating the repository +# signing key requires updating this value. +EXPECTED_KEY_FINGERPRINT = "D0F004A0025A1145C7807FCD0701EAC4D5E02107" + +# Accepted input shapes. rocm_version and repo_sub_folder are rendered into the +# generated repo files, package version fields, and (for the deb version string) +# other metadata, so they are constrained to characters that cannot alter the +# surrounding syntax. repo_base_url is concatenated into the repo files as well. +# Use \Z (not $) so a trailing newline is not accepted. +_VERSION_RE = re.compile(r"^[0-9][0-9A-Za-z.+~-]*\Z") +_SUB_FOLDER_RE = re.compile(r"^[0-9]{8}-[0-9A-Za-z._-]+\Z") + +# Release lines whose public repositories are signed. The nightly line is +# unsigned, so it ships no key and disables signature checking. +SIGNED_RELEASE_TYPES = ("prerelease", "release") +RELEASE_TYPES = ("prerelease", "release", "nightly") + +OS_PROFILES = { + "ubuntu2404": { + "family": "debian", + "pkg_type": "deb", + "description": "Ubuntu 24.04", + }, + "rhel8": { + "family": "rhel", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/yum.repos.d", + "description": "RHEL/Rocky/OL 8", + }, + "rhel10": { + "family": "rhel", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/yum.repos.d", + "description": "RHEL/Rocky/OL 10", + }, + "sles16": { + "family": "sles", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/zypp/repos.d", + "description": "SLES 16", + }, +} + + +# --- URL / signing derivation ------------------------------------------------ +# +# The public ROCm repositories do not share a single layout, so the repository +# URL is derived per release line: +# +# prerelease, release one repository per distro: +# // (deb) +# //x86_64/ (rpm) +# nightly one repository per package type, under a dated +# sub-folder: +# /deb// (deb) +# /rpm//x86_64/ (rpm) +# +# The signing key is at /gpg/rocm.gpg on every signed line. + + +def is_signed(release_type: str) -> bool: + """Whether this release line's public repository is signed.""" + return release_type in SIGNED_RELEASE_TYPES + + +def repo_baseurl( + repo_base_url: str, + pkg_type: str, + os_profile: str, + release_type: str, + repo_sub_folder: str = "", +) -> str: + """Return the repository baseurl baked into the repo file. + + The layout differs per release line (see the section comment above), so both + the target distro and the release line are required. deb returns the repo + root, beneath which apt resolves ``dists/stable/main/``; rpm returns the + directory containing ``repodata/``. ``repo_sub_folder`` is the dated segment + used by the nightly line. + """ + base = repo_base_url.rstrip("/") + # The layout is a property of the release line, not of whether it is signed; + # the two happen to correlate today but are independent. + if release_type == "nightly": + sub = f"{repo_sub_folder}/" if repo_sub_folder else "" + if pkg_type == "deb": + return f"{base}/deb/{sub}" + return f"{base}/rpm/{sub}x86_64/" + if pkg_type == "deb": + return f"{base}/{os_profile}/" + return f"{base}/{os_profile}/x86_64/" + + +def gpg_key_url(repo_base_url: str) -> str: + """Return the signing-key URL for a repo base url.""" + return f"{repo_base_url.rstrip('/')}/gpg/rocm.gpg" + + +def repo_metadata_url(baseurl: str, pkg_type: str) -> str: + """Return a repository index file that must exist under ``baseurl``. + + apt reads ``dists//Release``; dnf and zypper read + ``repodata/repomd.xml``. Fetching one proves the baseurl serves a repository. + """ + if pkg_type == "deb": + return f"{baseurl}dists/{DEB_SUITE}/Release" + return f"{baseurl}repodata/repomd.xml" + + +def verify_repo_url(baseurl: str, pkg_type: str) -> None: + """Fail the build if ``baseurl`` does not serve a package repository. + + The baseurl is baked into the package and is only exercised the first time a + user refreshes their package manager, so an unreachable one would otherwise + ship undetected. Transient failures are retried; a repository that is simply + not there fails immediately. + + The scheme is not re-checked here: this response is neither trusted nor + retained, and the signing key (which is) enforces https separately. + """ + url = repo_metadata_url(baseurl, pkg_type) + last_error: Exception | None = None + for attempt in range(1, REPO_VERIFY_ATTEMPTS + 1): + try: + print(f"Verifying repository (attempt {attempt}): {url}") + with urllib.request.urlopen(url, timeout=REPO_VERIFY_TIMEOUT_SEC) as resp: + status = getattr(resp, "status", None) or resp.getcode() + if status == 200: + print(f"Repository verified: {baseurl}") + return + last_error = RuntimeError(f"unexpected HTTP status {status}") + except urllib.error.HTTPError as e: + # A missing or forbidden index is a wrong URL, not a blip. Some + # object stores answer 403 rather than 404 for absent keys, so treat + # both as final; only rate limiting and timeouts are worth retrying. + if e.code not in RETRYABLE_HTTP_STATUS: + raise RuntimeError( + f"repository is not reachable at {baseurl} " + f"(checked {url}): HTTP {e.code}" + ) from e + last_error = e + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + last_error = e + if attempt < REPO_VERIFY_ATTEMPTS: + time.sleep(REPO_VERIFY_BACKOFF_SEC * attempt) + raise RuntimeError( + f"repository is not reachable at {baseurl} (checked {url}): {last_error}" + ) + + +def _require_https(url: str, what: str) -> None: + """Reject a non-https URL used to fetch trusted key material.""" + if urllib.parse.urlsplit(url).scheme != "https": + raise ValueError(f"{what} must use https to protect the key in transit: {url}") + + +def _valid_repo_base_url(url: str) -> bool: + """Whether a repo base URL is a well-formed http(s) URL with no control chars. + + The value is concatenated into the generated repo files, so whitespace or + control characters (e.g. a newline) that could inject extra directives are + rejected here. + """ + if any(c.isspace() or ord(c) < 0x20 for c in url): + return False + parts = urllib.parse.urlsplit(url) + return parts.scheme in ("http", "https") and bool(parts.netloc) + + +def rpm_version_release( + release_type: str, rocm_version: str, repo_sub_folder: str +) -> tuple[str, str]: + """Return (Version, Release) for the rpm package. + + Prerelease/release track the (rolling) ROCm version. Nightly is date-pinned + from the ``YYYYMMDD-`` sub-folder; an rpm Version cannot contain ``-``, + so the date becomes the Version and the id becomes the Release. + + The release line is part of the Release field so that two lines can never + produce an identically named package. Without it, installing the release + package over the prerelease one at the same ROCm version is a silent no-op: + the package manager reports success and leaves the old repository + configured. + """ + if release_type == "nightly": + date, _, ident = repo_sub_folder.partition("-") + return date, f"{ident or '1'}.nightly" + return rocm_version, f"1.{release_type}" + + +def deb_version(release_type: str, rocm_version: str, repo_sub_folder: str) -> str: + """Return the deb package version (nightly is date-pinned, no ``-``). + + Prerelease carries a ``~prerelease`` suffix. ``~`` sorts before the plain + version, so the release package upgrades over the prerelease one and the two + lines can never share a version (see ``rpm_version_release``). + """ + if release_type == "nightly": + return repo_sub_folder.replace("-", ".") + if release_type == "prerelease": + return f"{rocm_version}~prerelease" + return rocm_version + + +# --- signing key ------------------------------------------------------------- + + +def _fetch_signing_key(url: str) -> bytes: + """Fetch the armored signing key over https, size-bounded, with retries. + + Only transient network errors are retried; a scheme downgrade or an + over-size body fails immediately. + """ + # The fetched key is embedded and trusted, so it must travel over https. + _require_https(url, "signing key URL") + last_error: Exception | None = None + for attempt in range(1, GPG_FETCH_ATTEMPTS + 1): + try: + print(f"Fetching signing key (attempt {attempt}): {url}") + with urllib.request.urlopen(url, timeout=GPG_FETCH_TIMEOUT_SEC) as resp: + # urlopen follows redirects; re-check the scheme actually served + # so an https URL cannot be silently downgraded to http. + _require_https(resp.geturl(), "signing key URL (after redirects)") + # Read one byte past the cap to detect an over-size body. + key = resp.read(MAX_GPG_KEY_BYTES + 1) + if len(key) > MAX_GPG_KEY_BYTES: + raise ValueError( + f"signing key exceeds {MAX_GPG_KEY_BYTES} bytes: {url}" + ) + return key + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + last_error = e + if attempt < GPG_FETCH_ATTEMPTS: + time.sleep(GPG_FETCH_BACKOFF_SEC * attempt) + raise RuntimeError( + f"failed to fetch signing key after {GPG_FETCH_ATTEMPTS} attempts: {url}" + ) from last_error + + +def _run_gpg(argv: list[str], **kwargs) -> subprocess.CompletedProcess: + """Run gpg, reporting a missing binary by name instead of a traceback. + + Building any signed line needs gpg: the deb path to dearmor the keyring, and + both paths to check the key fingerprint. + """ + try: + return subprocess.run(argv, check=True, **kwargs) + except FileNotFoundError as e: + raise RuntimeError( + "gpg is required to build a signed release line " + "(install gnupg/gnupg2, or pass --gpg-key-file for an unsigned build)" + ) from e + + +def _key_fingerprint(armored: bytes) -> str: + """Return the primary-key fingerprint of an armored public key.""" + with tempfile.TemporaryDirectory() as home: + key_path = Path(home) / "key.asc" + key_path.write_bytes(armored) + result = _run_gpg( + ["gpg", "--homedir", home, "--with-colons", "--show-keys", str(key_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for line in result.stdout.decode(errors="replace").splitlines(): + if line.startswith("fpr:"): + return line.split(":")[9] + raise ValueError("no fingerprint found in the fetched signing key") + + +def _verify_key_fingerprint(armored: bytes) -> None: + """Reject a fetched key whose fingerprint is not the pinned AMD ROCm key.""" + fingerprint = _key_fingerprint(armored) + if fingerprint != EXPECTED_KEY_FINGERPRINT: + raise ValueError( + f"fetched signing key fingerprint {fingerprint} does not match the " + f"expected {EXPECTED_KEY_FINGERPRINT}" + ) + + +def load_signing_key(args: argparse.Namespace) -> bytes: + """Return the armored signing key bytes for a signed line. + + Read from ``--gpg-key-file`` when given (offline/test builds), otherwise + fetched from the repo's ``gpg/rocm.gpg`` at build time and verified against + the pinned fingerprint. Never written into the source tree. + """ + if args.gpg_key_file: + return args.gpg_key_file.read_bytes() + key = _fetch_signing_key(gpg_key_url(args.repo_base_url)) + _verify_key_fingerprint(key) + return key + + +def dearmor_key(armored: bytes) -> bytes: + """Convert an armored key to a binary keyring (for deb Signed-By).""" + result = _run_gpg( + ["gpg", "--dearmor"], + input=armored, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return result.stdout + + +# --- templating -------------------------------------------------------------- + + +def get_jinja_env() -> Environment: + # keep_trailing_newline so rendered config files end with a newline (POSIX + # text-file hygiene; Jinja strips the final newline by default). + return Environment( + loader=FileSystemLoader(str(SCRIPT_DIR)), + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + ) + + +def _render( + env: Environment, template_path: str, output_path: Path, context: dict +) -> None: + output_path.write_text(env.get_template(template_path).render(context)) + + +def build_context(args: argparse.Namespace, profile: dict) -> dict: + """Template fields shared by the repo file, spec, and deb metadata.""" + signed = is_signed(args.release_type) + return { + "pkg_name": PACKAGE_NAME, + "repo_id": REPO_ID, + "name": REPO_NAME, + "baseurl": repo_baseurl( + args.repo_base_url, + profile["pkg_type"], + args.os_profile, + args.release_type, + args.repo_sub_folder, + ), + "signed": signed, + "rocm_version": args.rocm_version, + "maintainer": MAINTAINER, + "description_short": "AMD ROCm repository configuration", + "description_long": ( + "Configures the system package manager to install AMD ROCm packages." + ), + "deb_keyring_path": DEB_KEYRING_PATH, + "rpm_gpg_key_path": RPM_GPG_KEY_PATH, + "rpm_repo_dir": profile.get("rpm_repo_dir", ""), + } + + +# --- rpm build --------------------------------------------------------------- + + +def build_rpm_package( + args: argparse.Namespace, profile: dict, context: dict, dest_dir: Path +) -> None: + env = get_jinja_env() + build_dir = dest_dir / "rpm" / PACKAGE_NAME + if build_dir.exists(): + shutil.rmtree(build_dir) + for subdir in ("SOURCES", "SPECS", "BUILD", "RPMS", "SRPMS"): + (build_dir / subdir).mkdir(parents=True, exist_ok=True) + sources_dir = build_dir / "SOURCES" + + _render( + env, + "template/repo/rpm/amdrocm.repo.j2", + sources_dir / f"{REPO_ID}.repo", + context, + ) + + if context["signed"]: + # rpm consumes the armored key as-is (gpgkey=file://). + (sources_dir / "RPM-GPG-KEY-rocm").write_bytes(load_signing_key(args)) + + version, release = rpm_version_release( + args.release_type, args.rocm_version, args.repo_sub_folder + ) + spec_context = { + **context, + "version": version, + "release": release, + "changelog_date": datetime.now(timezone.utc).strftime("%a %b %d %Y"), + } + spec_path = build_dir / "SPECS" / f"{PACKAGE_NAME}.spec" + _render(env, "template/repo/rpm/amdrocm-repo.spec.j2", spec_path, spec_context) + + print(f"Building rpm package: {PACKAGE_NAME} for {profile['description']}") + subprocess.run( + ["rpmbuild", "--define", f"_topdir {build_dir}", "-bb", str(spec_path)], + check=True, + ) + for rpm_file in (build_dir / "RPMS" / "noarch").glob("*.rpm"): + target = dest_dir / rpm_file.name + shutil.move(str(rpm_file), str(target)) + print(f"Package created: {target}") + + +# --- deb build --------------------------------------------------------------- + + +def build_deb_package( + args: argparse.Namespace, profile: dict, context: dict, dest_dir: Path +) -> None: + env = get_jinja_env() + package_dir = dest_dir / "deb" / PACKAGE_NAME + deb_dir = package_dir / "debian" + if package_dir.exists(): + shutil.rmtree(package_dir) + deb_dir.mkdir(parents=True, exist_ok=True) + + deb_context = { + **context, + "deb_version": deb_version( + args.release_type, args.rocm_version, args.repo_sub_folder + ), + "date": format_datetime(datetime.now(timezone.utc)), + } + + # Repository definition (deb822) and, for signed lines, the dearmored key, + # placed at the package root and mapped into the filesystem by debian/install. + _render( + env, + "template/repo/deb/amdrocm.sources.j2", + package_dir / f"{REPO_ID}.sources", + deb_context, + ) + if context["signed"]: + (package_dir / "rocm.gpg").write_bytes(dearmor_key(load_signing_key(args))) + + for name in ("control", "changelog", "rules", "postinst", "install"): + _render( + env, + f"template/repo/deb/{name}.j2", + deb_dir / name, + deb_context, + ) + (deb_dir / "rules").chmod(0o755) + (deb_dir / "postinst").chmod(0o755) + # Native source format: the package version carries no debian revision, so + # no separate upstream tarball is required. + (deb_dir / "source").mkdir(exist_ok=True) + (deb_dir / "source" / "format").write_text("3.0 (native)\n") + + print(f"Building deb package: {PACKAGE_NAME} for {profile['description']}") + subprocess.run( + ["dpkg-buildpackage", "-uc", "-us", "-b"], + cwd=package_dir, + check=True, + ) + # dpkg-buildpackage writes the .deb to the parent of the package directory. + for deb_file in package_dir.parent.glob("*.deb"): + target = dest_dir / deb_file.name + shutil.move(str(deb_file), str(target)) + print(f"Package created: {target}") + + +# --- CLI --------------------------------------------------------------------- + + +def list_profiles(pkg_type: str) -> list[str]: + """Return the OS profiles that build the given package type.""" + return [name for name, p in OS_PROFILES.items() if p["pkg_type"] == pkg_type] + + +def run_list_profiles(argv: list[str]) -> None: + """Print the OS profiles for a package type as a JSON array (CI matrix).""" + p = argparse.ArgumentParser( + prog="build_repo_package.py list-profiles", + description="Print the OS profiles for a package type as a JSON array.", + ) + p.add_argument("--pkg-type", required=True, choices=["deb", "rpm"]) + ns = p.parse_args(argv) + print(json.dumps(list_profiles(ns.pkg_type))) + + +def parse_args(argv=None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Build the amdrocm-repo package that configures ROCm repositories.", + ) + p.add_argument( + "--os-profile", + required=True, + choices=sorted(OS_PROFILES.keys()), + help="Target distro profile", + ) + p.add_argument( + "--release-type", + required=True, + choices=RELEASE_TYPES, + help="Release line to configure", + ) + p.add_argument( + "--repo-base-url", + required=True, + help="Public repository base URL (the packages-multi-arch base)", + ) + p.add_argument( + "--repo-sub-folder", + default="", + help="Dated sub-folder for the nightly line (YYYYMMDD-)", + ) + p.add_argument( + "--gpg-key-file", + type=Path, + default=None, + help="Signing key file to embed (defaults to fetching gpg/rocm.gpg from the repo)", + ) + p.add_argument( + "--rocm-version", + required=True, + help="ROCm version (e.g. 7.14.0)", + ) + p.add_argument( + "--dest-dir", + type=Path, + required=True, + help="Output directory for built packages", + ) + p.add_argument( + "--verify-repo-url", + action="store_true", + help=( + "Fail the build unless the configured repository is reachable. " + "No-op for the nightly line, whose dated sub-folder is published by " + "the same run that builds the package" + ), + ) + args = p.parse_args(argv) + if args.release_type == "nightly" and not args.repo_sub_folder: + p.error("--repo-sub-folder is required for --release-type nightly") + if args.repo_sub_folder and args.release_type != "nightly": + p.error("--repo-sub-folder is only valid for --release-type nightly") + # Validate values that are rendered into repo files, version fields, and + # maintainer metadata so they cannot alter the surrounding syntax. + if not _VERSION_RE.match(args.rocm_version): + p.error(f"--rocm-version has an unexpected format: {args.rocm_version!r}") + if not _valid_repo_base_url(args.repo_base_url): + p.error(f"--repo-base-url is not a valid http(s) URL: {args.repo_base_url!r}") + if args.repo_sub_folder and not _SUB_FOLDER_RE.match(args.repo_sub_folder): + p.error(f"--repo-sub-folder has an unexpected format: {args.repo_sub_folder!r}") + # rpmbuild requires an absolute _topdir; normalize so a relative --dest-dir works. + args.dest_dir = args.dest_dir.resolve() + return args + + +def main(argv=None) -> None: + if argv is None: + argv = sys.argv[1:] + if argv and argv[0] == "list-profiles": + run_list_profiles(argv[1:]) + return + args = parse_args(argv) + profile = OS_PROFILES[args.os_profile] + args.dest_dir.mkdir(parents=True, exist_ok=True) + context = build_context(args, profile) + # The nightly repository is published by the same run that builds this + # package, so its dated sub-folder does not exist yet and cannot be checked. + if args.verify_repo_url and args.release_type != "nightly": + verify_repo_url(context["baseurl"], profile["pkg_type"]) + if profile["pkg_type"] == "deb": + build_deb_package(args, profile, context, args.dest_dir) + else: + build_rpm_package(args, profile, context, args.dest_dir) + + +if __name__ == "__main__": + main() diff --git a/build_tools/packaging/linux/get_url_repo_params.py b/build_tools/packaging/linux/get_url_repo_params.py index 21b9c107f32..abfab2a35d5 100644 --- a/build_tools/packaging/linux/get_url_repo_params.py +++ b/build_tools/packaging/linux/get_url_repo_params.py @@ -15,6 +15,9 @@ get-repo-url Get full repo URL from components(release_type, native_package_type, repo_base_url, os_profile, repo_sub_folder). Prints repo_url=. extract-gfx-arch Extract and normalize GPU architecture from artifact group. Prints gfx_arch=. get-container-image Get container image for a given OS profile. Prints container_image=. + get-public-repo-base-url Get the public CDN base URL for a release line. Prints repo_base_url= (empty for ci/dev). + get-nightly-sub-folder Get the dated nightly sub-folder from a run id. Prints repo_sub_folder=YYYYMMDD-. + get-container-image-map Map a JSON array of OS profiles to container images. Prints images=. Usage: python build_tools/packaging/linux/get_url_repo_params.py get-base-url --from-url @@ -28,15 +31,17 @@ python build_tools/packaging/linux/get_url_repo_params.py get-base-url --from-url https://example.com/v2/whl python build_tools/packaging/linux/get_url_repo_params.py get-gpg-url --release-type prerelease --from-url https://rocm.prereleases.amd.com/packages/ubuntu2404 python build_tools/packaging/linux/get_url_repo_params.py get-repo-sub-folder --from-s3-prefix v3/packages/deb/20260204-12345 - python build_tools/packaging/linux/get_url_repo_params.py get-repo-url --release-type prerelease --native-package-type deb --repo-base-url https://x.com --os-profile ubuntu2404 --repo-sub-folder '' + python build_tools/packaging/linux/get_url_repo_params.py get-repo-url --release-type prerelease --native-package-type deb --repo-base-url https://example.com --os-profile ubuntu2404 --repo-sub-folder '' python build_tools/packaging/linux/get_url_repo_params.py extract-gfx-arch --artifact-group gfx94X-dcgpu python build_tools/packaging/linux/get_url_repo_params.py get-container-image --os-profile ubuntu2404 """ import argparse +import json import os import re import sys +from datetime import date, datetime, timezone from pathlib import Path from urllib.parse import urlparse @@ -145,13 +150,17 @@ def get_repo_url( ) -> str: """ Return the full repo URL for install tests. - - prerelease + deb: repo_base_url / os_profile - - prerelease + rpm: repo_base_url / os_profile / x86_64/ - - non-prerelease + deb: repo_base_url / deb / repo_sub_folder / - - non-prerelease + rpm: repo_base_url / rpm / repo_sub_folder / x86_64/ + + The public lines (prerelease, release) publish one repository per distro; + the per-run lines (dev, nightly, ci) publish one repository per package type + under a dated/run sub-folder. + - prerelease, release + deb: repo_base_url / os_profile + - prerelease, release + rpm: repo_base_url / os_profile / x86_64/ + - other + deb: repo_base_url / deb / repo_sub_folder / + - other + rpm: repo_base_url / rpm / repo_sub_folder / x86_64/ """ base = repo_base_url.rstrip("/") - if release_type == "prerelease": + if release_type in ("prerelease", "release"): if native_package_type == "deb": return f"{base}/{os_profile}" return f"{base}/{os_profile}/x86_64/" @@ -176,6 +185,73 @@ def cmd_repo_url(args: argparse.Namespace) -> int: return 0 +# --- public repository base URL --- + +# Public CDN base for each signed/served release line, using the flat +# packages-multi-arch layout (deb/, rpm/x86_64/, gpg/rocm.gpg beneath the base). +# Lines without a public repository (e.g. ci, dev) map to an empty string, which +# callers treat as "no amdrocm-repo package for this line". +_PUBLIC_REPO_BASE_URLS = { + "prerelease": "https://rocm.prereleases.amd.com/packages-multi-arch", + "release": "https://repo.amd.com/rocm/packages-multi-arch", + "nightly": "https://rocm.nightlies.amd.com/packages-multi-arch", +} + + +def get_public_repo_base_url(release_type: str) -> str: + """Return the public CDN base URL for a release line (empty if none).""" + return _PUBLIC_REPO_BASE_URLS.get(release_type.strip().lower(), "") + + +def cmd_public_repo_base_url(args: argparse.Namespace) -> int: + gha_set_output({"repo_base_url": get_public_repo_base_url(args.release_type)}) + return 0 + + +# --- nightly sub-folder --- + + +def nightly_sub_folder(run_id: str, *, today: date | None = None) -> str: + """Return the dated nightly sub-folder ``YYYYMMDD-``. + + ``today`` is injectable for deterministic tests; it defaults to the current + UTC date. + """ + day = today or datetime.now(timezone.utc).date() + return f"{day:%Y%m%d}-{run_id}" + + +def cmd_nightly_sub_folder(args: argparse.Namespace) -> int: + # Only the nightly line publishes into a dated sub-folder; every other line + # publishes at the repository root, so the value is empty there. + if args.release_type and args.release_type.strip().lower() != "nightly": + gha_set_output({"repo_sub_folder": ""}) + return 0 + gha_set_output({"repo_sub_folder": nightly_sub_folder(args.run_id)}) + return 0 + + +# --- container image map --- + + +def get_container_image_map(os_profiles: list[str]) -> dict[str, str]: + """Map each OS profile to its container image (for a build matrix).""" + return {profile: get_container_image(profile) for profile in os_profiles} + + +def cmd_container_image_map(args: argparse.Namespace) -> int: + try: + profiles = json.loads(args.os_profiles) + except json.JSONDecodeError as e: + print(f"Error: --os-profiles is not valid JSON: {e}", file=sys.stderr) + return 1 + if not isinstance(profiles, list) or not all(isinstance(p, str) for p in profiles): + print("Error: --os-profiles must be a JSON array of strings", file=sys.stderr) + return 1 + gha_set_output({"images": json.dumps(get_container_image_map(profiles))}) + return 0 + + # --- extract-gfx-arch --- @@ -387,6 +463,55 @@ def main(argv: list[str] | None = None) -> int: ) p_img.set_defaults(func=cmd_container_image) + # get-public-repo-base-url: public CDN base for a release line + p_pub = subparsers.add_parser( + "get-public-repo-base-url", + help="Print repo_base_url= for a release line (empty for lines without a public repo, e.g. ci/dev).", + ) + p_pub.add_argument( + "--release-type", + type=str, + required=True, + help="Release line (e.g. prerelease, release, nightly)", + ) + p_pub.set_defaults(func=cmd_public_repo_base_url) + + # get-nightly-sub-folder: dated sub-folder for the nightly line + p_sub = subparsers.add_parser( + "get-nightly-sub-folder", + help="Print repo_sub_folder=YYYYMMDD- for the nightly line.", + ) + p_sub.add_argument( + "--run-id", + type=str, + required=True, + help="Workflow run id to embed in the dated sub-folder", + ) + p_sub.add_argument( + "--release-type", + type=str, + default=None, + help=( + "If set, print an empty repo_sub_folder for any line other than " + "'nightly'. If omitted, always print the dated sub-folder." + ), + ) + p_sub.set_defaults(func=cmd_nightly_sub_folder) + + # get-container-image-map: os_profile -> image map for a build matrix + p_map = subparsers.add_parser( + "get-container-image-map", + help="Print images= mapping each OS profile (JSON array input) to its container image.", + ) + p_map.add_argument( + "--os-profiles", + type=str, + required=True, + metavar="JSON", + help='JSON array of OS profiles (e.g. ["rhel8","rhel10","sles16"])', + ) + p_map.set_defaults(func=cmd_container_image_map) + args = parser.parse_args(argv) return args.func(args) diff --git a/build_tools/packaging/linux/native_linux_package_install_test.py b/build_tools/packaging/linux/native_linux_package_install_test.py index a257f4bffdb..36c074513f5 100644 --- a/build_tools/packaging/linux/native_linux_package_install_test.py +++ b/build_tools/packaging/linux/native_linux_package_install_test.py @@ -136,6 +136,7 @@ import re import subprocess import sys +import time import traceback from argparse import ArgumentParser, Namespace from pathlib import Path @@ -174,6 +175,16 @@ def _env(key: str, default: str) -> str: # Relative path from install prefix to rdhc binary (script); overridable via ROCM_RDHC_REL_PATH RDHC_REL_PATH = _env("ROCM_RDHC_REL_PATH", "libexec/rocm-core/rdhc.py") +# amdrocm-repo package: the on-disk paths it installs (the package's public +# contract). Used by the install-via-package mode to confirm the repo config +# landed. These are independent of REPO_NAME (which names the manually-written +# repo file used by the default setup path). +AMDROCM_REPO_ID = "amdrocm" +AMDROCM_DEB_SOURCES = "/etc/apt/sources.list.d/amdrocm.sources" +AMDROCM_DEB_KEYRING = "/usr/share/keyrings/rocm.gpg" +AMDROCM_RPM_REPO_BASENAME = "amdrocm.repo" +AMDROCM_RPM_KEY = "/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" + # Pytest/CI only: becomes ``--rocm-version``. ENV_NATIVE_LINUX_INSTALL_ROCM_VERSION = "NATIVE_LINUX_INSTALL_ROCM_VERSION" @@ -184,6 +195,12 @@ def _env(key: str, default: str) -> str: ZYPP_CLEAN_TIMEOUT_SEC = 60 ZYPP_REFRESH_TIMEOUT_SEC = 120 DNF_CLEAN_TIMEOUT_SEC = 60 +DNF_MAKECACHE_TIMEOUT_SEC = 120 +REPO_PACKAGE_INSTALL_TIMEOUT_SEC = 120 +# Retry the post-install metadata refresh so a transient repo/CDN blip does not +# fail the (network-dependent) config-only gate. +REPO_REFRESH_ATTEMPTS = 3 +REPO_REFRESH_BACKOFF_SEC = 3 INSTALL_TIMEOUT_SEC = 1800 # 30 minutes ROCMINFO_TIMEOUT_SEC = 30 # rdhc.py ``--all`` runs the full ROCm deployment health check suite; 30s was too @@ -365,11 +382,14 @@ def __init__( gfx_arch: str | list[str] | None = None, rocm_version: str | None = None, gpg_key_url: str | None = None, + repo_package_dir: str | None = None, + repo_config_only: bool = False, ): """Initialize the native Linux package install test runner. Args: - repo_url: Full repository URL (constructed in YAML) + repo_url: Full repository URL (constructed in YAML). May be empty/None when + repo_package_dir is set (the amdrocm-repo package supplies the repo URL). os_profile: OS profile (e.g., ubuntu2404, rhel8, debian12, sles15, sles16, almalinux9, centos7, azl3) release_type: Type of release ('nightly' or 'prerelease') install_prefix: Installation prefix (default: /opt/rocm/core) @@ -382,11 +402,20 @@ def __init__( If unset: unversioned ``amdrocm`` / ``amdrocm-core-sdk`` (``gfx_arch`` alone does not add arch suffixes). gpg_key_url: GPG key URL + repo_package_dir: Directory holding the built ``amdrocm-repo`` package. When + set, the repository is configured by installing that package instead of + writing repo files directly. + repo_config_only: With ``repo_package_dir``, install the package and verify + the repo config, then stop (do not install ROCm). """ self.os_profile = os_profile.lower() self.package_type = self._derive_package_type(os_profile) - self.repo_url = repo_url.rstrip("/") + # repo_url may be empty/None in install-via-package mode (the package + # supplies it); tolerate both without raising. + self.repo_url = (repo_url or "").rstrip("/") self.release_type = release_type.lower() + self.repo_package_dir = repo_package_dir + self.repo_config_only = repo_config_only self.install_prefix = install_prefix self.gfx_arch_list = normalize_target_list( gfx_arch, lowercase=True, dedupe=True @@ -830,6 +859,184 @@ def install_rpm_packages(self) -> bool: print(f"\n[FAIL] Error during installation: {e}") return False + def _find_repo_package(self) -> Path | None: + """Locate the built amdrocm-repo package in repo_package_dir. + + Globs in Python (no shell) and requires exactly one match for this + package type. Returns the resolved path, or None on error. + """ + directory = Path(self.repo_package_dir) + if not directory.is_dir(): + print( + f"[FAIL] --repo-package-dir is not a directory: {self.repo_package_dir}" + ) + return None + pattern = ( + "amdrocm-repo*.deb" if self.package_type == "deb" else "amdrocm-repo*.rpm" + ) + matches = sorted(directory.glob(pattern)) + if not matches: + print(f"[FAIL] No {pattern} found in {self.repo_package_dir}") + return None + if len(matches) > 1: + print( + f"[FAIL] Multiple {pattern} found in {self.repo_package_dir}: {matches}" + ) + return None + return matches[0].resolve() + + def install_repo_package(self) -> bool: + """Install the amdrocm-repo package, then refresh the repo it configures. + + The package itself is unsigned, so the local install skips package + signature checks; the repository the package configures keeps its own + gpgcheck setting. The metadata refresh is scoped to the amdrocm repo. + + Returns: + True if the package installed and the repo metadata refreshed. + """ + print("\n" + "=" * 80) + print("CONFIGURING REPOSITORY VIA amdrocm-repo PACKAGE") + print("=" * 80) + + pkg = self._find_repo_package() + if pkg is None: + return False + print(f"\nRepo package: {pkg}") + + if self.package_type == "deb": + install_cmd = ["sudo", "apt", "install", "-y", str(pkg)] + elif self._is_sles(): + install_cmd = [ + "zypper", + "--non-interactive", + "--no-gpg-checks", + "install", + "-y", + str(pkg), + ] + else: + install_cmd = ["dnf", "install", "-y", "--nogpgcheck", str(pkg)] + + print(f"\nInstalling repo package: {' '.join(install_cmd)}") + try: + rc = _run_streaming(install_cmd, REPO_PACKAGE_INSTALL_TIMEOUT_SEC) + except subprocess.TimeoutExpired: + print("[FAIL] Installing amdrocm-repo package timed out") + return False + except OSError as e: + print(f"[FAIL] Error installing amdrocm-repo package: {e}") + return False + if rc != 0: + print(f"[FAIL] Failed to install amdrocm-repo package (exit code: {rc})") + return False + print("[PASS] amdrocm-repo package installed") + + # Refresh metadata for the repo the package just configured. The dnf and + # zypper refreshes are scoped to the amdrocm repo; apt has no clean + # per-source refresh, so the deb path runs a repo-wide `apt update` + # (a failing unrelated base-image source would also fail this step). + if self.package_type == "deb": + refresh_cmd = ["sudo", "apt", "update"] + refresh_timeout = APT_UPDATE_TIMEOUT_SEC + elif self._is_sles(): + refresh_cmd = [ + "zypper", + "--non-interactive", + "--gpg-auto-import-keys", + "refresh", + AMDROCM_REPO_ID, + ] + refresh_timeout = ZYPP_REFRESH_TIMEOUT_SEC + else: + refresh_cmd = [ + "dnf", + "makecache", + "--disablerepo=*", + f"--enablerepo={AMDROCM_REPO_ID}", + ] + refresh_timeout = DNF_MAKECACHE_TIMEOUT_SEC + + print(f"\nRefreshing repository metadata: {' '.join(refresh_cmd)}") + for attempt in range(1, REPO_REFRESH_ATTEMPTS + 1): + try: + rc = _run_streaming(refresh_cmd, refresh_timeout) + except subprocess.TimeoutExpired: + rc = None + print("[WARN] Repository metadata refresh timed out") + except OSError as e: + rc = None + print(f"[WARN] Error refreshing repository metadata: {e}") + if rc == 0: + print("[PASS] Repository metadata refreshed") + return True + if attempt < REPO_REFRESH_ATTEMPTS: + time.sleep(REPO_REFRESH_BACKOFF_SEC * attempt) + print( + f"Retrying repository metadata refresh " + f"({attempt + 1}/{REPO_REFRESH_ATTEMPTS})..." + ) + print("[FAIL] Failed to refresh repository metadata") + return False + + def assert_repo_configured(self) -> bool: + """Verify the amdrocm-repo package dropped the expected repo config. + + Checks the repo file exists (and looks like a ROCm repo) and, for signed + release lines, that the signing key was installed too. + + Returns: + True if the on-disk repo configuration is present and well-formed. + """ + print("\n" + "=" * 80) + print("VERIFYING REPOSITORY CONFIGURATION") + print("=" * 80) + + signed = self.release_type in ("prerelease", "release") + ok = True + + if self.package_type == "deb": + repo_file = Path(AMDROCM_DEB_SOURCES) + key_file = Path(AMDROCM_DEB_KEYRING) + # deb822 marker (independent of the repo host, which need not contain + # "rocm" for a mirror). + marker = "URIs:" + else: + repo_dir = ZYPP_REPOS_DIR if self._is_sles() else YUM_REPOS_DIR + repo_file = Path(repo_dir) / AMDROCM_RPM_REPO_BASENAME + key_file = Path(AMDROCM_RPM_KEY) + marker = "baseurl=" + + if repo_file.is_file(): + content = repo_file.read_text(encoding="utf-8", errors="replace") + print(f"[PASS] Repo file present: {repo_file}") + print(content) + # Match the marker at the start of a line so a file that only + # mentions it in a comment is not accepted as configured. + if not any(line.startswith(marker) for line in content.splitlines()): + print( + f"[FAIL] Repo file missing expected {marker!r} entry: {repo_file}" + ) + ok = False + else: + print(f"[FAIL] Repo file not found: {repo_file}") + ok = False + + if signed: + if key_file.is_file(): + print(f"[PASS] Signing key present: {key_file}") + else: + print(f"[FAIL] Signing key not found (signed line): {key_file}") + ok = False + else: + print("[INFO] Unsigned line: no signing key expected") + + if ok: + print("\n[PASS] Repository configuration verified") + else: + print("\n[FAIL] Repository configuration verification failed") + return ok + def run_repo_setup_and_install(self) -> bool: """Step 1: Repo setup and install. Run for both sanity (basic) and full test. @@ -861,9 +1068,23 @@ def run_repo_setup_and_install(self) -> bool: print( "GPU Architecture(s): (none — generic packages amdrocm, amdrocm-core-sdk)" ) - print(f"Repository URL: {self.repo_url}") + print(f"Repository URL: {self.repo_url or '(from amdrocm-repo package)'}") print(f"Packages (in order): {self.package_names}") + # Install-via-package mode: the amdrocm-repo package configures the repo. + if self.repo_package_dir: + if not self.install_repo_package(): + return False + if not self.assert_repo_configured(): + return False + if self.repo_config_only: + print("\n[PASS] Repo configured via amdrocm-repo package (config-only)") + return True + if self.package_type == "deb": + return self.install_deb_packages() + return self.install_rpm_packages() + + # Default: configure the repo directly, then install. if self.package_type == "deb": if not self.setup_deb_repository(): return False @@ -886,6 +1107,12 @@ def run_basic_verification(self) -> bool: print("STEP 2: BASIC INSTALL VERIFICATION") print("=" * 80) + if not self.install_prefix: + # Reachable when a non-config-only run omits --install-prefix; report + # the missing option instead of failing inside Path(). + print("\n[FAIL] --install-prefix is required to verify an installation") + return False + install_path = Path(self.install_prefix) if not install_path.exists(): print(f"\n[FAIL] Installation directory not found: {self.install_prefix}") @@ -1194,6 +1421,20 @@ def _build_argument_parser(*, exit_on_error: bool = True) -> ArgumentParser: metavar="DIR", help="Directory containing .deb or .rpm files. Required when --test-type is 'simulate'.", ) + parser.add_argument( + "--repo-package-dir", + type=str, + metavar="DIR", + help="Directory containing the built amdrocm-repo .deb/.rpm. When set, the repository is " + "configured by installing that package (instead of writing repo files); --repo-url and " + "--gpg-key-url become optional. Not valid with --test-type simulate.", + ) + parser.add_argument( + "--repo-config-only", + action="store_true", + help="With --repo-package-dir: install the amdrocm-repo package, verify the repo config, " + "and stop (do not install ROCm). --install-prefix is not required.", + ) parser.add_argument( "--pkg-type", type=str, @@ -1205,6 +1446,8 @@ def _build_argument_parser(*, exit_on_error: bool = True) -> ArgumentParser: def _validate_cli_args(parser: ArgumentParser, args: Namespace) -> None: if args.test_type == "simulate": + if args.repo_package_dir: + parser.error("--repo-package-dir is not valid with --test-type 'simulate'") if not args.packages_dir: parser.error("--packages-dir is required when --test-type is 'simulate'") if not args.pkg_type and not args.os_profile: @@ -1217,13 +1460,17 @@ def _validate_cli_args(parser: ArgumentParser, args: Namespace) -> None: except ValueError as e: parser.error(str(e)) return + if args.repo_config_only and not args.repo_package_dir: + parser.error("--repo-config-only requires --repo-package-dir") if not args.os_profile: parser.error( "--os-profile is required when --test-type is 'install', 'sanity', or 'full'" ) - if not args.repo_url: + # In install-via-package mode the amdrocm-repo package supplies the repo URL. + if not args.repo_url and not args.repo_package_dir: parser.error( - "--repo-url is required when --test-type is 'install', 'sanity', or 'full'" + "--repo-url is required when --test-type is 'install', 'sanity', or 'full' " + "(or provide --repo-package-dir to configure the repo via the amdrocm-repo package)" ) if args.rocm_version: try: @@ -1288,7 +1535,10 @@ def run_tests(args: Namespace) -> int: print(f"OS Profile: {args.os_profile}") print(f"Package Type (derived): {derived_package_type}") print(f"Release Type: {args.release_type}") - print(f"Repository URL: {args.repo_url}") + if args.repo_package_dir: + print(f"Repo package dir: {args.repo_package_dir}") + print(f"Repo config only: {args.repo_config_only}") + print(f"Repository URL: {args.repo_url or '(from amdrocm-repo package)'}") # Preview package-name rules before NativeLinuxPackageInstallTest is constructed. _norm = normalize_target_list(args.gfx_arch, lowercase=True, dedupe=True) if _norm: @@ -1331,12 +1581,14 @@ def run_tests(args: Namespace) -> int: test_runner = NativeLinuxPackageInstallTest( os_profile=args.os_profile, - repo_url=args.repo_url, + repo_url=args.repo_url or "", release_type=args.release_type, install_prefix=args.install_prefix, gfx_arch=args.gfx_arch, rocm_version=args.rocm_version, gpg_key_url=args.gpg_key_url, + repo_package_dir=args.repo_package_dir, + repo_config_only=args.repo_config_only, ) print("\n" + "=" * 80) @@ -1351,6 +1603,12 @@ def run_tests(args: Namespace) -> int: if not test_runner.run_repo_setup_and_install(): print("\n[FAIL] Step 1 (repo setup and install) failed.") return 1 + if args.repo_config_only: + print("\n" + "=" * 80) + print("[PASS] INSTALLATION TEST PASSED") + print("(repo-config-only: amdrocm-repo package configured the repo)") + print("=" * 80 + "\n") + return 0 if args.test_type == "install": print("\n" + "=" * 80) print("[PASS] INSTALLATION TEST PASSED") @@ -1381,7 +1639,10 @@ def run_tests(args: Namespace) -> int: def _argv_from_ci_env() -> list[str] | None: """Build CLI argv from workflow/container env (see ``test_native_linux_packages_install.yml``). - Required for sanity/full: OS_PROFILE, REPO_URL, RELEASE_TYPE, INSTALL_PREFIX. + Required for sanity/full: OS_PROFILE, RELEASE_TYPE, plus REPO_URL and + INSTALL_PREFIX — except in package mode (REPO_PACKAGE_DIR set), where the + repo is configured by the installed package so REPO_URL is not needed, and + when REPO_CONFIG_ONLY is set INSTALL_PREFIX is not needed either. Optional: GFX_ARCH, GPG_KEY_URL; ``NATIVE_LINUX_INSTALL_ROCM_VERSION`` maps to ``--rocm-version`` only when versioned package names are needed (omit for unversioned installs). """ @@ -1416,8 +1677,22 @@ def _argv_from_ci_env() -> list[str] | None: gfx_arch = gfx_raw.replace(";", " ").split() if gfx_raw else [] release_type = (os.environ.get("RELEASE_TYPE") or "").strip() install_prefix = (os.environ.get("INSTALL_PREFIX") or "").strip() + repo_package_dir = (os.environ.get("REPO_PACKAGE_DIR") or "").strip() + repo_config_only = (os.environ.get("REPO_CONFIG_ONLY") or "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) - if not (os_profile and repo_url and release_type and install_prefix): + # In install-via-package mode the amdrocm-repo package supplies the repo URL; + # in config-only mode there is no install/verify, so INSTALL_PREFIX is not + # needed either. + if not (os_profile and release_type): + return None + if not repo_url and not repo_package_dir: + return None + if not install_prefix and not repo_config_only: return None argv = [ @@ -1425,13 +1700,17 @@ def _argv_from_ci_env() -> list[str] | None: test_type, "--os-profile", os_profile, - "--repo-url", - repo_url, "--release-type", release_type, - "--install-prefix", - install_prefix, ] + if repo_url: + argv.extend(["--repo-url", repo_url]) + if install_prefix: + argv.extend(["--install-prefix", install_prefix]) + if repo_package_dir: + argv.extend(["--repo-package-dir", repo_package_dir]) + if repo_config_only: + argv.append("--repo-config-only") if gfx_arch: argv.extend(["--gfx-arch", *gfx_arch]) rocm_version = (os.environ.get(ENV_NATIVE_LINUX_INSTALL_ROCM_VERSION) or "").strip() @@ -1452,11 +1731,14 @@ def test_native_linux_package_install() -> None: if os.environ.get("GITHUB_ACTIONS") == "true": pytest.fail( "Missing required environment variables for native install test " - "(expected OS_PROFILE, REPO_URL, RELEASE_TYPE, INSTALL_PREFIX; " - "optional GFX_ARCH, NATIVE_LINUX_INSTALL_ROCM_VERSION; or for simulate: PACKAGES_DIR)." + "(expected OS_PROFILE, RELEASE_TYPE, plus REPO_URL and INSTALL_PREFIX " + "unless REPO_PACKAGE_DIR is set, and INSTALL_PREFIX is not needed with " + "REPO_CONFIG_ONLY; optional GFX_ARCH, NATIVE_LINUX_INSTALL_ROCM_VERSION; " + "or for simulate: PACKAGES_DIR)." ) pytest.skip( - "Set workflow env vars (OS_PROFILE, REPO_URL, RELEASE_TYPE, INSTALL_PREFIX); " + "Set workflow env vars (OS_PROFILE, RELEASE_TYPE, plus REPO_URL/INSTALL_PREFIX " + "unless REPO_PACKAGE_DIR/REPO_CONFIG_ONLY is set); " "optional GFX_ARCH and NATIVE_LINUX_INSTALL_ROCM_VERSION." ) diff --git a/build_tools/packaging/linux/publish_repo_package.py b/build_tools/packaging/linux/publish_repo_package.py new file mode 100644 index 00000000000..f1eeeb51114 --- /dev/null +++ b/build_tools/packaging/linux/publish_repo_package.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Publish a built amdrocm-repo package to S3 as a standalone per-distro file. + +The package is uploaded next to the native content packages, as a +``repo//`` sibling of the per-run ``packages//`` prefix, +rather than into the repository index. A client can then fetch it directly by +URL to configure the repository. The file becomes publicly reachable once the +release promotion step copies the per-run tree to the public CDN. + +The object name is fixed (``amdrocm-repo.``) so the download URL is stable +regardless of the built package's versioned filename. + +Usage: + python build_tools/packaging/linux/publish_repo_package.py \ + --file repo-package-out/amdrocm-repo-7.14.0-1.el10.noarch.rpm \ + --bucket therock-prerelease-artifacts \ + --prefix 12345-linux/packages/rpm \ + --os-profile rhel10 \ + --pkg-type rpm +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +# Fixed published object name; the extension is the package type. +PUBLISHED_NAME = "amdrocm-repo" + +# An os-profile becomes a path segment in the S3 key, so restrict it to a safe +# set of characters (no slashes, whitespace, or control characters). \Z (not $) +# so a trailing newline is not accepted. +_OS_PROFILE_RE = re.compile(r"^[A-Za-z0-9._-]+\Z") + + +def object_key(prefix: str, os_profile: str, pkg_type: str) -> str: + """Return the S3 key for the published bootstrap file. + + ``{prefix}/repo/{os_profile}/amdrocm-repo.{ext}`` — a sibling of the content + repository under the same per-run prefix, so it is not part of the repo + index (``dists/`` for deb, ``x86_64/repodata`` for rpm). + """ + return f"{prefix.rstrip('/')}/repo/{os_profile}/{PUBLISHED_NAME}.{pkg_type}" + + +def parse_args(argv=None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Publish an amdrocm-repo package to S3.", + ) + p.add_argument("--file", required=True, type=Path, help="Built package to upload") + p.add_argument("--bucket", required=True, help="Destination S3 bucket") + p.add_argument( + "--prefix", + required=True, + help="Per-run S3 prefix, e.g. -linux/packages/", + ) + p.add_argument( + "--os-profile", required=True, help="Target distro profile (path segment)" + ) + p.add_argument( + "--pkg-type", + required=True, + choices=["deb", "rpm"], + help="Package type, used as the published file extension", + ) + p.add_argument( + "--endpoint-url", + default=None, + help="S3 endpoint override (for testing against a local server)", + ) + args = p.parse_args(argv) + # Fail loudly rather than silently uploading nowhere if the bucket is empty + # (e.g. an unresolved release line without upload credentials). + if not args.bucket.strip(): + p.error("--bucket must be a non-empty bucket name") + if not _OS_PROFILE_RE.match(args.os_profile): + p.error(f"--os-profile has an unexpected format: {args.os_profile!r}") + if not args.file.is_file(): + p.error(f"--file is not a file: {args.file}") + return args + + +def publish(args: argparse.Namespace) -> str: + """Upload the file and confirm it landed. Returns the object key.""" + # Deferred import: boto3 is only needed for the actual upload, so key + # derivation stays importable without it. + import boto3 + + key = object_key(args.prefix, args.os_profile, args.pkg_type) + s3 = boto3.client("s3", endpoint_url=args.endpoint_url) + print(f"Uploading {args.file} -> s3://{args.bucket}/{key}") + s3.upload_file(str(args.file), args.bucket, key) + # Confirm the object exists so a job bounded by continue-on-error cannot + # silently succeed after a failed upload. + s3.head_object(Bucket=args.bucket, Key=key) + print(f"Published: s3://{args.bucket}/{key}") + return key + + +def main(argv=None) -> None: + publish(parse_args(argv)) + + +if __name__ == "__main__": + main() diff --git a/build_tools/packaging/linux/setup_repo_build_deps.sh b/build_tools/packaging/linux/setup_repo_build_deps.sh new file mode 100755 index 00000000000..eae3d2cc311 --- /dev/null +++ b/build_tools/packaging/linux/setup_repo_build_deps.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +# Install the system tools needed to build the amdrocm-repo package. +# +# Package mapping by --os-profile: +# +# - ubuntu* / debian* -> apt: debhelper-compat, dpkg-dev, build-essential +# - sles* -> zypper: rpm-build +# - else (RHEL-like) -> dnf: rpm-build +# +# gpg is installed for every profile: signed release lines need it to dearmor +# the deb keyring and to check the signing key fingerprint. +# +# Python is installed separately by setup_python_cmd.sh. +# +# Sample usage +# ------------ +# +# bash build_tools/packaging/linux/setup_repo_build_deps.sh --os-profile rhel10 + +set -euo pipefail + +OS_PROFILE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --os-profile) + OS_PROFILE="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$OS_PROFILE" ]]; then + echo "Error: --os-profile is required" >&2 + exit 1 +fi + +# Trim whitespace; lowercase for glob matching (e.g. SLES16 must match sles*) +OS_PROFILE="${OS_PROFILE#"${OS_PROFILE%%[![:space:]]*}"}" +OS_PROFILE="${OS_PROFILE%"${OS_PROFILE##*[![:space:]]}"}" +OS_PLC="${OS_PROFILE,,}" + +if [[ "$OS_PLC" == ubuntu* ]] || [[ "$OS_PLC" == debian* ]]; then + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update + sudo apt-get install -y \ + ca-certificates \ + gnupg \ + debhelper-compat \ + dpkg-dev \ + build-essential +elif [[ "$OS_PLC" == sles* ]]; then + zypper --non-interactive refresh + zypper --non-interactive install -y \ + ca-certificates \ + gpg2 \ + rpm-build +else + dnf install -y --allowerasing \ + ca-certificates \ + gnupg2 \ + rpm-build +fi diff --git a/build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 b/build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 new file mode 100644 index 00000000000..8f6cab95786 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/amdrocm.sources.j2 @@ -0,0 +1,10 @@ +Types: deb +URIs: {{ baseurl }} +Suites: stable +Components: main +Architectures: amd64 +{% if signed %} +Signed-By: {{ deb_keyring_path }} +{% else %} +Trusted: yes +{% endif %} diff --git a/build_tools/packaging/linux/template/repo/deb/changelog.j2 b/build_tools/packaging/linux/template/repo/deb/changelog.j2 new file mode 100644 index 00000000000..071194aa824 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/changelog.j2 @@ -0,0 +1,5 @@ +{{ pkg_name }} ({{ deb_version }}) stable; urgency=medium + + * ROCm {{ rocm_version }} repository configuration. + + -- {{ maintainer }} {{ date }} diff --git a/build_tools/packaging/linux/template/repo/deb/control.j2 b/build_tools/packaging/linux/template/repo/deb/control.j2 new file mode 100644 index 00000000000..3132c8d4da0 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/control.j2 @@ -0,0 +1,13 @@ +Source: {{ pkg_name }} +Section: admin +Priority: optional +Maintainer: {{ maintainer }} +Build-Depends: debhelper-compat (= 13) +Standards-Version: 4.6.2 +Homepage: https://github.com/ROCm/TheRock + +Package: {{ pkg_name }} +Architecture: all +Depends: ${misc:Depends} +Description: {{ description_short }} + {{ description_long }} diff --git a/build_tools/packaging/linux/template/repo/deb/install.j2 b/build_tools/packaging/linux/template/repo/deb/install.j2 new file mode 100644 index 00000000000..ca5871934ab --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/install.j2 @@ -0,0 +1,4 @@ +{{ repo_id }}.sources /etc/apt/sources.list.d/ +{% if signed %} +rocm.gpg /usr/share/keyrings/ +{% endif %} diff --git a/build_tools/packaging/linux/template/repo/deb/postinst.j2 b/build_tools/packaging/linux/template/repo/deb/postinst.j2 new file mode 100644 index 00000000000..90ad1d98f67 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/postinst.j2 @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +if [ "$1" = "configure" ]; then + echo "AMD ROCm repository configured." + echo "Run 'sudo apt-get update' to refresh package lists." +fi + +#DEBHELPER# + +exit 0 diff --git a/build_tools/packaging/linux/template/repo/deb/rules.j2 b/build_tools/packaging/linux/template/repo/deb/rules.j2 new file mode 100644 index 00000000000..cbe925d7587 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/deb/rules.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/make -f +%: + dh $@ diff --git a/build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 b/build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 new file mode 100644 index 00000000000..55dd2436774 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/rpm/amdrocm-repo.spec.j2 @@ -0,0 +1,29 @@ +Name: {{ pkg_name }} +Version: {{ version }} +Release: {{ release }}%{?dist} +BuildArch: noarch +Summary: {{ description_short }} +License: MIT +Vendor: Advanced Micro Devices, Inc +URL: https://github.com/ROCm/TheRock + +%description +{{ description_long }} + +%install +mkdir -p %{buildroot}{{ rpm_repo_dir }} +install -m 644 %{_sourcedir}/{{ repo_id }}.repo %{buildroot}{{ rpm_repo_dir }}/ +{% if signed %} +mkdir -p %{buildroot}/etc/pki/rpm-gpg +install -m 644 %{_sourcedir}/RPM-GPG-KEY-rocm %{buildroot}{{ rpm_gpg_key_path }} +{% endif %} + +%files +%config(noreplace) {{ rpm_repo_dir }}/{{ repo_id }}.repo +{% if signed %} +{{ rpm_gpg_key_path }} +{% endif %} + +%changelog +* {{ changelog_date }} {{ maintainer }} - {{ version }}-{{ release }} +- ROCm {{ rocm_version }} repository configuration. diff --git a/build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 b/build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 new file mode 100644 index 00000000000..1d34ea91c72 --- /dev/null +++ b/build_tools/packaging/linux/template/repo/rpm/amdrocm.repo.j2 @@ -0,0 +1,8 @@ +[{{ repo_id }}] +name={{ name }} +baseurl={{ baseurl }} +enabled=1 +gpgcheck={{ 1 if signed else 0 }} +{% if signed %} +gpgkey=file://{{ rpm_gpg_key_path }} +{% endif %} diff --git a/build_tools/packaging/linux/tests/get_url_repo_params_test.py b/build_tools/packaging/linux/tests/get_url_repo_params_test.py index 57ed8562784..6fc6742daa2 100644 --- a/build_tools/packaging/linux/tests/get_url_repo_params_test.py +++ b/build_tools/packaging/linux/tests/get_url_repo_params_test.py @@ -4,12 +4,15 @@ # Unit test coverage for get_url_repo_params.py: # get_base_url, get_gpg_key_url, gpg_key_url_needed_for_release_type, get_repo_sub_folder, -# get_repo_url, extract_gfx_arch, and main() subcommands. +# get_repo_url, extract_gfx_arch, get_public_repo_base_url, nightly_sub_folder, +# get_container_image_map, and main() subcommands. +import json import os import sys import tempfile import unittest +from datetime import date from pathlib import Path from unittest.mock import patch @@ -167,11 +170,11 @@ def test_prerelease_deb(self): get_url_repo_params.get_repo_url( release_type="prerelease", native_package_type="deb", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="ubuntu2404", repo_sub_folder="", ), - "https://x.com/ubuntu2404", + "https://example.com/ubuntu2404", ) def test_prerelease_rpm(self): @@ -180,11 +183,38 @@ def test_prerelease_rpm(self): get_url_repo_params.get_repo_url( release_type="prerelease", native_package_type="rpm", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="rhel8", repo_sub_folder="", ), - "https://x.com/rhel8/x86_64/", + "https://example.com/rhel8/x86_64/", + ) + + def test_release_deb(self): + # Test that release + deb yields base/os_profile. The release repository + # is served per distro, not as a flat deb/ tree. + self.assertEqual( + get_url_repo_params.get_repo_url( + release_type="release", + native_package_type="deb", + repo_base_url="https://example.com", + os_profile="ubuntu2404", + repo_sub_folder="", + ), + "https://example.com/ubuntu2404", + ) + + def test_release_rpm(self): + # Test that release + rpm yields base/os_profile/x86_64/ + self.assertEqual( + get_url_repo_params.get_repo_url( + release_type="release", + native_package_type="rpm", + repo_base_url="https://example.com", + os_profile="rhel10", + repo_sub_folder="", + ), + "https://example.com/rhel10/x86_64/", ) def test_nightly_deb(self): @@ -193,11 +223,11 @@ def test_nightly_deb(self): get_url_repo_params.get_repo_url( release_type="nightly", native_package_type="deb", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="ubuntu2404", repo_sub_folder="20260204-12345", ), - "https://x.com/deb/20260204-12345/", + "https://example.com/deb/20260204-12345/", ) def test_nightly_rpm(self): @@ -206,11 +236,11 @@ def test_nightly_rpm(self): get_url_repo_params.get_repo_url( release_type="nightly", native_package_type="rpm", - repo_base_url="https://x.com", + repo_base_url="https://example.com", os_profile="rhel8", repo_sub_folder="20260204-12345", ), - "https://x.com/rpm/20260204-12345/x86_64/", + "https://example.com/rpm/20260204-12345/x86_64/", ) def test_strips_trailing_slash_from_base(self): @@ -219,11 +249,11 @@ def test_strips_trailing_slash_from_base(self): get_url_repo_params.get_repo_url( release_type="prerelease", native_package_type="deb", - repo_base_url="https://x.com/", + repo_base_url="https://example.com/", os_profile="ubuntu2404", repo_sub_folder="", ), - "https://x.com/ubuntu2404", + "https://example.com/ubuntu2404", ) @@ -335,7 +365,7 @@ def test_get_repo_url_success(self): "--native-package-type", "deb", "--repo-base-url", - "https://x.com", + "https://example.com", "--os-profile", "ubuntu2404", "--repo-sub-folder", @@ -343,7 +373,7 @@ def test_get_repo_url_success(self): ] ) self.assertEqual(code, 0) - self.assertIn("repo_url=https://x.com/ubuntu2404", output) + self.assertIn("repo_url=https://example.com/ubuntu2404", output) def test_get_repo_url_error_returns_one(self): # Test that get-repo-url returns 1 and prints error when get_repo_url raises. @@ -359,7 +389,7 @@ def test_get_repo_url_error_returns_one(self): "--native-package-type", "deb", "--repo-base-url", - "https://x.com", + "https://example.com", "--os-profile", "ubuntu2404", "--repo-sub-folder", @@ -520,5 +550,144 @@ def test_get_container_image_subcommand(self): self.assertIn("container_image=ubuntu:24.04", output) +class GetPublicRepoBaseUrlTest(unittest.TestCase): + """Tests for get_public_repo_base_url() and its subcommand.""" + + def test_prerelease(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("prerelease"), + "https://rocm.prereleases.amd.com/packages-multi-arch", + ) + + def test_release(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("release"), + "https://repo.amd.com/rocm/packages-multi-arch", + ) + + def test_nightly(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("nightly"), + "https://rocm.nightlies.amd.com/packages-multi-arch", + ) + + def test_ci_and_dev_are_empty(self): + self.assertEqual(get_url_repo_params.get_public_repo_base_url("ci"), "") + self.assertEqual(get_url_repo_params.get_public_repo_base_url("dev"), "") + + def test_unknown_line_is_empty(self): + self.assertEqual(get_url_repo_params.get_public_repo_base_url("bogus"), "") + + def test_case_insensitive(self): + self.assertEqual( + get_url_repo_params.get_public_repo_base_url("PreRelease"), + "https://rocm.prereleases.amd.com/packages-multi-arch", + ) + + def test_subcommand_emits_url(self): + code, output = _run_main_with_output( + ["get-public-repo-base-url", "--release-type", "prerelease"] + ) + self.assertEqual(code, 0) + self.assertIn( + "repo_base_url=https://rocm.prereleases.amd.com/packages-multi-arch", + output, + ) + + def test_subcommand_emits_empty_for_ci(self): + code, output = _run_main_with_output( + ["get-public-repo-base-url", "--release-type", "ci"] + ) + self.assertEqual(code, 0) + self.assertIn("repo_base_url=", output) + self.assertNotIn("packages-multi-arch", output) + + +class NightlySubFolderTest(unittest.TestCase): + """Tests for nightly_sub_folder() and its subcommand.""" + + def test_formats_with_injected_date(self): + self.assertEqual( + get_url_repo_params.nightly_sub_folder("12345", today=date(2026, 7, 16)), + "20260716-12345", + ) + + def test_zero_padded_month_and_day(self): + self.assertEqual( + get_url_repo_params.nightly_sub_folder("7", today=date(2026, 1, 3)), + "20260103-7", + ) + + def test_subcommand_emits_sub_folder(self): + code, output = _run_main_with_output( + ["get-nightly-sub-folder", "--run-id", "12345"] + ) + self.assertEqual(code, 0) + # The date is the current UTC date; assert the structure, not the value. + self.assertRegex(output, r"repo_sub_folder=\d{8}-12345") + + def test_subcommand_emits_sub_folder_for_nightly(self): + code, output = _run_main_with_output( + ["get-nightly-sub-folder", "--release-type", "nightly", "--run-id", "12345"] + ) + self.assertEqual(code, 0) + self.assertRegex(output, r"repo_sub_folder=\d{8}-12345") + + def test_subcommand_emits_empty_for_other_lines(self): + # Only the nightly line publishes into a dated sub-folder; the other + # lines publish at the repository root. + for release_type in ("release", "prerelease", "ci"): + with self.subTest(release_type=release_type): + code, output = _run_main_with_output( + [ + "get-nightly-sub-folder", + "--release-type", + release_type, + "--run-id", + "12345", + ] + ) + self.assertEqual(code, 0) + self.assertIn("repo_sub_folder=", output) + self.assertNotRegex(output, r"repo_sub_folder=\S") + + +class GetContainerImageMapTest(unittest.TestCase): + """Tests for get_container_image_map() and its subcommand.""" + + def test_maps_rpm_profiles(self): + self.assertEqual( + get_url_repo_params.get_container_image_map(["rhel8", "sles16"]), + { + "rhel8": "registry.access.redhat.com/ubi8/ubi:8.10", + "sles16": "registry.suse.com/bci/bci-base:16.0", + }, + ) + + def test_subcommand_emits_json_map(self): + code, output = _run_main_with_output( + ["get-container-image-map", "--os-profiles", '["ubuntu2404"]'] + ) + self.assertEqual(code, 0) + # Recover the JSON value written after ``images=``. + line = next(l for l in output.splitlines() if l.startswith("images=")) + images = json.loads(line[len("images=") :]) + self.assertEqual( + images, {"ubuntu2404": "ghcr.io/rocm/no_rocm_image_ubuntu24_04:latest"} + ) + + def test_subcommand_rejects_invalid_json(self): + code, _ = _run_main_with_output( + ["get-container-image-map", "--os-profiles", "not-json"] + ) + self.assertEqual(code, 1) + + def test_subcommand_rejects_non_list_json(self): + code, _ = _run_main_with_output( + ["get-container-image-map", "--os-profiles", '{"a": 1}'] + ) + self.assertEqual(code, 1) + + if __name__ == "__main__": unittest.main() diff --git a/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py b/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py index 03de1c8f075..f0b42e860be 100644 --- a/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py +++ b/build_tools/packaging/linux/tests/native_linux_package_install_ut_test.py @@ -1556,5 +1556,429 @@ def test_kills_process_on_timeout(self, mock_popen): mock_proc.kill.assert_called_once() +@contextlib.contextmanager +def _clear_ci_env(): + """Remove CI env vars that _argv_from_ci_env reads so tests start clean.""" + keys = [ + "TEST_TYPE", + "OS_PROFILE", + "REPO_URL", + "GFX_ARCH", + "RELEASE_TYPE", + "INSTALL_PREFIX", + "REPO_PACKAGE_DIR", + "REPO_CONFIG_ONLY", + "GPG_KEY_URL", + "PACKAGES_DIR", + "SIMULATE_PKG_TYPE", + native_linux_package_install_test.ENV_NATIVE_LINUX_INSTALL_ROCM_VERSION, + ] + saved = {k: os.environ.pop(k, None) for k in keys} + try: + yield + finally: + for k, v in saved.items(): + if v is not None: + os.environ[k] = v + + +class RepoPackageModeCliTest(unittest.TestCase): + """CLI/constructor behavior for the install-via-package mode.""" + + def _parse(self, argv): + return native_linux_package_install_test.parse_cli_arguments( + argv, raise_instead_of_exit=True + ) + + def test_repo_package_dir_makes_repo_url_optional(self): + args = self._parse( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-package-dir", + "/pkgs", + "--install-prefix", + "/opt/rocm/core", + ] + ) + self.assertEqual(args.repo_package_dir, "/pkgs") + self.assertFalse(args.repo_config_only) + + def test_config_only_drops_repo_url_and_install_prefix(self): + args = self._parse( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-package-dir", + "/pkgs", + "--repo-config-only", + ] + ) + self.assertTrue(args.repo_config_only) + self.assertIsNone(args.repo_url) + self.assertIsNone(args.install_prefix) + + def test_config_only_requires_repo_package_dir(self): + with self.assertRaises(ValueError): + self._parse( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-config-only", + ] + ) + + def test_repo_url_still_required_without_repo_package_dir(self): + with self.assertRaises(ValueError): + self._parse(["--os-profile", "ubuntu2404", "--release-type", "prerelease"]) + + def test_repo_package_dir_rejected_with_simulate(self): + with self.assertRaises(ValueError): + self._parse( + [ + "--test-type", + "simulate", + "--packages-dir", + "/p", + "--os-profile", + "ubuntu2404", + "--repo-package-dir", + "/pkgs", + ] + ) + + def test_constructor_tolerates_none_repo_url(self): + runner = native_linux_package_install_test.NativeLinuxPackageInstallTest( + os_profile="rhel10", + repo_url=None, + release_type="nightly", + repo_package_dir="/pkgs", + repo_config_only=True, + ) + self.assertEqual(runner.repo_url, "") + self.assertEqual(runner.repo_package_dir, "/pkgs") + self.assertTrue(runner.repo_config_only) + + +class InstallRepoPackageTest(unittest.TestCase): + """Tests for install_repo_package() command construction (mocked subprocess).""" + + def _runner(self, os_profile, pkg_dir): + return native_linux_package_install_test.NativeLinuxPackageInstallTest( + os_profile=os_profile, + repo_url="", + release_type="prerelease", + repo_package_dir=str(pkg_dir), + ) + + @patch("native_linux_package_install_test._run_streaming") + def test_deb_installs_then_apt_update(self, mock_streaming): + mock_streaming.return_value = 0 + with tempfile.TemporaryDirectory() as d: + pkg = Path(d) / "amdrocm-repo_7.14.0_all.deb" + pkg.write_text("") + runner = self._runner("ubuntu2404", d) + with _suppress_script_output(): + self.assertTrue(runner.install_repo_package()) + install_cmd, refresh_cmd = (c.args[0] for c in mock_streaming.call_args_list) + self.assertEqual( + install_cmd, ["sudo", "apt", "install", "-y", str(pkg.resolve())] + ) + self.assertEqual(refresh_cmd, ["sudo", "apt", "update"]) + + @patch("native_linux_package_install_test._run_streaming") + def test_dnf_installs_nogpgcheck_then_scoped_makecache(self, mock_streaming): + mock_streaming.return_value = 0 + with tempfile.TemporaryDirectory() as d: + pkg = Path(d) / "amdrocm-repo-7.14.0-1.el10.noarch.rpm" + pkg.write_text("") + runner = self._runner("rhel10", d) + with _suppress_script_output(): + self.assertTrue(runner.install_repo_package()) + install_cmd, refresh_cmd = (c.args[0] for c in mock_streaming.call_args_list) + self.assertEqual( + install_cmd, ["dnf", "install", "-y", "--nogpgcheck", str(pkg.resolve())] + ) + self.assertEqual( + refresh_cmd, + ["dnf", "makecache", "--disablerepo=*", "--enablerepo=amdrocm"], + ) + + @patch("native_linux_package_install_test._run_streaming") + def test_sles_installs_no_gpg_checks_then_refresh(self, mock_streaming): + mock_streaming.return_value = 0 + with tempfile.TemporaryDirectory() as d: + pkg = Path(d) / "amdrocm-repo-7.14.0-1.noarch.rpm" + pkg.write_text("") + runner = self._runner("sles16", d) + with _suppress_script_output(): + self.assertTrue(runner.install_repo_package()) + install_cmd, refresh_cmd = (c.args[0] for c in mock_streaming.call_args_list) + self.assertEqual( + install_cmd, + [ + "zypper", + "--non-interactive", + "--no-gpg-checks", + "install", + "-y", + str(pkg.resolve()), + ], + ) + self.assertEqual( + refresh_cmd, + [ + "zypper", + "--non-interactive", + "--gpg-auto-import-keys", + "refresh", + "amdrocm", + ], + ) + + @patch("native_linux_package_install_test._run_streaming") + def test_missing_package_fails_without_running(self, mock_streaming): + with tempfile.TemporaryDirectory() as d: + runner = self._runner("ubuntu2404", d) # empty dir + with _suppress_script_output(): + self.assertFalse(runner.install_repo_package()) + mock_streaming.assert_not_called() + + @patch("native_linux_package_install_test._run_streaming") + def test_multiple_packages_fails_without_running(self, mock_streaming): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "amdrocm-repo_1_all.deb").write_text("") + (Path(d) / "amdrocm-repo_2_all.deb").write_text("") + runner = self._runner("ubuntu2404", d) + with _suppress_script_output(): + self.assertFalse(runner.install_repo_package()) + mock_streaming.assert_not_called() + + @patch("native_linux_package_install_test._run_streaming") + def test_install_failure_returns_false_without_refresh(self, mock_streaming): + mock_streaming.return_value = 1 # install fails + with tempfile.TemporaryDirectory() as d: + (Path(d) / "amdrocm-repo_1_all.deb").write_text("") + runner = self._runner("ubuntu2404", d) + with _suppress_script_output(): + self.assertFalse(runner.install_repo_package()) + self.assertEqual(mock_streaming.call_count, 1) # refresh not attempted + + +class AssertRepoConfiguredTest(unittest.TestCase): + """Tests for assert_repo_configured() (on-disk contract check).""" + + def _runner(self, os_profile, release_type, pkg_dir="/pkgs"): + return native_linux_package_install_test.NativeLinuxPackageInstallTest( + os_profile=os_profile, + repo_url="", + release_type=release_type, + repo_package_dir=pkg_dir, + ) + + def test_deb_signed_requires_sources_and_keyring(self): + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text( + "Types: deb\n" + "URIs: https://rocm.prereleases.amd.com/packages-multi-arch/ubuntu2404/\n" + "Signed-By: /usr/share/keyrings/rocm.gpg\n" + ) + keyring = Path(d) / "rocm.gpg" + keyring.write_text("key") + runner = self._runner("ubuntu2404", "prerelease") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING=str(keyring), + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + keyring.unlink() # signed line but keyring missing -> fail + with _suppress_script_output(): + self.assertFalse(runner.assert_repo_configured()) + + def test_deb_missing_sources_fails(self): + runner = self._runner("ubuntu2404", "prerelease") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES="/nonexistent/amdrocm.sources", + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertFalse(runner.assert_repo_configured()) + + def test_deb_nightly_needs_no_keyring(self): + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text("URIs: https://x/rocm/deb/\nTrusted: yes\n") + runner = self._runner("ubuntu2404", "nightly") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + def test_structurally_invalid_repo_file_fails(self): + # A repo file missing the expected deb822 marker (URIs:) is rejected, + # independent of whether the host string contains "rocm". + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text("not a valid deb822 sources file\n") + runner = self._runner("ubuntu2404", "nightly") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertFalse(runner.assert_repo_configured()) + + def test_mirror_host_without_rocm_still_passes(self): + # deb822 with a rocm-less mirror host is valid: the marker check must + # not reject it just because the URL lacks "rocm". + with tempfile.TemporaryDirectory() as d: + sources = Path(d) / "amdrocm.sources" + sources.write_text( + "Types: deb\n" + "URIs: https://mirror.example.com/packages-multi-arch/deb/20260716-1/\n" + "Suites: stable\n" + "Trusted: yes\n" + ) + runner = self._runner("ubuntu2404", "nightly") + with patch.multiple( + native_linux_package_install_test, + AMDROCM_DEB_SOURCES=str(sources), + AMDROCM_DEB_KEYRING="/nonexistent/rocm.gpg", + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + def test_rpm_signed_checks_yum_dir_and_key(self): + with tempfile.TemporaryDirectory() as d: + repo = Path(d) / "amdrocm.repo" + repo.write_text( + "[amdrocm]\nbaseurl=https://x/packages-multi-arch/rpm/x86_64/\n" + ) + key = Path(d) / "RPM-GPG-KEY-rocm" + key.write_text("key") + runner = self._runner("rhel10", "prerelease") + with patch.multiple( + native_linux_package_install_test, + YUM_REPOS_DIR=d, + AMDROCM_RPM_KEY=str(key), + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + def test_rpm_sles_uses_zypp_dir(self): + with tempfile.TemporaryDirectory() as d: + repo = Path(d) / "amdrocm.repo" + repo.write_text("[amdrocm]\nbaseurl=https://x/rocm/rpm/x86_64/\n") + key = Path(d) / "RPM-GPG-KEY-rocm" + key.write_text("key") + runner = self._runner("sles16", "prerelease") + with patch.multiple( + native_linux_package_install_test, + ZYPP_REPOS_DIR=d, + AMDROCM_RPM_KEY=str(key), + ): + with _suppress_script_output(): + self.assertTrue(runner.assert_repo_configured()) + + +class RunRepoSetupDispatchTest(unittest.TestCase): + """run_repo_setup_and_install dispatch: package mode vs config-only vs default.""" + + def _runner(self, **kw): + base = dict(os_profile="ubuntu2404", repo_url="", release_type="prerelease") + base.update(kw) + return native_linux_package_install_test.NativeLinuxPackageInstallTest(**base) + + def test_config_only_stops_before_rocm_install(self): + runner = self._runner(repo_package_dir="/pkgs", repo_config_only=True) + runner.install_repo_package = MagicMock(return_value=True) + runner.assert_repo_configured = MagicMock(return_value=True) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertTrue(runner.run_repo_setup_and_install()) + runner.install_repo_package.assert_called_once() + runner.assert_repo_configured.assert_called_once() + runner.install_deb_packages.assert_not_called() + + def test_package_mode_installs_rocm_when_not_config_only(self): + runner = self._runner(repo_package_dir="/pkgs", repo_config_only=False) + runner.install_repo_package = MagicMock(return_value=True) + runner.assert_repo_configured = MagicMock(return_value=True) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertTrue(runner.run_repo_setup_and_install()) + runner.install_deb_packages.assert_called_once() + + def test_failed_assert_short_circuits(self): + runner = self._runner(repo_package_dir="/pkgs", repo_config_only=False) + runner.install_repo_package = MagicMock(return_value=True) + runner.assert_repo_configured = MagicMock(return_value=False) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertFalse(runner.run_repo_setup_and_install()) + runner.install_deb_packages.assert_not_called() + + def test_default_path_uses_manual_setup(self): + runner = self._runner() # no repo_package_dir + runner.install_repo_package = MagicMock(return_value=True) + runner.setup_deb_repository = MagicMock(return_value=True) + runner.install_deb_packages = MagicMock(return_value=True) + with _suppress_script_output(): + self.assertTrue(runner.run_repo_setup_and_install()) + runner.setup_deb_repository.assert_called_once() + runner.install_repo_package.assert_not_called() + + +class ArgvFromCiEnvRepoPackageTest(unittest.TestCase): + """_argv_from_ci_env: install-via-package env mapping and relaxed requirements.""" + + def test_config_only_lane_needs_no_repo_url_or_install_prefix(self): + env = { + "TEST_TYPE": "sanity", + "OS_PROFILE": "ubuntu2404", + "RELEASE_TYPE": "prerelease", + "REPO_PACKAGE_DIR": "/pkgs", + "REPO_CONFIG_ONLY": "true", + } + with _clear_ci_env(), patch.dict(os.environ, env, clear=False): + argv = native_linux_package_install_test._argv_from_ci_env() + self.assertIsNotNone(argv) + self.assertIn("--repo-package-dir", argv) + self.assertEqual(argv[argv.index("--repo-package-dir") + 1], "/pkgs") + self.assertIn("--repo-config-only", argv) + self.assertNotIn("--repo-url", argv) + self.assertNotIn("--install-prefix", argv) + + def test_package_mode_with_install_prefix_no_repo_url(self): + env = { + "TEST_TYPE": "sanity", + "OS_PROFILE": "rhel10", + "RELEASE_TYPE": "prerelease", + "REPO_PACKAGE_DIR": "/pkgs", + "INSTALL_PREFIX": "/opt/rocm/core", + } + with _clear_ci_env(), patch.dict(os.environ, env, clear=False): + argv = native_linux_package_install_test._argv_from_ci_env() + self.assertIsNotNone(argv) + self.assertIn("--repo-package-dir", argv) + self.assertIn("--install-prefix", argv) + self.assertNotIn("--repo-url", argv) + self.assertNotIn("--repo-config-only", argv) + + if __name__ == "__main__": unittest.main() diff --git a/build_tools/tests/build_repo_package_test.py b/build_tools/tests/build_repo_package_test.py new file mode 100644 index 00000000000..4d351b02b9c --- /dev/null +++ b/build_tools/tests/build_repo_package_test.py @@ -0,0 +1,842 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for build_repo_package.py. + +build_repo_package.py lives in build_tools/packaging/linux/, which is not on the +path that conftest.py sets up (it only adds build_tools/). Add it explicitly. +Importing the module is side-effect-free thanks to its ``__main__`` guard. +""" + +import json +import sys +from pathlib import Path + +import pytest + +_LINUX_DIR = Path(__file__).resolve().parents[1] / "packaging" / "linux" +sys.path.insert(0, str(_LINUX_DIR)) + +import build_repo_package as brp # noqa: E402 + +PRERELEASE_BASE = "https://rocm.prereleases.amd.com/packages-multi-arch" +RELEASE_BASE = "https://repo.amd.com/rocm/packages-multi-arch" +NIGHTLY_BASE = "https://rocm.nightlies.amd.com/packages-multi-arch" +NIGHTLY_SUB = "20260716-12345" + + +def _args(*extra: str): + """Parse args with sensible defaults, overridable by appending flags.""" + base = [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--repo-base-url", + PRERELEASE_BASE, + "--rocm-version", + "7.14.0", + "--dest-dir", + "/tmp/out", + ] + return brp.parse_args(base + list(extra)) + + +def _render(template: str, context: dict) -> str: + return brp.get_jinja_env().get_template(template).render(context) + + +# --- URL construction --------------------------------------------------------- +# +# The release lines do not share a layout, so each one's exact URLs are pinned +# here: the public signed lines publish one repository per distro, and the +# nightly line publishes one per package type under a dated sub-folder. A URL +# that does not serve a repository only fails at the user's first metadata +# refresh, so these are pinned rather than derived. + +RELEASE_EXPECTED = { + "ubuntu2404": f"{RELEASE_BASE}/ubuntu2404/", + "rhel8": f"{RELEASE_BASE}/rhel8/x86_64/", + "rhel10": f"{RELEASE_BASE}/rhel10/x86_64/", + "sles16": f"{RELEASE_BASE}/sles16/x86_64/", +} + +PRERELEASE_EXPECTED = { + "ubuntu2404": f"{PRERELEASE_BASE}/ubuntu2404/", + "rhel8": f"{PRERELEASE_BASE}/rhel8/x86_64/", + "rhel10": f"{PRERELEASE_BASE}/rhel10/x86_64/", + "sles16": f"{PRERELEASE_BASE}/sles16/x86_64/", +} + +NIGHTLY_EXPECTED = { + "ubuntu2404": f"{NIGHTLY_BASE}/deb/{NIGHTLY_SUB}/", + "rhel8": f"{NIGHTLY_BASE}/rpm/{NIGHTLY_SUB}/x86_64/", + "rhel10": f"{NIGHTLY_BASE}/rpm/{NIGHTLY_SUB}/x86_64/", + "sles16": f"{NIGHTLY_BASE}/rpm/{NIGHTLY_SUB}/x86_64/", +} + + +def _baseurl(profile: str, release_type: str, base: str, sub: str = "") -> str: + return brp.repo_baseurl( + base, brp.OS_PROFILES[profile]["pkg_type"], profile, release_type, sub + ) + + +@pytest.mark.parametrize("profile,expected", sorted(RELEASE_EXPECTED.items())) +def test_release_baseurl_is_per_distro(profile, expected): + assert _baseurl(profile, "release", RELEASE_BASE) == expected + + +@pytest.mark.parametrize("profile,expected", sorted(PRERELEASE_EXPECTED.items())) +def test_prerelease_baseurl_is_per_distro(profile, expected): + assert _baseurl(profile, "prerelease", PRERELEASE_BASE) == expected + + +@pytest.mark.parametrize("profile,expected", sorted(NIGHTLY_EXPECTED.items())) +def test_nightly_baseurl_is_flat_and_dated(profile, expected): + assert _baseurl(profile, "nightly", NIGHTLY_BASE, NIGHTLY_SUB) == expected + + +def test_gpg_key_url_under_base(): + assert brp.gpg_key_url(PRERELEASE_BASE) == f"{PRERELEASE_BASE}/gpg/rocm.gpg" + + +def test_baseurl_tolerates_trailing_slash(): + assert ( + _baseurl("ubuntu2404", "release", RELEASE_BASE + "/") + == RELEASE_EXPECTED["ubuntu2404"] + ) + + +def test_signed_lines_are_per_distro_never_flat(): + # A flat /deb/ or /rpm/x86_64/ is not served on the release CDN, + # so a signed line must always carry the distro segment. + for release_type, base in ( + ("release", RELEASE_BASE), + ("prerelease", PRERELEASE_BASE), + ): + for profile in brp.OS_PROFILES: + url = _baseurl(profile, release_type, base) + assert f"/{profile}/" in url, f"{release_type} must be per-distro" + assert not url.endswith("/deb/") + assert "/rpm/x86_64/" not in url + + +def test_nightly_is_flat_and_carries_no_distro_segment(): + # The nightly repository has no per-distro tree, so it must stay flat. + for profile in brp.OS_PROFILES: + url = _baseurl(profile, "nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert f"/{profile}/" not in url + assert NIGHTLY_SUB in url + + +def test_gpg_key_url_is_not_the_domain_root(): + gpg = brp.gpg_key_url(PRERELEASE_BASE) + assert "/packages-multi-arch/gpg/rocm.gpg" in gpg + assert gpg != "https://rocm.prereleases.amd.com/gpg/rocm.gpg" + + +# --- repository reachability check -------------------------------------------- + + +def test_repo_metadata_url_targets_the_index(): + deb = brp.repo_metadata_url(RELEASE_EXPECTED["ubuntu2404"], "deb") + assert deb == f"{RELEASE_EXPECTED['ubuntu2404']}dists/{brp.DEB_SUITE}/Release" + rpm = brp.repo_metadata_url(RELEASE_EXPECTED["rhel10"], "rpm") + assert rpm == f"{RELEASE_EXPECTED['rhel10']}repodata/repomd.xml" + + +def test_verify_repo_url_accepts_a_served_repo(monkeypatch): + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(b"", RELEASE_EXPECTED["rhel10"], status=200), + ) + brp.verify_repo_url(RELEASE_EXPECTED["rhel10"], "rpm") # must not raise + + +def test_verify_repo_url_rejects_an_unserved_repo(monkeypatch): + # The flat shape on the release line 404/403s; the guard must fail the build + # rather than ship a package that cannot refresh. + def not_found(*a, **k): + raise brp.urllib.error.HTTPError( + f"{RELEASE_BASE}/rpm/x86_64/", 403, "Forbidden", {}, None + ) + + monkeypatch.setattr(brp.urllib.request, "urlopen", not_found) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + with pytest.raises(RuntimeError): + brp.verify_repo_url(f"{RELEASE_BASE}/rpm/x86_64/", "rpm") + + +def test_verify_repo_url_does_not_retry_a_missing_repo(monkeypatch): + # A wrong URL will not become right, so it must fail on the first attempt + # rather than spending the full retry budget. + calls = {"n": 0} + + def forbidden(*a, **k): + calls["n"] += 1 + raise brp.urllib.error.HTTPError( + f"{RELEASE_BASE}/rpm/x86_64/", 403, "Forbidden", {}, None + ) + + monkeypatch.setattr(brp.urllib.request, "urlopen", forbidden) + monkeypatch.setattr(brp.time, "sleep", lambda s: pytest.fail("must not back off")) + with pytest.raises(RuntimeError): + brp.verify_repo_url(f"{RELEASE_BASE}/rpm/x86_64/", "rpm") + assert calls["n"] == 1 + + +def test_verify_repo_url_retries_rate_limiting(monkeypatch): + calls = {"n": 0} + + def throttled(*a, **k): + calls["n"] += 1 + if calls["n"] < 2: + raise brp.urllib.error.HTTPError( + RELEASE_EXPECTED["rhel10"], 429, "Too Many Requests", {}, None + ) + return _FakeResponse(b"", RELEASE_EXPECTED["rhel10"], status=200) + + monkeypatch.setattr(brp.urllib.request, "urlopen", throttled) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + brp.verify_repo_url(RELEASE_EXPECTED["rhel10"], "rpm") + assert calls["n"] == 2 + + +def test_verify_repo_url_retries_then_succeeds(monkeypatch): + calls = {"n": 0} + + def flaky(*a, **k): + calls["n"] += 1 + if calls["n"] < 3: + raise brp.urllib.error.URLError("transient") + return _FakeResponse(b"", RELEASE_EXPECTED["rhel10"], status=200) + + monkeypatch.setattr(brp.urllib.request, "urlopen", flaky) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + brp.verify_repo_url(RELEASE_EXPECTED["rhel10"], "rpm") + assert calls["n"] == 3 + + +# --- reachability check wiring ------------------------------------------------- +# +# A guard that is never invoked is indistinguishable from no guard, so pin the +# call from main() rather than only testing the function in isolation. + + +def _main_argv(tmp_path, *extra: str) -> list[str]: + return [ + "--os-profile", + "rhel10", + "--release-type", + "release", + "--repo-base-url", + RELEASE_BASE, + "--rocm-version", + "7.14.0", + "--dest-dir", + str(tmp_path), + *extra, + ] + + +def test_verify_repo_url_is_off_by_default(): + assert _args().verify_repo_url is False + + +def test_main_verifies_the_repository_when_asked(monkeypatch, tmp_path): + checked = [] + monkeypatch.setattr( + brp, "verify_repo_url", lambda url, pkg_type: checked.append((url, pkg_type)) + ) + monkeypatch.setattr(brp, "build_rpm_package", lambda *a, **k: None) + brp.main(_main_argv(tmp_path, "--verify-repo-url")) + # The URL actually built for this profile is what gets checked. + assert checked == [(RELEASE_EXPECTED["rhel10"], "rpm")] + + +def test_main_skips_verification_for_nightly(monkeypatch, tmp_path): + # CI passes the flag unconditionally; the nightly repository is published by + # the same run, so its dated sub-folder cannot be checked at build time. + monkeypatch.setattr( + brp, + "verify_repo_url", + lambda *a: pytest.fail("nightly has nothing to verify yet"), + ) + monkeypatch.setattr(brp, "build_rpm_package", lambda *a, **k: None) + brp.main( + [ + "--os-profile", + "rhel10", + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + NIGHTLY_SUB, + "--rocm-version", + "7.14.0", + "--dest-dir", + str(tmp_path), + "--verify-repo-url", + ] + ) + + +def test_main_does_not_verify_by_default(monkeypatch, tmp_path): + monkeypatch.setattr( + brp, + "verify_repo_url", + lambda *a: pytest.fail("must not reach the network unless asked"), + ) + monkeypatch.setattr(brp, "build_rpm_package", lambda *a, **k: None) + brp.main(_main_argv(tmp_path)) + + +# --- context wiring ----------------------------------------------------------- + + +@pytest.mark.parametrize("profile", sorted(RELEASE_EXPECTED)) +def test_build_context_threads_profile_and_release_type(profile): + # The baseurl depends on both the target distro and the release line, so + # build_context must pass both through rather than defaulting either. + args = _args( + "--os-profile", + profile, + "--release-type", + "release", + "--repo-base-url", + RELEASE_BASE, + ) + ctx = brp.build_context(args, brp.OS_PROFILES[profile]) + assert ctx["baseurl"] == RELEASE_EXPECTED[profile] + + +# --- signing ------------------------------------------------------------------ + + +def test_signed_lines(): + assert brp.is_signed("prerelease") is True + assert brp.is_signed("release") is True + assert brp.is_signed("nightly") is False + + +# --- signing-key fetch hardening ---------------------------------------------- + + +class _FakeResponse: + """Minimal stand-in for a urlopen() result (context manager + read).""" + + def __init__(self, body: bytes, url: str, status: int = 200): + self._body = body + self._url = url + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def geturl(self) -> str: + return self._url + + def getcode(self) -> int: + return self.status + + def read(self, n: int = -1) -> bytes: + return self._body if n < 0 else self._body[:n] + + +def test_require_https_accepts_https(): + brp._require_https("https://example.com/gpg/rocm.gpg", "key URL") + + +def test_require_https_rejects_http(): + with pytest.raises(ValueError): + brp._require_https("http://example.com/gpg/rocm.gpg", "key URL") + + +def test_load_signing_key_rejects_non_https_base(monkeypatch): + # A non-https base must fail before any network fetch is attempted. + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: pytest.fail("must not fetch over a non-https URL"), + ) + args = _args() + args.repo_base_url = "http://rocm.example.com/packages-multi-arch" + with pytest.raises(ValueError): + brp.load_signing_key(args) + + +def test_load_signing_key_rejects_post_redirect_downgrade(monkeypatch): + # An https URL that redirects to http must be rejected after the fetch. + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(b"KEY", "http://evil.example/gpg/rocm.gpg"), + ) + with pytest.raises(ValueError): + brp.load_signing_key(_args()) + + +def test_load_signing_key_rejects_oversize_key(monkeypatch): + big = b"A" * (brp.MAX_GPG_KEY_BYTES + 1) + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(big, f"{PRERELEASE_BASE}/gpg/rocm.gpg"), + ) + with pytest.raises(ValueError): + brp.load_signing_key(_args()) + + +def test_load_signing_key_returns_fetched_key(monkeypatch): + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse( + b"ARMORED KEY", f"{PRERELEASE_BASE}/gpg/rocm.gpg" + ), + ) + # Fingerprint verification runs gpg on real key bytes; stub it here. + monkeypatch.setattr(brp, "_verify_key_fingerprint", lambda key: None) + assert brp.load_signing_key(_args()) == b"ARMORED KEY" + + +def test_load_signing_key_prefers_key_file(monkeypatch, tmp_path): + key = tmp_path / "rocm.gpg" + key.write_bytes(b"FILE KEY") + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: pytest.fail("must not fetch when --gpg-key-file is given"), + ) + args = _args() + args.gpg_key_file = key + assert brp.load_signing_key(args) == b"FILE KEY" + + +def test_load_signing_key_rejects_wrong_fingerprint(monkeypatch): + monkeypatch.setattr( + brp.urllib.request, + "urlopen", + lambda *a, **k: _FakeResponse(b"KEY", f"{PRERELEASE_BASE}/gpg/rocm.gpg"), + ) + monkeypatch.setattr(brp, "_key_fingerprint", lambda armored: "0" * 40) + with pytest.raises(ValueError): + brp.load_signing_key(_args()) + + +def test_verify_key_fingerprint_accepts_pinned(monkeypatch): + monkeypatch.setattr( + brp, "_key_fingerprint", lambda armored: brp.EXPECTED_KEY_FINGERPRINT + ) + brp._verify_key_fingerprint(b"whatever") # must not raise + + +def test_fetch_signing_key_retries_then_succeeds(monkeypatch): + calls = {"n": 0} + + def flaky(*a, **k): + calls["n"] += 1 + if calls["n"] < 3: + raise brp.urllib.error.URLError("transient") + return _FakeResponse(b"KEY", f"{PRERELEASE_BASE}/gpg/rocm.gpg") + + monkeypatch.setattr(brp.urllib.request, "urlopen", flaky) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) # no real backoff + assert brp._fetch_signing_key(f"{PRERELEASE_BASE}/gpg/rocm.gpg") == b"KEY" + assert calls["n"] == 3 + + +def test_fetch_signing_key_gives_up_after_attempts(monkeypatch): + def always_fail(*a, **k): + raise brp.urllib.error.URLError("down") + + monkeypatch.setattr(brp.urllib.request, "urlopen", always_fail) + monkeypatch.setattr(brp.time, "sleep", lambda s: None) + with pytest.raises(RuntimeError): + brp._fetch_signing_key(f"{PRERELEASE_BASE}/gpg/rocm.gpg") + + +# --- versioning --------------------------------------------------------------- + + +def test_rpm_version_rolling_for_prerelease(): + assert brp.rpm_version_release("prerelease", "7.14.0", "") == ( + "7.14.0", + "1.prerelease", + ) + + +def test_streams_never_share_a_package_version(): + # Two lines at the same ROCm version must not produce the same package: + # installing one over the other would otherwise report success and silently + # leave the original repository configured. + version = "7.14.0" + rpm_pre = brp.rpm_version_release("prerelease", version, "") + rpm_rel = brp.rpm_version_release("release", version, "") + assert rpm_pre != rpm_rel + assert brp.deb_version("prerelease", version, "") != brp.deb_version( + "release", version, "" + ) + + +def test_prerelease_sorts_before_release(): + # The release package must upgrade over the prerelease one, not be seen as a + # downgrade. deb: "~" sorts before the plain version. rpm: the release field + # differs and "prerelease" precedes "release". + assert brp.deb_version("prerelease", "7.14.0", "") == "7.14.0~prerelease" + assert brp.deb_version("release", "7.14.0", "") == "7.14.0" + assert brp.rpm_version_release("prerelease", "7.14.0", "")[1] < ( + brp.rpm_version_release("release", "7.14.0", "")[1] + ) + + +def test_rpm_version_nightly_splits_date_and_id(): + version, release = brp.rpm_version_release("nightly", "7.14.0", NIGHTLY_SUB) + assert version == "20260716" + assert release == "12345.nightly" + assert "-" not in version # rpm Version cannot contain a hyphen + + +def test_deb_version_nightly_has_no_hyphen(): + assert brp.deb_version("nightly", "7.14.0", NIGHTLY_SUB) == "20260716.12345" + + +def test_deb_version_rolling(): + assert brp.deb_version("release", "7.14.0", "") == "7.14.0" + + +# --- CLI validation ----------------------------------------------------------- + + +def test_release_type_rejects_unknown(): + with pytest.raises(SystemExit): + _args("--release-type", "dev") + + +def test_repo_base_url_required(): + with pytest.raises(SystemExit): + brp.parse_args( + [ + "--os-profile", + "ubuntu2404", + "--release-type", + "prerelease", + "--rocm-version", + "7.14.0", + "--dest-dir", + "/tmp/out", + ] + ) + + +def test_nightly_requires_sub_folder(): + with pytest.raises(SystemExit): + _args("--release-type", "nightly", "--repo-base-url", NIGHTLY_BASE) + + +def test_nightly_with_sub_folder_ok(): + args = _args( + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + NIGHTLY_SUB, + ) + assert args.repo_sub_folder == NIGHTLY_SUB + + +def test_sub_folder_rejected_for_non_nightly(): + with pytest.raises(SystemExit): + _args("--repo-sub-folder", NIGHTLY_SUB) # default release-type is prerelease + + +# --- input validation --------------------------------------------------------- + + +@pytest.mark.parametrize( + "bad_version", + [ + '7.14.0"; rm -rf /; echo "', # shell-injection payload + "7.14.0 evil", # whitespace + "7.14.0$(id)", # command substitution + "7.14.0\ngpgcheck=0", # embedded newline + "7.14.0\n", # trailing newline (regex must use \\Z, not $) + "", # empty + "-7.14.0", # must start with a digit + ], +) +def test_rocm_version_rejects_bad_format(bad_version): + with pytest.raises(SystemExit): + _args("--rocm-version", bad_version) + + +@pytest.mark.parametrize("good_version", ["7.14.0", "7.14.0~rc3", "8.0.1rc1", "7.0"]) +def test_rocm_version_accepts_valid(good_version): + assert _args("--rocm-version", good_version).rocm_version == good_version + + +@pytest.mark.parametrize( + "bad_url", + [ + f"{PRERELEASE_BASE}\ngpgcheck=0", # newline directive injection + f"{PRERELEASE_BASE} extra", # whitespace + "not a url", # no scheme/netloc + "ftp://example.com/repo", # unsupported scheme + ], +) +def test_repo_base_url_rejects_bad(bad_url): + with pytest.raises(SystemExit): + _args("--repo-base-url", bad_url) + + +def test_repo_base_url_accepts_https(): + assert _args("--repo-base-url", PRERELEASE_BASE).repo_base_url == PRERELEASE_BASE + + +def test_repo_sub_folder_rejects_bad_format(): + with pytest.raises(SystemExit): + _args( + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + "20260722-99\nX", # newline in the dated sub-folder + ) + + +def test_postinst_does_not_interpolate_version(): + # The postinst runs as root at install time; the version must not be rendered + # into it (defense-in-depth beyond input validation). + ctx = brp.build_context( + _args("--rocm-version", "9.9.9"), brp.OS_PROFILES["ubuntu2404"] + ) + out = _render("template/repo/deb/postinst.j2", ctx) + assert "9.9.9" not in out + assert "AMD ROCm repository configured." in out + + +# --- rpm .repo rendering ------------------------------------------------------ + + +def _rpm_repo(release_type: str, base: str, sub: str = "") -> str: + extra = [ + "--os-profile", + "rhel10", + "--release-type", + release_type, + "--repo-base-url", + base, + ] + if sub: + extra += ["--repo-sub-folder", sub] + args = _args(*extra) + return _render( + "template/repo/rpm/amdrocm.repo.j2", + brp.build_context(args, brp.OS_PROFILES["rhel10"]), + ) + + +def test_rpm_repo_signed_has_gpgcheck_and_gpgkey(): + out = _rpm_repo("prerelease", PRERELEASE_BASE) + assert "[amdrocm]" in out + assert "enabled=1" in out + assert "gpgcheck=1" in out + assert "gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" in out + assert f"baseurl={PRERELEASE_EXPECTED['rhel10']}" in out + + +def test_rpm_repo_nightly_unsigned_no_gpgkey(): + out = _rpm_repo("nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert "gpgcheck=0" in out + assert "gpgkey" not in out + + +def test_rpm_repo_ends_with_newline(): + assert _rpm_repo("prerelease", PRERELEASE_BASE).endswith("\n") + + +# --- deb .sources rendering --------------------------------------------------- + + +def _deb_sources(release_type: str, base: str, sub: str = "") -> str: + extra = [ + "--os-profile", + "ubuntu2404", + "--release-type", + release_type, + "--repo-base-url", + base, + ] + if sub: + extra += ["--repo-sub-folder", sub] + args = _args(*extra) + return _render( + "template/repo/deb/amdrocm.sources.j2", + brp.build_context(args, brp.OS_PROFILES["ubuntu2404"]), + ) + + +def test_deb_sources_signed_fields(): + out = _deb_sources("prerelease", PRERELEASE_BASE) + assert "Types: deb" in out + assert f"URIs: {PRERELEASE_EXPECTED['ubuntu2404']}" in out + # The suite is rendered from the constant the reachability check also uses, + # so the repo file and the verified index path cannot drift apart. + assert f"Suites: {brp.DEB_SUITE}" in out + assert "Suites: stable" in out + assert "Components: main" in out + assert "Architectures: amd64" in out + assert "Signed-By: /usr/share/keyrings/rocm.gpg" in out + assert "Trusted:" not in out + + +def test_deb_sources_nightly_trusted_not_signed(): + out = _deb_sources("nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert "Trusted: yes" in out + assert "Signed-By:" not in out + + +# --- rpm spec rendering ------------------------------------------------------- + + +def _spec(release_type: str, base: str, sub: str = "") -> str: + extra = [ + "--os-profile", + "rhel10", + "--release-type", + release_type, + "--repo-base-url", + base, + ] + if sub: + extra += ["--repo-sub-folder", sub] + args = _args(*extra) + ctx = brp.build_context(args, brp.OS_PROFILES["rhel10"]) + version, release = brp.rpm_version_release(release_type, args.rocm_version, sub) + return _render( + "template/repo/rpm/amdrocm-repo.spec.j2", + { + **ctx, + "version": version, + "release": release, + "changelog_date": "Mon Jul 20 2026", + }, + ) + + +def test_spec_name_and_repo_install(): + out = _spec("prerelease", PRERELEASE_BASE) + assert "Name: amdrocm-repo" in out + assert "Version: 7.14.0" in out + assert "/etc/yum.repos.d/amdrocm.repo" in out + + +def test_spec_signed_installs_key_nightly_does_not(): + signed = _spec("prerelease", PRERELEASE_BASE) + assert "/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm" in signed + unsigned = _spec("nightly", NIGHTLY_BASE, NIGHTLY_SUB) + assert "RPM-GPG-KEY-rocm" not in unsigned + + +def test_spec_has_no_conflicts(): + assert "Conflicts" not in _spec("prerelease", PRERELEASE_BASE) + + +# --- deb control / install rendering ------------------------------------------ + + +def test_control_has_no_key_runtime_deps(): + args = _args() + ctx = brp.build_context(args, brp.OS_PROFILES["ubuntu2404"]) + out = _render("template/repo/deb/control.j2", ctx) + assert "Package: amdrocm-repo" in out + assert "gnupg" not in out + assert "wget" not in out + assert "Conflicts" not in out + + +def test_install_maps_keyring_only_when_signed(): + signed = brp.build_context(_args(), brp.OS_PROFILES["ubuntu2404"]) + out_signed = _render("template/repo/deb/install.j2", signed) + assert "amdrocm.sources /etc/apt/sources.list.d/" in out_signed + assert "rocm.gpg /usr/share/keyrings/" in out_signed + + args_n = _args( + "--release-type", + "nightly", + "--repo-base-url", + NIGHTLY_BASE, + "--repo-sub-folder", + NIGHTLY_SUB, + ) + out_nightly = _render( + "template/repo/deb/install.j2", + brp.build_context(args_n, brp.OS_PROFILES["ubuntu2404"]), + ) + assert "rocm.gpg" not in out_nightly + + +# --- list-profiles -------------------------------------------------------- + + +def test_list_profiles_rpm(): + assert brp.list_profiles("rpm") == ["rhel8", "rhel10", "sles16"] + + +def test_list_profiles_deb(): + assert brp.list_profiles("deb") == ["ubuntu2404"] + + +def test_list_profiles_cli_emits_json(capsys): + brp.run_list_profiles(["--pkg-type", "rpm"]) + out = capsys.readouterr().out + assert json.loads(out) == ["rhel8", "rhel10", "sles16"] + + +def test_list_profiles_cli_rejects_invalid_pkg_type(capsys): + with pytest.raises(SystemExit): + brp.run_list_profiles(["--pkg-type", "foo"]) + + +def test_main_dispatches_list_profiles(capsys): + brp.main(["list-profiles", "--pkg-type", "deb"]) + out = capsys.readouterr().out + assert json.loads(out) == ["ubuntu2404"] + + +# --- dest-dir normalization ----------------------------------------------- + + +def test_relative_dest_dir_is_resolved_to_absolute(): + # rpmbuild needs an absolute _topdir, so a relative --dest-dir must be + # normalized to absolute at parse time. + args = brp.parse_args( + [ + "--os-profile", + "rhel10", + "--release-type", + "prerelease", + "--repo-base-url", + PRERELEASE_BASE, + "--rocm-version", + "7.14.0", + "--dest-dir", + "relout", + ] + ) + assert args.dest_dir.is_absolute() + assert args.dest_dir.name == "relout" + + +def test_absolute_dest_dir_stays_absolute(): + args = _args("--dest-dir", "/tmp/somewhere") + assert args.dest_dir.is_absolute() + assert args.dest_dir.name == "somewhere" diff --git a/build_tools/tests/publish_repo_package_test.py b/build_tools/tests/publish_repo_package_test.py new file mode 100644 index 00000000000..21d356573d0 --- /dev/null +++ b/build_tools/tests/publish_repo_package_test.py @@ -0,0 +1,151 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Unit tests for publish_repo_package.py. + +publish_repo_package.py lives in build_tools/packaging/linux/, which is not on +the path that conftest.py sets up (it only adds build_tools/). Add it +explicitly. Importing the module is side-effect-free (boto3 is a deferred import +inside ``publish``), so key derivation is testable without boto3 installed. +""" + +import sys +from pathlib import Path + +import pytest + +_LINUX_DIR = Path(__file__).resolve().parents[1] / "packaging" / "linux" +sys.path.insert(0, str(_LINUX_DIR)) + +import publish_repo_package as prp # noqa: E402 + +PREFIX = "12345-linux/packages" + + +# --- object key derivation ---------------------------------------------------- + + +def test_key_deb_profile(): + assert ( + prp.object_key(f"{PREFIX}/deb", "ubuntu2404", "deb") + == "12345-linux/packages/deb/repo/ubuntu2404/amdrocm-repo.deb" + ) + + +@pytest.mark.parametrize("profile", ["rhel8", "rhel10", "sles16"]) +def test_key_rpm_profiles(profile): + assert ( + prp.object_key(f"{PREFIX}/rpm", profile, "rpm") + == f"12345-linux/packages/rpm/repo/{profile}/amdrocm-repo.rpm" + ) + + +def test_key_is_sibling_not_in_repo_index(): + # The file must sit under repo//, never inside the content index + # (deb dists/, rpm x86_64/). + deb = prp.object_key(f"{PREFIX}/deb", "ubuntu2404", "deb") + rpm = prp.object_key(f"{PREFIX}/rpm", "rhel10", "rpm") + for key in (deb, rpm): + assert "/repo/" in key + assert "/dists/" not in deb + assert "/x86_64/" not in rpm + + +def test_key_tolerates_trailing_slash_prefix(): + assert ( + prp.object_key(f"{PREFIX}/rpm/", "rhel10", "rpm") + == "12345-linux/packages/rpm/repo/rhel10/amdrocm-repo.rpm" + ) + + +# --- CLI validation ----------------------------------------------------------- + + +def _args(tmp_path, **overrides): + pkg = tmp_path / "amdrocm-repo-7.14.0-1.el10.noarch.rpm" + pkg.write_bytes(b"PKG") + argv = [ + "--file", + str(overrides.get("file", pkg)), + "--bucket", + overrides.get("bucket", "therock-prerelease-artifacts"), + "--prefix", + overrides.get("prefix", f"{PREFIX}/rpm"), + "--os-profile", + overrides.get("os_profile", "rhel10"), + "--pkg-type", + overrides.get("pkg_type", "rpm"), + ] + return argv + + +def test_parse_args_ok(tmp_path): + args = prp.parse_args(_args(tmp_path)) + assert args.bucket == "therock-prerelease-artifacts" + assert args.pkg_type == "rpm" + + +def test_parse_args_rejects_empty_bucket(tmp_path): + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, bucket="")) + + +def test_parse_args_rejects_missing_file(tmp_path): + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, file=str(tmp_path / "nope.rpm"))) + + +def test_parse_args_rejects_bad_pkg_type(tmp_path): + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, pkg_type="tar")) + + +@pytest.mark.parametrize("bad", ["rhel10/../x", "a/b", "rhel 10", "rhel10\n", ""]) +def test_parse_args_rejects_bad_os_profile(tmp_path, bad): + # os_profile becomes an S3 key path segment; reject slashes/whitespace. + with pytest.raises(SystemExit): + prp.parse_args(_args(tmp_path, os_profile=bad)) + + +# --- publish (boto3 mocked) --------------------------------------------------- + + +class _FakeS3: + def __init__(self): + self.uploaded = [] + self.headed = [] + self.head_should_fail = False + + def upload_file(self, filename, bucket, key): + self.uploaded.append((filename, bucket, key)) + + def head_object(self, Bucket, Key): + self.headed.append((Bucket, Key)) + if self.head_should_fail: + raise RuntimeError("object not found") + + +def _install_fake_boto3(monkeypatch, fake): + import types + + fake_boto3 = types.SimpleNamespace(client=lambda service, **kw: fake) + monkeypatch.setitem(sys.modules, "boto3", fake_boto3) + + +def test_publish_uploads_then_confirms(tmp_path, monkeypatch): + fake = _FakeS3() + _install_fake_boto3(monkeypatch, fake) + args = prp.parse_args(_args(tmp_path)) + key = prp.publish(args) + assert key == "12345-linux/packages/rpm/repo/rhel10/amdrocm-repo.rpm" + assert fake.uploaded == [(str(args.file), "therock-prerelease-artifacts", key)] + assert fake.headed == [("therock-prerelease-artifacts", key)] + + +def test_publish_errors_when_head_fails(tmp_path, monkeypatch): + fake = _FakeS3() + fake.head_should_fail = True + _install_fake_boto3(monkeypatch, fake) + args = prp.parse_args(_args(tmp_path)) + with pytest.raises(RuntimeError): + prp.publish(args) diff --git a/docs/packaging/native_packaging.md b/docs/packaging/native_packaging.md index fb5c3f69e65..1596f189b69 100644 --- a/docs/packaging/native_packaging.md +++ b/docs/packaging/native_packaging.md @@ -6,6 +6,9 @@ distribution as Debian and RPM packages. This process involves both the reorganization of artifacts and the execution of package management tools +To configure a system package manager to install ROCm from a public repository, +see [rocm_repo_setup.md](rocm_repo_setup.md). + ## General Design The required packages and their details are stored in a JSON file diff --git a/docs/packaging/rocm_repo_setup.md b/docs/packaging/rocm_repo_setup.md new file mode 100644 index 00000000000..2872337d16f --- /dev/null +++ b/docs/packaging/rocm_repo_setup.md @@ -0,0 +1,199 @@ +# Configuring the ROCm package repository (`amdrocm-repo`) + +`amdrocm-repo` is a configuration package that points a system package manager +(`apt`, `dnf`, or `zypper`) at a public AMD ROCm repository. Once it is +installed, you can install and update ROCm packages with the native package +manager. + +It configures the repository only. The ROCm *content* packages and their upload +paths are documented in [native_packaging.md](native_packaging.md). + +## What the package installs + +`amdrocm-repo` is generated per OS profile. Every profile writes a single repo +definition and, for signed lines, the repository signing key: + +| OS profile | Package manager | Repo definition | Signing key | +| ----------------- | --------------- | ----------------------------------------- | ----------------------------------- | +| `ubuntu2404` | `apt` | `/etc/apt/sources.list.d/amdrocm.sources` | `/usr/share/keyrings/rocm.gpg` | +| `rhel8`, `rhel10` | `dnf` / `yum` | `/etc/yum.repos.d/amdrocm.repo` | `/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm` | +| `sles16` | `zypper` | `/etc/zypp/repos.d/amdrocm.repo` | `/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm` | + +The deb repo definition uses the deb822 `.sources` format +(`Suites: stable`, `Components: main`, `Architectures: amd64`) and references +the shipped keyring via `Signed-By`. The rpm repo definition references the +shipped key via `gpgkey=file://`. + +## Release lines + +The configured repository depends on the release line. All repositories serve +x86_64 (`amd64`) packages only. + +| Release line | Repository base | Signed | +| ------------ | ------------------------------------------------------ | ------ | +| `prerelease` | `https://rocm.prereleases.amd.com/packages-multi-arch` | yes | +| `release` | `https://repo.amd.com/rocm/packages-multi-arch` | yes | +| `nightly` | `https://rocm.nightlies.amd.com/packages-multi-arch` | no | + +The lines do not share a repository layout. The `prerelease` and `release` +repositories are published per distro; the `nightly` repository is published per +package type under a dated sub-folder. So the repo file resolves to: + +| Release line | deb `URIs` | rpm `baseurl` | +| ----------------------- | ----------------------- | ------------------------------ | +| `prerelease`, `release` | `//` | `//x86_64/` | +| `nightly` | `/deb//` | `/rpm//x86_64/` | + +The signing key is at `/gpg/rocm.gpg` on both signed lines. + +> **Nightly is unsigned.** The nightly repository is not signed, so the nightly +> `amdrocm-repo` disables signature verification (`gpgcheck=0` / apt +> `Trusted: yes`) and ships no key. Prefer a signed line (`prerelease` or +> `release`) for anything beyond testing. Nightly packages live in a dated +> sub-folder, so a nightly `amdrocm-repo` is pinned to its build date. + +## Download URLs + +On the `prerelease` and `nightly` lines, `amdrocm-repo` is published as a +standalone file per OS profile, alongside the packages it configures: + +- prerelease: `//repo//amdrocm-repo.` +- nightly: `///repo//amdrocm-repo.` + +where `` is `deb` or `rpm` and `` matches. For example, on the +prerelease line: + +| OS profile | Download URL | +| ------------ | ------------------------------------------------------------------------------------------- | +| `ubuntu2404` | `https://rocm.prereleases.amd.com/packages-multi-arch/deb/repo/ubuntu2404/amdrocm-repo.deb` | +| `rhel8` | `https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/rhel8/amdrocm-repo.rpm` | +| `rhel10` | `https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/rhel10/amdrocm-repo.rpm` | +| `sles16` | `https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/sles16/amdrocm-repo.rpm` | + +A distro's package is reachable once its line has published. The `release` line +is published through the ROCm release process rather than from this repository's +build, so its download location is not listed here. To use the release +repository before that package is available, build it locally with +`--release-type release` (see below). + +## Installing and using the repository + +Install the `amdrocm-repo` package for your distro, refresh the package +manager's metadata, then install ROCm packages. + +Ubuntu: + +```bash +curl -fsSL -O https://rocm.prereleases.amd.com/packages-multi-arch/deb/repo/ubuntu2404/amdrocm-repo.deb +sudo apt install ./amdrocm-repo.deb +sudo apt-get update +sudo apt install amdrocm +``` + +RHEL / Rocky / Oracle Linux: + +```bash +sudo dnf install https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/rhel10/amdrocm-repo.rpm +sudo dnf makecache +sudo dnf install amdrocm +``` + +SLES: + +```bash +sudo zypper install https://rocm.prereleases.amd.com/packages-multi-arch/rpm/repo/sles16/amdrocm-repo.rpm +sudo zypper refresh +sudo zypper install amdrocm +``` + +On a signed line, `zypper` prompts you to trust the repository signing key the +first time it refreshes; accept it to continue. + +Confirm the repository is configured by listing the `amdrocm*` packages it +publishes: + +```bash +apt list 'amdrocm*' # Ubuntu +dnf list 'amdrocm*' # RHEL / Rocky / Oracle Linux +zypper search amdrocm # SLES +``` + +The command returns the available packages (for example `amdrocm` and +`amdrocm-base`). An empty result means the repository is not configured, or its +metadata has not been refreshed. Install the components you need from that +list. + +## Troubleshooting + +- **404 when downloading the package.** The release line is not published for + your distro yet. Use the `prerelease` line, or build the package locally (see + below). +- **`amdrocm*` packages are not listed.** Refresh the package manager metadata + (`sudo apt-get update`, `sudo dnf makecache`, or `sudo zypper refresh`) after + installing `amdrocm-repo`. +- **`zypper` reports an untrusted signing key.** Accept the key when prompted on + the first refresh of a signed line. + +## Building the package locally + +`build_repo_package.py` builds `amdrocm-repo` for a single OS profile and +release line. It requires Python 3.12 or newer (note that some rpm base images, +such as RHEL 8, do not ship a system `python3` at that version), the `jinja2` +Python package, and the native packaging tools for the target format: + +- deb (`ubuntu2404`): `debhelper`, `dpkg-dev`, `build-essential` +- rpm (`rhel8`, `rhel10`, `sles16`): `rpm-build` + +For a signed line the builder also invokes `gpg` to prepare the key, and fetches +it over the network (see below), so `gpg` and `ca-certificates` must be present. + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile ubuntu2404 \ + --release-type prerelease \ + --repo-base-url https://rocm.prereleases.amd.com/packages-multi-arch \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out +``` + +For a signed line, the signing key is fetched from `/gpg/rocm.gpg` over +the network at build time. To build offline, or to pin a specific key, pass it +explicitly: + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile rhel10 \ + --release-type prerelease \ + --repo-base-url https://rocm.prereleases.amd.com/packages-multi-arch \ + --gpg-key-file ./rocm.gpg \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out +``` + +The nightly line is date-pinned; pass the dated sub-folder: + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile ubuntu2404 \ + --release-type nightly \ + --repo-base-url https://rocm.nightlies.amd.com/packages-multi-arch \ + --repo-sub-folder 20260722-123456789 \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out +``` + +Pass `--verify-repo-url` to fail the build unless the repository the package +configures is reachable. This catches a package that would install cleanly but +fail on the user's first metadata refresh: + +```bash +python3 build_tools/packaging/linux/build_repo_package.py \ + --os-profile rhel10 \ + --release-type release \ + --repo-base-url https://repo.amd.com/rocm/packages-multi-arch \ + --rocm-version 7.14.0 \ + --dest-dir ./repo-package-out \ + --verify-repo-url +``` + +See `build_repo_package.py --help` for the full option list. diff --git a/requirements-test.txt b/requirements-test.txt index ff1cc524f7e..dbdee1695a4 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,5 @@ boto3==1.39.15 +jinja2>=3.0.0 # Packaging scripts render templates; their tests import them. pytest==9.0.3 # This versions has subtests built-in. pytest-check==2.5.3 pytest-cov==6.0.0