diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 1ce70a65237..3ced5e0097c 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -14,11 +14,13 @@ steps: # clone azure-cli git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli python -m pip install -U pip - pip install azdev + # azdev 0.2.11b1 (decouples extension metadata from wheel==0.30.0) tho it is not on PyPI yet. + # Pinned to a specific azure-cli-dev-tools commit (not @dev) for reproducible builds + # until the pre-release is published; then switch this back to: + # pip install --upgrade "azdev==0.2.11b1" + pip install --upgrade "git+https://github.com/Azure/azure-cli-dev-tools.git@49390f30c6a822620d2fbdf88889c4445d22666f#egg=azdev" azdev --version azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug - # Installing setuptools with a version higher than 70.0.0 will not generate metadata.json - pip install setuptools==70.0.0 pip list -v az --version displayName: 'azdev setup' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 463347c6fc0..63009627c25 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,7 +9,7 @@ This checklist is used to make sure that common guidelines for a pull request ar ### General Guidelines - [ ] Have you run `azdev style ` locally? (`pip install azdev` required) -- [ ] Have you run `python scripts/ci/test_index.py -q` locally? (`pip install wheel==0.30.0` required) +- [ ] Have you run `python scripts/ci/test_index.py -q` locally? (`azdev` required; see `.azure-pipelines/templates/azdev_setup.yml` for the install command until `azdev==0.2.11b1` is on PyPI) - [ ] My extension version conforms to the [Extension version schema](https://github.com/Azure/azure-cli/blob/release/doc/extensions/versioning_guidelines.md) For new extensions: diff --git a/.gitignore b/.gitignore index 3e9d06ca5e3..5e83e3c36b9 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ celerybeat-schedule # virtualenv .venv +/.venv*/ .env .python-version venv/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a8a37960882..2b342bf2247 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -79,7 +79,12 @@ jobs: - bash: | #!/usr/bin/env bash set -ev - pip install wheel==0.30.0 requests packaging setuptools + # test_index.py imports scripts/ci/util.py, which reads wheel metadata via azdev's + # pkginfo-based metadata module (decoupled from wheel==0.30.0), so azdev must be + # installed here. Install from the merged azure-cli-dev-tools source until azdev + # 0.2.11b1 is published to PyPI, then switch this to: pip install --upgrade "azdev==0.2.11b1" + pip install --upgrade "git+https://github.com/Azure/azure-cli-dev-tools.git@49390f30c6a822620d2fbdf88889c4445d22666f#egg=azdev" + pip install requests packaging setuptools export CI="ADO" python ./scripts/ci/test_index.py -v displayName: "Verify Extensions Index" @@ -106,8 +111,6 @@ jobs: inputs: versionSpec: '$(python.version)' - template: .azure-pipelines/templates/azdev_setup.yml - - bash: pip install wheel==0.30.0 - displayName: 'Install wheel==0.30.0' - bash: | #!/usr/bin/env bash set -ev diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..4bb045817bc --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +markers = + e2e_packaging: End-to-end packaging tests for extension lifecycle and metadata behavior. + smoke: Fast smoke checks for harness readiness. + slow: Tests that build and install extensions and can take longer to run. diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py index 093ebaa7388..0f32423999c 100755 --- a/scripts/ci/test_index.py +++ b/scripts/ci/test_index.py @@ -9,6 +9,7 @@ from __future__ import print_function +import copy import glob import hashlib import json @@ -20,8 +21,6 @@ from packaging import version from util import SRC_PATH -from wheel.install import WHEEL_INFO_RE - from util import get_ext_metadata, get_whl_from_url, get_index_data @@ -39,6 +38,108 @@ def get_sha256sum(a_file): return sha256.hexdigest() +# Cosmetic packaging-format fields that the legacy wheel==0.30.0 metadata path +# emitted but that carry no extension behaviour (they describe the *build tool* +# and METADATA serialization, not the extension). The pkginfo-based path no +# longer reproduces them, so they are ignored when comparing generated metadata +# against the published index. Kept in sync with the tests/e2e/packaging +# metadata parity allowlist. +_METADATA_NOISE_TOP_LEVEL = ('generator', 'metadata_version', 'test_requires', 'license_file', + 'description_content_type', 'project_url') + + +def _canonical_requirement(raw): + """Reduce a requirement string to its semantic identity. + + ``run_requires`` in the published index is serialized inconsistently across + the years of tooling that produced it: a dependency may appear as + ``"jinja2 (~=2.10)"`` (PEP 314 form), ``"jinja2~=2.10"`` (PEP 508 form), or + both (wheel==0.30.0 emitted each requirement twice). These are all the *same* + dependency. Canonicalize to ``name + specifier (+ marker)`` so that only a + genuine dependency change (added/removed package, different version pin) is + treated as a difference -- spelling, parentheses and ordering are not. + """ + text = raw.replace('(', '').replace(')', '').strip() + try: + from packaging.requirements import Requirement + req = Requirement(text) + extras = ''.join(sorted('[{}]'.format(e) for e in req.extras)) + marker = ' ; {}'.format(req.marker) if req.marker else '' + return '{}{}{}{}'.format(req.name.lower(), extras, str(req.specifier), marker) + except Exception: # pragma: no cover - non-PEP-508 string, compare verbatim + return raw.strip() + + +def _is_extra_requirement(raw): + """True if a requirement is gated on an extra (e.g. ``; extra == "linting"``).""" + try: + from packaging.requirements import Requirement + marker = Requirement(raw.replace('(', '').replace(')', '').strip()).marker + return marker is not None and 'extra' in str(marker) + except Exception: # pragma: no cover - non-PEP-508 string + return 'extra' in (raw or '') + + +def _canonical_run_requires(run_requires): + """Flatten ``run_requires`` to a sorted set of canonical *runtime* requirements. + + Optional dependencies declared under an extra (e.g. a "linting" group) are + excluded: ``az extension add`` never installs extras, and wheel==0.30.0 and + the pkginfo path serialize them inconsistently (separate ``extra`` blocks + and/or ``; extra == "x"`` markers, sometimes both). Non-extra environment + markers (e.g. ``python_version``) are preserved so a genuinely conditional + runtime dependency is not conflated with an unconditional one. + """ + canon = set() + for entry in run_requires or []: + if not isinstance(entry, dict): + continue + # Skip whole optional-extra dependency groups. + if entry.get('extra'): + continue + condition = entry.get('environment') or '' + if 'extra' in condition: + continue + for raw in entry.get('requires') or []: + if _is_extra_requirement(raw): + continue + req = _canonical_requirement(raw) + canon.add('{} ; {}'.format(req, condition) if condition else req) + return sorted(canon) + + +def _without_metadata_noise(metadata): + cleaned = copy.deepcopy(metadata) + for key in _METADATA_NOISE_TOP_LEVEL: + cleaned.pop(key, None) + + cleaned.pop('extensions', None) + # `extras` lists optional dependency groups (e.g. "linting") that + # `az extension add` never installs, and that wheel==0.30.0 and the pkginfo + # path populate inconsistently. Ignore it; the real runtime dependency set + # is still compared via run_requires below. + cleaned.pop('extras', None) + + for key in ('run_requires', 'classifiers'): + if not cleaned.get(key): + cleaned.pop(key, None) + # Compare dependencies by their semantic identity, not their (historically + # inconsistent) string serialization. A real runtime dependency change + # still fails. + if isinstance(cleaned.get('run_requires'), list): + canonical = _canonical_run_requires(cleaned['run_requires']) + if canonical: + cleaned['run_requires'] = canonical + else: + cleaned.pop('run_requires', None) + # Leading/trailing whitespace in string values is a serialization artifact + # (e.g. a trailing space in a license string), never semantically meaningful. + for key, value in list(cleaned.items()): + if isinstance(value, str): + cleaned[key] = value.strip() + return cleaned + + def check_min_version(extension_name, metadata): if 'azext.minCliCoreVersion' not in metadata: try: @@ -85,14 +186,13 @@ def test_extension_filenames(self): "Extension name mismatch in extensions['{}']. " "Found an extension in the list with name " "{}".format(ext_name, item['metadata']['name'])) - # Due to https://github.com/pypa/wheel/issues/235 we prevent whls built with 0.31.0 or greater. - # 0.29.0, 0.30.0 are the two previous versions before that release. - parsed_filename = WHEEL_INFO_RE(item['filename']) - p = parsed_filename.groupdict() - self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename'])) - built_wheel = p.get('abi') == 'none' and p.get('plat') == 'any' - self.assertTrue(built_wheel, - "{} of {} not platform independent wheel. " + # Extensions must be published as platform-independent wheels, + # i.e. the filename ends in ``-none-any.whl`` (ABI tag ``none``, + # platform tag ``any``). This was previously derived from + # ``wheel.install.WHEEL_INFO_RE``; it is asserted directly here + # so the check no longer depends on the ``wheel`` package. + self.assertTrue(item['filename'].endswith('-none-any.whl'), + "{} of {} is not a platform-independent wheel. " "It should end in -none-any.whl".format(item['filename'], ext_name)) def test_extension_url_filename(self): @@ -158,6 +258,20 @@ def test_metadata(self): 'log-analytics': '0.2.1' } + # Extensions whose latest published wheel metadata is known to diverge + # from its index entry for historical reasons unrelated to the + # wheel->pkginfo tooling change (e.g. a hand-edited/older index field). + metadata_comparison_thresholds = { + # azure-cli-ml is deprecated; its 1.5.0 wheel ships an + # azext_metadata.json without azext.maxCliCoreVersion, while the + # index entry records 2.29.2. + 'azure-cli-ml': '1.5.0', + # stack-hci-vm's published index entry (<= 1.14.5) predates + # run_requires capture: the wheel declares runtime deps (kubernetes, + # pyyaml, websocket-client) that the stored index entry omits. + 'stack-hci-vm': '1.14.5', + } + extensions_dir = tempfile.mkdtemp() for ext_name, exts in self.index['extensions'].items(): # only test the latest version @@ -196,19 +310,20 @@ def test_metadata(self): else: raise ex - # Due to https://github.com/pypa/wheel/issues/195 we prevent whls built with 0.31.0 or greater. - # 0.29.0, 0.30.0 are the two previous versions before that release. - supported_generators = ['bdist_wheel (0.29.0)', 'bdist_wheel (0.30.0)'] - self.assertIn(metadata.get('generator'), supported_generators, - "{}: 'generator' should be one of {}. " - "Build the extension with a different version of the 'wheel' package " - "(e.g. `pip install wheel==0.30.0`). " - "This is due to https://github.com/pypa/wheel/issues/195".format(ext_name, - supported_generators)) - self.assertDictEqual(metadata, item['metadata'], - "Metadata for {} in index doesn't match the expected of: \n" - "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, - separators=(',', ': ')))) + # The pkginfo-based metadata path intentionally omits + # wheel==0.30.0-specific fields (e.g. `generator`, `document_names`). + # Ignore those known, benign differences and compare the rest. + try: + self.assertDictEqual(_without_metadata_noise(metadata), + _without_metadata_noise(item['metadata']), + "Metadata for {} in index doesn't match the expected of: \n" + "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, + separators=(',', ': ')))) + except AssertionError as ex: + threshold_version = metadata_comparison_thresholds.get(ext_name) + if threshold_version and version.parse(ext_version) <= version.parse(threshold_version): + continue + raise ex shutil.rmtree(extensions_dir) diff --git a/scripts/ci/util.py b/scripts/ci/util.py index ffc7d54797b..7e8fa8b9c16 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -5,7 +5,6 @@ import logging import os -import re import shlex import json import zipfile @@ -14,14 +13,6 @@ logger = logging.getLogger(__name__) -# copy from wheel==0.30.0 -WHEEL_INFO_RE = re.compile( - r"""^(?P(?P.+?)(-(?P\d.+?))?) - ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) - \.whl|\.dist-info)$""", - re.VERBOSE).match - - def get_repo_root(): current_dir = os.path.dirname(os.path.abspath(__file__)) while not os.path.exists(os.path.join(current_dir, 'CONTRIBUTING.rst')): @@ -54,27 +45,21 @@ def _get_azext_metadata(ext_dir): def get_ext_metadata(ext_dir, ext_file, ext_name): # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 - WHL_METADATA_FILENAME = 'metadata.json' - zip_ref = zipfile.ZipFile(ext_file, 'r') - zip_ref.extractall(ext_dir) - zip_ref.close() + # Read spec-defined wheel metadata via pkginfo so we don't depend on the + # legacy wheel-0.30.0 only ``metadata.json`` artifact. Imported lazily so + # that modules importing util for unrelated helpers (SRC_PATH, get_repo_root, + # get_index_data, ...) do not require azdev to be installed. + from azdev.operations.extensions.metadata import pkginfo_to_dict + generated_metadata = pkginfo_to_dict(ext_file) + with zipfile.ZipFile(ext_file, 'r') as zip_ref: + zip_ref.extractall(ext_dir) metadata = {} - dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] azext_metadata = _get_azext_metadata(ext_dir) - if not azext_metadata: raise ValueError('azext_metadata.json for Extension "{}" Metadata is missing'.format(ext_name)) - metadata.update(azext_metadata) - - for dist_info_dirname in dist_info_dirs: - parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) - if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): - whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) - if os.path.isfile(whl_metadata_filepath): - with open(whl_metadata_filepath) as f: - metadata.update(json.load(f)) + metadata.update(generated_metadata) return metadata diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/packaging/__init__.py b/tests/e2e/packaging/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/packaging/conftest.py b/tests/e2e/packaging/conftest.py new file mode 100644 index 00000000000..c3cc82a8861 --- /dev/null +++ b/tests/e2e/packaging/conftest.py @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from .fixtures import load_fixtures + + +@pytest.fixture(scope="session") +def repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +@pytest.fixture(scope="session") +def fixture_manifest(repo_root: Path) -> Path: + return repo_root / "tests" / "e2e" / "packaging" / "fixtures" / "manifest.json" + + +@pytest.fixture(scope="session") +def packaging_fixtures(repo_root: Path, fixture_manifest: Path): + return load_fixtures(repo_root=repo_root, manifest_path=fixture_manifest) + + +@pytest.fixture +def temp_extension_dir(tmp_path): + return tmp_path / "azext" diff --git a/tests/e2e/packaging/fixtures.py b/tests/e2e/packaging/fixtures.py new file mode 100644 index 00000000000..b0993007e56 --- /dev/null +++ b/tests/e2e/packaging/fixtures.py @@ -0,0 +1,39 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import List + + +@dataclass(frozen=True) +class PackagingFixture: + name: str + path: Path + kind: str + requires_build_isolation_off: bool + + +def load_fixtures(repo_root: Path, manifest_path: Path, include_disabled: bool = False) -> List[PackagingFixture]: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + + fixtures: List[PackagingFixture] = [] + for raw in data.get("fixtures", []): + if not include_disabled and not raw.get("enabled", False): + continue + + fixtures.append( + PackagingFixture( + name=raw["name"], + path=repo_root / raw["path"], + kind=raw.get("kind", "unspecified"), + requires_build_isolation_off=raw.get("requires_build_isolation_off", False), + ) + ) + + return fixtures diff --git a/tests/e2e/packaging/fixtures/manifest.json b/tests/e2e/packaging/fixtures/manifest.json new file mode 100644 index 00000000000..21c80963766 --- /dev/null +++ b/tests/e2e/packaging/fixtures/manifest.json @@ -0,0 +1,32 @@ +{ + "fixtures": [ + { + "name": "ssh", + "path": "src/ssh", + "kind": "baseline", + "requires_build_isolation_off": false, + "enabled": true + }, + { + "name": "containerapp", + "path": "src/containerapp", + "kind": "self_import_setup", + "requires_build_isolation_off": true, + "enabled": true + }, + { + "name": "account", + "path": "src/account", + "kind": "mid_complexity", + "requires_build_isolation_off": false, + "enabled": false + }, + { + "name": "aks-preview", + "path": "src/aks-preview", + "kind": "large_extension", + "requires_build_isolation_off": false, + "enabled": false + } + ] +} diff --git a/tests/e2e/packaging/run_local.ps1 b/tests/e2e/packaging/run_local.ps1 new file mode 100644 index 00000000000..5477074cb5f --- /dev/null +++ b/tests/e2e/packaging/run_local.ps1 @@ -0,0 +1,50 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +# +# Convenience runner for the e2e packaging tests. Resolves the Python +# interpreter portably in this order so it works the same way locally and in +# any contributor's checkout: +# +# 1. -Python argument (explicit override) +# 2. $env:VIRTUAL_ENV (the venv the developer has activated) +# 3. `python` on PATH (anything else, including a system install) +# +# Anything else (e.g. a hard-coded path to a personal venv) is a portability +# bug -- this script must not require a particular folder name to exist. + +[CmdletBinding()] +param( + [switch]$SmokeOnly, + [string]$Python +) + +$ErrorActionPreference = "Stop" + +if (-not $Python) { + if ($env:VIRTUAL_ENV) { + if ($IsWindows -or $env:OS -eq "Windows_NT") { + $Python = Join-Path $env:VIRTUAL_ENV "Scripts\python.exe" + } else { + $Python = Join-Path $env:VIRTUAL_ENV "bin/python" + } + } else { + $cmd = Get-Command python -ErrorAction SilentlyContinue + if (-not $cmd) { + throw "No Python interpreter found. Activate a venv or pass -Python ." + } + $Python = $cmd.Source + } +} + +if (-not (Test-Path $Python)) { + throw "Python interpreter not found at: $Python" +} + +$pytestArgs = @("-m", "pytest", "tests/e2e/packaging", "-q") +if ($SmokeOnly) { + $pytestArgs += @("-m", "smoke") +} + +& $Python @pytestArgs diff --git a/tests/e2e/packaging/runner.py b/tests/e2e/packaging/runner.py new file mode 100644 index 00000000000..dc1c0cbacd7 --- /dev/null +++ b/tests/e2e/packaging/runner.py @@ -0,0 +1,88 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional, Sequence + + +@dataclass(frozen=True) +class CommandResult: + command: Sequence[str] + returncode: int + stdout: str + stderr: str + + +class CommandError(RuntimeError): + def __init__(self, result: CommandResult): + formatted = ( + "Command failed with exit code {code}\n" + "command: {cmd}\n" + "stdout:\n{out}\n" + "stderr:\n{err}\n" + ).format( + code=result.returncode, + cmd=" ".join(result.command), + out=result.stdout, + err=result.stderr, + ) + super().__init__(formatted) + self.result = result + + +def _resolve_executable(name: str) -> str: + """Resolve `name` via PATH/PATHEXT. + + On Windows, tools like `az` are shipped as `az.cmd`; `subprocess.run` with a + list argv and no `shell=True` does not perform PATHEXT lookup, so a bare + `az` fails with FileNotFoundError. `shutil.which` honours PATHEXT and + returns the resolved path (e.g. `C:\\...\\az.cmd`) which subprocess can + launch directly. + """ + if os.path.sep in name or (sys.platform == "win32" and "/" in name): + return name + resolved = shutil.which(name) + return resolved or name + + +def run_command( + command: Sequence[str], + cwd: Optional[Path] = None, + env: Optional[Dict[str, str]] = None, + check: bool = True, +) -> CommandResult: + merged_env = dict(os.environ) + if env: + merged_env.update(env) + + resolved_command = [_resolve_executable(command[0]), *command[1:]] if command else [] + + completed = subprocess.run( + resolved_command, + cwd=str(cwd) if cwd else None, + env=merged_env, + text=True, + capture_output=True, + check=False, + ) + + result = CommandResult( + command=command, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + + if check and result.returncode != 0: + raise CommandError(result) + + return result diff --git a/tests/e2e/packaging/test_extension_metadata_parity.py b/tests/e2e/packaging/test_extension_metadata_parity.py new file mode 100644 index 00000000000..02a9503bb9d --- /dev/null +++ b/tests/e2e/packaging/test_extension_metadata_parity.py @@ -0,0 +1,248 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Parity tests for extension metadata produced without wheel 0.30.0. + +Builds an extension wheel on the active toolchain, derives an `index.json` +entry via `azdev.operations.extensions.util.get_ext_metadata`, and diffs it against +the published entry in `src/index.json`. Differences are split into: + +* schema_gaps — fields present in the published entry but missing from + the candidate. Must be empty. +* acceptable_noise — known, documented differences (e.g. `generator`, + `test_requires`). Must match the allowlist below. +* real_bugs — anything else. Allowed only when the source-tree version + is ahead of the latest published version. +""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[3] +INDEX_PATH = REPO_ROOT / "src" / "index.json" + + +def _azdev_available() -> bool: + try: + return ( + importlib.util.find_spec("azdev.operations.extensions.util") is not None + and importlib.util.find_spec("azdev.operations.extensions.metadata") is not None + ) + except ModuleNotFoundError: + return False + + +pytestmark = pytest.mark.e2e_packaging + + +# Kept independently from the prototype so drift between the two is caught. +# In sync with scripts/ci/test_index.py's _METADATA_NOISE_TOP_LEVEL. +ALLOWED_NOISE_TOP_LEVEL = { + "generator", "metadata_version", "test_requires", "license_file", + "description_content_type", "project_url", +} +# The whole python.details blob is informational wheel metadata not consumed by +# the extension index; ignore it rather than just document_names. +ALLOWED_NOISE_NESTED = {"extensions/python.details"} + + +@pytest.fixture(scope="module") +def azdev_metadata_api(): + if not _azdev_available(): + pytest.skip("azdev metadata module is not installed in this environment") + + from azdev.operations.extensions.util import get_ext_metadata + + return {"get_ext_metadata": get_ext_metadata} + + +def _build_wheel(ext_path: Path, out_dir: Path) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "wheel", + str(ext_path), + "--no-deps", + "-w", + str(out_dir), + ], + check=True, + capture_output=True, + text=True, + ) + wheels = sorted(out_dir.glob("*.whl")) + if not wheels: + raise RuntimeError("No wheel was built for {}".format(ext_path)) + return wheels[-1] + + +def _load_latest_index_entry(extension_name: str): + with INDEX_PATH.open(encoding="utf-8") as fh: + index = json.load(fh) + entries = index["extensions"].get(extension_name) + if not entries: + raise KeyError("Extension '{}' not found in src/index.json".format(extension_name)) + + from packaging.version import parse as parse_version + + latest = max(entries, key=lambda e: parse_version(e["metadata"]["version"])) + return latest["metadata"] + + +class _DiffReport: + def __init__(self, extension, version): + self.extension = extension + self.version = version + self.acceptable_noise = [] + self.schema_gaps = [] + self.real_bugs = [] + + +def _is_ignored_key(path): + if path and path[0] in ALLOWED_NOISE_TOP_LEVEL: + return True + for key in ALLOWED_NOISE_NESTED: + if "/".join(path[: len(key.split("/"))]) == key: + return True + return False + + +def _walk_diff(candidate, golden, path, out): + if isinstance(golden, dict) and isinstance(candidate, dict): + for key in sorted(set(golden) | set(candidate)): + _walk_diff(candidate.get(key), golden.get(key), path + (key,), out) + return + if candidate != golden: + out.append(path) + + +def _path_present(root, parts): + node = root + for p in parts: + if isinstance(node, dict) and p in node: + node = node[p] + else: + return False + return True + + +def _classify_diff(candidate, golden): + report = _DiffReport( + extension=str(golden.get("name", candidate.get("name", "?"))), + version=str(golden.get("version", candidate.get("version", "?"))), + ) + raw_diffs = [] + _walk_diff(candidate, golden, (), raw_diffs) + + for parts in raw_diffs: + if _is_ignored_key(parts): + report.acceptable_noise.append("/".join(parts) if parts else "(root)") + continue + + path_text = "/".join(parts) if parts else "(root)" + in_golden = _path_present(golden, parts) + in_candidate = _path_present(candidate, parts) + if in_golden and not in_candidate: + # Present in the published index but missing from the candidate. + report.schema_gaps.append(path_text) + else: + # A value mismatch, or a field the candidate adds that the index lacks. + report.real_bugs.append(path_text) + return report + + +def _run_or_skip(azdev_metadata_api, packaging_fixtures, fixture_name): + fixture = next((f for f in packaging_fixtures if f.name == fixture_name), None) + if fixture is None: + pytest.skip("Fixture '{}' not enabled in manifest".format(fixture_name)) + + get_ext_metadata = azdev_metadata_api["get_ext_metadata"] + with tempfile.TemporaryDirectory(prefix="e2e-metadata-") as tmp: + tmp_path = Path(tmp) + try: + wheel_path = _build_wheel(fixture.path, tmp_path / "wheel") + except subprocess.CalledProcessError as exc: + pytest.skip( + "Could not build wheel for '{}': {}\nstdout:\n{}\nstderr:\n{}".format( + fixture_name, exc, exc.stdout or "", exc.stderr or "" + ) + ) + # Anything beyond the build failing (metadata generation, index load, + # diff classification) is a real regression and must surface as a + # failure rather than a silent skip. + extract_dir = tmp_path / "extract" + extract_dir.mkdir(parents=True, exist_ok=True) + candidate = get_ext_metadata(str(extract_dir), str(wheel_path), fixture.name) + golden = _load_latest_index_entry(candidate["name"]) + report = _classify_diff(candidate, golden) + return candidate, report + + +@pytest.mark.parametrize("fixture_name", ["ssh"]) +def test_candidate_has_no_missing_fields_vs_index(azdev_metadata_api, packaging_fixtures, fixture_name): + _, report = _run_or_skip(azdev_metadata_api=azdev_metadata_api, packaging_fixtures=packaging_fixtures, fixture_name=fixture_name) + assert report.schema_gaps == [], ( + "Candidate metadata is missing fields present in src/index.json: {}".format( + report.schema_gaps + ) + ) + + +@pytest.mark.parametrize("fixture_name", ["ssh"]) +def test_candidate_diff_is_only_allowlisted_noise(azdev_metadata_api, packaging_fixtures, fixture_name): + _, report = _run_or_skip(azdev_metadata_api=azdev_metadata_api, packaging_fixtures=packaging_fixtures, fixture_name=fixture_name) + for entry in report.acceptable_noise: + top = entry.split("/", 1)[0] + assert ( + top in ALLOWED_NOISE_TOP_LEVEL or entry in ALLOWED_NOISE_NESTED + ), "Unexpected acceptable_noise key: {}".format(entry) + + +@pytest.mark.parametrize("fixture_name", ["ssh"]) +def test_candidate_divergence_explained_by_version_skew(azdev_metadata_api, packaging_fixtures, fixture_name): + candidate, report = _run_or_skip(azdev_metadata_api=azdev_metadata_api, packaging_fixtures=packaging_fixtures, fixture_name=fixture_name) + if not report.real_bugs: + return + + from packaging.version import parse as parse_version + + golden = _load_latest_index_entry(candidate["name"]) + candidate_v = parse_version(str(candidate["version"])) + golden_v = parse_version(str(golden["version"])) + if candidate_v != golden_v: + return + + details = [] + for key in report.real_bugs: + sub_c: object = candidate + sub_g: object = golden + for p in key.split("/"): + sub_c = sub_c[p] if isinstance(sub_c, dict) and p in sub_c else None + sub_g = sub_g[p] if isinstance(sub_g, dict) and p in sub_g else None + details.append( + " {key}:\n candidate: {c}\n golden: {g}".format( + key=key, + c=json.dumps(sub_c, sort_keys=True), + g=json.dumps(sub_g, sort_keys=True), + ) + ) + pytest.fail( + "Candidate metadata diverges from src/index.json without version skew " + "(candidate={cv}, golden={gv}):\n{detail}".format( + cv=candidate_v, gv=golden_v, detail="\n".join(details) + ) + ) diff --git a/tests/e2e/packaging/test_packaging_lifecycle.py b/tests/e2e/packaging/test_packaging_lifecycle.py new file mode 100644 index 00000000000..e604f5716d9 --- /dev/null +++ b/tests/e2e/packaging/test_packaging_lifecycle.py @@ -0,0 +1,288 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""End-to-end packaging lifecycle tests. + +Exercises the full build + install + remove flow for selected extension +fixtures, on both the `pip` / `az` and `azdev` surfaces, against whatever +wheel and setuptools versions are installed in the active env. +""" + +from __future__ import annotations + +import json +import importlib.util +import shutil +import sys +from functools import lru_cache +from pathlib import Path + +import pytest + +from .fixtures import PackagingFixture +from .runner import CommandError, run_command + + +def _get_fixture(packaging_fixtures, name: str) -> PackagingFixture: + for fixture in packaging_fixtures: + if fixture.name == name: + return fixture + raise AssertionError("Fixture '{}' was not found in packaging manifest".format(name)) + + +@lru_cache(maxsize=1) +def _ensure_wheel_available() -> None: + run_command([sys.executable, "-m", "pip", "install", "wheel"]) + + +def _build_wheel(ext_path: Path, wheel_dir: Path, no_build_isolation: bool = False, check: bool = True): + _ensure_wheel_available() + + wheel_dir.mkdir(parents=True, exist_ok=True) + + # Copy the fixture into the test tmp dir so the build cannot mutate the + # source tree or hit file locks left by an editable install of the same + # extension in the active env. + source_copy = wheel_dir.parent / ("src_" + ext_path.name) + if source_copy.exists(): + shutil.rmtree(source_copy, ignore_errors=True) + shutil.copytree( + ext_path, + source_copy, + ignore=shutil.ignore_patterns("build", "dist", "*.egg-info", "__pycache__"), + ) + + # --no-cache-dir bypasses pip's user-scoped wheel cache so each run + # exercises a real build. --use-pep517 forces the PEP 517 build interface + # so older pip defaults do not fall back to legacy `setup.py bdist_wheel`, + # which would bypass build isolation. + command = [ + sys.executable, + "-m", + "pip", + "wheel", + str(source_copy), + "--no-deps", + "--no-cache-dir", + "--use-pep517", + "-w", + str(wheel_dir), + ] + if no_build_isolation: + command.insert(5, "--no-build-isolation") + + result = run_command(command, check=check) + wheels = sorted(wheel_dir.glob("*.whl")) + return result, wheels + + +def _has_azure_cli_core() -> bool: + try: + return importlib.util.find_spec("azure.cli.core") is not None + except ModuleNotFoundError: + return False + + +# Stderr fragments azdev emits when no extension repo is configured. +_AZDEV_NOT_CONFIGURED_PATTERNS = ( + "please run `azdev setup`", + "extension repo path is empty", + "unable to retrieve extensions repo path", +) + + +def _is_azdev_not_configured(ex: CommandError) -> bool: + stderr = (ex.result.stderr or "").lower() + return any(p in stderr for p in _AZDEV_NOT_CONFIGURED_PATTERNS) + + +def _azdev_setup_skip_message() -> str: + repo = Path(__file__).resolve().parents[3] + return ( + "azdev is installed but no extension repo is configured. " + "Run: azdev setup -r \"{}\"".format(repo) + ) + + +def _require_azdev_ready_or_skip() -> None: + if not shutil.which("azdev"): + pytest.skip("azdev is not available in PATH") + + try: + run_command(["azdev", "--version"], check=True) + except CommandError as ex: + pytest.skip("azdev command exists but is not usable in this environment: {}".format(ex)) + + # `azdev extension list` does not require the extensions-repo config, so + # use a command that does: a dry-run `extension add` against a synthetic + # name. The repo-path check fires before the extension lookup, so an + # unconfigured azdev exits with a recognizable message regardless of + # whether the probe name exists. + probe = run_command( + ["azdev", "extension", "add", "__azdev_setup_probe__"], check=False + ) + if probe.returncode != 0: + stderr = (probe.stderr or "").lower() + if any(p in stderr for p in _AZDEV_NOT_CONFIGURED_PATTERNS): + pytest.skip(_azdev_setup_skip_message()) + + # Confirm extension list itself works (catches azdev installs that pass + # --version but choke later). + try: + run_command(["azdev", "extension", "list"], check=True) + except CommandError as ex: + if _is_azdev_not_configured(ex): + pytest.skip(_azdev_setup_skip_message()) + pytest.skip( + "azdev is not usable for extension operations in this environment: {}".format(ex) + ) + + +@pytest.mark.e2e_packaging +@pytest.mark.smoke +def test_extension_wheel_builds_without_wheel_pin(packaging_fixtures, tmp_path): + """Build the ssh extension via `pip wheel --use-pep517` on the active toolchain.""" + ssh_fixture = _get_fixture(packaging_fixtures, "ssh") + _, wheels = _build_wheel(ssh_fixture.path, tmp_path / "wheelhouse") + assert wheels, "No wheel produced for '{}'".format(ssh_fixture.path) + wheel_path = wheels[-1] # get the latest wheel + assert wheel_path.name.endswith(".whl") + assert "ssh" in wheel_path.name.lower() + + +@pytest.mark.e2e_packaging +@pytest.mark.slow +def test_az_extension_lifecycle_add_show_list_remove(packaging_fixtures, tmp_path, temp_extension_dir): + """Add a built wheel via `az extension add`, verify it, then remove it.""" + if not shutil.which("az"): + pytest.skip("az is not available in PATH") + + ssh_fixture = _get_fixture(packaging_fixtures, "ssh") + _, wheels = _build_wheel(ssh_fixture.path, tmp_path / "wheelhouse") + assert wheels, "No wheel produced for '{}'".format(ssh_fixture.path) + wheel_path = wheels[-1] + + temp_extension_dir.mkdir(parents=True, exist_ok=True) + env = {"AZURE_EXTENSION_DIR": str(temp_extension_dir)} + installed_dir = temp_extension_dir / "ssh" + + run_command(["az", "extension", "add", "--source", str(wheel_path), "--yes"], env=env) + + # Assert on AZURE_EXTENSION_DIR contents rather than `az extension list`: + # an `azdev setup -r ` editable install of any extension leaks into + # `az extension list` regardless of AZURE_EXTENSION_DIR. + assert installed_dir.exists(), ( + "Expected `az extension add` to create {} but it does not exist".format(installed_dir) + ) + + show_result = run_command( + ["az", "extension", "show", "--name", "ssh", "--output", "json"], env=env + ) + show_payload = json.loads(show_result.stdout) + assert show_payload.get("name") == "ssh" + + run_command(["az", "extension", "remove", "--name", "ssh"], env=env) + + assert not installed_dir.exists(), ( + "Expected `az extension remove` to delete {} but it still exists".format(installed_dir) + ) + + +@pytest.mark.e2e_packaging +@pytest.mark.slow +def test_self_importing_setup_py_fails_under_default_build_isolation(packaging_fixtures, tmp_path): + """`containerapp/setup.py` self-imports before the wheel is built, so it + must fail under default build isolation with a ModuleNotFoundError. + """ + containerapp_fixture = _get_fixture(packaging_fixtures, "containerapp") + result, _ = _build_wheel( + containerapp_fixture.path, + tmp_path / "wheelhouse-default-isolation", + no_build_isolation=False, + check=False, + ) + + assert result.returncode != 0, "Expected containerapp build to fail with default isolation" + combined_output = (result.stdout + "\n" + result.stderr).lower() + assert "azext_containerapp" in combined_output or "modulenotfounderror" in combined_output + + +@pytest.mark.e2e_packaging +@pytest.mark.slow +def test_self_importing_setup_py_builds_with_no_build_isolation_workaround(packaging_fixtures, tmp_path): + """Same fixture as the previous test, but with `--no-build-isolation` it builds.""" + if not _has_azure_cli_core(): + pytest.skip("azure.cli.core is not available in this environment") + + containerapp_fixture = _get_fixture(packaging_fixtures, "containerapp") + _, wheels = _build_wheel( + containerapp_fixture.path, + tmp_path / "wheelhouse-no-build-isolation", + no_build_isolation=True, + check=True, + ) + + assert wheels, "Expected containerapp wheel to be produced with --no-build-isolation" + + +@pytest.mark.e2e_packaging +@pytest.mark.slow +def test_azdev_extension_lifecycle_add_show_list_remove(packaging_fixtures, temp_extension_dir): + """Add, show, list and remove an extension via `azdev extension ...`.""" + _require_azdev_ready_or_skip() + + temp_extension_dir.mkdir(parents=True, exist_ok=True) + env = {"AZURE_EXTENSION_DIR": str(temp_extension_dir)} + + ssh_fixture = _get_fixture(packaging_fixtures, "ssh") + + def _azdev_install_state(name: str) -> str: + result = run_command(["azdev", "extension", "list", "-o", "json"], env=env) + entries = json.loads(result.stdout) + match = next((e for e in entries if e.get("name") == name), None) + assert match is not None, "Extension '{}' not found in azdev extension list".format(name) + return match.get("install") or "" + + # Ensure clean state before this test starts. + run_command(["azdev", "extension", "remove", ssh_fixture.name], env=env, check=False) + + run_command(["azdev", "extension", "add", ssh_fixture.name], env=env) + + show_result = run_command(["azdev", "extension", "show", "--mod-name", ssh_fixture.name], env=env) + assert ssh_fixture.name in (show_result.stdout + show_result.stderr).lower() + + assert _azdev_install_state(ssh_fixture.name), ( + "Expected '{}' to be installed after `azdev extension add`".format(ssh_fixture.name) + ) + + run_command(["azdev", "extension", "remove", ssh_fixture.name], env=env) + + assert not _azdev_install_state(ssh_fixture.name), ( + "Expected '{}' to be uninstalled after `azdev extension remove`".format(ssh_fixture.name) + ) + + +@pytest.mark.e2e_packaging +@pytest.mark.slow +def test_azdev_extension_build_produces_wheel(packaging_fixtures, tmp_path): + """Build an extension wheel via `azdev extension build`. + + Exercises azdev's own build path (decoupled from `wheel==0.30.0`), as + opposed to the direct `pip wheel` build covered elsewhere. + """ + _require_azdev_ready_or_skip() + + ssh_fixture = _get_fixture(packaging_fixtures, "ssh") + dist_dir = tmp_path / "azdev-dist" + + run_command(["azdev", "extension", "build", ssh_fixture.name, "--dist-dir", str(dist_dir)]) + + wheels = sorted(dist_dir.glob("*.whl")) + assert wheels, "Expected `azdev extension build` to produce a wheel in {}".format(dist_dir) + assert any("ssh" in wheel.name.lower() for wheel in wheels), ( + "Expected an ssh wheel from `azdev extension build`, found: {}".format( + [wheel.name for wheel in wheels] + ) + ) diff --git a/tests/e2e/packaging/test_packaging_smoke.py b/tests/e2e/packaging/test_packaging_smoke.py new file mode 100644 index 00000000000..88aca60eaff --- /dev/null +++ b/tests/e2e/packaging/test_packaging_smoke.py @@ -0,0 +1,61 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Smoke tests for the e2e packaging harness. + +Run unconditionally on every invocation so a broken harness fails fast and +visibly instead of silently skipping every lifecycle test downstream. +""" + +from __future__ import annotations + +import shutil + +import pytest + +from .runner import CommandError, run_command + + +@pytest.mark.e2e_packaging +@pytest.mark.smoke +def test_fixture_manifest_has_expected_core_fixtures(packaging_fixtures): + names = {f.name for f in packaging_fixtures} + assert "ssh" in names + assert "containerapp" in names + + +@pytest.mark.e2e_packaging +@pytest.mark.smoke +def test_fixture_paths_exist(packaging_fixtures): + missing = [f.path for f in packaging_fixtures if not f.path.exists()] + assert not missing, "Missing fixture paths: {}".format(", ".join(str(path) for path in missing)) + + +@pytest.mark.e2e_packaging +@pytest.mark.smoke +def test_az_cli_is_installed_and_usable(): + if not shutil.which("az"): + pytest.skip("az is not available in PATH") + + result = run_command(["az", "--version"], check=True) + assert "azure-cli" in result.stdout.lower() + + +@pytest.mark.e2e_packaging +@pytest.mark.smoke +def test_azdev_is_installed_and_usable(): + if not shutil.which("azdev"): + pytest.skip("azdev is not available in PATH") + + try: + result = run_command(["azdev", "--version"], check=True) + except CommandError as ex: + pytest.skip("azdev command exists but is not usable in this environment: {}".format(ex)) + + # `azdev --version` prints just the version number first (e.g. "0.2.10"). + first_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else "" + assert first_line and first_line[0].isdigit() and "." in first_line, ( + "Unexpected azdev --version output: {!r}".format(result.stdout) + )