From f6c3f7e92518c42307dc432a78265e7149a41ce8 Mon Sep 17 00:00:00 2001 From: Neil Lydick Date: Wed, 30 Sep 2026 02:48:03 -0700 Subject: [PATCH] fix(ml): preserve extension identity with modern wheel metadata Write the package identity into the built Azure CLI metadata so older metadata parsers can identify ML wheels built with modern setuptools. Keep source metadata, runtime dependencies, and minimum CLI unchanged. Add regression coverage for wheel identity, legacy readers, CLI JSON output, and incremental builds, with a Python 3.12 CI gate. Authored-by: GitHub Copilot SDK (version not exposed) Model: GPT-6 Astra (gpt-6-astra) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: da39d407-596a-4078-a372-19ac7fb8699e --- azure-pipelines.yml | 7 ++ src/machinelearningservices/CHANGELOG.rst | 1 + src/machinelearningservices/HISTORY.rst | 1 + src/machinelearningservices/README.rst | 16 +++ src/machinelearningservices/setup.py | 22 ++++ tests/e2e/packaging/fixtures/manifest.json | 7 ++ .../e2e/packaging/test_packaging_lifecycle.py | 110 ++++++++++++++++++ 7 files changed, 164 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 17529da6fc4..78b496c76ea 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -119,6 +119,13 @@ jobs: ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) WHEELS_OUTPUT_DIR: $(Build.ArtifactStagingDirectory)/wheels-$(python.version) + - bash: | + set -ev + source ./env/bin/activate + python -m pytest tests/e2e/packaging/test_packaging_lifecycle.py -k ml_metadata -q + displayName: 'Validate ML extension metadata compatibility' + condition: and(succeeded(), eq(variables['python.version'], '3.12')) + - task: PublishPipelineArtifact@1 displayName: 'Publish wheels artifacts from tests' inputs: diff --git a/src/machinelearningservices/CHANGELOG.rst b/src/machinelearningservices/CHANGELOG.rst index a29f158a1bd..09473247353 100644 --- a/src/machinelearningservices/CHANGELOG.rst +++ b/src/machinelearningservices/CHANGELOG.rst @@ -1,5 +1,6 @@ ## Unreleased +- Preserve installed extension version detection on older Azure CLI versions when using modern wheel metadata. - Improve SSH connection argument handling and endpoint validation for jobs and compute instances. - Honor repeatable `az ml job connect-ssh --ssh-args` options. diff --git a/src/machinelearningservices/HISTORY.rst b/src/machinelearningservices/HISTORY.rst index fb641fb81db..c0cd7f2195c 100644 --- a/src/machinelearningservices/HISTORY.rst +++ b/src/machinelearningservices/HISTORY.rst @@ -5,6 +5,7 @@ Release History Unreleased ++++++++++ +* Preserve installed extension version detection on older Azure CLI versions when using modern wheel metadata. * Improve SSH connection argument handling and endpoint validation for jobs and compute instances. * Honor repeatable ``az ml job connect-ssh --ssh-args`` options. diff --git a/src/machinelearningservices/README.rst b/src/machinelearningservices/README.rst index 8e62fb0808e..c68cb7c40e9 100644 --- a/src/machinelearningservices/README.rst +++ b/src/machinelearningservices/README.rst @@ -18,3 +18,19 @@ Install this extension using the below CLI command: .. code-block:: bash az extension add --name ml + + +Packaging compatibility +------------------------------------------------------------------------------ + +The wheel build writes the package name and version into the bundled +``azext_metadata.json`` using the package definition in ``setup.py``. This lets +older Azure CLI metadata readers identify the installed extension even when +they cannot parse the wheel's newer metadata format. The source metadata file +is not modified, and build-tool downgrades are not required. + +Run the packaging regression tests from the repository root: + +.. code-block:: bash + + python -m pytest tests/e2e/packaging/test_packaging_lifecycle.py -k ml_metadata diff --git a/src/machinelearningservices/setup.py b/src/machinelearningservices/setup.py index 71dab2fa0da..eb050f7ef9f 100644 --- a/src/machinelearningservices/setup.py +++ b/src/machinelearningservices/setup.py @@ -6,8 +6,11 @@ # -------------------------------------------------------------------------------------------- +import json +import os from io import open from setuptools import setup, find_packages +from setuptools.command.build_py import build_py # HISTORY.rst entry. VERSION = '2.45.0' @@ -40,6 +43,24 @@ with open("CHANGELOG.rst", encoding="utf-8") as f: changelog = f.read() + +class BuildPyWithExtensionMetadata(build_py): + def run(self): + super().run() + self.execute(self._write_extension_metadata, (), "Writing Azure CLI extension identity metadata") + + def _write_extension_metadata(self): + metadata_path = os.path.join(self.build_lib, "azext_mlv2", "azext_metadata.json") + with open(metadata_path, encoding="utf-8") as metadata_file: + metadata = json.load(metadata_file) + # Older CLI pkginfo versions cannot read metadata emitted by modern setuptools. + metadata["name"] = self.distribution.get_name() + metadata["version"] = self.distribution.get_version() + with open(metadata_path, "w", encoding="utf-8") as metadata_file: + json.dump(metadata, metadata_file, indent=2) + metadata_file.write("\n") + + setup( name='ml', version=VERSION, @@ -54,4 +75,5 @@ packages=find_packages(), install_requires=DEPENDENCIES, package_data={'azext_mlv2': ['azext_metadata.json']}, + cmdclass={'build_py': BuildPyWithExtensionMetadata}, ) diff --git a/tests/e2e/packaging/fixtures/manifest.json b/tests/e2e/packaging/fixtures/manifest.json index 21c80963766..c3a57eb3ed6 100644 --- a/tests/e2e/packaging/fixtures/manifest.json +++ b/tests/e2e/packaging/fixtures/manifest.json @@ -14,6 +14,13 @@ "requires_build_isolation_off": true, "enabled": true }, + { + "name": "machinelearningservices", + "path": "src/machinelearningservices", + "kind": "legacy_cli_metadata", + "requires_build_isolation_off": true, + "enabled": true + }, { "name": "account", "path": "src/account", diff --git a/tests/e2e/packaging/test_packaging_lifecycle.py b/tests/e2e/packaging/test_packaging_lifecycle.py index e604f5716d9..dc821d7793f 100644 --- a/tests/e2e/packaging/test_packaging_lifecycle.py +++ b/tests/e2e/packaging/test_packaging_lifecycle.py @@ -16,8 +16,12 @@ import importlib.util import shutil import sys +import zipfile +from email.parser import BytesParser from functools import lru_cache from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch import pytest @@ -85,6 +89,112 @@ def _has_azure_cli_core() -> bool: return False +def _read_ml_wheel_metadata(wheel_path): + with zipfile.ZipFile(wheel_path) as wheel: + metadata_files = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + assert len(metadata_files) == 1 + package_metadata = BytesParser().parsebytes(wheel.read(metadata_files[0])) + extension_metadata = json.loads(wheel.read("azext_mlv2/azext_metadata.json")) + return package_metadata, extension_metadata + + +@pytest.fixture(scope="module") +def ml_metadata_wheel(packaging_fixtures, tmp_path_factory): + fixture = _get_fixture(packaging_fixtures, "machinelearningservices") + source_metadata = fixture.path / "azext_mlv2" / "azext_metadata.json" + wheel_dir = tmp_path_factory.mktemp("ml-metadata") / "wheels" + _, wheels = _build_wheel( + fixture.path, wheel_dir, no_build_isolation=fixture.requires_build_isolation_off + ) + assert len(wheels) == 1 + source_copy = wheel_dir.parent / ("src_" + fixture.path.name) + assert (source_copy / "azext_mlv2" / "azext_metadata.json").read_bytes() == source_metadata.read_bytes() + return wheels[0] + + +@pytest.fixture(scope="module") +def ml_metadata_install(ml_metadata_wheel, tmp_path_factory): + if not _has_azure_cli_core(): + pytest.skip("azure.cli.core is not available in this environment") + + root = tmp_path_factory.mktemp("ml-metadata-install") + extension_dir = root / "extensions" + installed_dir = extension_dir / "ml" + run_command([ + sys.executable, "-m", "pip", "install", "--target", + str(installed_dir), str(ml_metadata_wheel), + ]) + return installed_dir, { + "AZURE_CONFIG_DIR": str(root / "config"), + "AZURE_EXTENSION_DIR": str(extension_dir), + "AZURE_EXTENSION_SYS_DIR": str(root / "system-extensions"), + "AZURE_EXTENSION_DEV_SOURCES": "", + "AZURE_EXTENSION_USE_DYNAMIC_INSTALL": "no", + "AZURE_CORE_COLLECT_TELEMETRY": "no", + } + + +@pytest.mark.e2e_packaging +def test_ml_metadata_matches_package_identity(ml_metadata_wheel): + package_metadata, extension_metadata = _read_ml_wheel_metadata(ml_metadata_wheel) + assert package_metadata["Name"] == "ml" + assert package_metadata["Version"] + assert extension_metadata["name"] == package_metadata["Name"] + assert extension_metadata["version"] == package_metadata["Version"] + assert "azext.minCliCoreVersion" in extension_metadata + + +@pytest.mark.e2e_packaging +def test_ml_metadata_supports_legacy_reader(ml_metadata_wheel, ml_metadata_install): + from azure.cli.core.extension import WheelExtension + + package_metadata, _ = _read_ml_wheel_metadata(ml_metadata_wheel) + installed_dir, _ = ml_metadata_install + # pkginfo 1.8.2 leaves name/version unset for Metadata-Version: 2.4. + unreadable_metadata = SimpleNamespace(name=None, version=None) + with patch("azure.cli.core.extension.pkginfo.Wheel", return_value=unreadable_metadata): + extension = WheelExtension("ml", str(installed_dir)) + assert extension.version == package_metadata["Version"] + assert extension.metadata["name"] == "ml" + + +@pytest.mark.e2e_packaging +@pytest.mark.parametrize( + "arguments,version_path", + [ + (["extension", "show", "--name", "ml"], ["version"]), + (["version"], ["extensions", "ml"]), + ], +) +def test_ml_metadata_cli_reports_version(ml_metadata_wheel, ml_metadata_install, arguments, version_path): + package_metadata, _ = _read_ml_wheel_metadata(ml_metadata_wheel) + _, env = ml_metadata_install + result = run_command( + [sys.executable, "-m", "azure.cli", *arguments, "--output", "json"], env=env + ) + value = json.loads(result.stdout) + for key in version_path: + value = value[key] + assert value == package_metadata["Version"] + + +@pytest.mark.e2e_packaging +def test_ml_metadata_refreshes_on_incremental_build(ml_metadata_wheel, tmp_path): + source_copy = ml_metadata_wheel.parent.parent / "src_machinelearningservices" + run_command( + [ + sys.executable, "setup.py", "egg_info", "--tag-build", ".dev1", + "bdist_wheel", "--dist-dir", str(tmp_path), + ], + cwd=source_copy, + ) + wheels = list(tmp_path.glob("*.whl")) + assert len(wheels) == 1 + package_metadata, extension_metadata = _read_ml_wheel_metadata(wheels[0]) + assert package_metadata["Version"].endswith(".dev1") + assert extension_metadata["version"] == package_metadata["Version"] + + # Stderr fragments azdev emits when no extension repo is configured. _AZDEV_NOT_CONFIGURED_PATTERNS = ( "please run `azdev setup`",