fix(ml): preserve extension identity with modern wheel metadata

azure-pipelines.yml

@@ -119,6 +119,13 @@ jobs:
         ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
         WHEELS_OUTPUT_DIR: $(Build.ArtifactStagingDirectory)/wheels-$(python.version)
 
+    - bash: |
+        set -ev
+        source ./env/bin/activate
+        python -m pytest tests/e2e/packaging/test_packaging_lifecycle.py -k ml_metadata -q
+      displayName: 'Validate ML extension metadata compatibility'
+      condition: and(succeeded(), eq(variables['python.version'], '3.12'))
+
     - task: PublishPipelineArtifact@1
       displayName: 'Publish wheels artifacts from tests'
       inputs:

src/machinelearningservices/CHANGELOG.rst

@@ -1,5 +1,6 @@
 ## Unreleased
 
+- Preserve installed extension version detection on older Azure CLI versions when using modern wheel metadata.
 - Improve SSH connection argument handling and endpoint validation for jobs and compute instances.
 - Honor repeatable `az ml job connect-ssh --ssh-args` options.
 

src/machinelearningservices/HISTORY.rst

@@ -5,6 +5,7 @@ Release History
 
 Unreleased
 ++++++++++
+* Preserve installed extension version detection on older Azure CLI versions when using modern wheel metadata.
 * Improve SSH connection argument handling and endpoint validation for jobs and compute instances.
 * Honor repeatable ``az ml job connect-ssh --ssh-args`` options.
 

src/machinelearningservices/README.rst

@@ -18,3 +18,19 @@ Install this extension using the below CLI command:
     .. code-block:: bash
 
         az extension add --name ml
+
+
+Packaging compatibility
+------------------------------------------------------------------------------
+
+The wheel build writes the package name and version into the bundled
+``azext_metadata.json`` using the package definition in ``setup.py``. This lets
+older Azure CLI metadata readers identify the installed extension even when
+they cannot parse the wheel's newer metadata format. The source metadata file
+is not modified, and build-tool downgrades are not required.
+
+Run the packaging regression tests from the repository root:
+
+.. code-block:: bash
+
+    python -m pytest tests/e2e/packaging/test_packaging_lifecycle.py -k ml_metadata

src/machinelearningservices/setup.py

@@ -6,8 +6,11 @@
 # --------------------------------------------------------------------------------------------
 
 
+import json
+import os
 from io import open
 from setuptools import setup, find_packages
+from setuptools.command.build_py import build_py
 
 # HISTORY.rst entry.
 VERSION = '2.45.0'
@@ -40,6 +43,24 @@
 with open("CHANGELOG.rst", encoding="utf-8") as f:
     changelog = f.read()
 
+
+class BuildPyWithExtensionMetadata(build_py):
+    def run(self):
+        super().run()
+        self.execute(self._write_extension_metadata, (), "Writing Azure CLI extension identity metadata")
+
+    def _write_extension_metadata(self):
+        metadata_path = os.path.join(self.build_lib, "azext_mlv2", "azext_metadata.json")
+        with open(metadata_path, encoding="utf-8") as metadata_file:
+            metadata = json.load(metadata_file)
+        # Older CLI pkginfo versions cannot read metadata emitted by modern setuptools.
+        metadata["name"] = self.distribution.get_name()
+        metadata["version"] = self.distribution.get_version()
+        with open(metadata_path, "w", encoding="utf-8") as metadata_file:
+            json.dump(metadata, metadata_file, indent=2)
+            metadata_file.write("\n")
+
+
 setup(
     name='ml',
     version=VERSION,
@@ -54,4 +75,5 @@
     packages=find_packages(),
     install_requires=DEPENDENCIES,
     package_data={'azext_mlv2': ['azext_metadata.json']},
+    cmdclass={'build_py': BuildPyWithExtensionMetadata},
 )

tests/e2e/packaging/fixtures/manifest.json

@@ -14,6 +14,13 @@
       "requires_build_isolation_off": true,
       "enabled": true
     },
+    {
+      "name": "machinelearningservices",
+      "path": "src/machinelearningservices",
+      "kind": "legacy_cli_metadata",
+      "requires_build_isolation_off": true,
+      "enabled": true
+    },
     {
       "name": "account",
       "path": "src/account",

tests/e2e/packaging/test_packaging_lifecycle.py

@@ -16,8 +16,12 @@
 import importlib.util
 import shutil
 import sys
+import zipfile
+from email.parser import BytesParser
 from functools import lru_cache
 from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
 
 import pytest
 
@@ -85,6 +89,112 @@ def _has_azure_cli_core() -> bool:
         return False
 
 
+def _read_ml_wheel_metadata(wheel_path):
+    with zipfile.ZipFile(wheel_path) as wheel:
+        metadata_files = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")]
+        assert len(metadata_files) == 1
+        package_metadata = BytesParser().parsebytes(wheel.read(metadata_files[0]))
+        extension_metadata = json.loads(wheel.read("azext_mlv2/azext_metadata.json"))
+    return package_metadata, extension_metadata
+
+
+@pytest.fixture(scope="module")
+def ml_metadata_wheel(packaging_fixtures, tmp_path_factory):
+    fixture = _get_fixture(packaging_fixtures, "machinelearningservices")
+    source_metadata = fixture.path / "azext_mlv2" / "azext_metadata.json"
+    wheel_dir = tmp_path_factory.mktemp("ml-metadata") / "wheels"
+    _, wheels = _build_wheel(
+        fixture.path, wheel_dir, no_build_isolation=fixture.requires_build_isolation_off
+    )
+    assert len(wheels) == 1
+    source_copy = wheel_dir.parent / ("src_" + fixture.path.name)
+    assert (source_copy / "azext_mlv2" / "azext_metadata.json").read_bytes() == source_metadata.read_bytes()
+    return wheels[0]
+
+
+@pytest.fixture(scope="module")
+def ml_metadata_install(ml_metadata_wheel, tmp_path_factory):
+    if not _has_azure_cli_core():
+        pytest.skip("azure.cli.core is not available in this environment")
+
+    root = tmp_path_factory.mktemp("ml-metadata-install")
+    extension_dir = root / "extensions"
+    installed_dir = extension_dir / "ml"
+    run_command([
+        sys.executable, "-m", "pip", "install", "--target",
+        str(installed_dir), str(ml_metadata_wheel),
+    ])
+    return installed_dir, {
+        "AZURE_CONFIG_DIR": str(root / "config"),
+        "AZURE_EXTENSION_DIR": str(extension_dir),
+        "AZURE_EXTENSION_SYS_DIR": str(root / "system-extensions"),
+        "AZURE_EXTENSION_DEV_SOURCES": "",
+        "AZURE_EXTENSION_USE_DYNAMIC_INSTALL": "no",
+        "AZURE_CORE_COLLECT_TELEMETRY": "no",
+    }
+
+
+@pytest.mark.e2e_packaging
+def test_ml_metadata_matches_package_identity(ml_metadata_wheel):
+    package_metadata, extension_metadata = _read_ml_wheel_metadata(ml_metadata_wheel)
+    assert package_metadata["Name"] == "ml"
+    assert package_metadata["Version"]
+    assert extension_metadata["name"] == package_metadata["Name"]
+    assert extension_metadata["version"] == package_metadata["Version"]
+    assert "azext.minCliCoreVersion" in extension_metadata
+
+
+@pytest.mark.e2e_packaging
+def test_ml_metadata_supports_legacy_reader(ml_metadata_wheel, ml_metadata_install):
+    from azure.cli.core.extension import WheelExtension
+
+    package_metadata, _ = _read_ml_wheel_metadata(ml_metadata_wheel)
+    installed_dir, _ = ml_metadata_install
+    # pkginfo 1.8.2 leaves name/version unset for Metadata-Version: 2.4.
+    unreadable_metadata = SimpleNamespace(name=None, version=None)
+    with patch("azure.cli.core.extension.pkginfo.Wheel", return_value=unreadable_metadata):
+        extension = WheelExtension("ml", str(installed_dir))
+        assert extension.version == package_metadata["Version"]
+        assert extension.metadata["name"] == "ml"
+
+
+@pytest.mark.e2e_packaging
+@pytest.mark.parametrize(
+    "arguments,version_path",
+    [
+        (["extension", "show", "--name", "ml"], ["version"]),
+        (["version"], ["extensions", "ml"]),
+    ],
+)
+def test_ml_metadata_cli_reports_version(ml_metadata_wheel, ml_metadata_install, arguments, version_path):
+    package_metadata, _ = _read_ml_wheel_metadata(ml_metadata_wheel)
+    _, env = ml_metadata_install
+    result = run_command(
+        [sys.executable, "-m", "azure.cli", *arguments, "--output", "json"], env=env
+    )
+    value = json.loads(result.stdout)
+    for key in version_path:
+        value = value[key]
+    assert value == package_metadata["Version"]
+
+
+@pytest.mark.e2e_packaging
+def test_ml_metadata_refreshes_on_incremental_build(ml_metadata_wheel, tmp_path):
+    source_copy = ml_metadata_wheel.parent.parent / "src_machinelearningservices"
+    run_command(
+        [
+            sys.executable, "setup.py", "egg_info", "--tag-build", ".dev1",
+            "bdist_wheel", "--dist-dir", str(tmp_path),
+        ],
+        cwd=source_copy,
+    )
+    wheels = list(tmp_path.glob("*.whl"))
+    assert len(wheels) == 1
+    package_metadata, extension_metadata = _read_ml_wheel_metadata(wheels[0])
+    assert package_metadata["Version"].endswith(".dev1")
+    assert extension_metadata["version"] == package_metadata["Version"]
+
+
 # Stderr fragments azdev emits when no extension repo is configured.
 _AZDEV_NOT_CONFIGURED_PATTERNS = (
     "please run `azdev setup`",