diff --git a/HISTORY.rst b/HISTORY.rst index bf7b6f596..bc6a53e30 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ Release History =============== +0.2.3b1 ++++++++ +* extract metadata generation logic and decouple from wheel 0.30.0 + 0.2.2 ++++++ * Update dependency `azure-cli-diff-tool` to `0.1.0`. @@ -11,7 +15,7 @@ Release History * `azdev extension cal-next-version`: Adjust `minor` or `patch` update for previous preview versioning pattern. 0.2.0 -+++++ +++++++ * `azdev generated-breaking-change-report`: Support multi-line upcoming breaking change announcement 0.1.99 diff --git a/azdev/__init__.py b/azdev/__init__.py index 4322e44ee..24bae1bbd 100644 --- a/azdev/__init__.py +++ b/azdev/__init__.py @@ -4,4 +4,4 @@ # license information. # ----------------------------------------------------------------------------- -__VERSION__ = '0.2.2' +__VERSION__ = '0.2.3b1' diff --git a/azdev/operations/code_gen.py b/azdev/operations/code_gen.py index e8dafc48d..25f7f57c9 100644 --- a/azdev/operations/code_gen.py +++ b/azdev/operations/code_gen.py @@ -297,6 +297,9 @@ def _create_package(prefix, repo_path, is_ext, name='test', display_name=None, d _generate_files(env, kwargs, test_files, dest_path) if is_ext: - result = pip_cmd('install -e {}'.format(new_package_path), "Installing `{}{}`...".format(prefix, name)) + result = pip_cmd( + f'install -e {new_package_path} --config-settings editable_mode=compat', + f"Installing `{prefix}{name}`..." + ) if result.error: raise result.error # pylint: disable=raising-bad-type diff --git a/azdev/operations/extensions/__init__.py b/azdev/operations/extensions/__init__.py index 55097253c..9068982f8 100644 --- a/azdev/operations/extensions/__init__.py +++ b/azdev/operations/extensions/__init__.py @@ -45,7 +45,10 @@ def add_extension(extensions): raise CLIError('extension(s) not found: {}'.format(' '.join(extensions))) for path in paths_to_add: - result = pip_cmd('install -e {}'.format(path), "Adding extension '{}'...".format(path)) + result = pip_cmd( + f'install -e {path} --config-settings editable_mode=compat --no-build-isolation', + f"Adding extension '{path}'..." + ) if result.error: raise result.error # pylint: disable=raising-bad-type diff --git a/azdev/operations/extensions/metadata.py b/azdev/operations/extensions/metadata.py new file mode 100644 index 000000000..a5fac1336 --- /dev/null +++ b/azdev/operations/extensions/metadata.py @@ -0,0 +1,515 @@ +# pylint: disable=C0325,R1725,W0612,R1704,W0718,R0914,E1101,R0912,R0915,R1705 + +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +""" +Tools for generate metadata for index.json. +""" + +import email.parser +import os.path +import re +import textwrap +import zipfile + +from collections import OrderedDict, namedtuple + +from importlib.metadata import entry_points +from packaging.requirements import Requirement # pip install packagin + +METADATA_VERSION = "2.0" + +PLURAL_FIELDS = {"classifier": "classifiers", + "provides_dist": "provides", + "provides_extra": "extras"} + +SKIP_FIELDS = set() + +CONTACT_FIELDS = (({"email": "author_email", "name": "author"}, + "author"), + ({"email": "maintainer_email", "name": "maintainer"}, + "maintainer")) + +# commonly filled out as "UNKNOWN" by distutils: +UNKNOWN_FIELDS = {"author", "author_email", "platform", "home_page", "license"} + +# Wheel itself is probably the only program that uses non-extras markers +# in METADATA/PKG-INFO. Support its syntax with the extra at the end only. +EXTRA_RE = re.compile(r"""^(?P.*?)(;\s*(?P.*?)(extra == '(?P.*?)')?)$""") +KEYWORDS_RE = re.compile("[\0-,]+") + +MayRequiresKey = namedtuple('MayRequiresKey', ('condition', 'extra')) + + +class OrderedDefaultDict(OrderedDict): + def __init__(self, *args, **kwargs): + if not args: + self.default_factory = None + else: + if not (args[0] is None or callable(args[0])): + raise TypeError('first argument must be callable or None') + self.default_factory = args[0] + args = args[1:] + super(OrderedDefaultDict, self).__init__(*args, **kwargs) + + def __missing__(self, key): + if self.default_factory is None: + raise KeyError(key) + self[key] = default = self.default_factory() + return default + + +def read_pkg_info(path): + with zipfile.ZipFile(path, 'r') as zf: + for file_name in zf.namelist(): + if file_name.endswith("METADATA"): + with zf.open(file_name, "r") as metadata_file: + content = email.parser.Parser().parsestr(metadata_file.read().decode("utf-8")) + return content + return {} + + +def unique(iterable): + """ + Yield unique values in iterable, preserving order. + """ + seen = set() + for value in iterable: + if value not in seen: + seen.add(value) + yield value + + +def handle_requires(metadata, pkg_info, key): + """ + Place the runtime requirements from pkg_info into metadata. + Ensures requirements are in standard format without spaces around ~= + """ + # may_requires = DefaultDict(list) + may_requires = OrderedDefaultDict(list) + for value in sorted(pkg_info.get_all(key)): + extra_match = EXTRA_RE.search(value) + if extra_match: + groupdict = extra_match.groupdict() + condition = groupdict['condition'] + extra = groupdict['extra'] + package = groupdict['package'] + if condition.endswith(' and '): + condition = condition[:-5] + else: + condition, extra = None, None + package = value + + # Standardize the package requirement format by removing spaces around ~= + if ' ~=' in package: + package = package.replace(' ~=', '~=') + + key = MayRequiresKey(condition, extra) + may_requires[key].append(package) + + if may_requires: + metadata['run_requires'] = [] + + def sort_key(item): + # Both condition and extra could be None, which can't be compared + # against strings in Python 3. + key, value = item + if key.condition is None: + return '' + return key.condition + + for key, value in sorted(may_requires.items(), key=sort_key): + may_requirement = OrderedDict((('requires', value),)) + # may_requirement = defaultdict((('requires', value),)) + if key.extra: + may_requirement['extra'] = key.extra + if key.condition: + may_requirement['environment'] = key.condition + metadata['run_requires'].append(may_requirement) + + if 'extras' not in metadata: + metadata['extras'] = [] + metadata['extras'].extend([key.extra for key in may_requires.keys() if key.extra]) + + +def get_wheel_generator(whl_path): + """ + Extract the Generator value from the WHEEL file in a .whl package. + + Args: + whl_path (str): Path to the .whl file. + + Returns: + str: The Generator value, or None if not found. + """ + try: + # Ensure the file is a valid zip file + if not zipfile.is_zipfile(whl_path): + print(f"The file {whl_path} is not a valid zip file.") + return "bdist_wheel" + + # Open the .whl file + with zipfile.ZipFile(whl_path, 'r') as zf: + # Locate the WHEEL file + wheel_file = None + for file_name in zf.namelist(): + if ".dist-info/WHEEL" in file_name: + wheel_file = file_name + break + + if not wheel_file: + print(f"WHEEL file not found in {whl_path}.") + return "bdist_wheel" + + # Read and parse the WHEEL file + with zf.open(wheel_file) as wf: + for line in wf: + decoded_line = line.decode("utf-8").strip() + if decoded_line.startswith("Generator:"): + _, generator_value = decoded_line.split(":", 1) + return generator_value.strip() + + print(f"Generator key not found in {wheel_file}.") + return "bdist_wheel" + + except Exception as e: + print(f"An error occurred while processing {whl_path}: {e}") + return "bdist_wheel" + + +def pkginfo_to_dict(path, distribution=None): + """ + Convert PKG-INFO to a prototype Metadata 2.0 (PEP 426) dict. + + The description is included under the key ['description'] rather than + being written to a separate file. + + path: path to wheel file + distribution: optional distutils Distribution() + """ + # metadata = DefaultDict( + # lambda: DefaultDict( lambda: DefaultDict(defaultdict))) + + metadata = OrderedDefaultDict( + lambda: OrderedDefaultDict(lambda: OrderedDefaultDict(OrderedDict))) + + metadata["generator"] = "bdist_wheel (0.30.0)" + try: + pkg_info = read_pkg_info(path) + except Exception: + with open(path, 'rb') as pkg_info_file: + pkg_info = email.parser.Parser().parsestr(pkg_info_file.read().decode('utf-8')) + + # description = None + + if pkg_info['Summary']: + metadata['summary'] = pkginfo_unicode(pkg_info, 'Summary') + del pkg_info['Summary'] + + # if pkg_info['Description']: + # description = dedent_description(pkg_info) + # del pkg_info['Description'] + # else: + # payload = pkg_info.get_payload() + # if isinstance(payload, bytes): + # # Avoid a Python 2 Unicode error. + # # We still suffer ? glyphs on Python 3. + # payload = payload.decode('utf-8') + # if payload: + # description = payload + + # if description: + # pkg_info['description'] = description + + for key in sorted(unique(k.lower() for k in pkg_info.keys())): + low_key = key.replace('-', '_') + + if low_key in SKIP_FIELDS: + continue + + if low_key in UNKNOWN_FIELDS and pkg_info.get(key) == 'UNKNOWN': + continue + + if low_key in sorted(PLURAL_FIELDS): + metadata[PLURAL_FIELDS[low_key]] = pkg_info.get_all(key) + + elif low_key == "requires_dist": + handle_requires(metadata, pkg_info, key) + + elif low_key == 'provides_extra': + if 'extras' not in metadata: + metadata['extras'] = [] + metadata['extras'].extend(pkg_info.get_all(key)) + + elif low_key == 'home_page': + metadata['extensions']['python.details']['project_urls'] = {'Home': pkg_info[key]} + + elif low_key == 'keywords': + metadata['keywords'] = KEYWORDS_RE.split(pkg_info[key]) + + else: + metadata[low_key] = pkg_info[key] + + metadata['metadata_version'] = METADATA_VERSION + + if 'extras' in metadata: + metadata['extras'] = sorted(set(metadata['extras'])) + + # include more information if distribution is available + if distribution: + for requires, attr in (('test_requires', 'tests_require'),): + try: + requirements = getattr(distribution, attr) + if isinstance(requirements, list): + new_requirements = sorted(convert_requirements(requirements)) + metadata[requires] = [{'requires': new_requirements}] + except AttributeError: + pass + + # handle contacts + contacts = [] + for contact_type, role in CONTACT_FIELDS: + contact = OrderedDict() + # contact = defaultdict() + for key in sorted(contact_type): + if contact_type[key] in metadata: + contact[key] = metadata.pop(contact_type[key]) + if contact: + contact['role'] = role + contacts.append(contact) + if contacts: + metadata['extensions']['python.details']['contacts'] = contacts + + # handle document_names + # check for DESCRIPTION.rst and LICENSE.txt file in the wheel package + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path, 'r') as zf: + has_description = any('DESCRIPTION.rst' in name for name in zf.namelist()) + has_license = any('LICENSE.txt' in name for name in zf.namelist()) + + if has_description or has_license: + document_names = metadata['extensions']['python.details'].setdefault('document_names', {}) + if has_description: + document_names['description'] = 'DESCRIPTION.rst' + if has_license: + document_names['license'] = 'LICENSE.txt' + + # convert entry points to exports + try: + with open(os.path.join(os.path.dirname(path), "entry_points.txt"), "r"): + ep_map = entry_points() + exports = OrderedDict() + # exports = defaultdict() + for group, items in sorted(ep_map.items()): + exports[group] = OrderedDict() + # exports[group] = defaultdict() + for item in sorted(map(str, items.values())): + name, export = item.split(' = ', 1) + exports[group][name] = export + if exports: + metadata['extensions']['python.exports'] = exports + except IOError: + pass + + # copy console_scripts entry points to commands + if 'python.exports' in metadata['extensions']: + for (ep_script, wrap_script) in (('console_scripts', 'wrap_console'), + ('gui_scripts', 'wrap_gui')): + if ep_script in metadata['extensions']['python.exports']: + metadata['extensions']['python.commands'][wrap_script] = \ + metadata['extensions']['python.exports'][ep_script] + + return recursive_ordered_to_dict(metadata) + + +def recursive_ordered_to_dict(obj): + """ + Recursively process a data structure to convert OrderedDict and dict to sorted dict. + + Args: + obj: The input object to process (can be OrderedDict, dict, list, or any type). + + Returns: + dict: A dict-transformed version of the input object, with keys sorted. + """ + if isinstance(obj, (OrderedDict, dict)): + # Convert to dict and recursively process values, sorting keys + return {k: recursive_ordered_to_dict(v) for k, v in sorted(obj.items())} + elif isinstance(obj, list): + # Recursively process each item in the list + return [recursive_ordered_to_dict(item) for item in obj] + else: + # Return the object as-is for non-iterable types + return obj + + +def requires_to_requires_dist(requirement): + """Compose the version predicates for requirement in PEP 345 fashion.""" + requires_dist = [] + for op, ver in requirement.specs: + requires_dist.append(op + ver) + if not requires_dist: + return '' + return " (%s)" % ','.join(sorted(requires_dist)) + + +def convert_requirements(requirements): + """Yield Requires-Dist: strings for parsed requirements strings.""" + for req in requirements: + parsed_requirement = Requirement(req) + # parsed_requirement = pkg_resources.Requirement.parse(req) + spec = requires_to_requires_dist(parsed_requirement) + extras = ",".join(parsed_requirement.extras) + if extras: + extras = "[%s]" % extras + yield (parsed_requirement.project_name + extras + spec) + + +def safe_extra(extra): + """Mimics pkg_resources.safe_extra functionality. + Convert an arbitrary string to a standard 'extra' name + + Any runs of non-alphanumeric characters are replaced with a single '_', + and the result is always lowercased. + """ + return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() + + +def generate_requirements(extras_require): + """ + Convert requirements from a setup()-style dictionary to ('Requires-Dist', 'requirement') + and ('Provides-Extra', 'extra') tuples. + + extras_require is a dictionary of {extra: [requirements]} as passed to setup(), + using the empty extra {'': [requirements]} to hold install_requires. + """ + for extra, depends in extras_require.items(): + condition = '' + if extra and ':' in extra: # setuptools extra:condition syntax + extra, condition = extra.split(':', 1) + extra = safe_extra(extra) + # extra = pkg_resources.safe_extra(extra) + if extra: + yield ('Provides-Extra', extra) + if condition: + condition += " and " + condition += "extra == '%s'" % extra + if condition: + condition = '; ' + condition + for new_req in convert_requirements(depends): + yield ('Requires-Dist', new_req + condition) + + +def split_sections(s): + """Mimics pkg_resources.split_sections. + + Split a string or iterable thereof into (section, content) pairs. + + Each ``section`` is a stripped version of the section header ("[section]"), + and each ``content`` is a list of stripped lines excluding blank lines and + comment-only lines. If there are any such lines before the first section + header, they're returned in a first ``section`` of ``None``. + """ + + def yield_lines(content): + """Yields non-blank, non-comment lines from the input.""" + for line in content: + line = line.strip() + if line and not line.startswith("#"): + yield line + + section = None + content = [] + for line in yield_lines(s): + if line.startswith("[") and line.endswith("]"): + if section or content: + yield section, content + section = line[1:-1].strip() + content = [] + elif line.startswith("[") and not line.endswith("]"): + raise ValueError("Invalid section heading", line) + else: + content.append(line) + + # wrap up the last segment + if section or content: + yield section, content + + +def pkginfo_to_metadata(egg_info_path, pkginfo_path): + """ + Convert .egg-info directory with PKG-INFO to the Metadata 1.3 aka + old-draft Metadata 2.0 format. + """ + pkg_info = read_pkg_info(pkginfo_path) + pkg_info.replace_header('Metadata-Version', '2.0') + requires_path = os.path.join(egg_info_path, 'requires.txt') + if os.path.exists(requires_path): + with open(requires_path) as requires_file: + requires = requires_file.read() + for extra, reqs in sorted(split_sections(requires), key=lambda x: x[0] or ''): + # for extra, reqs in sorted(pkg_resources.split_sections(requires), key=lambda x: x[0] or ''): + for item in generate_requirements({extra: reqs}): + pkg_info[item[0]] = item[1] + + description = pkg_info['Description'] + if description: + pkg_info.set_payload(dedent_description(pkg_info)) + del pkg_info['Description'] + + return pkg_info + + +def pkginfo_unicode(pkg_info, field): + """Hack to coax Unicode out of an email Message() - Python 3.3+""" + text = pkg_info[field] + field = field.lower() + if not isinstance(text, str): + if not hasattr(pkg_info, 'raw_items'): # Python 3.2 + return str(text) + for item in pkg_info.raw_items(): + if item[0].lower() == field: + text = item[1].encode('ascii', 'surrogateescape') \ + .decode('utf-8') + break + + return text + + +def dedent_description(pkg_info): + """ + Dedent and convert pkg_info['Description'] to Unicode. + """ + description = pkg_info['Description'] + + # Python 3 Unicode handling, sorta. + surrogates = False + if not isinstance(description, str): + surrogates = True + description = pkginfo_unicode(pkg_info, 'Description') + + description_lines = description.splitlines() + description_dedent = '\n'.join( + # if the first line of long_description is blank, + # the first line here will be indented. + (description_lines[0].lstrip(), + textwrap.dedent('\n'.join(description_lines[1:])), + '\n')) + + if surrogates: + description_dedent = description_dedent \ + .encode("utf8") \ + .decode("ascii", "surrogateescape") + + return description_dedent + + +if __name__ == "__main__": + import sys + import pprint + + pprint.pprint(pkginfo_to_dict(sys.argv[1])) diff --git a/azdev/operations/extensions/util.py b/azdev/operations/extensions/util.py index ead988fd8..f51942e21 100644 --- a/azdev/operations/extensions/util.py +++ b/azdev/operations/extensions/util.py @@ -12,6 +12,7 @@ from knack.util import CLIError from azdev.utilities import EXTENSION_PREFIX +from azdev.operations.extensions.metadata import pkginfo_to_dict WHEEL_INFO_RE = re.compile( @@ -44,8 +45,7 @@ def _get_azext_metadata(ext_dir): def get_ext_metadata(ext_dir, ext_file, ext_name): - # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 - WHL_METADATA_FILENAME = 'metadata.json' + generated_metadata = pkginfo_to_dict(ext_file) with zipfile.ZipFile(ext_file, 'r') as zip_ref: zip_ref.extractall(ext_dir) metadata = {} @@ -56,10 +56,7 @@ def get_ext_metadata(ext_dir, ext_file, ext_name): for dist_info_dirname in dist_info_dirs: parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): - whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) - if os.path.isfile(whl_metadata_filepath): - with open(whl_metadata_filepath) as f: - metadata.update(json.load(f)) + metadata.update(generated_metadata) return metadata diff --git a/azdev/operations/setup.py b/azdev/operations/setup.py index 3782908ee..a85e82f54 100644 --- a/azdev/operations/setup.py +++ b/azdev/operations/setup.py @@ -49,7 +49,10 @@ def _install_extensions(ext_paths): # install specified extensions for path in ext_paths or []: - result = pip_cmd('install -e {}'.format(path), "Adding extension '{}'...".format(path)) + result = pip_cmd( + f'install -e {path} --config-settings editable_mode=compat', + f"Adding extension '{path}'..." + ) if result.error: raise result.error # pylint: disable=raising-bad-type @@ -90,38 +93,40 @@ def _install_cli(cli_path, deps=None): # Resolve dependencies from setup.py files. # command modules have dependency on azure-cli-core so install this first pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-telemetry')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-telemetry')), "Installing `azure-cli-telemetry`..." ) pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-core')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-core')), "Installing `azure-cli-core`..." ) # azure cli has dependencies on the above packages so install this one last pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli')), "Installing `azure-cli`..." ) pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-testsdk')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-testsdk')), "Installing `azure-cli-testsdk`..." ) else: # First install packages without dependencies, # then resolve dependencies from requirements.*.txt file. pip_cmd( - "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli-telemetry')), + "install -e {} --no-deps --config-settings editable_mode=compat".format( + os.path.join(cli_src, 'azure-cli-telemetry')), "Installing `azure-cli-telemetry`..." ) pip_cmd( - "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli-core')), + "install -e {} --no-deps --config-settings editable_mode=compat".format( + os.path.join(cli_src, 'azure-cli-core')), "Installing `azure-cli-core`..." ) pip_cmd( - "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli')), + "install -e {} --no-deps --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli')), "Installing `azure-cli`..." ) @@ -129,7 +134,7 @@ def _install_cli(cli_path, deps=None): # azure-cli package for running commands. # Here we need to install with dependencies for azdev test. pip_cmd( - "install -e {}".format(os.path.join(cli_src, 'azure-cli-testsdk')), + "install -e {} --config-settings editable_mode=compat".format(os.path.join(cli_src, 'azure-cli-testsdk')), "Installing `azure-cli-testsdk`..." ) import platform diff --git a/azdev/operations/tests/test_metadata.py b/azdev/operations/tests/test_metadata.py new file mode 100644 index 000000000..3b8f5deb8 --- /dev/null +++ b/azdev/operations/tests/test_metadata.py @@ -0,0 +1,263 @@ +# pylint: disable=C0301 + +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import json +import os +import re +import zipfile +import requests + +from deepdiff import DeepDiff +from azdev.operations.extensions.metadata import pkginfo_to_dict + + +def download_wheel(url, dest_dir): + """ + Download wheel file from URL to destination directory + """ + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + filename = os.path.basename(url) + dest_path = os.path.join(dest_dir, filename) + + response = requests.get(url) + response.raise_for_status() + + with open(dest_path, 'wb') as f: + f.write(response.content) + + return dest_path + + +def clean_metadata(original_metadata): + """ + Remove specified keys from the metadata. + :param original_metadata: Original metadata + :return: Cleaned metadata + """ + keys_to_remove = { + "azext.isPreview", + "azext.minCliCoreVersion", + "azext.isExperimental", + "azext.isExprimental", + "azext.maxCliCoreVersion" + } + return {k: v for k, v in original_metadata.items() if k not in keys_to_remove} + + +# copy from wheel==0.30.0 +WHEEL_INFO_RE = re.compile( + r"""^(?P(?P.+?)(-(?P\d.+?))?) + ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl|\.dist-info)$""", + re.VERBOSE).match + + +def _get_extension_modname(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L153 + EXTENSIONS_MOD_PREFIX = 'azext_' + pos_mods = [n for n in os.listdir(ext_dir) + if n.startswith(EXTENSIONS_MOD_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] + if len(pos_mods) != 1: + raise AssertionError("Expected 1 module to load starting with " + "'{}': got {}".format(EXTENSIONS_MOD_PREFIX, pos_mods)) + return pos_mods[0] + + +def _get_azext_metadata(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L109 + AZEXT_METADATA_FILENAME = 'azext_metadata.json' + azext_metadata = None + ext_modname = _get_extension_modname(ext_dir=ext_dir) + azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) + if os.path.isfile(azext_metadata_filepath): + with open(azext_metadata_filepath) as f: + azext_metadata = json.load(f) + return azext_metadata + + +def get_ext_metadata(ext_dir, ext_file, ext_name): + generated_metadata = pkginfo_to_dict(ext_file) + print(f"generated_metadata from python wheel package: \n{generated_metadata}") + + with zipfile.ZipFile(ext_file, 'r') as zip_ref: + zip_ref.extractall(ext_dir) + + metadata = {} + # dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] + + azext_metadata = _get_azext_metadata(ext_dir) + print(f"azext_metadata from python wheel package: \n{azext_metadata}") + + if not azext_metadata: + raise ValueError('azext_metadata.json for Extension "{}" Metadata is missing'.format(ext_name)) + + metadata.update(azext_metadata) + + metadata.update(generated_metadata) + # for dist_info_dirname in dist_info_dirs: + # parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) + # if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'): + # metadata.update(generated_metadata) + return metadata + + +def compare_metadata(wheel_url, expected_metadata): + """ + Compare metadata between wheel file and expected metadata + """ + temp_dir = 'temp_wheels' + + try: + print(f"Metadata from index.json: \n{expected_metadata}") + # Download the wheel + print(f"Downloading wheel from {wheel_url}") + ext_file = download_wheel(wheel_url, temp_dir) + ext_name = os.path.basename(wheel_url) + + # Get metadata from wheel + wheel_metadata = get_ext_metadata(temp_dir, ext_file, ext_name) + + # Compare metadata + print(f"Metadata from python wheel package: \n{wheel_metadata}") + diff = DeepDiff(expected_metadata, wheel_metadata, ignore_order=True) + + if diff: + print("Metadata mismatch found:") + print(f"Differences:\n{diff}") + return False + + print("Metadata built from python wheel package matches metadata from index.json.") + return True + + finally: + # Cleanup + if os.path.exists(temp_dir): + import shutil + shutil.rmtree(temp_dir) + + +def test_wheel(): + """ + Test specific wheel metadata consistency + """ + wheel_url = [ + "https://azurecliext.blob.core.windows.net/release/azure_cli_ml-1.41.0-py3-none-any.whl", + "https://azurecliprod.blob.core.windows.net/cli-extensions/alias-0.5.2-py2.py3-none-any.whl" + ] + metadata_from_index = [ + { + "azext.minCliCoreVersion": "2.3.1", + "classifiers": [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9" + ], + "description_content_type": "text/x-rst", + "extensions": { + "python.details": { + "contacts": [ + { + "email": "azpycli@microsoft.com", + "name": "Microsoft Corporation", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst", + "license": "LICENSE.txt" + }, + "project_urls": { + "Home": "https://docs.microsoft.com/python/api/overview/azure/ml/?view=azure-ml-py" + } + } + }, + "extras": [], + "generator": "bdist_wheel (0.30.0)", + "license": "Proprietary https://aka.ms/azureml-preview-sdk-license ", + "metadata_version": "2.0", + "name": "azure-cli-ml", + "requires_python": ">=3.5,<4", + "run_requires": [ + { + "requires": [ + "adal (>=1.2.1)", + "azureml-cli-common (~=1.41)", + "cryptography (<=3.3.2)", + "docker (>=3.7.2)", + "msrest (>=0.6.6)", + "pyyaml (>=5.1.0)", + "requests (>=2.21.0)" + ] + } + ], + "summary": "Microsoft Azure Command-Line Tools AzureML Command Module", + "version": "1.41.0" + }, + { + "azext.isPreview": True, + "azext.minCliCoreVersion": "2.0.50.dev0", + "classifiers": [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "License :: OSI Approved :: MIT License" + ], + "extensions": { + "python.details": { + "contacts": [ + { + "email": "t-chwong@microsoft.com", + "name": "Ernest Wong", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst" + }, + "project_urls": { + "Home": "https://github.com/Azure/azure-cli-extensions" + } + } + }, + "extras": [], + "generator": "bdist_wheel (0.30.0)", + "license": "MIT", + "metadata_version": "2.0", + "name": "alias", + "run_requires": [ + { + "requires": [ + "jinja2 (~=2.10)" + ] + } + ], + "summary": "Support for command aliases", + "version": "0.5.2" + } + ] + + for idx, url in enumerate(wheel_url): + assert compare_metadata(url, metadata_from_index[idx]), "Metadata comparison failed" + + +if __name__ == "__main__": + test_wheel() diff --git a/azure-pipelines-cli.yml b/azure-pipelines-cli.yml index ae5f55d0c..b567ecbc1 100644 --- a/azure-pipelines-cli.yml +++ b/azure-pipelines-cli.yml @@ -103,6 +103,7 @@ jobs: azdev setup -c ./azure-cli -r ./azure-cli-extensions azdev --version + az --version python -m pytest azdev/ --ignore=azdev/mod_templates --junitxml=junit/test-results.xml --cov=azdev --cov-report=xml - task: PublishTestResults@2 @@ -280,6 +281,7 @@ jobs: set -ev . scripts/ci/install.sh azdev --version + az --version displayName: 'Azdev Setup' - bash: | set -ev diff --git a/setup.py b/setup.py index f211a0249..ffbb6d15c 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ 'azure-cli-diff-tool~=0.1.0', 'packaging', 'tqdm', - 'wheel==0.30.0', + 'setuptools>=64.0.0', 'microsoft-security-utilities-secret-masker~=1.0.0b4' ], package_data={