From 7e399fd1c6c385d5ba322eec7f14d806d43b56d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Wed, 27 Aug 2025 01:00:24 +0200 Subject: [PATCH 01/22] Document important PackagesList methods --- conan/api/model/list.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 8faedec405d..0e37cc75612 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -265,6 +265,7 @@ def add_configurations(self, confs): pass def refs(self): + """ Get all the recipe references in the list.""" result = {} for ref, ref_dict in self.recipes.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): @@ -277,6 +278,7 @@ def refs(self): @staticmethod def prefs(ref, recipe_bundle): + """ Get all the package references for a given recipe reference given a bundle.""" result = {} for package_id, pkg_bundle in recipe_bundle.get("packages", {}).items(): prevs = pkg_bundle.get("revisions", {}) From 23538dcaad962b9b1b52d2d11d1a56501106951d Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 28 Aug 2025 09:57:51 +0200 Subject: [PATCH 02/22] wip --- conan/api/model/list.py | 23 +++++++++++++++-- conan/cli/commands/remove.py | 50 ++++++++++++++++++------------------ conan/cli/commands/upload.py | 37 ++++++++++++++------------ 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index d16ad9ccc50..32c676b2142 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -104,7 +104,7 @@ def load_graph(graphfile, graph_recipes=None, graph_binaries=None, context=None) ) mpkglist = MultiPackagesList._define_graph(graph, graph_recipes, graph_binaries, - context=base_context) + context=base_context) if context == "build-only": host = MultiPackagesList._define_graph(graph, graph_recipes, graph_binaries, context="host") @@ -266,7 +266,6 @@ def add_configurations(self, confs): pass def refs(self): - """ Get all the recipe references in the list.""" result = {} for ref, ref_dict in self.recipes.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): @@ -277,6 +276,26 @@ def refs(self): result[recipe] = rrev_dict return result + def items(self) -> dict[RecipeReference, dict[PkgReference, dict]]: + """ Get all the recipe references in the package list.""" + result = {} + for ref, ref_dict in self.recipes.items(): + for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): + t = rrev_dict.get("timestamp") + recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this + if t is not None: + recipe.timestamp = t + + pref_dict = {} + for package_id, pkg_bundle in rrev_dict.get("packages", {}).items(): + prevs = pkg_bundle.get("revisions", {}) + for prev, prev_bundle in prevs.items(): + t = prev_bundle.pop("timestamp", None) + pref = PkgReference(recipe, package_id, prev, t) + pref_dict[pref] = prev_bundle + result[recipe] = pref_dict + return result.items() + @staticmethod def prefs(ref, recipe_bundle): """ Get all the package references for a given recipe reference given a bundle.""" diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 5e488b9b030..1c6f77f678e 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -93,35 +93,35 @@ def confirmation(message): multi_package_list.add(cache_name, package_list) # TODO: This iteration and removal of not-confirmed is ugly and complicated, improve it - for ref, ref_bundle in package_list.refs().items(): - ref_dict = package_list.recipes[str(ref)]["revisions"] - packages = ref_bundle.get("packages") - if packages is None: - if confirmation(f"Remove the recipe and all the packages of '{ref.repr_notime()}'?"): - if not args.dry_run: - conan_api.remove.recipe(ref, remote=remote) - else: + result = {} + for ref, ref_info in package_list.recipes.items(): + result_ref = {} + for rrev, rrev_info in ref_info["revisions"].items(): + packages = rrev_info.get("packages") + if packages is None: + if confirmation(f"Remove the recipe and all the packages of '{ref}#{rrev}'?"): + if not args.dry_run: + conan_api.remove.recipe(ref, remote=remote) + + prefs = package_list.prefs(ref, ref_bundle) + if not prefs: + ConanOutput().info(f"No binaries to remove for '{ref.repr_notime()}'") ref_dict.pop(ref.revision) if not ref_dict: package_list.recipes.pop(str(ref)) - continue - prefs = package_list.prefs(ref, ref_bundle) - if not prefs: - ConanOutput().info(f"No binaries to remove for '{ref.repr_notime()}'") - ref_dict.pop(ref.revision) - if not ref_dict: - package_list.recipes.pop(str(ref)) - continue + continue - for pref, _ in prefs.items(): - if confirmation(f"Remove the package '{pref.repr_notime()}'?"): - if not args.dry_run: - conan_api.remove.package(pref, remote=remote) - else: - pref_dict = packages[pref.package_id]["revisions"] - pref_dict.pop(pref.revision) - if not pref_dict: - packages.pop(pref.package_id) + for pref, _ in prefs.items(): + if confirmation(f"Remove the package '{pref.repr_notime()}'?"): + if not args.dry_run: + conan_api.remove.package(pref, remote=remote) + else: + pref_dict = packages[pref.package_id]["revisions"] + pref_dict.pop(pref.revision) + if not pref_dict: + packages.pop(pref.package_id) + result[ref] = result_ref + package_list.recipes = result return { "results": multi_package_list.serialize(), diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 0338002c9dd..9f858b5c315 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -121,20 +121,23 @@ def upload(conan_api: ConanAPI, parser, *args): def _ask_confirm_upload(conan_api, package_list): ui = UserInput(conan_api.config.get("core:non_interactive")) - for ref, bundle in package_list.refs().items(): - msg = "Are you sure you want to upload recipe '%s'?" % ref.repr_notime() - ref_dict = package_list.recipes[str(ref)]["revisions"] - if not ui.request_boolean(msg): - ref_dict.pop(ref.revision) - # clean up empy refs - if not ref_dict: - package_list.recipes.pop(str(ref)) - else: - for pref, prev_bundle in package_list.prefs(ref, bundle).items(): - msg = "Are you sure you want to upload package '%s'?" % pref.repr_notime() - pkgs_dict = ref_dict[ref.revision]["packages"] - if not ui.request_boolean(msg): - pref_dict = pkgs_dict[pref.package_id]["revisions"] - pref_dict.pop(pref.revision) - if not pref_dict: - pkgs_dict.pop(pref.package_id) + result = {} + for ref, ref_info in package_list.recipes.items(): + result_ref = {} + for rrev, rrev_info in ref_info["revisions"].items(): + msg = f"Are you sure you want to upload recipe '{ref}#{rrev}'?" + if ui.request_boolean(msg): + result_rrev = {} + if rrev_info.get("timestamp"): + result_rrev["timestamp"] = rrev_info["timestamp"] + for pkg_id, pkg_id_info in rrev_info["packages"].items(): + for prev, prev_info in pkg_id_info["revisions"].items(): + msg = (f"Are you sure you want to upload package " + f"'{ref}#{rrev}:{pkg_id}#{prev}'?") + if ui.request_boolean(msg): + pkg_info = result_rrev.setdefault("packages", {}).setdefault(pkg_id, {}) + pkg_info.setdefault("revisions", {})[prev] = prev_info + pkg_info["info"] = pkg_id_info["info"] + result_ref.setdefault("revisions", {})[rrev] = result_rrev + result[ref] = result_ref + package_list.recipes = result From 211f7e7110a90d126c6cc04ad4b9d542b89be7bd Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 2 Sep 2025 13:23:55 +0200 Subject: [PATCH 03/22] wip --- conan/cli/commands/remove.py | 44 ++++++++++--------- conan/cli/commands/upload.py | 6 +-- .../list/test_combined_pkglist_flows.py | 4 +- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 1c6f77f678e..be0a60f5344 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -1,5 +1,5 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, MultiPackagesList +from conan.api.model import ListPattern, MultiPackagesList, RecipeReference, PkgReference from conan.api.output import cli_out_write, ConanOutput from conan.api.input import UserInput from conan.cli import make_abs_path @@ -96,31 +96,33 @@ def confirmation(message): result = {} for ref, ref_info in package_list.recipes.items(): result_ref = {} - for rrev, rrev_info in ref_info["revisions"].items(): + for rrev, rrev_info in ref_info.get("revisions", {}).items(): + full_ref = RecipeReference.loads(ref) + full_ref.revision = rrev packages = rrev_info.get("packages") if packages is None: if confirmation(f"Remove the recipe and all the packages of '{ref}#{rrev}'?"): if not args.dry_run: - conan_api.remove.recipe(ref, remote=remote) + conan_api.remove.recipe(full_ref, remote=remote) + result_ref.setdefault("revisions", {})[rrev] = rrev_info + else: + result_rrev = {} + for pkg_id, pkg_id_info in packages.items(): + package_revisions = pkg_id_info.get("revisions") + if package_revisions is None: + ConanOutput().info(f"No binaries to remove for '{full_ref.repr_notime()}'") + continue + for prev, prev_info in package_revisions.items(): + if confirmation(f"Remove the package '{ref}#{rrev}:{pkg_id}#{prev}'?"): + if not args.dry_run: + pref = PkgReference(full_ref, pkg_id, prev) + conan_api.remove.package(pref, remote=remote) + result_rrev.setdefault("packages", {})[pkg_id] = pkg_id_info + if result_rrev: + result_ref.setdefault("revisions", {})[rrev] = result_rrev - prefs = package_list.prefs(ref, ref_bundle) - if not prefs: - ConanOutput().info(f"No binaries to remove for '{ref.repr_notime()}'") - ref_dict.pop(ref.revision) - if not ref_dict: - package_list.recipes.pop(str(ref)) - continue - - for pref, _ in prefs.items(): - if confirmation(f"Remove the package '{pref.repr_notime()}'?"): - if not args.dry_run: - conan_api.remove.package(pref, remote=remote) - else: - pref_dict = packages[pref.package_id]["revisions"] - pref_dict.pop(pref.revision) - if not pref_dict: - packages.pop(pref.package_id) - result[ref] = result_ref + if result_ref: + result[ref] = result_ref package_list.recipes = result return { diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 9f858b5c315..652e162ae56 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -124,14 +124,14 @@ def _ask_confirm_upload(conan_api, package_list): result = {} for ref, ref_info in package_list.recipes.items(): result_ref = {} - for rrev, rrev_info in ref_info["revisions"].items(): + for rrev, rrev_info in ref_info.get("revisions", {}).items(): msg = f"Are you sure you want to upload recipe '{ref}#{rrev}'?" if ui.request_boolean(msg): result_rrev = {} if rrev_info.get("timestamp"): result_rrev["timestamp"] = rrev_info["timestamp"] - for pkg_id, pkg_id_info in rrev_info["packages"].items(): - for prev, prev_info in pkg_id_info["revisions"].items(): + for pkg_id, pkg_id_info in rrev_info.get("packages", {}).items(): + for prev, prev_info in pkg_id_info.get("revisions", {}).items(): msg = (f"Are you sure you want to upload package " f"'{ref}#{rrev}:{pkg_id}#{prev}'?") if ui.request_boolean(msg): diff --git a/test/integration/command/list/test_combined_pkglist_flows.py b/test/integration/command/list/test_combined_pkglist_flows.py index 6eff4ebc742..99f69597bdf 100644 --- a/test/integration/command/list/test_combined_pkglist_flows.py +++ b/test/integration/command/list/test_combined_pkglist_flows.py @@ -372,7 +372,7 @@ def test_remove_packages_no_revisions(self, client, remote): # It is necessary to do *#* for actually removing something remote = "-r=default" if remote else "" client.run(f"list *#*:* {remote} --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json {remote} -c") + client.run(f"remove --list=pkglist.json {remote} -c --format=json") assert "No binaries to remove for 'zli/1.0.0#f034dc90894493961d92dd32a9ee3b78'" in client.out assert "No binaries to remove for 'zlib/1.0.0@user/channel" \ "#ffd4bc45820ddb320ab224685b9ba3fb" in client.out @@ -383,7 +383,7 @@ def test_remove_packages(self, client, remote): remote = "-r=default" if remote else "" client.run(f"list *#*:*#* {remote} --format=json", redirect_stdout="pkglist.json") client.run(f"remove --list=pkglist.json {remote} -c") - + print(client.out) assert "Removed recipe and all binaries" not in client.out assert "zli/1.0.0#f034dc90894493961d92dd32a9ee3b78: Removed binaries" in client.out assert "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb: " \ From 90e5588a5e10b82fca9fbc7ed834b60727a147fd Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 2 Sep 2025 13:47:45 +0200 Subject: [PATCH 04/22] proposal draft --- conan/api/subapi/cache.py | 4 +-- conan/api/subapi/download.py | 4 +-- conan/cli/commands/remove.py | 4 +-- .../list/test_combined_pkglist_flows.py | 31 ++++++++++++++++--- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 862530b324a..c0cfb6fd071 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -109,14 +109,14 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, for f in backup_files: remove(f) - for ref, ref_bundle in package_list.refs().items(): + for ref, packages in package_list.items(): ConanOutput(ref.repr_notime()).verbose("Cleaning recipe cache contents") ref_layout = cache.recipe_layout(ref) if source: rmdir(ref_layout.source()) if download: rmdir(ref_layout.download_export()) - for pref, _ in package_list.prefs(ref, ref_bundle).items(): + for pref, _ in packages.items(): ConanOutput(pref).verbose("Cleaning package cache contents") pref_layout = cache.pkg_layout(pref) if build: diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index f6b1763c607..f3486e83ae9 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -83,9 +83,9 @@ def download_full(self, package_list: PackagesList, remote: Remote, """Download the recipes and packages specified in the ``package_list`` from the remote, parallelized based on ``core.download:parallel``""" def _download_pkglist(pkglist): - for ref, recipe_bundle in pkglist.refs().items(): + for ref, packages in pkglist.items(): self.recipe(ref, remote, metadata) - for pref, _ in pkglist.prefs(ref, recipe_bundle).items(): + for pref, _ in packages.items(): self.package(pref, remote, metadata) t = time.time() diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index be0a60f5344..8c02d73d0d8 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -92,7 +92,6 @@ def confirmation(message): multi_package_list = MultiPackagesList() multi_package_list.add(cache_name, package_list) - # TODO: This iteration and removal of not-confirmed is ugly and complicated, improve it result = {} for ref, ref_info in package_list.recipes.items(): result_ref = {} @@ -117,10 +116,9 @@ def confirmation(message): if not args.dry_run: pref = PkgReference(full_ref, pkg_id, prev) conan_api.remove.package(pref, remote=remote) - result_rrev.setdefault("packages", {})[pkg_id] = pkg_id_info + result_rrev.setdefault("packages", {})[pkg_id] = pkg_id_info if result_rrev: result_ref.setdefault("revisions", {})[rrev] = result_rrev - if result_ref: result[ref] = result_ref package_list.recipes = result diff --git a/test/integration/command/list/test_combined_pkglist_flows.py b/test/integration/command/list/test_combined_pkglist_flows.py index 99f69597bdf..daa5df00824 100644 --- a/test/integration/command/list/test_combined_pkglist_flows.py +++ b/test/integration/command/list/test_combined_pkglist_flows.py @@ -351,19 +351,29 @@ def client(self): def test_remove_nothing_only_refs(self, client): # It is necessary to do *#* for actually removing something client.run(f"list * --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json -c") + client.run(f"remove --list=pkglist.json -c --format=json") assert "Nothing to remove, package list do not contain recipe revisions" in client.out + result = json.loads(client.stdout) + assert result["Local Cache"] == {} # Nothing was removed @pytest.mark.parametrize("remote", [False, True]) def test_remove_all(self, client, remote): # It is necessary to do *#* for actually removing something remote = "-r=default" if remote else "" client.run(f"list *#* {remote} --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json {remote} -c") + client.run(f"remove --list=pkglist.json {remote} -c --dry-run") assert "zli/1.0.0#f034dc90894493961d92dd32a9ee3b78:" \ " Removed recipe and all binaries" in client.out assert "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb:" \ " Removed recipe and all binaries" in client.out + + client.run(f"remove --list=pkglist.json {remote} -c --format=json") + result = json.loads(client.stdout) + origin = "Local Cache" if not remote else "default" + assert len(result[origin]["zli/1.0.0"]["revisions"]) == 1 + assert len(result[origin]["zlib/1.0.0@user/channel"]["revisions"]) == 1 + assert "packages" not in client.stdout # Packages are not listed at all + client.run(f"list * {remote}") assert "There are no matching recipe references" in client.out @@ -376,18 +386,31 @@ def test_remove_packages_no_revisions(self, client, remote): assert "No binaries to remove for 'zli/1.0.0#f034dc90894493961d92dd32a9ee3b78'" in client.out assert "No binaries to remove for 'zlib/1.0.0@user/channel" \ "#ffd4bc45820ddb320ab224685b9ba3fb" in client.out + result = json.loads(client.stdout) + origin = "Local Cache" if not remote else "default" + assert result[origin] == {} # Nothing was removed @pytest.mark.parametrize("remote", [False, True]) def test_remove_packages(self, client, remote): # It is necessary to do *#* for actually removing something remote = "-r=default" if remote else "" client.run(f"list *#*:*#* {remote} --format=json", redirect_stdout="pkglist.json") - client.run(f"remove --list=pkglist.json {remote} -c") - print(client.out) + client.run(f"remove --list=pkglist.json {remote} -c --dry-run") assert "Removed recipe and all binaries" not in client.out assert "zli/1.0.0#f034dc90894493961d92dd32a9ee3b78: Removed binaries" in client.out assert "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb: " \ "Removed binaries" in client.out + + client.run(f"remove --list=pkglist.json {remote} -c --format=json") + result = json.loads(client.stdout) + origin = "Local Cache" if not remote else "default" + zli_revs = result[origin]["zli/1.0.0"]["revisions"] + zli_uc_revs = result[origin]["zlib/1.0.0@user/channel"]["revisions"] + assert len(zli_revs) == 1 + assert len(zli_uc_revs) == 1 + assert len(zli_revs["f034dc90894493961d92dd32a9ee3b78"]["packages"]) == 1 + assert len(zli_uc_revs["ffd4bc45820ddb320ab224685b9ba3fb"]["packages"]) == 1 + client.run(f"list *:* {remote}") assert "zli/1.0.0" in client.out assert "zlib/1.0.0@user/channel" in client.out From eb300982b11a908e303fb9b6618e3f4c3b42c5f3 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 4 Sep 2025 18:21:28 +0200 Subject: [PATCH 05/22] wip --- conan/api/model/list.py | 48 ++++++++++++++----------- conan/internal/cache/integrity_check.py | 4 +-- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 67f816f8382..e89f7907d1d 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -1,9 +1,11 @@ +import copy import fnmatch import json import os from json import JSONDecodeError from conan.api.model import RecipeReference, PkgReference +from conan.api.output import ConanOutput from conan.errors import ConanException from conan.internal.errors import NotFoundException from conan.internal.model.version_range import VersionRange @@ -193,7 +195,7 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): class PackagesList: """ A collection of recipes, revisions and packages.""" def __init__(self): - self.recipes = {} + self._data = {} def merge(self, other): def recursive_dict_update(d, u): # TODO: repeated from conandata.py @@ -203,15 +205,15 @@ def recursive_dict_update(d, u): # TODO: repeated from conandata.py else: d[k] = v return d - recursive_dict_update(self.recipes, other.recipes) + recursive_dict_update(self._data, other.recipes) def keep_outer(self, other): - if not self.recipes: + if not self._data: return for ref, info in other.recipes.items(): - if self.recipes.get(ref, {}) == info: - self.recipes.pop(ref) + if self._data.get(ref, {}) == info: + self._data.pop(ref) def split(self): """ @@ -219,24 +221,24 @@ def split(self): This can be useful to parallelize things like upload, parallelizing per-reference """ result = [] - for r, content in self.recipes.items(): + for r, content in self._data.items(): subpkglist = PackagesList() - subpkglist.recipes[r] = content + subpkglist._data[r] = content result.append(subpkglist) return result def only_recipes(self) -> None: """ Filter out all the packages and package revisions, keep only the recipes and - recipe revisions in self.recipes. + recipe revisions in self._data. """ - for ref, ref_dict in self.recipes.items(): + for ref, ref_dict in self._data.items(): for rrev_dict in ref_dict.get("revisions", {}).values(): rrev_dict.pop("packages", None) def add_refs(self, refs): # RREVS alreday come in ASCENDING order, so upload does older revisions first for ref in refs: - ref_dict = self.recipes.setdefault(str(ref), {}) + ref_dict = self._data.setdefault(str(ref), {}) if ref.revision: revs_dict = ref_dict.setdefault("revisions", {}) rev_dict = revs_dict.setdefault(ref.revision, {}) @@ -245,7 +247,7 @@ def add_refs(self, refs): def add_prefs(self, rrev, prefs): # Prevs already come in ASCENDING order, so upload does older revisions first - revs_dict = self.recipes[str(rrev)]["revisions"] + revs_dict = self._data[str(rrev)]["revisions"] rev_dict = revs_dict[rrev.revision] packages_dict = rev_dict.setdefault("packages", {}) @@ -259,15 +261,18 @@ def add_prefs(self, rrev, prefs): def add_configurations(self, confs): for pref, conf in confs.items(): - rev_dict = self.recipes[str(pref.ref)]["revisions"][pref.ref.revision] + rev_dict = self._data[str(pref.ref)]["revisions"][pref.ref.revision] try: rev_dict["packages"][pref.package_id]["info"] = conf except KeyError: # If package_id does not exist, do nothing, only add to existing prefs pass def refs(self): + kk + ConanOutput().warning("PackageLists.refs() non-public, non-documented method will be " + "removed, use .items() instead", warn_tag="deprecated") result = {} - for ref, ref_dict in self.recipes.items(): + for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): t = rrev_dict.get("timestamp") recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this @@ -276,29 +281,32 @@ def refs(self): result[recipe] = rrev_dict return result - def items(self) -> dict[RecipeReference, dict[PkgReference, dict]]: + def items(self): """ Get all the recipe references in the package list.""" result = {} - for ref, ref_dict in self.recipes.items(): + for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): - t = rrev_dict.get("timestamp") recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this + t = rrev_dict.get("timestamp") if t is not None: recipe.timestamp = t - pref_dict = {} + packages = rrev_dict.pop("packages", {}) for package_id, pkg_bundle in rrev_dict.get("packages", {}).items(): prevs = pkg_bundle.get("revisions", {}) for prev, prev_bundle in prevs.items(): t = prev_bundle.pop("timestamp", None) pref = PkgReference(recipe, package_id, prev, t) - pref_dict[pref] = prev_bundle + pref_dict.setdefault("packages", {})[pref] = prev_bundle result[recipe] = pref_dict return result.items() @staticmethod def prefs(ref, recipe_bundle): """ Get all the package references for a given recipe reference given a bundle.""" + kk + ConanOutput().warning("PackageLists.prefs() non-public, non-documented method will be " + "removed, use .items() instead", warn_tag="deprecated") result = {} for package_id, pkg_bundle in recipe_bundle.get("packages", {}).items(): prevs = pkg_bundle.get("revisions", {}) @@ -310,13 +318,13 @@ def prefs(ref, recipe_bundle): def serialize(self): """ Serialize the instance to a dictionary.""" - return self.recipes.copy() + return copy.deepcopy(self._data) @staticmethod def deserialize(data): """ Loads the data from a serialized dictionary.""" result = PackagesList() - result.recipes = data + result._data = copy.deepcopy(data) return result diff --git a/conan/internal/cache/integrity_check.py b/conan/internal/cache/integrity_check.py index 43b526d11f6..f88f325fd1b 100644 --- a/conan/internal/cache/integrity_check.py +++ b/conan/internal/cache/integrity_check.py @@ -22,9 +22,9 @@ def __init__(self, cache): def check(self, pkg_list): corrupted = False - for ref, recipe_bundle in pkg_list.refs().items(): + for ref, ref_info in pkg_list.items(): corrupted = self._recipe_corrupted(ref) or corrupted - for pref, prev_bundle in pkg_list.prefs(ref, recipe_bundle).items(): + for pref, _ in ref_info.get("packages", {}).items(): corrupted = self._package_corrupted(pref) or corrupted if corrupted: raise ConanException("There are corrupted artifacts, check the error logs") From 38af8488692667da0ec16954226cf954d6d2e85c Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 5 Sep 2025 14:30:50 +0200 Subject: [PATCH 06/22] dirty, but to see if tests pass --- conan/api/model/list.py | 18 +++++++++--------- conan/api/subapi/cache.py | 10 +++++----- conan/api/subapi/download.py | 14 +++++++------- conan/api/subapi/list.py | 12 ++++++------ conan/api/subapi/upload.py | 4 ++-- conan/cli/commands/download.py | 2 +- conan/cli/commands/remove.py | 10 ++++++---- conan/cli/commands/upload.py | 11 ++++++----- conan/internal/api/upload.py | 4 ++-- conan/internal/api/uploader.py | 16 ++++++++-------- conan/internal/cache/integrity_check.py | 4 ++-- conan/internal/rest/download_cache.py | 10 +++------- conan/internal/rest/pkg_sign.py | 4 ++-- .../command/upload/test_upload_bundle.py | 2 +- test/unittests/model/test_list.py | 6 +++--- 15 files changed, 63 insertions(+), 64 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index e89f7907d1d..3fdef56baf1 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -197,6 +197,9 @@ class PackagesList: def __init__(self): self._data = {} + def __bool__(self): + return bool(self._data) + def merge(self, other): def recursive_dict_update(d, u): # TODO: repeated from conandata.py for k, v in u.items(): @@ -205,13 +208,13 @@ def recursive_dict_update(d, u): # TODO: repeated from conandata.py else: d[k] = v return d - recursive_dict_update(self._data, other.recipes) + recursive_dict_update(self._data, other._data) def keep_outer(self, other): if not self._data: return - for ref, info in other.recipes.items(): + for ref, info in other._data.items(): if self._data.get(ref, {}) == info: self._data.pop(ref) @@ -281,25 +284,22 @@ def refs(self): result[recipe] = rrev_dict return result - def items(self): + def walk(self): """ Get all the recipe references in the package list.""" - result = {} for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this t = rrev_dict.get("timestamp") if t is not None: recipe.timestamp = t - - packages = rrev_dict.pop("packages", {}) + packages = {} for package_id, pkg_bundle in rrev_dict.get("packages", {}).items(): prevs = pkg_bundle.get("revisions", {}) for prev, prev_bundle in prevs.items(): t = prev_bundle.pop("timestamp", None) pref = PkgReference(recipe, package_id, prev, t) - pref_dict.setdefault("packages", {})[pref] = prev_bundle - result[recipe] = pref_dict - return result.items() + packages[pref] = prev_bundle + yield recipe, rrev_dict, packages @staticmethod def prefs(ref, recipe_bundle): diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index c0cfb6fd071..6fb8cbe31c1 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -109,7 +109,7 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, for f in backup_files: remove(f) - for ref, packages in package_list.items(): + for ref, _, packages in package_list.walk(): ConanOutput(ref.repr_notime()).verbose("Cleaning recipe cache contents") ref_layout = cache.recipe_layout(ref) if source: @@ -135,7 +135,7 @@ def save(self, package_list, tgz_path, no_source=False): compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) tar_files: dict[str, str] = {} # {path_in_tar: abs_path} - for ref, ref_bundle in package_list.refs().items(): + for ref, ref_bundle, packages in package_list.walk(): ref_layout = cache.recipe_layout(ref) recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable @@ -152,7 +152,7 @@ def save(self, package_list, tgz_path, no_source=False): if os.path.exists(path): tar_files[f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}"] = path - for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): + for pref, pref_bundle in packages.items(): pref_layout = cache.pkg_layout(pref) pkg_folder = pref_layout.package() folder = os.path.relpath(pkg_folder, cache_folder) @@ -194,7 +194,7 @@ def restore(self, path): # After unzipping the files, we need to update the DB that references these files out = ConanOutput() package_list = PackagesList.deserialize(json.loads(pkglist)) - for ref, ref_bundle in package_list.refs().items(): + for ref, ref_bundle, packages in package_list.walk(): ref.timestamp = revision_timestamp_now() ref_bundle["timestamp"] = ref.timestamp try: @@ -207,7 +207,7 @@ def restore(self, path): # In the case of recipes, they are always "in place", so just checking it assert rel_path == recipe_folder, f"{rel_path}!={recipe_folder}" out.info(f"Restore: {ref} in {recipe_folder}") - for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): + for pref, pref_bundle in packages.items(): pref.timestamp = revision_timestamp_now() pref_bundle["timestamp"] = pref.timestamp try: diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index f3d02bda9d5..5c9861d1dbf 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -83,19 +83,19 @@ def download_full(self, package_list: PackagesList, remote: Remote, """Download the recipes and packages specified in the ``package_list`` from the remote, parallelized based on ``core.download:parallel``""" def _download_pkglist(pkglist): - for ref, packages in pkglist.items(): + for ref, ref_dict, packages in pkglist.walk(): self.recipe(ref, remote, metadata) - recipe_bundle.pop("files", None) - recipe_bundle.pop("upload-urls", None) - for pref, _ in packages.items(): + ref_dict.pop("files", None) + ref_dict.pop("upload-urls", None) + for pref, pkg_dict in packages.items(): self.package(pref, remote, metadata) - pref_bundle.pop("files", None) - pref_bundle.pop("upload-urls", None) + pkg_dict.pop("files", None) + pkg_dict.pop("upload-urls", None) t = time.time() parallel = self._conan_api.config.get("core.download:parallel", default=1, check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None - if not thread_pool or len(package_list.refs()) <= 1: + if not thread_pool or len(package_list._data) <= 1: _download_pkglist(package_list) else: ConanOutput().subtitle(f"Downloading with {parallel} parallel threads") diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index d582960c7b6..e920423fb4d 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -306,7 +306,7 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): pkglist.add_prefs(ref, [pref]) pkglist.add_configurations({pref: candidate.binary_config}) # Add the diff data - rev_dict = pkglist.recipes[str(pref.ref)]["revisions"][pref.ref.revision] + rev_dict = pkglist._data[str(pref.ref)]["revisions"][pref.ref.revision] rev_dict["packages"][pref.package_id]["diff"] = candidate.serialize() remote = candidate.remote.name if candidate.remote else "Local Cache" rev_dict["packages"][pref.package_id]["remote"] = remote @@ -319,7 +319,7 @@ def find_remotes(self, package_list, remotes): result = MultiPackagesList() for r in remotes: result_pkg_list = PackagesList() - for ref, recipe_bundle in package_list.refs().items(): + for ref, recipe_bundle, packages in package_list.walk(): ref_no_rev = copy.copy(ref) # TODO: Improve ugly API ref_no_rev.revision = None try: @@ -329,7 +329,7 @@ def find_remotes(self, package_list, remotes): if ref not in revs: # not found continue result_pkg_list.add_refs([ref]) - for pref, pref_bundle in package_list.prefs(ref, recipe_bundle).items(): + for pref, pref_bundle in packages.items(): pref_no_rev = copy.copy(pref) # TODO: Improve ugly API pref_no_rev.revision = None try: @@ -340,7 +340,7 @@ def find_remotes(self, package_list, remotes): result_pkg_list.add_prefs(ref, [pref]) info = recipe_bundle["packages"][pref.package_id]["info"] result_pkg_list.add_configurations({pref: info}) - if result_pkg_list.recipes: + if result_pkg_list: result.add(r.name, result_pkg_list) return result @@ -371,9 +371,9 @@ def outdated(self, deps_graph, remotes): remote_ref_list = self.select(ref_pattern, package_query=None, remote=remote) except NotFoundException: continue - if not remote_ref_list.recipes: + if not remote_ref_list: continue - str_latest_ref = list(remote_ref_list.recipes.keys())[-1] + str_latest_ref = list(remote_ref_list._data.keys())[-1] recipe_ref = RecipeReference.loads(str_latest_ref) if (node_info["latest_remote"] is None or node_info["latest_remote"]["ref"] < recipe_ref): diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 31d076c6705..94e3cd7b7a1 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -38,7 +38,7 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, enabled_rem A ``force_upload`` key will be added to the entries that will be uploaded. """ app = ConanApp(self._conan_api) - for ref, bundle in package_list.refs().items(): + for ref, _, bundle in package_list.walk(): layout = app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = app.loader.load_basic(conanfile_path, remotes=enabled_remotes) @@ -130,7 +130,7 @@ def _upload_pkglist(pkglist, subtitle=lambda _: None): ConanOutput().title(f"Uploading to remote {remote.name}") parallel = self._conan_api.config.get("core.upload:parallel", default=1, check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None - if not thread_pool or len(package_list.recipes) <= 1: + if not thread_pool or len(package_list._data) <= 1: _upload_pkglist(package_list, subtitle=ConanOutput().subtitle) else: ConanOutput().subtitle(f"Uploading with {parallel} parallel threads") diff --git a/conan/cli/commands/download.py b/conan/cli/commands/download.py index a7c72a0ccbd..386d9f25d37 100644 --- a/conan/cli/commands/download.py +++ b/conan/cli/commands/download.py @@ -56,7 +56,7 @@ def download(conan_api: ConanAPI, parser, *args): ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, args.package_query, remote) - if package_list.recipes: + if package_list: conan_api.download.download_full(package_list, remote, args.metadata) else: ConanOutput().warning(f"No packages were downloaded because the package list is empty.") diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 8c02d73d0d8..e176bd96adf 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -1,5 +1,6 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, MultiPackagesList, RecipeReference, PkgReference +from conan.api.model import ListPattern, MultiPackagesList, RecipeReference, PkgReference, \ + PackagesList from conan.api.output import cli_out_write, ConanOutput from conan.api.input import UserInput from conan.cli import make_abs_path @@ -81,7 +82,7 @@ def confirmation(message): listfile = make_abs_path(args.list) multi_package_list = MultiPackagesList.load(listfile) package_list = multi_package_list[cache_name] - refs_to_remove = package_list.refs() + refs_to_remove = list(package_list.walk()) if not refs_to_remove: # the package list might contain only refs, no revs ConanOutput().warning("Nothing to remove, package list do not contain recipe revisions") else: @@ -93,7 +94,7 @@ def confirmation(message): multi_package_list.add(cache_name, package_list) result = {} - for ref, ref_info in package_list.recipes.items(): + for ref, ref_info in package_list.serialize().items(): result_ref = {} for rrev, rrev_info in ref_info.get("revisions", {}).items(): full_ref = RecipeReference.loads(ref) @@ -121,7 +122,8 @@ def confirmation(message): result_ref.setdefault("revisions", {})[rrev] = result_rrev if result_ref: result[ref] = result_ref - package_list.recipes = result + package_list = PackagesList.deserialize(result) + multi_package_list.add(cache_name, package_list) return { "results": multi_package_list.serialize(), diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 652e162ae56..2810e090e54 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -1,5 +1,5 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, MultiPackagesList +from conan.api.model import ListPattern, MultiPackagesList, PackagesList from conan.api.output import ConanOutput from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument @@ -97,10 +97,10 @@ def upload(conan_api: ConanAPI, parser, *args): ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) - if package_list.recipes: + if package_list: # If only if search with "*" we ask for confirmation if not args.list and not args.confirm and "*" in args.pattern: - _ask_confirm_upload(conan_api, package_list) + package_list = _ask_confirm_upload(conan_api, package_list) conan_api.upload.upload_full(package_list, remote, enabled_remotes, args.check, args.force, args.metadata, args.dry_run) @@ -122,7 +122,7 @@ def upload(conan_api: ConanAPI, parser, *args): def _ask_confirm_upload(conan_api, package_list): ui = UserInput(conan_api.config.get("core:non_interactive")) result = {} - for ref, ref_info in package_list.recipes.items(): + for ref, ref_info in package_list.serialize().items(): result_ref = {} for rrev, rrev_info in ref_info.get("revisions", {}).items(): msg = f"Are you sure you want to upload recipe '{ref}#{rrev}'?" @@ -140,4 +140,5 @@ def _ask_confirm_upload(conan_api, package_list): pkg_info["info"] = pkg_id_info["info"] result_ref.setdefault("revisions", {})[rrev] = result_rrev result[ref] = result_ref - package_list.recipes = result + package_list = PackagesList.deserialize(result) + return package_list diff --git a/conan/internal/api/upload.py b/conan/internal/api/upload.py index 0093c083a12..82e4c94823e 100644 --- a/conan/internal/api/upload.py +++ b/conan/internal/api/upload.py @@ -4,12 +4,12 @@ def add_urls(package_list, remote): router = ClientV2Router(remote.url.rstrip("/")) - for ref, bundle in package_list.refs().items(): + for ref, bundle, packages in package_list.walk(): for f, fp in bundle.get("files", {}).items(): bundle.setdefault("upload-urls", {})[f] = { 'url': router.recipe_file(ref, f), 'checksum': sha1sum(fp) } - for pref, prev_bundle in package_list.prefs(ref, bundle).items(): + for pref, prev_bundle in packages.items(): for f, fp in prev_bundle.get("files", {}).items(): prev_bundle.setdefault("upload-urls", {})[f] = { 'url': router.package_file(pref, f), 'checksum': sha1sum(fp) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index cb1706e4f5f..f893b53d6f1 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -29,9 +29,9 @@ def __init__(self, app: ConanApp): self._app = app def check(self, upload_bundle, remote, force): - for ref, recipe_bundle in upload_bundle.refs().items(): + for ref, recipe_bundle, packages in upload_bundle.walk(): self._check_upstream_recipe(ref, recipe_bundle, remote, force) - for pref, prev_bundle in upload_bundle.prefs(ref, recipe_bundle).items(): + for pref, prev_bundle in packages.items(): self._check_upstream_package(pref, prev_bundle, remote, force) def _check_upstream_recipe(self, ref, ref_bundle, remote, force): @@ -85,7 +85,7 @@ def __init__(self, app: ConanApp, global_conf): def prepare(self, pkg_list, enabled_remotes): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) - for ref, bundle in pkg_list.refs().items(): + for ref, bundle, packages in pkg_list.walk(): layout = self._app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = self._app.loader.load_basic(conanfile_path) @@ -101,7 +101,7 @@ def prepare(self, pkg_list, enabled_remotes): bundle.pop("upload-urls", None) if bundle.get("upload"): self._prepare_recipe(ref, bundle, conanfile, enabled_remotes) - for pref, prev_bundle in pkg_list.prefs(ref, bundle).items(): + for pref, prev_bundle in packages.items(): prev_bundle.pop("files", None) # If defined from a previous upload prev_bundle.pop("upload-urls", None) if prev_bundle.get("upload"): @@ -227,10 +227,10 @@ def __init__(self, app: ConanApp): self._app = app def upload(self, upload_data, remote): - for ref, bundle in upload_data.refs().items(): + for ref, bundle, packages in upload_data.walk(): if bundle.get("upload"): self.upload_recipe(ref, bundle, remote) - for pref, prev_bundle in upload_data.prefs(ref, bundle).items(): + for pref, prev_bundle in packages.items(): if prev_bundle.get("upload"): self.upload_package(pref, prev_bundle, remote) @@ -317,7 +317,7 @@ def _metadata_files(folder, metadata): def gather_metadata(package_list, cache, metadata): - for rref, recipe_bundle in package_list.refs().items(): + for rref, recipe_bundle, packages in package_list.walk(): if metadata or recipe_bundle["upload"]: metadata_folder = cache.recipe_layout(rref).metadata() files = _metadata_files(metadata_folder, metadata) @@ -326,7 +326,7 @@ def gather_metadata(package_list, cache, metadata): recipe_bundle.setdefault("files", {}).update(files) recipe_bundle["upload"] = True - for pref, pkg_bundle in package_list.prefs(rref, recipe_bundle).items(): + for pref, pkg_bundle in packages.items(): if metadata or pkg_bundle["upload"]: metadata_folder = cache.pkg_layout(pref).metadata() files = _metadata_files(metadata_folder, metadata) diff --git a/conan/internal/cache/integrity_check.py b/conan/internal/cache/integrity_check.py index f88f325fd1b..13fcfc96b36 100644 --- a/conan/internal/cache/integrity_check.py +++ b/conan/internal/cache/integrity_check.py @@ -22,9 +22,9 @@ def __init__(self, cache): def check(self, pkg_list): corrupted = False - for ref, ref_info in pkg_list.items(): + for ref, _, packages in pkg_list.walk(): corrupted = self._recipe_corrupted(ref) or corrupted - for pref, _ in ref_info.get("packages", {}).items(): + for pref, _ in packages.items(): corrupted = self._package_corrupted(pref) or corrupted if corrupted: raise ConanException("There are corrupted artifacts, check the error logs") diff --git a/conan/internal/rest/download_cache.py b/conan/internal/rest/download_cache.py index 9d9c916fbdc..91f3fa7fd8b 100644 --- a/conan/internal/rest/download_cache.py +++ b/conan/internal/rest/download_cache.py @@ -70,15 +70,11 @@ def has_excluded_urls(backup_urls): for excluded_url in excluded_urls) for url in backup_urls) - def should_upload_sources(package): - return any(prev.get("upload") for prev in package["revisions"].values()) - all_refs = set() if package_list is not None: - for k, ref in package_list.refs().items(): - packages = ref.get("packages", {}).values() - if not only_upload or ref.get("upload") or any(should_upload_sources(p) for p in packages): - all_refs.add(str(k)) + for ref, ref_info, packages in package_list.walk(): + if not only_upload or ref_info.get("upload") or any(p.get("upload") for p in packages.values()): + all_refs.add(str(ref)) path_backups_contents = [] diff --git a/conan/internal/rest/pkg_sign.py b/conan/internal/rest/pkg_sign.py index baa8e8c72cc..0a448ac573c 100644 --- a/conan/internal/rest/pkg_sign.py +++ b/conan/internal/rest/pkg_sign.py @@ -29,10 +29,10 @@ def _sign(ref, files, folder): for f in os.listdir(metadata_sign): files[f"{METADATA}/sign/{f}"] = os.path.join(metadata_sign, f) - for rref, recipe_bundle in upload_data.refs().items(): + for rref, recipe_bundle, packages in upload_data.walk(): if recipe_bundle["upload"]: _sign(rref, recipe_bundle["files"], self._cache.recipe_layout(rref).download_export()) - for pref, pkg_bundle in upload_data.prefs(rref, recipe_bundle).items(): + for pref, pkg_bundle in packages.items(): if pkg_bundle["upload"]: _sign(pref, pkg_bundle["files"], self._cache.pkg_layout(pref).download_package()) diff --git a/test/integration/command/upload/test_upload_bundle.py b/test/integration/command/upload/test_upload_bundle.py index 3e3bf7c4b68..fa2ca46d07f 100644 --- a/test/integration/command/upload/test_upload_bundle.py +++ b/test/integration/command/upload/test_upload_bundle.py @@ -36,7 +36,7 @@ def upload_bundle(conan_api, parser, *args, **kwargs): ref_pattern = ListPattern(args.reference, package_id="*") package_list = conan_api.list.select(ref_pattern) - if not package_list.recipes: + if not package_list: raise ConanException("No recipes found matching pattern '{}'".format(args.reference)) # Check if the recipes/packages are in the remote diff --git a/test/unittests/model/test_list.py b/test/unittests/model/test_list.py index 96799f43afa..ae68ce70564 100644 --- a/test/unittests/model/test_list.py +++ b/test/unittests/model/test_list.py @@ -2,8 +2,7 @@ def test_package_list_only_recipes(): - pl = PackagesList() - pl.recipes = { + data = { "foobar/0.1.0": {'revisions': {'85eb0587a3c12b90216c72070e9eef3e': {'timestamp': 1740151190.975, @@ -42,8 +41,9 @@ def test_package_list_only_recipes(): 'compiler.version': '11', 'os': 'Linux'}, 'options': {'shared': 'True'}}}}}}} } + pl = PackagesList.deserialize(data) pl.only_recipes() - assert pl.recipes == {'foobar/0.1.0': { + assert pl.serialize() == {'foobar/0.1.0': { 'revisions': {'85eb0587a3c12b90216c72070e9eef3e': {'timestamp': 1740151190.975}}}, 'qux/0.2.1': {'revisions': { '71c3c11b98a6f2ae11f0f391f5e62e2b': {'timestamp': 1740151186.976}}}} From 75e5601991d3524908e83d516a013e3efe16b85a Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 5 Sep 2025 15:58:37 +0200 Subject: [PATCH 07/22] fix test --- conan/api/subapi/upload.py | 4 ++-- test/functional/only_source_test.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 94e3cd7b7a1..1b1117b2385 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -38,14 +38,14 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, enabled_rem A ``force_upload`` key will be added to the entries that will be uploaded. """ app = ConanApp(self._conan_api) - for ref, _, bundle in package_list.walk(): + for ref, ref_info, _ in package_list.walk(): layout = app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = app.loader.load_basic(conanfile_path, remotes=enabled_remotes) if conanfile.upload_policy == "skip": ConanOutput().info(f"{ref}: Skipping upload of binaries, " "because upload_policy='skip'") - bundle["packages"] = {} + ref_info["packages"] = {} UploadUpstreamChecker(app).check(package_list, remote, force) diff --git a/test/functional/only_source_test.py b/test/functional/only_source_test.py index cd6ccc811f7..d1b518f59b6 100644 --- a/test/functional/only_source_test.py +++ b/test/functional/only_source_test.py @@ -147,5 +147,6 @@ def test_build_policy_missing(): assert "pkg/1.0: Building package from source" not in c.out c.run("upload * -r=default -c") + print(c.out) assert "Uploading package" not in c.out assert "pkg/1.0: Skipping upload of binaries, because upload_policy='skip'" in c.out From 02508f8feb9f85c6c4f9e2c7267715cc0db36e70 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 5 Sep 2025 15:58:55 +0200 Subject: [PATCH 08/22] remove print --- test/functional/only_source_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/only_source_test.py b/test/functional/only_source_test.py index d1b518f59b6..cd6ccc811f7 100644 --- a/test/functional/only_source_test.py +++ b/test/functional/only_source_test.py @@ -147,6 +147,5 @@ def test_build_policy_missing(): assert "pkg/1.0: Building package from source" not in c.out c.run("upload * -r=default -c") - print(c.out) assert "Uploading package" not in c.out assert "pkg/1.0: Skipping upload of binaries, because upload_policy='skip'" in c.out From 300f44221b1eadceb82265b4ddfe91fda0a3b4bc Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 9 Sep 2025 16:56:53 +0200 Subject: [PATCH 09/22] review --- conan/api/model/list.py | 22 +++++++++++-------- conan/api/subapi/download.py | 2 +- conan/api/subapi/upload.py | 2 +- conan/cli/commands/list.py | 2 +- conan/cli/commands/remove.py | 2 +- conan/cli/commands/search.py | 5 ++--- conan/internal/api/upload.py | 12 +++++----- .../command/upload/test_upload_bundle.py | 20 ++++++++--------- 8 files changed, 35 insertions(+), 32 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 3fdef56baf1..4e85fa9ecaa 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -201,6 +201,7 @@ def __bool__(self): return bool(self._data) def merge(self, other): + assert isinstance(other, PackagesList) def recursive_dict_update(d, u): # TODO: repeated from conandata.py for k, v in u.items(): if isinstance(v, dict): @@ -211,6 +212,7 @@ def recursive_dict_update(d, u): # TODO: repeated from conandata.py recursive_dict_update(self._data, other._data) def keep_outer(self, other): + assert isinstance(other, PackagesList) if not self._data: return @@ -271,7 +273,6 @@ def add_configurations(self, confs): pass def refs(self): - kk ConanOutput().warning("PackageLists.refs() non-public, non-documented method will be " "removed, use .items() instead", warn_tag="deprecated") result = {} @@ -285,7 +286,12 @@ def refs(self): return result def walk(self): - """ Get all the recipe references in the package list.""" + """ Iterate the contents of the package list. + Every iteration returns [RecipeReference, dict, dict] + The first dictionary is the information directly belonging to the recipe-revision. + The second dictionary contains PkgReference as keys, and a dictionariy with the values + belonging to that specific package reference. + """ for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this @@ -293,18 +299,16 @@ def walk(self): if t is not None: recipe.timestamp = t packages = {} - for package_id, pkg_bundle in rrev_dict.get("packages", {}).items(): - prevs = pkg_bundle.get("revisions", {}) - for prev, prev_bundle in prevs.items(): - t = prev_bundle.pop("timestamp", None) + for package_id, pkg_info in rrev_dict.get("packages", {}).items(): + prevs = pkg_info.get("revisions", {}) + for prev, prev_info in prevs.items(): + t = prev_info.pop("timestamp", None) pref = PkgReference(recipe, package_id, prev, t) - packages[pref] = prev_bundle + packages[pref] = prev_info yield recipe, rrev_dict, packages @staticmethod def prefs(ref, recipe_bundle): - """ Get all the package references for a given recipe reference given a bundle.""" - kk ConanOutput().warning("PackageLists.prefs() non-public, non-documented method will be " "removed, use .items() instead", warn_tag="deprecated") result = {} diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index 5c9861d1dbf..7ea7b429697 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -95,7 +95,7 @@ def _download_pkglist(pkglist): t = time.time() parallel = self._conan_api.config.get("core.download:parallel", default=1, check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None - if not thread_pool or len(package_list._data) <= 1: + if not thread_pool or len(package_list._data) <= 1: # FIXME: Iteration when multiple rrevs _download_pkglist(package_list) else: ConanOutput().subtitle(f"Downloading with {parallel} parallel threads") diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 1b1117b2385..f5dabe72f7a 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -130,7 +130,7 @@ def _upload_pkglist(pkglist, subtitle=lambda _: None): ConanOutput().title(f"Uploading to remote {remote.name}") parallel = self._conan_api.config.get("core.upload:parallel", default=1, check_type=int) thread_pool = ThreadPool(parallel) if parallel > 1 else None - if not thread_pool or len(package_list._data) <= 1: + if not thread_pool or len(package_list._data) <= 1: # FIXME: Iteration when multiple rrevs _upload_pkglist(package_list, subtitle=ConanOutput().subtitle) else: ConanOutput().subtitle(f"Uploading with {parallel} parallel threads") diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 6b5100b2ae1..fba9bfb12a7 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -57,7 +57,7 @@ def print_serial(item, indent=None, color_index=None): def print_list_text(results): """ Do a little format modification to serialized - list bundle, so it looks prettier on text output + package-list, so it looks prettier on text output """ info = results["results"] diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index e176bd96adf..0b2387f686e 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -11,7 +11,7 @@ def summary_remove_list(results): """ Do a little format modification to serialized - list bundle, so it looks prettier on text output + package-list so it looks prettier on text output """ cli_out_write("Remove summary:") info = results["results"] diff --git a/conan/cli/commands/search.py b/conan/cli/commands/search.py index 7a34033502c..08ca30c1660 100644 --- a/conan/cli/commands/search.py +++ b/conan/cli/commands/search.py @@ -7,7 +7,6 @@ from conan.errors import ConanException -# FIXME: "conan search" == "conan list (*) -r="*"" --> implement @conan_alias_command?? @conan_command(group="Consumer", formatters={"text": print_list_text, "json": print_list_json}) def search(conan_api: ConanAPI, parser, *args): @@ -31,11 +30,11 @@ def search(conan_api: ConanAPI, parser, *args): results = OrderedDict() for remote in remotes: try: - list_bundle = conan_api.list.select(ref_pattern, package_query=None, remote=remote) + pkglist = conan_api.list.select(ref_pattern, package_query=None, remote=remote) except Exception as e: results[remote.name] = {"error": str(e)} else: - results[remote.name] = list_bundle.serialize() + results[remote.name] = pkglist.serialize() return { "results": results } diff --git a/conan/internal/api/upload.py b/conan/internal/api/upload.py index 82e4c94823e..632fd13ef62 100644 --- a/conan/internal/api/upload.py +++ b/conan/internal/api/upload.py @@ -4,13 +4,13 @@ def add_urls(package_list, remote): router = ClientV2Router(remote.url.rstrip("/")) - for ref, bundle, packages in package_list.walk(): - for f, fp in bundle.get("files", {}).items(): - bundle.setdefault("upload-urls", {})[f] = { + for ref, ref_info, packages in package_list.walk(): + for f, fp in ref_info.get("files", {}).items(): + ref_info.setdefault("upload-urls", {})[f] = { 'url': router.recipe_file(ref, f), 'checksum': sha1sum(fp) } - for pref, prev_bundle in packages.items(): - for f, fp in prev_bundle.get("files", {}).items(): - prev_bundle.setdefault("upload-urls", {})[f] = { + for pref, pref_info in packages.items(): + for f, fp in pref_info.get("files", {}).items(): + pref_info.setdefault("upload-urls", {})[f] = { 'url': router.package_file(pref, f), 'checksum': sha1sum(fp) } diff --git a/test/integration/command/upload/test_upload_bundle.py b/test/integration/command/upload/test_upload_bundle.py index fa2ca46d07f..66f55a9bded 100644 --- a/test/integration/command/upload/test_upload_bundle.py +++ b/test/integration/command/upload/test_upload_bundle.py @@ -6,10 +6,10 @@ from conan.test.utils.tools import TestClient -def test_upload_bundle(): - """ Test how a custom command can create an upload bundle and print it +def test_upload_pkg_list(): + """ Test how a custom command can create an upload pkglist and print it """ - c = TestClient(default_server_user=True) + c = TestClient(default_server_user=True, light=True) mycommand = textwrap.dedent(""" import json import os @@ -19,9 +19,9 @@ def test_upload_bundle(): from conan.api.output import cli_out_write @conan_command(group="custom commands") - def upload_bundle(conan_api, parser, *args, **kwargs): + def upload_pkglist(conan_api, parser, *args, **kwargs): \""" - create an upload bundle + create an upload piglist \""" parser.add_argument('reference', help="Recipe reference or package reference, can contain * as " @@ -46,11 +46,11 @@ def upload_bundle(conan_api, parser, *args, **kwargs): """) command_file_path = os.path.join(c.cache_folder, 'extensions', - 'commands', 'cmd_upload_bundle.py') + 'commands', 'cmd_upload_pkglist.py') c.save({command_file_path: mycommand}) c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) c.run("create .") - c.run('upload-bundle "*" -r=default', redirect_stdout="mybundle.json") - bundle = c.load("mybundle.json") - bundle = json.loads(bundle) - assert bundle["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"]["upload"] is True + c.run('upload-pkglist "*" -r=default', redirect_stdout="mypkglist.json") + pkglist = c.load("mypkglist.json") + pkglist = json.loads(pkglist) + assert pkglist["pkg/0.1"]["revisions"]["485dad6cb11e2fa99d9afbe44a57a164"]["upload"] is True From 1f777b771651cfc23f34fec681854548e8a55071 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 11 Sep 2025 12:18:22 +0200 Subject: [PATCH 10/22] walk()->items() + accessor --- conan/api/model/list.py | 10 ++++++++-- conan/api/subapi/cache.py | 8 +++++--- conan/api/subapi/download.py | 3 ++- conan/api/subapi/list.py | 5 +++-- conan/api/subapi/upload.py | 4 ++-- conan/cli/commands/remove.py | 2 +- conan/internal/api/upload.py | 3 ++- conan/internal/api/uploader.py | 16 ++++++++++------ conan/internal/cache/integrity_check.py | 2 +- conan/internal/rest/download_cache.py | 3 ++- conan/internal/rest/pkg_sign.py | 3 ++- 11 files changed, 38 insertions(+), 21 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 4e85fa9ecaa..155b1df894a 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -285,7 +285,7 @@ def refs(self): result[recipe] = rrev_dict return result - def walk(self): + def items(self): """ Iterate the contents of the package list. Every iteration returns [RecipeReference, dict, dict] The first dictionary is the information directly belonging to the recipe-revision. @@ -305,7 +305,13 @@ def walk(self): t = prev_info.pop("timestamp", None) pref = PkgReference(recipe, package_id, prev, t) packages[pref] = prev_info - yield recipe, rrev_dict, packages + yield recipe, packages + + def recipe_info(self, ref): + """ gives read/write access to the dictionary containing a specific RecipeReference + information + """ + return self._data[str(ref)]["revisions"][ref.revision] @staticmethod def prefs(ref, recipe_bundle): diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 6fb8cbe31c1..23502b647d5 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -109,7 +109,7 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, for f in backup_files: remove(f) - for ref, _, packages in package_list.walk(): + for ref, packages in package_list.items(): ConanOutput(ref.repr_notime()).verbose("Cleaning recipe cache contents") ref_layout = cache.recipe_layout(ref) if source: @@ -135,10 +135,11 @@ def save(self, package_list, tgz_path, no_source=False): compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) tar_files: dict[str, str] = {} # {path_in_tar: abs_path} - for ref, ref_bundle, packages in package_list.walk(): + for ref, packages in package_list.items(): ref_layout = cache.recipe_layout(ref) recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable + ref_bundle = package_list.recipe_info(ref) ref_bundle["recipe_folder"] = recipe_folder out.info(f"Saving {ref}: {recipe_folder}") # Package only selected folders, not DOWNLOAD one @@ -194,7 +195,8 @@ def restore(self, path): # After unzipping the files, we need to update the DB that references these files out = ConanOutput() package_list = PackagesList.deserialize(json.loads(pkglist)) - for ref, ref_bundle, packages in package_list.walk(): + for ref, packages in package_list.items(): + ref_bundle = package_list.recipe_info(ref) ref.timestamp = revision_timestamp_now() ref_bundle["timestamp"] = ref.timestamp try: diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index 7ea7b429697..96d39109522 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -83,8 +83,9 @@ def download_full(self, package_list: PackagesList, remote: Remote, """Download the recipes and packages specified in the ``package_list`` from the remote, parallelized based on ``core.download:parallel``""" def _download_pkglist(pkglist): - for ref, ref_dict, packages in pkglist.walk(): + for ref, packages in pkglist.items(): self.recipe(ref, remote, metadata) + ref_dict = pkglist.recipe_info(ref) ref_dict.pop("files", None) ref_dict.pop("upload-urls", None) for pref, pkg_dict in packages.items(): diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index e920423fb4d..57ffe682cbf 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -306,7 +306,7 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): pkglist.add_prefs(ref, [pref]) pkglist.add_configurations({pref: candidate.binary_config}) # Add the diff data - rev_dict = pkglist._data[str(pref.ref)]["revisions"][pref.ref.revision] + rev_dict = pkglist.recipe_info(ref) rev_dict["packages"][pref.package_id]["diff"] = candidate.serialize() remote = candidate.remote.name if candidate.remote else "Local Cache" rev_dict["packages"][pref.package_id]["remote"] = remote @@ -319,7 +319,7 @@ def find_remotes(self, package_list, remotes): result = MultiPackagesList() for r in remotes: result_pkg_list = PackagesList() - for ref, recipe_bundle, packages in package_list.walk(): + for ref, packages in package_list.items(): ref_no_rev = copy.copy(ref) # TODO: Improve ugly API ref_no_rev.revision = None try: @@ -338,6 +338,7 @@ def find_remotes(self, package_list, remotes): continue if pref in prevs: result_pkg_list.add_prefs(ref, [pref]) + recipe_bundle = package_list.recipe_info(ref) info = recipe_bundle["packages"][pref.package_id]["info"] result_pkg_list.add_configurations({pref: info}) if result_pkg_list: diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index f5dabe72f7a..8b7aa6c1a43 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -38,14 +38,14 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, enabled_rem A ``force_upload`` key will be added to the entries that will be uploaded. """ app = ConanApp(self._conan_api) - for ref, ref_info, _ in package_list.walk(): + for ref, _ in package_list.items(): layout = app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = app.loader.load_basic(conanfile_path, remotes=enabled_remotes) if conanfile.upload_policy == "skip": ConanOutput().info(f"{ref}: Skipping upload of binaries, " "because upload_policy='skip'") - ref_info["packages"] = {} + package_list.recipe_info(ref)["packages"] = {} UploadUpstreamChecker(app).check(package_list, remote, force) diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 0b2387f686e..8451a3c544d 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -82,7 +82,7 @@ def confirmation(message): listfile = make_abs_path(args.list) multi_package_list = MultiPackagesList.load(listfile) package_list = multi_package_list[cache_name] - refs_to_remove = list(package_list.walk()) + refs_to_remove = list(package_list.items()) if not refs_to_remove: # the package list might contain only refs, no revs ConanOutput().warning("Nothing to remove, package list do not contain recipe revisions") else: diff --git a/conan/internal/api/upload.py b/conan/internal/api/upload.py index 632fd13ef62..033e00b892b 100644 --- a/conan/internal/api/upload.py +++ b/conan/internal/api/upload.py @@ -4,7 +4,8 @@ def add_urls(package_list, remote): router = ClientV2Router(remote.url.rstrip("/")) - for ref, ref_info, packages in package_list.walk(): + for ref, packages in package_list.items(): + ref_info = package_list.recipe_info(ref) for f, fp in ref_info.get("files", {}).items(): ref_info.setdefault("upload-urls", {})[f] = { 'url': router.recipe_file(ref, f), 'checksum': sha1sum(fp) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index f893b53d6f1..81bdedbbe14 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -28,9 +28,10 @@ class UploadUpstreamChecker: def __init__(self, app: ConanApp): self._app = app - def check(self, upload_bundle, remote, force): - for ref, recipe_bundle, packages in upload_bundle.walk(): - self._check_upstream_recipe(ref, recipe_bundle, remote, force) + def check(self, package_list, remote, force): + for ref, packages in package_list.items(): + recipe_info = package_list.recipe_info(ref) + self._check_upstream_recipe(ref, recipe_info, remote, force) for pref, prev_bundle in packages.items(): self._check_upstream_package(pref, prev_bundle, remote, force) @@ -85,7 +86,7 @@ def __init__(self, app: ConanApp, global_conf): def prepare(self, pkg_list, enabled_remotes): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) - for ref, bundle, packages in pkg_list.walk(): + for ref, packages in pkg_list.items(): layout = self._app.cache.recipe_layout(ref) conanfile_path = layout.conanfile() conanfile = self._app.loader.load_basic(conanfile_path) @@ -97,6 +98,7 @@ def prepare(self, pkg_list, enabled_remotes): "Failing because conf 'core.scm:local_url!=allow'") # Just in case it was defined from a previous run + bundle = pkg_list.recipe_info(ref) bundle.pop("files", None) bundle.pop("upload-urls", None) if bundle.get("upload"): @@ -227,7 +229,8 @@ def __init__(self, app: ConanApp): self._app = app def upload(self, upload_data, remote): - for ref, bundle, packages in upload_data.walk(): + for ref, packages in upload_data.items(): + bundle = upload_data.recipe_info(ref) if bundle.get("upload"): self.upload_recipe(ref, bundle, remote) for pref, prev_bundle in packages.items(): @@ -317,7 +320,8 @@ def _metadata_files(folder, metadata): def gather_metadata(package_list, cache, metadata): - for rref, recipe_bundle, packages in package_list.walk(): + for rref, packages in package_list.items(): + recipe_bundle = package_list.recipe_info(rref) if metadata or recipe_bundle["upload"]: metadata_folder = cache.recipe_layout(rref).metadata() files = _metadata_files(metadata_folder, metadata) diff --git a/conan/internal/cache/integrity_check.py b/conan/internal/cache/integrity_check.py index 13fcfc96b36..4a400376e60 100644 --- a/conan/internal/cache/integrity_check.py +++ b/conan/internal/cache/integrity_check.py @@ -22,7 +22,7 @@ def __init__(self, cache): def check(self, pkg_list): corrupted = False - for ref, _, packages in pkg_list.walk(): + for ref, packages in pkg_list.items(): corrupted = self._recipe_corrupted(ref) or corrupted for pref, _ in packages.items(): corrupted = self._package_corrupted(pref) or corrupted diff --git a/conan/internal/rest/download_cache.py b/conan/internal/rest/download_cache.py index 81bdd820f2d..ee4fa4958c8 100644 --- a/conan/internal/rest/download_cache.py +++ b/conan/internal/rest/download_cache.py @@ -72,7 +72,8 @@ def has_excluded_urls(backup_urls): all_refs = set() if package_list is not None: - for ref, ref_info, packages in package_list.walk(): + for ref, packages in package_list.items(): + ref_info = package_list.recipe_info(ref) if not only_upload or ref_info.get("upload") or any(p.get("upload") for p in packages.values()): all_refs.add(str(ref)) diff --git a/conan/internal/rest/pkg_sign.py b/conan/internal/rest/pkg_sign.py index 0a448ac573c..9169839d560 100644 --- a/conan/internal/rest/pkg_sign.py +++ b/conan/internal/rest/pkg_sign.py @@ -29,7 +29,8 @@ def _sign(ref, files, folder): for f in os.listdir(metadata_sign): files[f"{METADATA}/sign/{f}"] = os.path.join(metadata_sign, f) - for rref, recipe_bundle, packages in upload_data.walk(): + for rref, packages in upload_data.items(): + recipe_bundle = upload_data.recipe_info(rref) if recipe_bundle["upload"]: _sign(rref, recipe_bundle["files"], self._cache.recipe_layout(rref).download_export()) for pref, pkg_bundle in packages.items(): From 65264201482aa775332593ff42dbe84abe063e35 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 11 Sep 2025 12:20:52 +0200 Subject: [PATCH 11/22] remove private ._data access --- conan/api/subapi/list.py | 2 +- test/integration/command/test_outdated.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index 57ffe682cbf..a8acacffb81 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -374,7 +374,7 @@ def outdated(self, deps_graph, remotes): continue if not remote_ref_list: continue - str_latest_ref = list(remote_ref_list._data.keys())[-1] + str_latest_ref = list(remote_ref_list.serialize().keys())[-1] recipe_ref = RecipeReference.loads(str_latest_ref) if (node_info["latest_remote"] is None or node_info["latest_remote"]["ref"] < recipe_ref): diff --git a/test/integration/command/test_outdated.py b/test/integration/command/test_outdated.py index c5294cf7372..4c94e38bc7a 100644 --- a/test/integration/command/test_outdated.py +++ b/test/integration/command/test_outdated.py @@ -9,7 +9,7 @@ @pytest.fixture def create_libs(): - tc = TestClient(default_server_user=True) + tc = TestClient(default_server_user=True, light=True) tc.save({"conanfile.py": GenConanfile()}) tc.run("create . --name=zlib --version=1.0") tc.run("create . --name=zlib --version=2.0") From 4f524ae3e964fb08b7fb5a0969060d8638619495 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 14 Sep 2025 18:04:02 +0200 Subject: [PATCH 12/22] wip --- conan/api/model/list.py | 98 ++++++++++++++++++--------- conan/api/subapi/cache.py | 14 ++-- conan/api/subapi/download.py | 5 +- conan/api/subapi/list.py | 31 +++++---- conan/api/subapi/upload.py | 2 +- conan/cli/commands/export.py | 2 +- conan/cli/commands/remove.py | 57 +++++++--------- conan/cli/commands/upload.py | 36 ++++------ conan/internal/api/upload.py | 5 +- conan/internal/api/uploader.py | 22 +++--- conan/internal/rest/download_cache.py | 5 +- conan/internal/rest/pkg_sign.py | 5 +- 12 files changed, 157 insertions(+), 125 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index 155b1df894a..b5f9c25b19d 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -152,11 +152,11 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): continue pyref = RecipeReference.loads(pyref) if any(r == "*" or r == pyrecipe for r in recipes): - cache_list.add_refs([pyref]) + cache_list.add_ref(pyref) pyremote = pyreq["remote"] if pyremote: remote_list = pkglist.lists.setdefault(pyremote, PackagesList()) - remote_list.add_refs([pyref]) + remote_list.add_ref(pyref) recipe = node["recipe"] if recipe in (RECIPE_EDITABLE, RECIPE_CONSUMER, RECIPE_VIRTUAL, RECIPE_PLATFORM): @@ -167,18 +167,18 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): ref.timestamp = node["rrev_timestamp"] recipe = recipe.lower() if any(r == "*" or r == recipe for r in recipes): - cache_list.add_refs([ref]) + cache_list.add_ref(ref) remote = node["remote"] if remote: remote_list = pkglist.lists.setdefault(remote, PackagesList()) - remote_list.add_refs([ref]) + remote_list.add_ref(ref) pref = PkgReference(ref, node["package_id"], node["prev"], node["prev_timestamp"]) binary_remote = node["binary_remote"] if binary_remote: remote_list = pkglist.lists.setdefault(binary_remote, PackagesList()) - remote_list.add_refs([ref]) # Binary listed forces recipe listed - remote_list.add_prefs(ref, [pref]) + remote_list.add_ref(ref) # Binary listed forces recipe listed + remote_list.add_pref(pref) binary = node["binary"] if binary in (BINARY_SKIP, BINARY_INVALID, BINARY_MISSING): @@ -186,9 +186,9 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): binary = binary.lower() if any(b == "*" or b == binary for b in binaries): - cache_list.add_refs([ref]) # Binary listed forces recipe listed - cache_list.add_prefs(ref, [pref]) - cache_list.add_configurations({pref: node["info"]}) + cache_list.add_ref(ref) # Binary listed forces recipe listed + cache_list.add_pref(pref) + cache_list.add_configuration(pref, node["info"]) return pkglist @@ -241,36 +241,61 @@ def only_recipes(self) -> None: rrev_dict.pop("packages", None) def add_refs(self, refs): + ConanOutput().warning("PackageLists.add_refs() non-public, non-documented method will be " + "removed, use .add_ref() instead", warn_tag="deprecated") # RREVS alreday come in ASCENDING order, so upload does older revisions first for ref in refs: - ref_dict = self._data.setdefault(str(ref), {}) - if ref.revision: - revs_dict = ref_dict.setdefault("revisions", {}) - rev_dict = revs_dict.setdefault(ref.revision, {}) - if ref.timestamp: - rev_dict["timestamp"] = ref.timestamp + self.add_ref(ref) + + def add_ref(self, ref: RecipeReference) -> None: + """ + Adds a new RecipeReference to a pacakge list + """ + ref_dict = self._data.setdefault(str(ref), {}) + if ref.revision: + revs_dict = ref_dict.setdefault("revisions", {}) + rev_dict = revs_dict.setdefault(ref.revision, {}) + if ref.timestamp: + rev_dict["timestamp"] = ref.timestamp def add_prefs(self, rrev, prefs): + ConanOutput().warning("PackageLists.add_prrefs() non-public, non-documented method will be " + "removed, use .add_pref() instead", warn_tag="deprecated") # Prevs already come in ASCENDING order, so upload does older revisions first - revs_dict = self._data[str(rrev)]["revisions"] - rev_dict = revs_dict[rrev.revision] - packages_dict = rev_dict.setdefault("packages", {}) + for p in prefs: + self.add_pref(p) - for pref in prefs: - package_dict = packages_dict.setdefault(pref.package_id, {}) - if pref.revision: - prevs_dict = package_dict.setdefault("revisions", {}) - prev_dict = prevs_dict.setdefault(pref.revision, {}) - if pref.timestamp: - prev_dict["timestamp"] = pref.timestamp + def add_pref(self, pref: PkgReference) -> None: + """ + Add a PkgReferene to an already existing RecipeReference inside a package list + """ + # Prevs already come in ASCENDING order, so upload does older revisions first + rev_dict = self.recipe_dict(pref.ref) + packages_dict = rev_dict.setdefault("packages", {}) + package_dict = packages_dict.setdefault(pref.package_id, {}) + if pref.revision: + prevs_dict = package_dict.setdefault("revisions", {}) + prev_dict = prevs_dict.setdefault(pref.revision, {}) + if pref.timestamp: + prev_dict["timestamp"] = pref.timestamp def add_configurations(self, confs): - for pref, conf in confs.items(): - rev_dict = self._data[str(pref.ref)]["revisions"][pref.ref.revision] - try: - rev_dict["packages"][pref.package_id]["info"] = conf - except KeyError: # If package_id does not exist, do nothing, only add to existing prefs - pass + ConanOutput().warning("PackageLists.add_configurations() non-public, non-documented method " + "will be removed, use .add_configuration() instead", + warn_tag="deprecated") + for k, v in confs.items(): + self.add_configuration(k, v) + + def add_configuration(self, pref: PkgReference, conf: dict) -> None: + """ + Add the configuration information for the binary for an already existing PkgReference + in the package list + """ + rev_dict = self.recipe_dict(pref.ref) + try: + rev_dict["packages"][pref.package_id]["info"] = conf + except KeyError: # If package_id does not exist, do nothing, only add to existing prefs + pass def refs(self): ConanOutput().warning("PackageLists.refs() non-public, non-documented method will be " @@ -302,17 +327,24 @@ def items(self): for package_id, pkg_info in rrev_dict.get("packages", {}).items(): prevs = pkg_info.get("revisions", {}) for prev, prev_info in prevs.items(): - t = prev_info.pop("timestamp", None) + t = prev_info.get("timestamp") pref = PkgReference(recipe, package_id, prev, t) packages[pref] = prev_info yield recipe, packages - def recipe_info(self, ref): + def recipe_dict(self, ref: RecipeReference): """ gives read/write access to the dictionary containing a specific RecipeReference information """ return self._data[str(ref)]["revisions"][ref.revision] + def package_dict(self, pref: PkgReference): + """ gives read/write access to the dictionary containing a specific PkgReference + information + """ + ref_dict = self.recipe_dict(pref.ref) + return ref_dict["packages"][pref.package_id]["revisions"][pref.revision] + @staticmethod def prefs(ref, recipe_bundle): ConanOutput().warning("PackageLists.prefs() non-public, non-documented method will be " diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 23502b647d5..b897eb7c63a 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -139,7 +139,7 @@ def save(self, package_list, tgz_path, no_source=False): ref_layout = cache.recipe_layout(ref) recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable - ref_bundle = package_list.recipe_info(ref) + ref_bundle = package_list.recipe_dict(ref) ref_bundle["recipe_folder"] = recipe_folder out.info(f"Saving {ref}: {recipe_folder}") # Package only selected folders, not DOWNLOAD one @@ -153,19 +153,20 @@ def save(self, package_list, tgz_path, no_source=False): if os.path.exists(path): tar_files[f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}"] = path - for pref, pref_bundle in packages.items(): + for pref in packages: pref_layout = cache.pkg_layout(pref) pkg_folder = pref_layout.package() folder = os.path.relpath(pkg_folder, cache_folder) folder = folder.replace("\\", "/") # make win paths portable - pref_bundle["package_folder"] = folder + pkg_dict = package_list.package_dict(pref) + pkg_dict["package_folder"] = folder out.info(f"Saving {pref}: {folder}") tar_files[folder] = os.path.join(cache_folder, folder) if os.path.exists(pref_layout.metadata()): metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder) metadata_folder = metadata_folder.replace("\\", "/") # make paths portable - pref_bundle["metadata_folder"] = metadata_folder + pkg_dict["metadata_folder"] = metadata_folder out.info(f"Saving {pref} metadata: {metadata_folder}") tar_files[metadata_folder] = os.path.join(cache_folder, metadata_folder) @@ -196,7 +197,7 @@ def restore(self, path): out = ConanOutput() package_list = PackagesList.deserialize(json.loads(pkglist)) for ref, packages in package_list.items(): - ref_bundle = package_list.recipe_info(ref) + ref_bundle = package_list.recipe_dict(ref) ref.timestamp = revision_timestamp_now() ref_bundle["timestamp"] = ref.timestamp try: @@ -209,8 +210,9 @@ def restore(self, path): # In the case of recipes, they are always "in place", so just checking it assert rel_path == recipe_folder, f"{rel_path}!={recipe_folder}" out.info(f"Restore: {ref} in {recipe_folder}") - for pref, pref_bundle in packages.items(): + for pref in packages: pref.timestamp = revision_timestamp_now() + pref_bundle = package_list.package_dict(pref) pref_bundle["timestamp"] = pref.timestamp try: pkg_layout = cache.pkg_layout(pref) diff --git a/conan/api/subapi/download.py b/conan/api/subapi/download.py index 96d39109522..849e52f68fa 100644 --- a/conan/api/subapi/download.py +++ b/conan/api/subapi/download.py @@ -85,11 +85,12 @@ def download_full(self, package_list: PackagesList, remote: Remote, def _download_pkglist(pkglist): for ref, packages in pkglist.items(): self.recipe(ref, remote, metadata) - ref_dict = pkglist.recipe_info(ref) + ref_dict = pkglist.recipe_dict(ref) ref_dict.pop("files", None) ref_dict.pop("upload-urls", None) - for pref, pkg_dict in packages.items(): + for pref in packages: self.package(pref, remote, metadata) + pkg_dict = pkglist.package_dict(pref) pkg_dict.pop("files", None) pkg_dict.pop("upload-urls", None) diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index a8acacffb81..7927753144c 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -203,7 +203,8 @@ def select(self, pattern: ListPattern, package_query=None, remote: Remote = None # Show only the recipe references if pattern.package_id is None and pattern.rrev is None: - select_bundle.add_refs(refs) + for r in refs: + select_bundle.add_ref(r) return select_bundle def msg_format(msg, item, total): @@ -225,7 +226,8 @@ def msg_format(msg, item, total): if lru and pattern.package_id is None: # Filter LRUs rrevs = [r for r in rrevs if app.cache.get_recipe_lru(r) < limit_time] - select_bundle.add_refs(rrevs) + for rr in rrevs: + select_bundle.add_ref(rr) if pattern.package_id is None: # Stop if not displaying binaries continue @@ -266,8 +268,11 @@ def msg_format(msg, item, total): if lru: # Filter LRUs prefs = [r for r in prefs if app.cache.get_package_lru(r) < limit_time] - select_bundle.add_prefs(rrev, prefs) - select_bundle.add_configurations(packages) + for p in prefs: + select_bundle.add_pref(p) + conf = packages.get(p) + if conf: + select_bundle.add_configuration(p, conf) return select_bundle def explain_missing_binaries(self, ref, conaninfo, remotes): @@ -295,7 +300,7 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): candidates.sort() pkglist = PackagesList() - pkglist.add_refs([ref]) + pkglist.add_ref(ref) # Return the closest matches, stop adding when distance is increased candidate_distance = None for candidate in candidates: @@ -303,10 +308,10 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): break candidate_distance = candidate.distance pref = candidate.pref - pkglist.add_prefs(ref, [pref]) - pkglist.add_configurations({pref: candidate.binary_config}) + pkglist.add_pref(pref) + pkglist.add_configuration(pref, candidate.binary_config) # Add the diff data - rev_dict = pkglist.recipe_info(ref) + rev_dict = pkglist.recipe_dict(ref) rev_dict["packages"][pref.package_id]["diff"] = candidate.serialize() remote = candidate.remote.name if candidate.remote else "Local Cache" rev_dict["packages"][pref.package_id]["remote"] = remote @@ -328,8 +333,8 @@ def find_remotes(self, package_list, remotes): continue if ref not in revs: # not found continue - result_pkg_list.add_refs([ref]) - for pref, pref_bundle in packages.items(): + result_pkg_list.add_ref(ref) + for pref, pkg_info in packages.items(): pref_no_rev = copy.copy(pref) # TODO: Improve ugly API pref_no_rev.revision = None try: @@ -337,10 +342,8 @@ def find_remotes(self, package_list, remotes): except NotFoundException: continue if pref in prevs: - result_pkg_list.add_prefs(ref, [pref]) - recipe_bundle = package_list.recipe_info(ref) - info = recipe_bundle["packages"][pref.package_id]["info"] - result_pkg_list.add_configurations({pref: info}) + result_pkg_list.add_pref(pref) + result_pkg_list.add_configuration(pref, pkg_info) if result_pkg_list: result.add(r.name, result_pkg_list) return result diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 8b7aa6c1a43..4b1b1b6eeaa 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -45,7 +45,7 @@ def check_upstream(self, package_list: PackagesList, remote: Remote, enabled_rem if conanfile.upload_policy == "skip": ConanOutput().info(f"{ref}: Skipping upload of binaries, " "because upload_policy='skip'") - package_list.recipe_info(ref)["packages"] = {} + package_list.recipe_dict(ref)["packages"] = {} UploadUpstreamChecker(app).check(package_list, remote, force) diff --git a/conan/cli/commands/export.py b/conan/cli/commands/export.py index c052fe1cd4a..75d8961951f 100644 --- a/conan/cli/commands/export.py +++ b/conan/cli/commands/export.py @@ -58,7 +58,7 @@ def export(conan_api, parser, *args): conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) exported_list = PackagesList() - exported_list.add_refs([ref]) + exported_list.add_ref(ref) pkglist = MultiPackagesList() pkglist.add("Local Cache", exported_list) diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 8451a3c544d..6954a83584e 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -1,6 +1,5 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, MultiPackagesList, RecipeReference, PkgReference, \ - PackagesList +from conan.api.model import ListPattern, MultiPackagesList, PackagesList from conan.api.output import cli_out_write, ConanOutput from conan.api.input import UserInput from conan.cli import make_abs_path @@ -93,37 +92,31 @@ def confirmation(message): multi_package_list = MultiPackagesList() multi_package_list.add(cache_name, package_list) - result = {} - for ref, ref_info in package_list.serialize().items(): - result_ref = {} - for rrev, rrev_info in ref_info.get("revisions", {}).items(): - full_ref = RecipeReference.loads(ref) - full_ref.revision = rrev - packages = rrev_info.get("packages") - if packages is None: - if confirmation(f"Remove the recipe and all the packages of '{ref}#{rrev}'?"): + result = PackagesList() + for ref, packages in package_list.items(): + ref_dict = package_list.recipe_dict(ref).copy() + packages_dict = ref_dict.pop("packages", None) + if packages_dict is None: + if confirmation(f"Remove the recipe and all the packages of '{ref.repr_notime()}'?"): + if not args.dry_run: + conan_api.remove.recipe(ref, remote=remote) + result.add_ref(ref) + result.recipe_dict(ref).update(ref_dict) # it doesn't contain "packages" + else: + if not packages: # weird, there is inner package-ids but without prevs + ConanOutput().info(f"No binaries to remove for '{ref.repr_notime()}'") + continue + for pref, pkg_id_info in packages.items(): + if confirmation(f"Remove the package '{pref.repr_notime()}'?"): if not args.dry_run: - conan_api.remove.recipe(full_ref, remote=remote) - result_ref.setdefault("revisions", {})[rrev] = rrev_info - else: - result_rrev = {} - for pkg_id, pkg_id_info in packages.items(): - package_revisions = pkg_id_info.get("revisions") - if package_revisions is None: - ConanOutput().info(f"No binaries to remove for '{full_ref.repr_notime()}'") - continue - for prev, prev_info in package_revisions.items(): - if confirmation(f"Remove the package '{ref}#{rrev}:{pkg_id}#{prev}'?"): - if not args.dry_run: - pref = PkgReference(full_ref, pkg_id, prev) - conan_api.remove.package(pref, remote=remote) - result_rrev.setdefault("packages", {})[pkg_id] = pkg_id_info - if result_rrev: - result_ref.setdefault("revisions", {})[rrev] = result_rrev - if result_ref: - result[ref] = result_ref - package_list = PackagesList.deserialize(result) - multi_package_list.add(cache_name, package_list) + conan_api.remove.package(pref, remote=remote) + result.add_ref(ref) + result.recipe_dict(ref).update(ref_dict) # it doesn't contain "packages" + result.add_pref(pref) + result.add_configuration(pref, pkg_id_info) + pkg_dict = package_list.package_dict(pref) + result.package_dict(pref).update(pkg_dict) + multi_package_list.add(cache_name, result) return { "results": multi_package_list.serialize(), diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 2810e090e54..91a2fcd692b 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -121,24 +121,18 @@ def upload(conan_api: ConanAPI, parser, *args): def _ask_confirm_upload(conan_api, package_list): ui = UserInput(conan_api.config.get("core:non_interactive")) - result = {} - for ref, ref_info in package_list.serialize().items(): - result_ref = {} - for rrev, rrev_info in ref_info.get("revisions", {}).items(): - msg = f"Are you sure you want to upload recipe '{ref}#{rrev}'?" - if ui.request_boolean(msg): - result_rrev = {} - if rrev_info.get("timestamp"): - result_rrev["timestamp"] = rrev_info["timestamp"] - for pkg_id, pkg_id_info in rrev_info.get("packages", {}).items(): - for prev, prev_info in pkg_id_info.get("revisions", {}).items(): - msg = (f"Are you sure you want to upload package " - f"'{ref}#{rrev}:{pkg_id}#{prev}'?") - if ui.request_boolean(msg): - pkg_info = result_rrev.setdefault("packages", {}).setdefault(pkg_id, {}) - pkg_info.setdefault("revisions", {})[prev] = prev_info - pkg_info["info"] = pkg_id_info["info"] - result_ref.setdefault("revisions", {})[rrev] = result_rrev - result[ref] = result_ref - package_list = PackagesList.deserialize(result) - return package_list + result = PackagesList() + for ref, packages in package_list.items(): + msg = f"Are you sure you want to upload recipe '{ref.repr_notime()}'?" + if ui.request_boolean(msg): + result.add_ref(ref) + ref_dict = package_list.recipe_dict(ref).copy() + ref_dict.pop("packages", None) + result.recipe_dict(ref).update(ref_dict) + for pref, pkg_id_info in packages.items(): + msg = f"Are you sure you want to upload package '{pref.repr_notime()}'?" + if ui.request_boolean(msg): + result.add_pref(pref) + result.add_configuration(pref, pkg_id_info) + result.package_dict(pref).update(package_list.package_dict(pref)) + return result diff --git a/conan/internal/api/upload.py b/conan/internal/api/upload.py index 033e00b892b..c127204cb1d 100644 --- a/conan/internal/api/upload.py +++ b/conan/internal/api/upload.py @@ -5,12 +5,13 @@ def add_urls(package_list, remote): router = ClientV2Router(remote.url.rstrip("/")) for ref, packages in package_list.items(): - ref_info = package_list.recipe_info(ref) + ref_info = package_list.recipe_dict(ref) for f, fp in ref_info.get("files", {}).items(): ref_info.setdefault("upload-urls", {})[f] = { 'url': router.recipe_file(ref, f), 'checksum': sha1sum(fp) } - for pref, pref_info in packages.items(): + for pref in packages: + pref_info = package_list.package_dict(pref) for f, fp in pref_info.get("files", {}).items(): pref_info.setdefault("upload-urls", {})[f] = { 'url': router.package_file(pref, f), 'checksum': sha1sum(fp) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 81bdedbbe14..6ebb4d54ba7 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -30,10 +30,11 @@ def __init__(self, app: ConanApp): def check(self, package_list, remote, force): for ref, packages in package_list.items(): - recipe_info = package_list.recipe_info(ref) + recipe_info = package_list.recipe_dict(ref) self._check_upstream_recipe(ref, recipe_info, remote, force) - for pref, prev_bundle in packages.items(): - self._check_upstream_package(pref, prev_bundle, remote, force) + for pref in packages: + pkg_dict = package_list.package_dict(pref) + self._check_upstream_package(pref, pkg_dict, remote, force) def _check_upstream_recipe(self, ref, ref_bundle, remote, force): output = ConanOutput(scope=str(ref)) @@ -98,12 +99,13 @@ def prepare(self, pkg_list, enabled_remotes): "Failing because conf 'core.scm:local_url!=allow'") # Just in case it was defined from a previous run - bundle = pkg_list.recipe_info(ref) + bundle = pkg_list.recipe_dict(ref) bundle.pop("files", None) bundle.pop("upload-urls", None) if bundle.get("upload"): self._prepare_recipe(ref, bundle, conanfile, enabled_remotes) - for pref, prev_bundle in packages.items(): + for pref in packages: + prev_bundle = pkg_list.package_dict(pref) prev_bundle.pop("files", None) # If defined from a previous upload prev_bundle.pop("upload-urls", None) if prev_bundle.get("upload"): @@ -230,10 +232,11 @@ def __init__(self, app: ConanApp): def upload(self, upload_data, remote): for ref, packages in upload_data.items(): - bundle = upload_data.recipe_info(ref) + bundle = upload_data.recipe_dict(ref) if bundle.get("upload"): self.upload_recipe(ref, bundle, remote) - for pref, prev_bundle in packages.items(): + for pref in packages: + prev_bundle = upload_data.package_dict(pref) if prev_bundle.get("upload"): self.upload_package(pref, prev_bundle, remote) @@ -321,7 +324,7 @@ def _metadata_files(folder, metadata): def gather_metadata(package_list, cache, metadata): for rref, packages in package_list.items(): - recipe_bundle = package_list.recipe_info(rref) + recipe_bundle = package_list.recipe_dict(rref) if metadata or recipe_bundle["upload"]: metadata_folder = cache.recipe_layout(rref).metadata() files = _metadata_files(metadata_folder, metadata) @@ -330,7 +333,8 @@ def gather_metadata(package_list, cache, metadata): recipe_bundle.setdefault("files", {}).update(files) recipe_bundle["upload"] = True - for pref, pkg_bundle in packages.items(): + for pref in packages: + pkg_bundle = package_list.package_dict(pref) if metadata or pkg_bundle["upload"]: metadata_folder = cache.pkg_layout(pref).metadata() files = _metadata_files(metadata_folder, metadata) diff --git a/conan/internal/rest/download_cache.py b/conan/internal/rest/download_cache.py index ee4fa4958c8..5e366084470 100644 --- a/conan/internal/rest/download_cache.py +++ b/conan/internal/rest/download_cache.py @@ -73,8 +73,9 @@ def has_excluded_urls(backup_urls): all_refs = set() if package_list is not None: for ref, packages in package_list.items(): - ref_info = package_list.recipe_info(ref) - if not only_upload or ref_info.get("upload") or any(p.get("upload") for p in packages.values()): + ref_info = package_list.recipe_dict(ref) + if (not only_upload or ref_info.get("upload") + or any(package_list.package_dict(p).get("upload") for p in packages)): all_refs.add(str(ref)) path_backups_contents = [] diff --git a/conan/internal/rest/pkg_sign.py b/conan/internal/rest/pkg_sign.py index 9169839d560..96d1eb1072a 100644 --- a/conan/internal/rest/pkg_sign.py +++ b/conan/internal/rest/pkg_sign.py @@ -30,10 +30,11 @@ def _sign(ref, files, folder): files[f"{METADATA}/sign/{f}"] = os.path.join(metadata_sign, f) for rref, packages in upload_data.items(): - recipe_bundle = upload_data.recipe_info(rref) + recipe_bundle = upload_data.recipe_dict(rref) if recipe_bundle["upload"]: _sign(rref, recipe_bundle["files"], self._cache.recipe_layout(rref).download_export()) - for pref, pkg_bundle in packages.items(): + for pref in packages: + pkg_bundle = upload_data.package_dict(pref) if pkg_bundle["upload"]: _sign(pref, pkg_bundle["files"], self._cache.pkg_layout(pref).download_package()) From de798733a1f165791a5a2b1aff48576801b4f83e Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 14 Sep 2025 19:20:25 +0200 Subject: [PATCH 13/22] wip --- conan/api/model/list.py | 28 +++++++++++----------------- conan/api/subapi/list.py | 13 +++++-------- conan/cli/commands/remove.py | 3 +-- conan/cli/commands/upload.py | 3 +-- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index b5f9c25b19d..be302c0cda8 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -187,8 +187,7 @@ def _define_graph(graph, graph_recipes=None, graph_binaries=None, context=None): binary = binary.lower() if any(b == "*" or b == binary for b in binaries): cache_list.add_ref(ref) # Binary listed forces recipe listed - cache_list.add_pref(pref) - cache_list.add_configuration(pref, node["info"]) + cache_list.add_pref(pref, node["info"]) return pkglist @@ -265,7 +264,7 @@ def add_prefs(self, rrev, prefs): for p in prefs: self.add_pref(p) - def add_pref(self, pref: PkgReference) -> None: + def add_pref(self, pref: PkgReference, pkg_info: dict = None) -> None: """ Add a PkgReferene to an already existing RecipeReference inside a package list """ @@ -278,24 +277,19 @@ def add_pref(self, pref: PkgReference) -> None: prev_dict = prevs_dict.setdefault(pref.revision, {}) if pref.timestamp: prev_dict["timestamp"] = pref.timestamp + if pkg_info is not None: + package_dict["info"] = pkg_info def add_configurations(self, confs): ConanOutput().warning("PackageLists.add_configurations() non-public, non-documented method " - "will be removed, use .add_configuration() instead", + "will be removed, use .add_pref() instead", warn_tag="deprecated") - for k, v in confs.items(): - self.add_configuration(k, v) - - def add_configuration(self, pref: PkgReference, conf: dict) -> None: - """ - Add the configuration information for the binary for an already existing PkgReference - in the package list - """ - rev_dict = self.recipe_dict(pref.ref) - try: - rev_dict["packages"][pref.package_id]["info"] = conf - except KeyError: # If package_id does not exist, do nothing, only add to existing prefs - pass + for pref, conf in confs.items(): + rev_dict = self.recipe_dict(pref.ref) + try: + rev_dict["packages"][pref.package_id]["info"] = conf + except KeyError: # If package_id does not exist, do nothing, only add to existing prefs + pass def refs(self): ConanOutput().warning("PackageLists.refs() non-public, non-documented method will be " diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index 7927753144c..d304e205492 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -268,11 +268,10 @@ def msg_format(msg, item, total): if lru: # Filter LRUs prefs = [r for r in prefs if app.cache.get_package_lru(r) < limit_time] + select_bundle.recipe_dict(rrev)["packages"] = {} for p in prefs: - select_bundle.add_pref(p) - conf = packages.get(p) - if conf: - select_bundle.add_configuration(p, conf) + pkg_info = packages.get(PkgReference(p.ref, p.package_id)) + select_bundle.add_pref(p, pkg_info) return select_bundle def explain_missing_binaries(self, ref, conaninfo, remotes): @@ -308,8 +307,7 @@ def explain_missing_binaries(self, ref, conaninfo, remotes): break candidate_distance = candidate.distance pref = candidate.pref - pkglist.add_pref(pref) - pkglist.add_configuration(pref, candidate.binary_config) + pkglist.add_pref(pref, candidate.binary_config) # Add the diff data rev_dict = pkglist.recipe_dict(ref) rev_dict["packages"][pref.package_id]["diff"] = candidate.serialize() @@ -342,8 +340,7 @@ def find_remotes(self, package_list, remotes): except NotFoundException: continue if pref in prevs: - result_pkg_list.add_pref(pref) - result_pkg_list.add_configuration(pref, pkg_info) + result_pkg_list.add_pref(pref, pkg_info) if result_pkg_list: result.add(r.name, result_pkg_list) return result diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 6954a83584e..1a775a87ca8 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -112,8 +112,7 @@ def confirmation(message): conan_api.remove.package(pref, remote=remote) result.add_ref(ref) result.recipe_dict(ref).update(ref_dict) # it doesn't contain "packages" - result.add_pref(pref) - result.add_configuration(pref, pkg_id_info) + result.add_pref(pref, pkg_id_info) pkg_dict = package_list.package_dict(pref) result.package_dict(pref).update(pkg_dict) multi_package_list.add(cache_name, result) diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 91a2fcd692b..3f9d871add2 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -132,7 +132,6 @@ def _ask_confirm_upload(conan_api, package_list): for pref, pkg_id_info in packages.items(): msg = f"Are you sure you want to upload package '{pref.repr_notime()}'?" if ui.request_boolean(msg): - result.add_pref(pref) - result.add_configuration(pref, pkg_id_info) + result.add_pref(pref, pkg_id_info) result.package_dict(pref).update(package_list.package_dict(pref)) return result From 3540999990f5bf2b4c61d676fb782c4f66da0ab9 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 14 Sep 2025 19:22:38 +0200 Subject: [PATCH 14/22] wip --- conan/api/subapi/list.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index d304e205492..6040357a814 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -268,8 +268,10 @@ def msg_format(msg, item, total): if lru: # Filter LRUs prefs = [r for r in prefs if app.cache.get_package_lru(r) < limit_time] + # Packages dict has been listed, even if empty select_bundle.recipe_dict(rrev)["packages"] = {} for p in prefs: + # the "packages" dict is not using the package-revision pkg_info = packages.get(PkgReference(p.ref, p.package_id)) select_bundle.add_pref(p, pkg_info) return select_bundle From b2cfd62953c99a2863914e1e113aea2ea643b5e4 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 14 Sep 2025 19:27:56 +0200 Subject: [PATCH 15/22] wip --- conan/api/subapi/cache.py | 2 +- conan/internal/cache/integrity_check.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b897eb7c63a..2aca229a799 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -116,7 +116,7 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, rmdir(ref_layout.source()) if download: rmdir(ref_layout.download_export()) - for pref, _ in packages.items(): + for pref in packages: ConanOutput(pref).verbose("Cleaning package cache contents") pref_layout = cache.pkg_layout(pref) if build: diff --git a/conan/internal/cache/integrity_check.py b/conan/internal/cache/integrity_check.py index 4a400376e60..e335cb0b3a1 100644 --- a/conan/internal/cache/integrity_check.py +++ b/conan/internal/cache/integrity_check.py @@ -24,7 +24,7 @@ def check(self, pkg_list): corrupted = False for ref, packages in pkg_list.items(): corrupted = self._recipe_corrupted(ref) or corrupted - for pref, _ in packages.items(): + for pref in packages: corrupted = self._package_corrupted(pref) or corrupted if corrupted: raise ConanException("There are corrupted artifacts, check the error logs") From 71e45b93010d8e185649453b8ded8531cbf96dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Wed, 17 Sep 2025 14:19:26 +0200 Subject: [PATCH 16/22] Some last changes to the decoumentation --- conan/api/model/list.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index be302c0cda8..ca21429a13d 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -3,6 +3,7 @@ import json import os from json import JSONDecodeError +from typing import Iterable from conan.api.model import RecipeReference, PkgReference from conan.api.output import ConanOutput @@ -197,6 +198,7 @@ def __init__(self): self._data = {} def __bool__(self): + """ Whether the package list contains any recipe""" return bool(self._data) def merge(self, other): @@ -221,7 +223,7 @@ def keep_outer(self, other): def split(self): """ - Returns a list of PackageList, splitted one per reference. + Returns a list of PackageList, split one per reference. This can be useful to parallelize things like upload, parallelizing per-reference """ result = [] @@ -266,7 +268,7 @@ def add_prefs(self, rrev, prefs): def add_pref(self, pref: PkgReference, pkg_info: dict = None) -> None: """ - Add a PkgReferene to an already existing RecipeReference inside a package list + Add a PkgReference to an already existing RecipeReference inside a package list """ # Prevs already come in ASCENDING order, so upload does older revisions first rev_dict = self.recipe_dict(pref.ref) @@ -304,12 +306,12 @@ def refs(self): result[recipe] = rrev_dict return result - def items(self): + def items(self) -> Iterable[tuple[RecipeReference, dict[PkgReference, dict]]]: """ Iterate the contents of the package list. - Every iteration returns [RecipeReference, dict, dict] + The first dictionary is the information directly belonging to the recipe-revision. - The second dictionary contains PkgReference as keys, and a dictionariy with the values - belonging to that specific package reference. + The second dictionary contains PkgReference as keys, and a dictionary with the values + belonging to that specific package reference (settings, options, etc.). """ for ref, ref_dict in self._data.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): @@ -327,13 +329,13 @@ def items(self): yield recipe, packages def recipe_dict(self, ref: RecipeReference): - """ gives read/write access to the dictionary containing a specific RecipeReference - information + """ Gives read/write access to the dictionary containing a specific RecipeReference + information. """ return self._data[str(ref)]["revisions"][ref.revision] def package_dict(self, pref: PkgReference): - """ gives read/write access to the dictionary containing a specific PkgReference + """ Gives read/write access to the dictionary containing a specific PkgReference information """ ref_dict = self.recipe_dict(pref.ref) From 9726b404f79bf7e5405e6c246960ec96269aae44 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 17 Sep 2025 15:42:43 +0200 Subject: [PATCH 17/22] Update conan/api/model/list.py Co-authored-by: Carlos Zoido --- conan/api/model/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index ca21429a13d..ef3b0b5a1ed 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -260,7 +260,7 @@ def add_ref(self, ref: RecipeReference) -> None: rev_dict["timestamp"] = ref.timestamp def add_prefs(self, rrev, prefs): - ConanOutput().warning("PackageLists.add_prrefs() non-public, non-documented method will be " + ConanOutput().warning("PackageLists.add_prefs() non-public, non-documented method will be " "removed, use .add_pref() instead", warn_tag="deprecated") # Prevs already come in ASCENDING order, so upload does older revisions first for p in prefs: From e8c6222b6008d5ecebb286e0e170fac145667060 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 17 Sep 2025 15:42:57 +0200 Subject: [PATCH 18/22] Update conan/api/model/list.py Co-authored-by: Carlos Zoido --- conan/api/model/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index ef3b0b5a1ed..ea3ac793988 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -250,7 +250,7 @@ def add_refs(self, refs): def add_ref(self, ref: RecipeReference) -> None: """ - Adds a new RecipeReference to a pacakge list + Adds a new RecipeReference to a package list """ ref_dict = self._data.setdefault(str(ref), {}) if ref.revision: From 4704ec75507f366511d970b56fe063981dbabe83 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 17 Sep 2025 15:43:12 +0200 Subject: [PATCH 19/22] Update test/integration/command/upload/test_upload_bundle.py Co-authored-by: Carlos Zoido --- test/integration/command/upload/test_upload_bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/command/upload/test_upload_bundle.py b/test/integration/command/upload/test_upload_bundle.py index 66f55a9bded..e7effa3b539 100644 --- a/test/integration/command/upload/test_upload_bundle.py +++ b/test/integration/command/upload/test_upload_bundle.py @@ -21,7 +21,7 @@ def test_upload_pkg_list(): @conan_command(group="custom commands") def upload_pkglist(conan_api, parser, *args, **kwargs): \""" - create an upload piglist + create an upload pkglist \""" parser.add_argument('reference', help="Recipe reference or package reference, can contain * as " From 8011560e66486b3b0186e3366b444ef7cc27d49c Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 Sep 2025 11:59:07 +0200 Subject: [PATCH 20/22] review, normalized package list name --- conan/api/model/list.py | 2 +- conan/cli/commands/list.py | 2 +- conan/cli/commands/remove.py | 2 +- conan/cli/commands/upload.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conan/api/model/list.py b/conan/api/model/list.py index ea3ac793988..b3430c94e08 100644 --- a/conan/api/model/list.py +++ b/conan/api/model/list.py @@ -242,7 +242,7 @@ def only_recipes(self) -> None: rrev_dict.pop("packages", None) def add_refs(self, refs): - ConanOutput().warning("PackageLists.add_refs() non-public, non-documented method will be " + ConanOutput().warning("PackagesLists.add_refs() non-public, non-documented method will be " "removed, use .add_ref() instead", warn_tag="deprecated") # RREVS alreday come in ASCENDING order, so upload does older revisions first for ref in refs: diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index fba9bfb12a7..1e7fcbc7d7f 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -57,7 +57,7 @@ def print_serial(item, indent=None, color_index=None): def print_list_text(results): """ Do a little format modification to serialized - package-list, so it looks prettier on text output + package list, so it looks prettier on text output """ info = results["results"] diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 1a775a87ca8..20350d825be 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -10,7 +10,7 @@ def summary_remove_list(results): """ Do a little format modification to serialized - package-list so it looks prettier on text output + package list so it looks prettier on text output """ cli_out_write("Remove summary:") info = results["results"] diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 3f9d871add2..8211d9bc53f 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -10,7 +10,7 @@ def summary_upload_list(results): """ Do a little format modification to serialized - list bundle, so it looks prettier on text output + package list, so it looks prettier on text output """ ConanOutput().subtitle("Upload summary") info = results["results"] From 843ef221419bd606ffdb22ad288a55605ac10fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= <5364255+AbrilRBS@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:36:21 +0200 Subject: [PATCH 21/22] Remove last pkglist in help strings --- conan/cli/commands/audit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conan/cli/commands/audit.py b/conan/cli/commands/audit.py index 9bbe1fff558..8f689f19a00 100644 --- a/conan/cli/commands/audit.py +++ b/conan/cli/commands/audit.py @@ -111,9 +111,9 @@ def audit_list(conan_api: ConanAPI, parser, subparser, *args): """ input_group = subparser.add_mutually_exclusive_group(required=True) input_group.add_argument("reference", help="Reference to list vulnerabilities for", nargs="?") - input_group.add_argument("-l", "--list", help="pkglist file to list vulnerabilities for") - input_group.add_argument("-s", "--sbom", help="sbom file to list vulnerabilities for") - input_group.add_argument("-lock", "--lockfile", help="lockfile file to list vulnerabilities for") + input_group.add_argument("-l", "--list", help="Package list file to list vulnerabilities for") + input_group.add_argument("-s", "--sbom", help="SBOM file to list vulnerabilities for") + input_group.add_argument("-lock", "--lockfile", help="Lockfile file to list vulnerabilities for") subparser.add_argument("-r", "--remote", help="Remote to use for listing") _add_provider_arg(subparser) args = parser.parse_args(*args) From 75c083c507c68a5224b93f57d73b064343e47350 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 22 Sep 2025 14:02:43 +0200 Subject: [PATCH 22/22] Update conan/cli/commands/audit.py --- conan/cli/commands/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/cli/commands/audit.py b/conan/cli/commands/audit.py index 8f689f19a00..2ceb8d99c1e 100644 --- a/conan/cli/commands/audit.py +++ b/conan/cli/commands/audit.py @@ -113,7 +113,7 @@ def audit_list(conan_api: ConanAPI, parser, subparser, *args): input_group.add_argument("reference", help="Reference to list vulnerabilities for", nargs="?") input_group.add_argument("-l", "--list", help="Package list file to list vulnerabilities for") input_group.add_argument("-s", "--sbom", help="SBOM file to list vulnerabilities for") - input_group.add_argument("-lock", "--lockfile", help="Lockfile file to list vulnerabilities for") + input_group.add_argument("-lock", "--lockfile", help="Path to the lockfile to check for vulnerabilities") subparser.add_argument("-r", "--remote", help="Remote to use for listing") _add_provider_arg(subparser) args = parser.parse_args(*args)