From 4cd281112cdd16cf4f5ec80d23ac9bb12f6a580e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 10:55:31 +0200 Subject: [PATCH 01/24] Proof of concept --- conan/api/subapi/cache.py | 15 ++-- conan/internal/api/migrations.py | 2 + conan/internal/api/uploader.py | 28 ++------ conan/internal/cache/home_paths.py | 4 ++ conan/internal/rest/remote_manager.py | 14 ++-- conan/internal/util/compression.py | 69 +++++++++++++++++++ conan/internal/util/files.py | 11 --- test/unittests/util/files/tar_extract_test.py | 6 +- 8 files changed, 97 insertions(+), 52 deletions(-) create mode 100644 conan/internal/util/compression.py diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b0d3ede2d10..ffa05bf3b66 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -18,6 +18,7 @@ from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove +from conan.internal.util.compression import tar_extract class CacheAPI: @@ -133,6 +134,8 @@ def save(self, package_list, tgz_path, no_source=False): mkdir(os.path.dirname(tgz_path)) name = os.path.basename(tgz_path) compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) + # compress_fn = self._load_compress_plugin() + with open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) @@ -182,14 +185,10 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - - with open(path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - fileobj = the_tar.extractfile("pkglist.json") - pkglist = fileobj.read() - the_tar.extraction_filter = (lambda member, _: member) # fully_trusted (Py 3.14) - the_tar.extractall(path=cache_folder) - the_tar.close() + tar_extract(self.conan_api.cache_folder, path, cache_folder) + # Retrieve the package list from the already extracted archive + with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: + pkglist = file_handler.read() # After unzipping the files, we need to update the DB that references these files out = ConanOutput() diff --git a/conan/internal/api/migrations.py b/conan/internal/api/migrations.py index fc54424e940..da3faf4d3cd 100644 --- a/conan/internal/api/migrations.py +++ b/conan/internal/api/migrations.py @@ -4,6 +4,7 @@ from conan.api.output import ConanOutput from conan.internal.default_settings import migrate_settings_file +from conan.internal.util.compression import migrate_compression_plugin from conans.migrations import Migrator from conan.internal.util.dates import timestamp_now from conan.internal.util.files import load, save @@ -51,6 +52,7 @@ def _apply_migrations(self, old_version): # Update profile plugin from conan.internal.api.profile.profile_loader import migrate_profile_plugin migrate_profile_plugin(self.cache_folder) + migrate_compression_plugin(self.cache_folder) if old_version and old_version < "2.0.14-": _migrate_pkg_db_lru(self.cache_folder, old_version) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index b075e4f07c7..f8921fa1c5e 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,8 +12,9 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) +from conan.internal.util.compression import tar_compress from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, - set_dirty_context_manager, mkdir, human_size) + mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -156,8 +157,8 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref) + tgz = tar_compress(self._app.cache_folder, tgz_files, tgz_name, download_export_folder, + compresslevel=compresslevel, ref=ref) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -203,8 +204,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref) + tgz_path = tar_compress(self._app.cache_folder, tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, + compresslevel=compresslevel, ref=pref) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -271,23 +272,6 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None): - t1 = time.time() - # FIXME, better write to disk sequentially and not keep tgz contents in memory - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref)).info(f"Compressing {name}") - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) - for filename, abs_path in sorted(files.items()): - # recursive is False in case it is a symlink to a folder - tgz.add(abs_path, filename, recursive=False) - tgz.close() - - duration = time.time() - t1 - ConanOutput().debug(f"{name} compressed in {duration} time") - return tgz_path - - def _total_size(cache_files): total_size = 0 for file in cache_files.values(): diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index 5674fbba145..5d27bfd5a0e 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -90,3 +90,7 @@ def settings_path_user(self): @property def config_version_path(self): return os.path.join(self._home, "config_version.json") + + @property + def compression_plugin_path(self): + return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py") diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index d87f3d849bb..eeaf5e939b7 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -17,7 +17,8 @@ from conan.api.model import RecipeReference from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME -from conan.internal.util.files import mkdir, tar_extract +from conan.internal.util.files import mkdir +from conan.internal.util.compression import tar_extract class RemoteManager: @@ -81,7 +82,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref)) + uncompress_file(self._home_folder, tgz_file, export_folder, scope=str(ref)) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -123,7 +124,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref)) + uncompress_file(self._home_folder, tgz_file, export_sources_folder, scope=str(ref)) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -171,7 +172,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref)) + uncompress_file(self._home_folder, tgz_file, package_folder, scope=str(pref.ref)) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -281,15 +282,14 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None): +def uncompress_file(cache_folder, src_path, dest_folder, scope=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - with open(src_path, mode='rb') as file_handler: - tar_extract(file_handler, dest_folder) + tar_extract(cache_folder, src_path, dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py new file mode 100644 index 00000000000..046e32d1407 --- /dev/null +++ b/conan/internal/util/compression.py @@ -0,0 +1,69 @@ +import os +from conan.internal.cache.home_paths import HomePaths +from conan.internal.loader import load_python_file +from conan.internal.errors import ConanException + +def tar_extract(cache_folder, src_path, destination_dir): + compress_plugin = _load_compress_plugin(cache_folder) + compress_plugin.tar_extract(src_path, destination_dir) + +def tar_compress(cache_folder, files, name, dest_dir, compresslevel=None, ref=None): + compress_plugin = _load_compress_plugin(cache_folder) + return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) + +def _load_compress_plugin(cache_folder): + compression_plugin_path = HomePaths(cache_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + # TODO + raise ConanException("The 'compression.py' plugin file doesn't exist. If you want " + "to disable it, edit its contents instead of removing it") + + mod, _ = load_python_file(compression_plugin_path) + if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") + return mod + + +_default_compression_plugin = """\ +# This file was generated by Conan. Remove this comment if you edit this file or Conan +# will destroy your changes. + +import os +import time +import tarfile +from conan.api.output import ConanOutput + +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None): + t1 = time.time() + # FIXME, better write to disk sequentially and not keep tgz contents in memory + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref)).info(f"Compressing {name}") + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) + for filename, abs_path in sorted(files.items()): + # recursive is False in case it is a symlink to a folder + tgz.add(abs_path, filename, recursive=False) + tgz.close() + + duration = time.time() - t1 + ConanOutput().debug(f"{name} compressed in {duration} time") + return tgz_path + + +def tar_extract(src_path, destination_dir): + with open(src_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() +""" + +def migrate_compression_plugin(cache_folder): + from conan.internal.api.migrations import update_file + + profile_plugin_file = HomePaths(cache_folder).compression_plugin_path + update_file(profile_plugin_file, _default_compression_plugin) + diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 98746479ae0..01b9a76fe64 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -6,7 +6,6 @@ import shutil import stat import sys -import tarfile import time from contextlib import contextmanager @@ -256,16 +255,6 @@ def mkdir(path): os.makedirs(path) -def tar_extract(fileobj, destination_dir): - the_tar = tarfile.open(fileobj=fileobj) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() - - def merge_directories(src, dst): from conan.tools.files import copy copy(None, pattern="*", src=src, dst=dst) diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 6f031a56a13..9c71f5d07b8 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -50,13 +50,11 @@ def check_files(destination_dir): with chdir(working_dir): # Unpack and check destination_dir = os.path.join(self.tmp_folder, "dest") - with open(self.tgz_file, 'rb') as file_handler: - tar_extract(file_handler, destination_dir) + tar_extract(self.tgz_file, destination_dir) check_files(destination_dir) # Unpack and check (now we have a symlinked local folder) os.symlink(temp_folder(), "folder") destination_dir = os.path.join(self.tmp_folder, "dest2") - with open(self.tgz_file, 'rb') as file_handler: - tar_extract(file_handler, destination_dir) + tar_extract(self.tgz_file, destination_dir) check_files(destination_dir) From 156e036a882a3b1d7c084024eea67c603092fed0 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 11:44:29 +0200 Subject: [PATCH 02/24] Simplify --- conan/api/subapi/cache.py | 6 +- conan/internal/api/migrations.py | 2 - conan/internal/api/uploader.py | 25 ++---- conan/internal/rest/remote_manager.py | 10 +-- conan/internal/util/compression.py | 84 ++++++++++--------- conan/test/utils/test_files.py | 2 +- .../integration/command/upload/upload_test.py | 2 +- .../util/files/strip_root_extract_test.py | 2 +- test/unittests/util/files/tar_extract_test.py | 5 +- 9 files changed, 61 insertions(+), 77 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index ffa05bf3b66..b27b9c06565 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -6,7 +6,6 @@ from conan.api.model import PackagesList from conan.api.output import ConanOutput -from conan.internal.api.uploader import gzopen_without_timestamps from conan.internal.cache.cache import PkgCache from conan.internal.cache.conan_reference_layout import EXPORT_SRC_FOLDER, EXPORT_FOLDER, SRC_FOLDER, \ METADATA, DOWNLOAD_EXPORT_FOLDER @@ -18,7 +17,7 @@ from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove -from conan.internal.util.compression import tar_extract +from conan.internal.util.compression import gzopen_without_timestamps, tar_extract class CacheAPI: @@ -134,7 +133,6 @@ def save(self, package_list, tgz_path, no_source=False): mkdir(os.path.dirname(tgz_path)) name = os.path.basename(tgz_path) compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) - # compress_fn = self._load_compress_plugin() with open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, @@ -185,7 +183,7 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(self.conan_api.cache_folder, path, cache_folder) + tar_extract(path, cache_folder, cache_folder=self.conan_api.cache_folder) # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/internal/api/migrations.py b/conan/internal/api/migrations.py index da3faf4d3cd..fc54424e940 100644 --- a/conan/internal/api/migrations.py +++ b/conan/internal/api/migrations.py @@ -4,7 +4,6 @@ from conan.api.output import ConanOutput from conan.internal.default_settings import migrate_settings_file -from conan.internal.util.compression import migrate_compression_plugin from conans.migrations import Migrator from conan.internal.util.dates import timestamp_now from conan.internal.util.files import load, save @@ -52,7 +51,6 @@ def _apply_migrations(self, old_version): # Update profile plugin from conan.internal.api.profile.profile_loader import migrate_profile_plugin migrate_profile_plugin(self.cache_folder) - migrate_compression_plugin(self.cache_folder) if old_version and old_version < "2.0.14-": _migrate_pkg_db_lru(self.cache_folder, old_version) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index f8921fa1c5e..06cf59da330 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,9 +12,9 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.compression import tar_compress from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, mkdir, human_size) +from conan.internal.util.compression import tar_compress UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -157,8 +157,8 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = tar_compress(self._app.cache_folder, tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref) + tgz = tar_compress(tgz_files, tgz_name, download_export_folder, + compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -204,8 +204,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = tar_compress(self._app.cache_folder, tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref) + tgz_path = tar_compress(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, + compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -257,21 +257,6 @@ def upload_package(self, pref, prev_bundle, remote): output.debug(f"Upload {pref} in {duration} time") -def gzopen_without_timestamps(name, fileobj, compresslevel=None): - """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was - setted in Gzip file causing md5 to change. Not possible using the - previous tarfile open because arguments are not passed to GzipFile constructor - """ - compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 - fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) - # Format is forced because in Python3.8, it changed and it generates different tarfiles - # with different checksums, which break hashes of tgzs - # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions - t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) - t._extfileobj = False - return t - - def _total_size(cache_files): total_size = 0 for file in cache_files.values(): diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index eeaf5e939b7..9ebff0a07ab 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -82,7 +82,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(self._home_folder, tgz_file, export_folder, scope=str(ref)) + uncompress_file(tgz_file, export_folder, scope=str(ref), cache_folder=self._home_folder) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -124,7 +124,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(self._home_folder, tgz_file, export_sources_folder, scope=str(ref)) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), cache_folder=self._home_folder) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -172,7 +172,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(self._home_folder, tgz_file, package_folder, scope=str(pref.ref)) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), cache_folder=self._home_folder) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -282,14 +282,14 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(cache_folder, src_path, dest_folder, scope=None): +def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - tar_extract(cache_folder, src_path, dest_folder) + tar_extract(src_path, dest_folder, cache_folder=cache_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index 046e32d1407..d72bfa023e2 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -1,39 +1,34 @@ -import os from conan.internal.cache.home_paths import HomePaths from conan.internal.loader import load_python_file from conan.internal.errors import ConanException -def tar_extract(cache_folder, src_path, destination_dir): - compress_plugin = _load_compress_plugin(cache_folder) - compress_plugin.tar_extract(src_path, destination_dir) +import os +import gzip +import time +import tarfile +from conan.api.output import ConanOutput +from conan.internal.util.files import set_dirty_context_manager -def tar_compress(cache_folder, files, name, dest_dir, compresslevel=None, ref=None): +def tar_extract(src_path, destination_dir, cache_folder=None): compress_plugin = _load_compress_plugin(cache_folder) - return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) - -def _load_compress_plugin(cache_folder): - compression_plugin_path = HomePaths(cache_folder).compression_plugin_path - if not os.path.exists(compression_plugin_path): - # TODO - raise ConanException("The 'compression.py' plugin file doesn't exist. If you want " - "to disable it, edit its contents instead of removing it") - - mod, _ = load_python_file(compression_plugin_path) - if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): - raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - return mod + if compress_plugin: + return compress_plugin.tar_extract(src_path, destination_dir) + with open(src_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() -_default_compression_plugin = """\ -# This file was generated by Conan. Remove this comment if you edit this file or Conan -# will destroy your changes. -import os -import time -import tarfile -from conan.api.output import ConanOutput +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): + compress_plugin = _load_compress_plugin(cache_folder) + if compress_plugin: + return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) @@ -50,20 +45,29 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None): return tgz_path -def tar_extract(src_path, destination_dir): - with open(src_path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() -""" +def gzopen_without_timestamps(name, fileobj, compresslevel=None): + """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was + setted in Gzip file causing md5 to change. Not possible using the + previous tarfile open because arguments are not passed to GzipFile constructor + """ + compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 + fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) + # Format is forced because in Python3.8, it changed and it generates different tarfiles + # with different checksums, which break hashes of tgzs + # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions + t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) + t._extfileobj = False + return t -def migrate_compression_plugin(cache_folder): - from conan.internal.api.migrations import update_file - profile_plugin_file = HomePaths(cache_folder).compression_plugin_path - update_file(profile_plugin_file, _default_compression_plugin) +def _load_compress_plugin(cache_folder): + if not cache_folder: + return None + compression_plugin_path = HomePaths(cache_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + return None + mod, _ = load_python_file(compression_plugin_path) + if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") + return mod diff --git a/conan/test/utils/test_files.py b/conan/test/utils/test_files.py index 7b3f011116d..55b7c173fab 100644 --- a/conan/test/utils/test_files.py +++ b/conan/test/utils/test_files.py @@ -8,7 +8,7 @@ import time from io import BytesIO -from conan.internal.api.uploader import gzopen_without_timestamps +from conan.internal.util.compression import gzopen_without_timestamps from conan.tools.files.files import untargz from conan.internal.subsystems import get_cased_path from conan.errors import ConanException diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 63368208747..1bb1044d534 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -6,6 +6,7 @@ import unittest from collections import OrderedDict +from conan.internal.util.compression import gzopen_without_timestamps import pytest from mock import patch from requests import Response @@ -13,7 +14,6 @@ from conan.errors import ConanException from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.api.uploader import gzopen_without_timestamps from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ GenConanfile, TestRequester, TestingResponse diff --git a/test/unittests/util/files/strip_root_extract_test.py b/test/unittests/util/files/strip_root_extract_test.py index 1e219be7802..0f86c4b1165 100644 --- a/test/unittests/util/files/strip_root_extract_test.py +++ b/test/unittests/util/files/strip_root_extract_test.py @@ -3,7 +3,7 @@ import unittest import zipfile -from conan.internal.api.uploader import gzopen_without_timestamps +from conan.internal.util.compression import gzopen_without_timestamps from conan.tools.files.files import untargz, unzip from conan.errors import ConanException from conan.internal.model.manifest import gather_files diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 9c71f5d07b8..f00b1b825a6 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -2,12 +2,11 @@ import platform import tarfile import unittest - import pytest -from conan.internal.api.uploader import gzopen_without_timestamps +from conan.internal.util.compression import gzopen_without_timestamps, tar_extract from conan.test.utils.test_files import temp_folder -from conan.internal.util.files import tar_extract, save, gather_files, chdir +from conan.internal.util.files import save, gather_files, chdir class TarExtractTest(unittest.TestCase): From c7b5dff762d692e7e77b2f8dc3ae78c4ff813c6d Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 12:29:31 +0200 Subject: [PATCH 03/24] Added tar_compressor --- conan/api/subapi/cache.py | 17 ++++++++--------- conan/internal/util/compression.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b27b9c06565..32e9759efca 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -1,8 +1,6 @@ import json import os import shutil -import tarfile -from io import BytesIO from conan.api.model import PackagesList from conan.api.output import ConanOutput @@ -17,7 +15,7 @@ from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now from conan.internal.util.files import rmdir, mkdir, remove -from conan.internal.util.compression import gzopen_without_timestamps, tar_extract +from conan.internal.util.compression import tar_compressor, tar_extract class CacheAPI: @@ -135,8 +133,8 @@ def save(self, package_list, tgz_path, no_source=False): compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) with open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, - compresslevel=compresslevel) + tgz = tar_compressor(name, fileobj=tgz_handle, compresslevel=compresslevel, + cache_path=self.conan_api.cache_folder) for ref, ref_bundle in package_list.refs().items(): ref_layout = cache.recipe_layout(ref) recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) @@ -170,11 +168,12 @@ def save(self, package_list, tgz_path, no_source=False): out.info(f"Saving {pref} metadata: {metadata_folder}") tgz.add(os.path.join(cache_folder, metadata_folder), metadata_folder, recursive=True) + # Create pgklist.json to add it to the tgz serialized = json.dumps(package_list.serialize(), indent=2) - info = tarfile.TarInfo(name="pkglist.json") - data = serialized.encode('utf-8') - info.size = len(data) - tgz.addfile(tarinfo=info, fileobj=BytesIO(data)) + pkglist_path = os.path.join(cache_folder, "pkglist.json") + with open(pkglist_path, "w") as file_handler: + file_handler.write(serialized) + tgz.add(pkglist_path, "pkglist.json", recursive=False) tgz.close() def restore(self, path): diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index d72bfa023e2..e237b39ce13 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -44,6 +44,13 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_fold ConanOutput().debug(f"{name} compressed in {duration} time") return tgz_path +def tar_compressor(name, fileobj, compresslevel, cache_path=None): + compress_plugin = _load_compress_plugin(cache_path) + if compress_plugin: + return compress_plugin.TarCompressor(name, fileobj, compresslevel) + else: + return gzopen_without_timestamps(name, fileobj, compresslevel) + def gzopen_without_timestamps(name, fileobj, compresslevel=None): """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was @@ -71,3 +78,14 @@ def _load_compress_plugin(cache_folder): if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") return mod + + +""" +Plugin `compression.py` interface: + + def tar_extract(src_path, destination_dir) -> None + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None) -> str + class TarCompressor(name, fileobj, compresslevel) + def add(self, abs_path, filename, recursive=True) -> None + def close() -> None +""" From be30ba2494347d974d4cc1567dccc5d9be9fb8bc Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 16 May 2025 13:05:10 +0200 Subject: [PATCH 04/24] Extra simplify --- conan/internal/api/uploader.py | 48 ++++++++++++++++--- conan/internal/rest/remote_manager.py | 11 +++-- conan/internal/util/compression.py | 8 ++-- conan/internal/util/files.py | 11 +++++ conan/test/utils/test_files.py | 2 +- .../integration/command/upload/upload_test.py | 2 +- .../util/files/strip_root_extract_test.py | 2 +- test/unittests/util/files/tar_extract_test.py | 11 +++-- 8 files changed, 75 insertions(+), 20 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 06cf59da330..2d020f39827 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,9 +12,9 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) +from conan.internal.util.compression import load_compress_plugin from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, - mkdir, human_size) -from conan.internal.util.compression import tar_compress + set_dirty_context_manager, mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" UPLOAD_POLICY_SKIP = "skip-upload" @@ -157,8 +157,8 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = tar_compress(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) + tgz = compress_files(tgz_files, tgz_name, download_export_folder, + compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -204,8 +204,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = tar_compress(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) + tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, + compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -257,6 +257,42 @@ def upload_package(self, pref, prev_bundle, remote): output.debug(f"Upload {pref} in {duration} time") +def gzopen_without_timestamps(name, fileobj, compresslevel=None): + """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was + setted in Gzip file causing md5 to change. Not possible using the + previous tarfile open because arguments are not passed to GzipFile constructor + """ + compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 + fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) + # Format is forced because in Python3.8, it changed and it generates different tarfiles + # with different checksums, which break hashes of tgzs + # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions + t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) + t._extfileobj = False + return t + + +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): + compress_plugin = load_compress_plugin(cache_folder) + if compress_plugin: + return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) + + t1 = time.time() + # FIXME, better write to disk sequentially and not keep tgz contents in memory + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref)).info(f"Compressing {name}") + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) + for filename, abs_path in sorted(files.items()): + # recursive is False in case it is a symlink to a folder + tgz.add(abs_path, filename, recursive=False) + tgz.close() + + duration = time.time() - t1 + ConanOutput().debug(f"{name} compressed in {duration} time") + return tgz_path + + def _total_size(cache_files): total_size = 0 for file in cache_files.values(): diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 9ebff0a07ab..19da098dbe6 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -15,10 +15,10 @@ from conan.internal.model.info import load_binary_info from conan.api.model import PkgReference from conan.api.model import RecipeReference +from conan.internal.util.compression import load_compress_plugin from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME -from conan.internal.util.files import mkdir -from conan.internal.util.compression import tar_extract +from conan.internal.util.files import mkdir, tar_extract class RemoteManager: @@ -289,7 +289,12 @@ def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): if big_file: hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - tar_extract(src_path, dest_folder, cache_folder=cache_folder) + + compression_plugin = load_compress_plugin(cache_folder) + if compression_plugin: + compression_plugin.tar_extract(src_path, dest_folder) + else: + tar_extract(src_path, dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index e237b39ce13..e811c84f46e 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -10,7 +10,7 @@ from conan.internal.util.files import set_dirty_context_manager def tar_extract(src_path, destination_dir, cache_folder=None): - compress_plugin = _load_compress_plugin(cache_folder) + compress_plugin = load_compress_plugin(cache_folder) if compress_plugin: return compress_plugin.tar_extract(src_path, destination_dir) @@ -25,7 +25,7 @@ def tar_extract(src_path, destination_dir, cache_folder=None): def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): - compress_plugin = _load_compress_plugin(cache_folder) + compress_plugin = load_compress_plugin(cache_folder) if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) @@ -45,7 +45,7 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_fold return tgz_path def tar_compressor(name, fileobj, compresslevel, cache_path=None): - compress_plugin = _load_compress_plugin(cache_path) + compress_plugin = load_compress_plugin(cache_path) if compress_plugin: return compress_plugin.TarCompressor(name, fileobj, compresslevel) else: @@ -67,7 +67,7 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def _load_compress_plugin(cache_folder): +def load_compress_plugin(cache_folder): if not cache_folder: return None compression_plugin_path = HomePaths(cache_folder).compression_plugin_path diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 01b9a76fe64..98746479ae0 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -6,6 +6,7 @@ import shutil import stat import sys +import tarfile import time from contextlib import contextmanager @@ -255,6 +256,16 @@ def mkdir(path): os.makedirs(path) +def tar_extract(fileobj, destination_dir): + the_tar = tarfile.open(fileobj=fileobj) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() + + def merge_directories(src, dst): from conan.tools.files import copy copy(None, pattern="*", src=src, dst=dst) diff --git a/conan/test/utils/test_files.py b/conan/test/utils/test_files.py index 55b7c173fab..7b3f011116d 100644 --- a/conan/test/utils/test_files.py +++ b/conan/test/utils/test_files.py @@ -8,7 +8,7 @@ import time from io import BytesIO -from conan.internal.util.compression import gzopen_without_timestamps +from conan.internal.api.uploader import gzopen_without_timestamps from conan.tools.files.files import untargz from conan.internal.subsystems import get_cased_path from conan.errors import ConanException diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index 1bb1044d534..63368208747 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -6,7 +6,6 @@ import unittest from collections import OrderedDict -from conan.internal.util.compression import gzopen_without_timestamps import pytest from mock import patch from requests import Response @@ -14,6 +13,7 @@ from conan.errors import ConanException from conan.api.model import PkgReference from conan.api.model import RecipeReference +from conan.internal.api.uploader import gzopen_without_timestamps from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ GenConanfile, TestRequester, TestingResponse diff --git a/test/unittests/util/files/strip_root_extract_test.py b/test/unittests/util/files/strip_root_extract_test.py index 0f86c4b1165..1e219be7802 100644 --- a/test/unittests/util/files/strip_root_extract_test.py +++ b/test/unittests/util/files/strip_root_extract_test.py @@ -3,7 +3,7 @@ import unittest import zipfile -from conan.internal.util.compression import gzopen_without_timestamps +from conan.internal.api.uploader import gzopen_without_timestamps from conan.tools.files.files import untargz, unzip from conan.errors import ConanException from conan.internal.model.manifest import gather_files diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index f00b1b825a6..6f031a56a13 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -2,11 +2,12 @@ import platform import tarfile import unittest + import pytest -from conan.internal.util.compression import gzopen_without_timestamps, tar_extract +from conan.internal.api.uploader import gzopen_without_timestamps from conan.test.utils.test_files import temp_folder -from conan.internal.util.files import save, gather_files, chdir +from conan.internal.util.files import tar_extract, save, gather_files, chdir class TarExtractTest(unittest.TestCase): @@ -49,11 +50,13 @@ def check_files(destination_dir): with chdir(working_dir): # Unpack and check destination_dir = os.path.join(self.tmp_folder, "dest") - tar_extract(self.tgz_file, destination_dir) + with open(self.tgz_file, 'rb') as file_handler: + tar_extract(file_handler, destination_dir) check_files(destination_dir) # Unpack and check (now we have a symlinked local folder) os.symlink(temp_folder(), "folder") destination_dir = os.path.join(self.tmp_folder, "dest2") - tar_extract(self.tgz_file, destination_dir) + with open(self.tgz_file, 'rb') as file_handler: + tar_extract(file_handler, destination_dir) check_files(destination_dir) From 1295d4f02c2736267410dee51faba4c134b585f0 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 19 May 2025 11:23:52 +0200 Subject: [PATCH 05/24] Cache the plugin load --- conan/api/subapi/cache.py | 89 +++++++++++++++--------------- conan/internal/util/compression.py | 21 ++----- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 32e9759efca..d019477b511 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -1,6 +1,7 @@ import json import os import shutil +import tempfile from conan.api.model import PackagesList from conan.api.output import ConanOutput @@ -14,8 +15,8 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now -from conan.internal.util.files import rmdir, mkdir, remove -from conan.internal.util.compression import tar_compressor, tar_extract +from conan.internal.util.files import rmdir, mkdir, remove, save +from conan.internal.util.compression import tar_compress, tar_extract class CacheAPI: @@ -129,52 +130,50 @@ def save(self, package_list, tgz_path, no_source=False): cache_folder = cache.store # Note, this is not the home, but the actual package cache out = ConanOutput() mkdir(os.path.dirname(tgz_path)) - name = os.path.basename(tgz_path) compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) + tar_files: dict[str,str] = {} # {path_in_tar: abs_path} - with open(tgz_path, "wb") as tgz_handle: - tgz = tar_compressor(name, fileobj=tgz_handle, compresslevel=compresslevel, - cache_path=self.conan_api.cache_folder) - for ref, ref_bundle in package_list.refs().items(): - ref_layout = cache.recipe_layout(ref) - recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) - recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable - ref_bundle["recipe_folder"] = recipe_folder - out.info(f"Saving {ref}: {recipe_folder}") - # Package only selected folders, not DOWNLOAD one - for f in (EXPORT_FOLDER, EXPORT_SRC_FOLDER, SRC_FOLDER): - if f == SRC_FOLDER and no_source: - continue - path = os.path.join(cache_folder, recipe_folder, f) - if os.path.exists(path): - tgz.add(path, f"{recipe_folder}/{f}", recursive=True) - path = os.path.join(cache_folder, recipe_folder, DOWNLOAD_EXPORT_FOLDER, METADATA) + for ref, ref_bundle in package_list.refs().items(): + ref_layout = cache.recipe_layout(ref) + recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder) + recipe_folder = recipe_folder.replace("\\", "/") # make win paths portable + ref_bundle["recipe_folder"] = recipe_folder + out.info(f"Saving {ref}: {recipe_folder}") + # Package only selected folders, not DOWNLOAD one + for f in (EXPORT_FOLDER, EXPORT_SRC_FOLDER, SRC_FOLDER): + if f == SRC_FOLDER and no_source: + continue + path = os.path.join(cache_folder, recipe_folder, f) if os.path.exists(path): - tgz.add(path, f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}", - recursive=True) - - for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): - pref_layout = cache.pkg_layout(pref) - pkg_folder = pref_layout.package() - folder = os.path.relpath(pkg_folder, cache_folder) - folder = folder.replace("\\", "/") # make win paths portable - pref_bundle["package_folder"] = folder - out.info(f"Saving {pref}: {folder}") - tgz.add(os.path.join(cache_folder, folder), folder, recursive=True) - if os.path.exists(pref_layout.metadata()): - metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder) - metadata_folder = metadata_folder.replace("\\", "/") # make paths portable - pref_bundle["metadata_folder"] = metadata_folder - out.info(f"Saving {pref} metadata: {metadata_folder}") - tgz.add(os.path.join(cache_folder, metadata_folder), metadata_folder, - recursive=True) - # Create pgklist.json to add it to the tgz - serialized = json.dumps(package_list.serialize(), indent=2) - pkglist_path = os.path.join(cache_folder, "pkglist.json") - with open(pkglist_path, "w") as file_handler: - file_handler.write(serialized) - tgz.add(pkglist_path, "pkglist.json", recursive=False) - tgz.close() + tar_files[f"{recipe_folder}/{f}"] = path + path = os.path.join(cache_folder, recipe_folder, DOWNLOAD_EXPORT_FOLDER, METADATA) + if os.path.exists(path): + tar_files[f"{recipe_folder}/{DOWNLOAD_EXPORT_FOLDER}/{METADATA}"] = path + + for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items(): + pref_layout = cache.pkg_layout(pref) + pkg_folder = pref_layout.package() + folder = os.path.relpath(pkg_folder, cache_folder) + folder = folder.replace("\\", "/") # make win paths portable + pref_bundle["package_folder"] = folder + out.info(f"Saving {pref}: {folder}") + tar_files[folder] = os.path.join(cache_folder, folder) + + if os.path.exists(pref_layout.metadata()): + metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder) + metadata_folder = metadata_folder.replace("\\", "/") # make paths portable + pref_bundle["metadata_folder"] = metadata_folder + out.info(f"Saving {pref} metadata: {metadata_folder}") + tar_files[metadata_folder] = os.path.join(cache_folder, metadata_folder) + + # Create a temporary file in order to reuse compress_files functionality + serialized = json.dumps(package_list.serialize(), indent=2) + pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") + save(pkglist_path, serialized) + tar_files["pkglist.json"] = pkglist_path + tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, + recursive=True, ref=None, cache_folder=self.conan_api.cache_folder) + remove(pkglist_path) def restore(self, path): if not os.path.isfile(path): diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index e811c84f46e..6a4f3e993d0 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -1,3 +1,4 @@ +from functools import lru_cache from conan.internal.cache.home_paths import HomePaths from conan.internal.loader import load_python_file from conan.internal.errors import ConanException @@ -24,7 +25,7 @@ def tar_extract(src_path, destination_dir, cache_folder=None): the_tar.close() -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_folder=None): +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, cache_folder=None): compress_plugin = load_compress_plugin(cache_folder) if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) @@ -37,21 +38,13 @@ def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, cache_fold tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False in case it is a symlink to a folder - tgz.add(abs_path, filename, recursive=False) + tgz.add(abs_path, filename, recursive=recursive) tgz.close() duration = time.time() - t1 ConanOutput().debug(f"{name} compressed in {duration} time") return tgz_path -def tar_compressor(name, fileobj, compresslevel, cache_path=None): - compress_plugin = load_compress_plugin(cache_path) - if compress_plugin: - return compress_plugin.TarCompressor(name, fileobj, compresslevel) - else: - return gzopen_without_timestamps(name, fileobj, compresslevel) - - def gzopen_without_timestamps(name, fileobj, compresslevel=None): """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was setted in Gzip file causing md5 to change. Not possible using the @@ -67,6 +60,7 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t +@lru_cache(maxsize=1) def load_compress_plugin(cache_folder): if not cache_folder: return None @@ -83,9 +77,6 @@ def load_compress_plugin(cache_folder): """ Plugin `compression.py` interface: - def tar_extract(src_path, destination_dir) -> None - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None) -> str - class TarCompressor(name, fileobj, compresslevel) - def add(self, abs_path, filename, recursive=True) -> None - def close() -> None + def tar_extract(src_path: str, destination_dir: str) -> None + def tar_compress(files: List[str], name: str, dest_dir: str, compresslevel=None, ref: str=None, cache_folder:str, recursive: bool = False) -> str """ From cc4d3add4326f1de0d4e22579e3c9c34b46e177a Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 19 May 2025 17:23:54 +0200 Subject: [PATCH 06/24] WIP --- conan/api/subapi/config.py | 2 ++ conan/internal/util/compression.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index f7342e475b2..5485181fcd4 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,6 +3,7 @@ import platform import textwrap import yaml +from conan.internal.util.compression import load_compress_plugin from jinja2 import Environment, FileSystemLoader from conan import conan_version @@ -31,6 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) + self.compress_plugin = load_compress_plugin(conan_api.cache_folder) def home(self): return self.conan_api.cache_folder diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index 6a4f3e993d0..d1090beb299 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -60,7 +60,6 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -@lru_cache(maxsize=1) def load_compress_plugin(cache_folder): if not cache_folder: return None From f094d9aec5633d1caf77b584ac0ea8b5aff8b5e1 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 20 May 2025 13:51:25 +0200 Subject: [PATCH 07/24] Moved plugin load to ConfigAPI --- conan/api/subapi/cache.py | 4 ++-- conan/api/subapi/config.py | 11 +++++++++-- conan/internal/api/uploader.py | 12 +++++++----- conan/internal/rest/remote_manager.py | 13 +++++++------ conan/internal/util/compression.py | 10 ++++------ 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index d019477b511..a0a5fd99086 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -172,7 +172,7 @@ def save(self, package_list, tgz_path, no_source=False): save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, - recursive=True, ref=None, cache_folder=self.conan_api.cache_folder) + recursive=True, ref=None, compress_plugin=self.conan_api.config.compress_plugin) remove(pkglist_path) def restore(self, path): @@ -181,7 +181,7 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(path, cache_folder, cache_folder=self.conan_api.cache_folder) + tar_extract(path, cache_folder, compress_plugin=self.conan_api.config.compress_plugin) # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 5485181fcd4..3ccb3b0545e 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,7 +3,7 @@ import platform import textwrap import yaml -from conan.internal.util.compression import load_compress_plugin +from conan.internal.util.compression import load_compression_plugin from jinja2 import Environment, FileSystemLoader from conan import conan_version @@ -32,7 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) - self.compress_plugin = load_compress_plugin(conan_api.cache_folder) + self._compress_plugin = None def home(self): return self.conan_api.cache_folder @@ -238,4 +238,11 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() + self._compress_plugin = None self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) + + @property + def compression_plugin(self): + if not self._compress_plugin: + self._compress_plugin = load_compression_plugin(self.conan_api.cache_folder) + return self._compress_plugin diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 1a9bb70ab5c..3aaf93ca5bb 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,7 +12,7 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.compression import load_compress_plugin +from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) @@ -158,7 +158,8 @@ def add_tgz(tgz_name, tgz_files): elif tgz_files: compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref, cache_folder=self._app.cache_folder) + compresslevel=compresslevel, ref=ref, + compress_plugin=self._app.conan_api.config.compress_plugin) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -205,7 +206,8 @@ def _compress_package_files(self, layout, pref): tgz_files = {f: path for f, path in files.items()} compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref, cache_folder=self._app.cache_folder) + compresslevel=compresslevel, ref=pref, + compress_plugin=self._app.conan_api.config.compress_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -272,8 +274,8 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, cache_folder=None): - compress_plugin = load_compress_plugin(cache_folder) +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, + compress_plugin=None): if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 19da098dbe6..4eda8e1b154 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -1,3 +1,4 @@ +from gzip import compress import os import shutil from typing import List @@ -15,7 +16,7 @@ from conan.internal.model.info import load_binary_info from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.util.compression import load_compress_plugin +from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME from conan.internal.util.files import mkdir, tar_extract @@ -28,6 +29,7 @@ def __init__(self, cache, auth_manager, home_folder): self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache, home_folder) self._home_folder = home_folder + self._compression_plugin = load_compression_plugin(home_folder) # TODO: should use the instantiated one in ConfigAPI def _local_folder_remote(self, remote): if remote.remote_type == LOCAL_RECIPES_INDEX: @@ -82,7 +84,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), cache_folder=self._home_folder) + uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._compression_plugin) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -124,7 +126,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), cache_folder=self._home_folder) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._compression_plugin) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -172,7 +174,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), cache_folder=self._home_folder) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._compression_plugin) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -282,7 +284,7 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): +def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB @@ -290,7 +292,6 @@ def uncompress_file(src_path, dest_folder, scope=None, cache_folder=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - compression_plugin = load_compress_plugin(cache_folder) if compression_plugin: compression_plugin.tar_extract(src_path, dest_folder) else: diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index d1090beb299..f8ddedf55e5 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -10,8 +10,7 @@ from conan.api.output import ConanOutput from conan.internal.util.files import set_dirty_context_manager -def tar_extract(src_path, destination_dir, cache_folder=None): - compress_plugin = load_compress_plugin(cache_folder) +def tar_extract(src_path, destination_dir, compress_plugin=None): if compress_plugin: return compress_plugin.tar_extract(src_path, destination_dir) @@ -25,15 +24,14 @@ def tar_extract(src_path, destination_dir, cache_folder=None): the_tar.close() -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, cache_folder=None): - compress_plugin = load_compress_plugin(cache_folder) +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compress_plugin=None): if compress_plugin: return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref)).info(f"Compressing {name}") + ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): @@ -60,7 +58,7 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def load_compress_plugin(cache_folder): +def load_compression_plugin(cache_folder): if not cache_folder: return None compression_plugin_path = HomePaths(cache_folder).compression_plugin_path From fee5fab5cfeb655bf3dd83d91ad5211f2923097a Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 20 May 2025 16:23:01 +0200 Subject: [PATCH 08/24] Pass config_api to remote_manager and local_recipe_index --- conan/api/subapi/cache.py | 4 ++-- conan/api/subapi/config.py | 10 +++++----- conan/internal/api/uploader.py | 12 +++++------- conan/internal/conan_app.py | 6 +++--- conan/internal/rest/remote_manager.py | 13 ++++++------- .../internal/rest/rest_client_local_recipe_index.py | 4 ++-- conan/internal/util/compression.py | 12 ++++++------ 7 files changed, 29 insertions(+), 32 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index a0a5fd99086..b78ecd8e196 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -172,7 +172,7 @@ def save(self, package_list, tgz_path, no_source=False): save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, - recursive=True, ref=None, compress_plugin=self.conan_api.config.compress_plugin) + recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) def restore(self, path): @@ -181,7 +181,7 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(path, cache_folder, compress_plugin=self.conan_api.config.compress_plugin) + tar_extract(path, cache_folder, compression_plugin=self.conan_api.config.compression_plugin) # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 3ccb3b0545e..bc4ab0be776 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -32,7 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) - self._compress_plugin = None + self._compression_plugin = None def home(self): return self.conan_api.cache_folder @@ -238,11 +238,11 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() - self._compress_plugin = None + self._compression_plugin = None self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) @property def compression_plugin(self): - if not self._compress_plugin: - self._compress_plugin = load_compression_plugin(self.conan_api.cache_folder) - return self._compress_plugin + if not self._compression_plugin: + self._compression_plugin = load_compression_plugin(self.conan_api.cache_folder) + return self._compression_plugin diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 3aaf93ca5bb..e3cca0ea177 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -12,7 +12,6 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) @@ -159,7 +158,7 @@ def add_tgz(tgz_name, tgz_files): compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz = compress_files(tgz_files, tgz_name, download_export_folder, compresslevel=compresslevel, ref=ref, - compress_plugin=self._app.conan_api.config.compress_plugin) + compression_plugin=self._app.conan_api.config.compression_plugin) result[tgz_name] = tgz add_tgz(EXPORT_TGZ_NAME, files) @@ -207,7 +206,7 @@ def _compress_package_files(self, layout, pref): compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, compresslevel=compresslevel, ref=pref, - compress_plugin=self._app.conan_api.config.compress_plugin) + compression_plugin=self._app.conan_api.config.compression_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -274,10 +273,9 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, - compress_plugin=None): - if compress_plugin: - return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) +def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): + if compression_plugin: + return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 0ca4df974e2..9e1aa07a053 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -52,7 +52,7 @@ def __init__(self, conan_api): localdb = LocalDB(cache_folder) auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, global_conf) # Handle remote connections - self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder) + self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, self.conan_api.config) global_editables = conan_api.local.editable_packages ws_editables = conan_api.workspace.editable_packages self.editable_packages = global_editables.update_copy(ws_editables) @@ -81,10 +81,10 @@ class LocalRecipesIndexApp: - loader (for the export phase of local-recipes-index) The others are internally use by other collaborators """ - def __init__(self, cache_folder): + def __init__(self, cache_folder, config_api): self.global_conf = ConfDefinition() self.cache = PkgCache(cache_folder, self.global_conf) - self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder) + self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder, config_api=config_api) editable_packages = EditablePackages() self.proxy = ConanProxy(self, editable_packages) self.range_resolver = RangeResolver(self, self.global_conf, editable_packages) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 4eda8e1b154..d3894accdb4 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -16,7 +16,6 @@ from conan.internal.model.info import load_binary_info from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.util.compression import load_compression_plugin from conan.internal.util.files import rmdir, human_size from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME from conan.internal.util.files import mkdir, tar_extract @@ -24,16 +23,16 @@ class RemoteManager: """ Will handle the remotes to get recipes, packages etc """ - def __init__(self, cache, auth_manager, home_folder): + def __init__(self, cache, auth_manager, home_folder, config_api): self._cache = cache self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache, home_folder) self._home_folder = home_folder - self._compression_plugin = load_compression_plugin(home_folder) # TODO: should use the instantiated one in ConfigAPI + self._config_api = config_api def _local_folder_remote(self, remote): if remote.remote_type == LOCAL_RECIPES_INDEX: - return RestApiClientLocalRecipesIndex(remote, self._home_folder) + return RestApiClientLocalRecipesIndex(remote, self._home_folder, self._config_api) def check_credentials(self, remote, force_auth=False): self._call_remote(remote, "check_credentials", force_auth) @@ -84,7 +83,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._compression_plugin) + uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -126,7 +125,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._compression_plugin) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -174,7 +173,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._compression_plugin) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._config_api.compression_plugin) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) diff --git a/conan/internal/rest/rest_client_local_recipe_index.py b/conan/internal/rest/rest_client_local_recipe_index.py index 33082ca1356..83af84a263a 100644 --- a/conan/internal/rest/rest_client_local_recipe_index.py +++ b/conan/internal/rest/rest_client_local_recipe_index.py @@ -58,14 +58,14 @@ class RestApiClientLocalRecipesIndex: a local folder assuming the conan-center-index repo layout """ - def __init__(self, remote, home_folder): + def __init__(self, remote, home_folder, config_api): self._remote = remote local_recipes_index_path = HomePaths(home_folder).local_recipes_index_path local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name, ".conan") repo_folder = self._remote.url from conan.internal.conan_app import LocalRecipesIndexApp - self._app = LocalRecipesIndexApp(local_recipes_index_path) + self._app = LocalRecipesIndexApp(local_recipes_index_path, config_api) self._hook_manager = HookManager(HomePaths(local_recipes_index_path).hooks_path) self._layout = _LocalRecipesIndexLayout(repo_folder) diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py index f8ddedf55e5..9ebe0af96ce 100644 --- a/conan/internal/util/compression.py +++ b/conan/internal/util/compression.py @@ -10,9 +10,9 @@ from conan.api.output import ConanOutput from conan.internal.util.files import set_dirty_context_manager -def tar_extract(src_path, destination_dir, compress_plugin=None): - if compress_plugin: - return compress_plugin.tar_extract(src_path, destination_dir) +def tar_extract(src_path, destination_dir, compression_plugin=None): + if compression_plugin: + return compression_plugin.tar_extract(src_path, destination_dir) with open(src_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) @@ -24,9 +24,9 @@ def tar_extract(src_path, destination_dir, compress_plugin=None): the_tar.close() -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compress_plugin=None): - if compress_plugin: - return compress_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) +def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): + if compression_plugin: + return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory From 452f7746b5d793cdd32e3f66e3c26dcaf73d45fa Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 20 May 2025 16:34:52 +0200 Subject: [PATCH 09/24] Restore previous tar_extract fixing tests --- conan/internal/rest/remote_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index d3894accdb4..10f6617d068 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -294,7 +294,8 @@ def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None): if compression_plugin: compression_plugin.tar_extract(src_path, dest_folder) else: - tar_extract(src_path, dest_folder) + with open(src_path, mode='rb') as file_handler: + tar_extract(file_handler, dest_folder) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) From c1a73204c088773d4f1f4b7be23bfc4336adb79b Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Thu, 22 May 2025 17:17:27 +0200 Subject: [PATCH 10/24] Removed compression.py module and minimize diff --- conan/api/subapi/cache.py | 18 ++- conan/api/subapi/config.py | 13 ++- conan/internal/api/uploader.py | 4 +- conan/internal/util/compression.py | 79 ------------- .../extensions/test_compression_plugin.py | 106 ++++++++++++++++++ 5 files changed, 132 insertions(+), 88 deletions(-) delete mode 100644 conan/internal/util/compression.py create mode 100644 test/integration/extensions/test_compression_plugin.py diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index b78ecd8e196..025f2b56d2a 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -5,6 +5,7 @@ from conan.api.model import PackagesList from conan.api.output import ConanOutput +from conan.internal.api.uploader import compress_files from conan.internal.cache.cache import PkgCache from conan.internal.cache.conan_reference_layout import EXPORT_SRC_FOLDER, EXPORT_FOLDER, SRC_FOLDER, \ METADATA, DOWNLOAD_EXPORT_FOLDER @@ -15,8 +16,7 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.util.dates import revision_timestamp_now -from conan.internal.util.files import rmdir, mkdir, remove, save -from conan.internal.util.compression import tar_compress, tar_extract +from conan.internal.util.files import rmdir, mkdir, remove, save, tar_extract class CacheAPI: @@ -171,8 +171,9 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - tar_compress(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, - recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) + print(tar_files) + compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, + recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) def restore(self, path): @@ -181,7 +182,14 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - tar_extract(path, cache_folder, compression_plugin=self.conan_api.config.compression_plugin) + + compression_plugin = self.conan_api.config.compression_plugin + if compression_plugin: + compression_plugin.tar_extract(path, cache_folder) + else: + with open(path, mode='rb') as file_handler: + tar_extract(file_handler, cache_folder) + # Retrieve the package list from the already extracted archive with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: pkglist = file_handler.read() diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index bc4ab0be776..9540da485d9 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -3,7 +3,7 @@ import platform import textwrap import yaml -from conan.internal.util.compression import load_compression_plugin +from conan.internal.loader import load_python_file from jinja2 import Environment, FileSystemLoader from conan import conan_version @@ -243,6 +243,15 @@ def reinit(self): @property def compression_plugin(self): + def load_compression_plugin(): + compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path + if not os.path.exists(compression_plugin_path): + return None + mod, _ = load_python_file(compression_plugin_path) + if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): + raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") + return mod if not self._compression_plugin: - self._compression_plugin = load_compression_plugin(self.conan_api.cache_folder) + self._compression_plugin = load_compression_plugin() return self._compression_plugin + diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index e3cca0ea177..31c14ef2a49 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -275,12 +275,12 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): if compression_plugin: - return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) + return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref, recursive) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}") + ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): diff --git a/conan/internal/util/compression.py b/conan/internal/util/compression.py deleted file mode 100644 index 9ebe0af96ce..00000000000 --- a/conan/internal/util/compression.py +++ /dev/null @@ -1,79 +0,0 @@ -from functools import lru_cache -from conan.internal.cache.home_paths import HomePaths -from conan.internal.loader import load_python_file -from conan.internal.errors import ConanException - -import os -import gzip -import time -import tarfile -from conan.api.output import ConanOutput -from conan.internal.util.files import set_dirty_context_manager - -def tar_extract(src_path, destination_dir, compression_plugin=None): - if compression_plugin: - return compression_plugin.tar_extract(src_path, destination_dir) - - with open(src_path, mode='rb') as file_handler: - the_tar = tarfile.open(fileobj=file_handler) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() - - -def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): - if compression_plugin: - return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref) - - t1 = time.time() - # FIXME, better write to disk sequentially and not keep tgz contents in memory - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name}") - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) - for filename, abs_path in sorted(files.items()): - # recursive is False in case it is a symlink to a folder - tgz.add(abs_path, filename, recursive=recursive) - tgz.close() - - duration = time.time() - t1 - ConanOutput().debug(f"{name} compressed in {duration} time") - return tgz_path - -def gzopen_without_timestamps(name, fileobj, compresslevel=None): - """ !! Method overrided by laso to pass mtime=0 (!=None) to avoid time.time() was - setted in Gzip file causing md5 to change. Not possible using the - previous tarfile open because arguments are not passed to GzipFile constructor - """ - compresslevel = compresslevel if compresslevel is not None else 9 # default Gzip = 9 - fileobj = gzip.GzipFile(name, "w", compresslevel, fileobj, mtime=0) - # Format is forced because in Python3.8, it changed and it generates different tarfiles - # with different checksums, which break hashes of tgzs - # PAX_FORMAT is the default for Py38, lets make it explicit for older Python versions - t = tarfile.TarFile.taropen(name, "w", fileobj, format=tarfile.PAX_FORMAT) - t._extfileobj = False - return t - - -def load_compression_plugin(cache_folder): - if not cache_folder: - return None - compression_plugin_path = HomePaths(cache_folder).compression_plugin_path - if not os.path.exists(compression_plugin_path): - return None - - mod, _ = load_python_file(compression_plugin_path) - if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): - raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - return mod - - -""" -Plugin `compression.py` interface: - - def tar_extract(src_path: str, destination_dir: str) -> None - def tar_compress(files: List[str], name: str, dest_dir: str, compresslevel=None, ref: str=None, cache_folder:str, recursive: bool = False) -> str -""" diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py new file mode 100644 index 00000000000..6dc5ec21b5c --- /dev/null +++ b/test/integration/extensions/test_compression_plugin.py @@ -0,0 +1,106 @@ +import os +import textwrap + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +def test_compression_plugin_not_existing(): + """Test that the compression plugin is not used if it does not exist""" + c = TestClient() + c.save({"conanfile.py": GenConanfile("pkg", "1.0")}) + c.run("create .") + c.run("cache save 'pkg/*'") + assert "Compressing conan_cache_save.tgz\n" in c.out + c.run("cache restore conan_cache_save.tgz") + # Default decompress does not have any output + assert "Decompressing conan_cache_save.tgz" not in c.out + + +def test_compression_plugin_not_valid(): + """Test an error is raised if the compression plugin is not valid""" + + c = TestClient() + compression_plugin = textwrap.dedent( + """ + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): + return None + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'", assert_error=True) + assert ( + "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions" + in c.out + ) + + +def test_compression_plugin_correctly_load(): + """Test that the compression plugin is correctly loaded and used on: + - cache save/restore + - remote upload/download + """ + c = TestClient(default_server_user=True) + + compression_plugin = textwrap.dedent( + """ + import os + import tarfile + from conan.api.output import ConanOutput + + # xz compression + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (xz)") + kwargs = {"preset": compresslevel} if compresslevel else {} + with tarfile.open(tgz_path, f"w:xz", **kwargs) as tgz: + for filename, abs_path in sorted(files.items()): + tgz.add(abs_path, filename, recursive=True) + return tgz_path + + def tar_extract(src_path, destination_dir, *args, **kwargs): + ConanOutput().info(f"Decompressing {os.path.basename(src_path)} using compression plugin (xz)") + with open(src_path, mode='rb') as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, path: member) + the_tar.extractall(path=destination_dir) + the_tar.close() + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'") + assert "Compressing conan_cache_save.tgz using compression plugin (xz)" in c.out + c.run("remove pkg/* -c") + c.run("cache restore conan_cache_save.tgz") + assert "Decompressing conan_cache_save.tgz using compression plugin (xz)" in c.out + c.run("list pkg/1.0") + assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out + + # Remove pre existing tgz to force a recompression + c.run("remove pkg/* -c") + c.run("create .") + # Check the plugin is also used on remote interactions + c.run("upload * -r=default -c") + assert "Compressing conan_package.tgz using compression plugin (xz)" in c.out + assert "pkg/1.0: Uploading recipe" in c.out + c.run("remove pkg/* -c") + c.run("download 'pkg/*' -r=default") + assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out From a240c8c619766eb8a14d9f77343ce3821f604169 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Thu, 22 May 2025 17:25:30 +0200 Subject: [PATCH 11/24] Remove debug print --- conan/api/subapi/cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 025f2b56d2a..1198ebe5924 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -171,7 +171,6 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - print(tar_files) compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) From cf7de747bf3f66ce25ae95e7f9accfd4af8654ff Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 23 May 2025 10:03:37 +0200 Subject: [PATCH 12/24] Applied thread suggestions --- conan/api/subapi/config.py | 6 ++---- .../command/cache/test_cache_save_restore.py | 4 ++++ .../extensions/test_compression_plugin.py | 12 ------------ 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 9540da485d9..9577d86c962 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -243,15 +243,13 @@ def reinit(self): @property def compression_plugin(self): - def load_compression_plugin(): + if not self._compression_plugin: compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): return None mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") - return mod - if not self._compression_plugin: - self._compression_plugin = load_compression_plugin() + self._compression_plugin = mod return self._compression_plugin diff --git a/test/integration/command/cache/test_cache_save_restore.py b/test/integration/command/cache/test_cache_save_restore.py index a6bc91b495d..113a0dd0ac5 100644 --- a/test/integration/command/cache/test_cache_save_restore.py +++ b/test/integration/command/cache/test_cache_save_restore.py @@ -158,9 +158,13 @@ def test_cache_save_excluded_folders(): # exclude source c.run("cache save * --no-source") + # Check default compression function is being used and not compression.py plugin one + assert "Compressing conan_cache_save.tgz\n" in c.out c3 = TestClient() shutil.copy2(cache_path, c3.current_folder) c3.run("cache restore conan_cache_save.tgz") + # Default decompress does not have any output + assert "Decompressing conan_cache_save.tgz" not in c3.out ref_layout = c3.get_latest_ref_layout(ref) assert not os.path.exists(os.path.join(ref_layout.source(), "mysrc.c")) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 6dc5ec21b5c..cee4c3b7079 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -5,18 +5,6 @@ from conan.test.utils.tools import TestClient -def test_compression_plugin_not_existing(): - """Test that the compression plugin is not used if it does not exist""" - c = TestClient() - c.save({"conanfile.py": GenConanfile("pkg", "1.0")}) - c.run("create .") - c.run("cache save 'pkg/*'") - assert "Compressing conan_cache_save.tgz\n" in c.out - c.run("cache restore conan_cache_save.tgz") - # Default decompress does not have any output - assert "Decompressing conan_cache_save.tgz" not in c.out - - def test_compression_plugin_not_valid(): """Test an error is raised if the compression plugin is not valid""" From 557f8e001f42110087dbf760d1ffd2a64783f35e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 23 May 2025 11:31:38 +0200 Subject: [PATCH 13/24] Remove created pkglist.json on cache restore after usage --- conan/api/subapi/cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 1198ebe5924..024fa6de68b 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -190,8 +190,11 @@ def restore(self, path): tar_extract(file_handler, cache_folder) # Retrieve the package list from the already extracted archive - with open(os.path.join(cache_folder, "pkglist.json")) as file_handler: + pkglist_path = os.path.join(cache_folder, "pkglist.json") + with open(pkglist_path) as file_handler: pkglist = file_handler.read() + # Delete the pkglist.json file to keep cache clean + remove(pkglist_path) # After unzipping the files, we need to update the DB that references these files out = ConanOutput() From f82d764a2d86b4af76b105488de7834893782aa3 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 26 May 2025 11:09:27 +0200 Subject: [PATCH 14/24] Remove unused and avoid rechecking FS --- conan/api/subapi/config.py | 7 ++++--- conan/internal/rest/remote_manager.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index 9577d86c962..c509753e832 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -32,7 +32,6 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) - self._compression_plugin = None def home(self): return self.conan_api.cache_folder @@ -238,14 +237,16 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() - self._compression_plugin = None + if hasattr(self, "_compression_plugin"): + del self._compression_plugin self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) @property def compression_plugin(self): - if not self._compression_plugin: + if not hasattr(self, "_compression_plugin"): compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): + self._compression_plugin = None return None mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 10f6617d068..3dce1518b02 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -1,4 +1,3 @@ -from gzip import compress import os import shutil from typing import List From ac1fc45695fec2acd78f30588448c25a9c9a19fa Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 26 May 2025 19:47:44 +0200 Subject: [PATCH 15/24] Added test to check extract failure and issue #18259 test --- conan/api/subapi/config.py | 10 +- conan/internal/rest/remote_manager.py | 2 +- conan/internal/util/files.py | 19 +-- .../extensions/test_compression_plugin.py | 108 ++++++++++++++++++ test/unittests/util/files/tar_extract_test.py | 2 + 5 files changed, 128 insertions(+), 13 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index c509753e832..d03c275b0c2 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -32,6 +32,7 @@ def __init__(self, conan_api): self._new_config = None self._cli_core_confs = None self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path) + self._compression_plugin = None def home(self): return self.conan_api.cache_folder @@ -237,17 +238,16 @@ def reinit(self): if self._new_config is not None: self._new_config.clear() self._populate_global_conf() - if hasattr(self, "_compression_plugin"): - del self._compression_plugin + self._compression_plugin = None self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path) @property def compression_plugin(self): - if not hasattr(self, "_compression_plugin"): + if self._compression_plugin is None: compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): - self._compression_plugin = None - return None + self._compression_plugin = False # Avoid FS re-check + return False mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 3dce1518b02..14b45d089d0 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -282,7 +282,7 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None): +def uncompress_file(src_path, dest_folder, scope="", compression_plugin=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 98746479ae0..2cd89b90cb5 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -257,13 +257,18 @@ def mkdir(path): def tar_extract(fileobj, destination_dir): - the_tar = tarfile.open(fileobj=fileobj) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() + try: + the_tar = tarfile.open(fileobj=fileobj) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() + except tarfile.ReadError: + raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}. The file compression is not recogniced.\n" + "This file could have been compressed using a `compression` plugin.\n" + "If your organization uses this plugin, ensure it is correctly installed on your environment.") def merge_directories(src, dst): diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index cee4c3b7079..31a1e206e9e 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,6 +1,7 @@ import os import textwrap +from conan.internal.util.files import tar_extract from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -92,3 +93,110 @@ def tar_extract(src_path, destination_dir, *args, **kwargs): c.run("remove pkg/* -c") c.run("download 'pkg/*' -r=default") assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out + + +def test_compression_plugin_tar_not_compatible_with_builtin(): + """ + Test that built in tar_extract function fails when uncompressing a non compatible file (a file + which has been compressed using the compression plugin with a different algorithm than the built-in one). + """ + c = TestClient(default_server_user=True) + + compression_plugin = textwrap.dedent( + """ + import os + import zipfile + from conan.api.output import ConanOutput + + # zip compression + def tar_extract(src_path, destination_dir): + # extract tar using zipfile library + ConanOutput().info(f"Decompressing {src_path} using compression plugin (zip)") + with zipfile.ZipFile(src_path, 'r') as zip_ref: + zip_ref.extractall(destination_dir) + + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): + # compress files using zipfile library taking into account recursive + zip_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (zip)") + with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: + for filename, abs_path in sorted(files.items()): + if recursive: + arcname = os.path.relpath(abs_path, start=os.path.dirname(abs_path)) + zipf.write(abs_path, arcname) + else: + zipf.write(abs_path, filename) + return zip_path + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'") + c.run("remove pkg/* -c") + os.unlink(os.path.join(c.cache_folder, "extensions", "plugins", "compression.py")) + c.run("cache restore conan_cache_save.tgz", assert_error=True) + assert ( + "Error while extracting conan_cache_save.tgz. The file compression is not recogniced.\n" + "This file could have been compressed using a `compression` plugin.\n" + "If your organization uses this plugin, ensure it is correctly installed on your environment." + ) in c.out + + +# https://github.com/conan-io/conan/issues/18259 +def test_compress_in_subdirectory(): + c = TestClient(default_server_user=True) + compression_plugin = textwrap.dedent( + """ + import os + import tarfile + from conan.api.output import ConanOutput + def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): + # compress files using tarfile putting all content in a `conan/` subfolder + tgz_path = os.path.join(dest_dir, name) + ConanOutput(scope=str(ref or "")).info(f"Compressing {name} in conan subfolder") + with open(tgz_path, "wb") as tgz_handle: + tgz = tarfile.open(name, "w", fileobj=tgz_handle) + for filename, abs_path in sorted(files.items()): + tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) + tgz.close() + return tgz_path + + def tar_extract(src_path, destination_dir, *args, **kwargs): + ConanOutput().info(f"Decompressing {src_path} in conan subfolder") + with open(src_path, mode="rb") as file_handler: + the_tar = tarfile.open(fileobj=file_handler) + the_tar.extraction_filter = (lambda member, path: member) + for member in the_tar.getmembers(): + if member.name.startswith("conan/"): + member.name = member.name[len("conan/"):] # Strip 'conan/' prefix + the_tar.extract(member, path=destination_dir) + the_tar.close() + """ + ) + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*'") + c.run("remove pkg/* -c") + c.run("cache restore conan_cache_save.tgz") + with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler: + destination_dir = os.path.join(c.cache_folder, "extracted") + tar_extract(file_handler, destination_dir) + assert os.listdir(destination_dir) == ["conan"] + assert os.path.exists(os.path.join(destination_dir, "conan", "pkglist.json")) + + diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 6f031a56a13..9ca622dbb9d 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -60,3 +60,5 @@ def check_files(destination_dir): with open(self.tgz_file, 'rb') as file_handler: tar_extract(file_handler, destination_dir) check_files(destination_dir) + + From 181a7366336882124b7a333eae689f0bcc92d611 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 11:24:54 +0200 Subject: [PATCH 16/24] Added config to compression.py interface and renamed parameters --- conan/api/subapi/cache.py | 3 +- conan/api/subapi/config.py | 5 +- conan/internal/api/uploader.py | 20 ++++-- conan/internal/rest/remote_manager.py | 13 ++-- .../extensions/test_compression_plugin.py | 70 +++++++++---------- 5 files changed, 58 insertions(+), 53 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 024fa6de68b..ca2cf67c907 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -184,7 +184,8 @@ def restore(self, path): compression_plugin = self.conan_api.config.compression_plugin if compression_plugin: - compression_plugin.tar_extract(path, cache_folder) + compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder, + config=self.conan_api.config.global_conf) else: with open(path, mode='rb') as file_handler: tar_extract(file_handler, cache_folder) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index d03c275b0c2..7b8aba3eb2c 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -247,10 +247,9 @@ def compression_plugin(self): compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path if not os.path.exists(compression_plugin_path): self._compression_plugin = False # Avoid FS re-check - return False + return None mod, _ = load_python_file(compression_plugin_path) if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"): raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions") self._compression_plugin = mod - return self._compression_plugin - + return self._compression_plugin or None diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 31c14ef2a49..44585516704 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -155,9 +155,8 @@ def add_tgz(tgz_name, tgz_files): if os.path.isfile(tgz): result[tgz_name] = tgz elif tgz_files: - compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref, + config=self._global_conf, ref=ref, compression_plugin=self._app.conan_api.config.compression_plugin) result[tgz_name] = tgz @@ -203,9 +202,8 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} - compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref, + config=self._global_conf, ref=pref, compression_plugin=self._app.conan_api.config.compression_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -273,15 +271,23 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None): +def compress_files(files, name, dest_dir, config=None, ref=None, recursive=False, compression_plugin=None): + tgz_path = os.path.join(dest_dir, name) if compression_plugin: - return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref, recursive) + compression_plugin.tar_compress( + archive_path=tgz_path, + files=files, + recursive=recursive, + config=config, + ref=ref, + ) + return tgz_path t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory - tgz_path = os.path.join(dest_dir, name) ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False by default in case it is a symlink to a folder diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 14b45d089d0..01e7bfaa552 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -82,7 +82,7 @@ def get_recipe(self, ref, remote, metadata=None): tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) if tgz_file: - uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) + uncompress_file(tgz_file, export_folder, scope=str(ref), config_api=self._config_api) mkdir(export_folder) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) @@ -124,7 +124,7 @@ def get_recipe_sources(self, ref, layout, remote): self._signer.verify(ref, download_folder, files=zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin) + uncompress_file(tgz_file, export_sources_folder, scope=str(ref), config_api=self._config_api) def get_package(self, pref, remote, metadata=None): output = ConanOutput(scope=str(pref.ref)) @@ -172,7 +172,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) package_folder = layout.package() - uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._config_api.compression_plugin) + uncompress_file(tgz_file, package_folder, scope=str(pref.ref), config_api=self._config_api) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing for file_name, file_path in zipped_files.items(): # copy CONANINFO and CONANMANIFEST shutil.move(file_path, os.path.join(package_folder, file_name)) @@ -282,7 +282,7 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -def uncompress_file(src_path, dest_folder, scope="", compression_plugin=None): +def uncompress_file(src_path, dest_folder, scope="", config_api=None): try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB @@ -290,8 +290,9 @@ def uncompress_file(src_path, dest_folder, scope="", compression_plugin=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - if compression_plugin: - compression_plugin.tar_extract(src_path, dest_folder) + if config_api and config_api.compression_plugin: + config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder, + config=config_api.global_conf) else: with open(src_path, mode='rb') as file_handler: tar_extract(file_handler, dest_folder) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 31a1e206e9e..0bb21e3414b 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -12,8 +12,8 @@ def test_compression_plugin_not_valid(): c = TestClient() compression_plugin = textwrap.dedent( """ - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): - return None + def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + pass """ ) @@ -47,21 +47,21 @@ def test_compression_plugin_correctly_load(): from conan.api.output import ConanOutput # xz compression - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (xz)") + def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + name = os.path.basename(archive_path) + ConanOutput().info(f"Compressing {name} using compression plugin (xz)") + compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None kwargs = {"preset": compresslevel} if compresslevel else {} - with tarfile.open(tgz_path, f"w:xz", **kwargs) as tgz: + with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) - return tgz_path - def tar_extract(src_path, destination_dir, *args, **kwargs): - ConanOutput().info(f"Decompressing {os.path.basename(src_path)} using compression plugin (xz)") - with open(src_path, mode='rb') as file_handler: + def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") + with open(archive_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) the_tar.extraction_filter = (lambda member, path: member) - the_tar.extractall(path=destination_dir) + the_tar.extractall(path=dest_dir) the_tar.close() """ ) @@ -109,24 +109,24 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): from conan.api.output import ConanOutput # zip compression - def tar_extract(src_path, destination_dir): - # extract tar using zipfile library - ConanOutput().info(f"Decompressing {src_path} using compression plugin (zip)") - with zipfile.ZipFile(src_path, 'r') as zip_ref: - zip_ref.extractall(destination_dir) - - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False): + def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): # compress files using zipfile library taking into account recursive - zip_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (zip)") - with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: + name = os.path.basename(archive_path) + compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + ConanOutput().info(f"Compressing {name} using compression plugin (zip)") + with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: for filename, abs_path in sorted(files.items()): if recursive: arcname = os.path.relpath(abs_path, start=os.path.dirname(abs_path)) zipf.write(abs_path, arcname) else: zipf.write(abs_path, filename) - return zip_path + + def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + # extract tar using zipfile library + ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (zip)") + with zipfile.ZipFile(archive_path, 'r') as zip_ref: + zip_ref.extractall(dest_dir) """ ) @@ -158,26 +158,25 @@ def test_compress_in_subdirectory(): import os import tarfile from conan.api.output import ConanOutput - def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs): + def tar_compress(archive_path, files, recursive, *args, **kwargs): # compress files using tarfile putting all content in a `conan/` subfolder - tgz_path = os.path.join(dest_dir, name) - ConanOutput(scope=str(ref or "")).info(f"Compressing {name} in conan subfolder") - with open(tgz_path, "wb") as tgz_handle: + name = os.path.basename(archive_path) + ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") + with open(archive_path, "wb") as tgz_handle: tgz = tarfile.open(name, "w", fileobj=tgz_handle) for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) tgz.close() - return tgz_path - def tar_extract(src_path, destination_dir, *args, **kwargs): - ConanOutput().info(f"Decompressing {src_path} in conan subfolder") - with open(src_path, mode="rb") as file_handler: + def tar_extract(archive_path, dest_dir, *args, **kwargs): + ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") + with open(archive_path, mode="rb") as file_handler: the_tar = tarfile.open(fileobj=file_handler) the_tar.extraction_filter = (lambda member, path: member) for member in the_tar.getmembers(): if member.name.startswith("conan/"): member.name = member.name[len("conan/"):] # Strip 'conan/' prefix - the_tar.extract(member, path=destination_dir) + the_tar.extract(member, path=dest_dir) the_tar.close() """ ) @@ -194,9 +193,8 @@ def tar_extract(src_path, destination_dir, *args, **kwargs): c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler: - destination_dir = os.path.join(c.cache_folder, "extracted") - tar_extract(file_handler, destination_dir) - assert os.listdir(destination_dir) == ["conan"] - assert os.path.exists(os.path.join(destination_dir, "conan", "pkglist.json")) - + dest_dir = os.path.join(c.cache_folder, "extracted") + tar_extract(file_handler, dest_dir) + assert os.listdir(dest_dir) == ["conan"] + assert os.path.exists(os.path.join(dest_dir, "conan", "pkglist.json")) From 3b32ec20f991b1bee1c5b6e9f4e4c606062e241f Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 11:54:31 +0200 Subject: [PATCH 17/24] Fix invokation --- conan/api/subapi/cache.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index ca2cf67c907..afa6d1195de 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -130,7 +130,6 @@ def save(self, package_list, tgz_path, no_source=False): cache_folder = cache.store # Note, this is not the home, but the actual package cache out = ConanOutput() mkdir(os.path.dirname(tgz_path)) - compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int) tar_files: dict[str,str] = {} # {path_in_tar: abs_path} for ref, ref_bundle in package_list.refs().items(): @@ -171,7 +170,7 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, + compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), config=self.conan_api.config, recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) From b52bce2ffd0be2376a427dd06e7ec643ff9226a4 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 12:24:20 +0200 Subject: [PATCH 18/24] Rename config for conf --- conan/api/subapi/cache.py | 4 ++-- conan/internal/api/uploader.py | 10 +++++----- conan/internal/rest/remote_manager.py | 2 +- .../extensions/test_compression_plugin.py | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index afa6d1195de..753229ae15f 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -170,7 +170,7 @@ def save(self, package_list, tgz_path, no_source=False): pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json") save(pkglist_path, serialized) tar_files["pkglist.json"] = pkglist_path - compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), config=self.conan_api.config, + compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), conf=self.conan_api.config, recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin) remove(pkglist_path) @@ -184,7 +184,7 @@ def restore(self, path): compression_plugin = self.conan_api.config.compression_plugin if compression_plugin: compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder, - config=self.conan_api.config.global_conf) + conf=self.conan_api.config.global_conf) else: with open(path, mode='rb') as file_handler: tar_extract(file_handler, cache_folder) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 44585516704..5b0738ca509 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -156,7 +156,7 @@ def add_tgz(tgz_name, tgz_files): result[tgz_name] = tgz elif tgz_files: tgz = compress_files(tgz_files, tgz_name, download_export_folder, - config=self._global_conf, ref=ref, + conf=self._global_conf, ref=ref, compression_plugin=self._app.conan_api.config.compression_plugin) result[tgz_name] = tgz @@ -203,7 +203,7 @@ def _compress_package_files(self, layout, pref): if not os.path.isfile(package_tgz): tgz_files = {f: path for f, path in files.items()} tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - config=self._global_conf, ref=pref, + conf=self._global_conf, ref=pref, compression_plugin=self._app.conan_api.config.compression_plugin) assert tgz_path == package_tgz assert os.path.exists(package_tgz) @@ -271,14 +271,14 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): return t -def compress_files(files, name, dest_dir, config=None, ref=None, recursive=False, compression_plugin=None): +def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): tgz_path = os.path.join(dest_dir, name) if compression_plugin: compression_plugin.tar_compress( archive_path=tgz_path, files=files, recursive=recursive, - config=config, + conf=conf, ref=ref, ) return tgz_path @@ -287,7 +287,7 @@ def compress_files(files, name, dest_dir, config=None, ref=None, recursive=False # FIXME, better write to disk sequentially and not keep tgz contents in memory ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): # recursive is False by default in case it is a symlink to a folder diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 01e7bfaa552..a95214826a7 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -292,7 +292,7 @@ def uncompress_file(src_path, dest_folder, scope="", config_api=None): if config_api and config_api.compression_plugin: config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder, - config=config_api.global_conf) + conf=config_api.global_conf) else: with open(src_path, mode='rb') as file_handler: tar_extract(file_handler, dest_folder) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 0bb21e3414b..cb179d020fd 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -12,7 +12,7 @@ def test_compression_plugin_not_valid(): c = TestClient() compression_plugin = textwrap.dedent( """ - def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): pass """ ) @@ -47,16 +47,16 @@ def test_compression_plugin_correctly_load(): from conan.api.output import ConanOutput # xz compression - def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): name = os.path.basename(archive_path) ConanOutput().info(f"Compressing {name} using compression plugin (xz)") - compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None kwargs = {"preset": compresslevel} if compresslevel else {} with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) - def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") with open(archive_path, mode='rb') as file_handler: the_tar = tarfile.open(fileobj=file_handler) @@ -109,10 +109,10 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): from conan.api.output import ConanOutput # zip compression - def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): # compress files using zipfile library taking into account recursive name = os.path.basename(archive_path) - compresslevel = config.get("core.gzip:compresslevel", check_type=int) if config else None + compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None ConanOutput().info(f"Compressing {name} using compression plugin (zip)") with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=compresslevel) as zipf: for filename, abs_path in sorted(files.items()): @@ -122,7 +122,7 @@ def tar_compress(archive_path, files, recursive, config=None, *args, **kwargs): else: zipf.write(abs_path, filename) - def tar_extract(archive_path, dest_dir, config=None, *args, **kwargs): + def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): # extract tar using zipfile library ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (zip)") with zipfile.ZipFile(archive_path, 'r') as zip_ref: From 91da7a4b5e5b8478201d05273fcf23ddb9f3a433 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Tue, 27 May 2025 14:23:29 +0200 Subject: [PATCH 19/24] Added ref on test --- test/integration/extensions/test_compression_plugin.py | 4 ++-- test/unittests/util/files/tar_extract_test.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index cb179d020fd..43b64b369dd 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -47,9 +47,9 @@ def test_compression_plugin_correctly_load(): from conan.api.output import ConanOutput # xz compression - def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): + def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): name = os.path.basename(archive_path) - ConanOutput().info(f"Compressing {name} using compression plugin (xz)") + ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None kwargs = {"preset": compresslevel} if compresslevel else {} with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: diff --git a/test/unittests/util/files/tar_extract_test.py b/test/unittests/util/files/tar_extract_test.py index 9ca622dbb9d..6f031a56a13 100644 --- a/test/unittests/util/files/tar_extract_test.py +++ b/test/unittests/util/files/tar_extract_test.py @@ -60,5 +60,3 @@ def check_files(destination_dir): with open(self.tgz_file, 'rb') as file_handler: tar_extract(file_handler, destination_dir) check_files(destination_dir) - - From 989fde99776b773973b79e2a3ad6b2bd8a5e0679 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 20 Jun 2025 09:47:50 +0200 Subject: [PATCH 20/24] Move to different approach: tar encapsulation respecting extensions --- conan/api/subapi/cache.py | 14 ++--- conan/internal/api/uploader.py | 20 +++++-- conan/internal/rest/remote_manager.py | 13 ++--- conan/internal/util/files.py | 52 ++++++++++++++----- .../extensions/test_compression_plugin.py | 37 ++++++------- 5 files changed, 90 insertions(+), 46 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index 753229ae15f..addab4b6a85 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -181,13 +181,13 @@ def restore(self, path): cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf) cache_folder = cache.store # Note, this is not the home, but the actual package cache - compression_plugin = self.conan_api.config.compression_plugin - if compression_plugin: - compression_plugin.tar_extract(archive_path=path, dest_dir=cache_folder, - conf=self.conan_api.config.global_conf) - else: - with open(path, mode='rb') as file_handler: - tar_extract(file_handler, cache_folder) + with open(path, mode="rb") as file_handler: + tar_extract( + fileobj=file_handler, + destination_dir=cache_folder, + compression_plugin=self.conan_api.config.compression_plugin, + conf=self.conan_api.config.global_conf, + ) # Retrieve the package list from the already extracted archive pkglist_path = os.path.join(cache_folder, "pkglist.json") diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 5b0738ca509..abd1ce3322d 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -4,6 +4,7 @@ import shutil import tarfile import time +from pathlib import Path from conan.internal.conan_app import ConanApp from conan.api.output import ConanOutput @@ -12,7 +13,7 @@ from conan.errors import ConanException from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) -from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, +from conan.internal.util.files import (COMPRESSED_PLUGIN_TAR_NAME, clean_dirty, is_dirty, gather_files, remove, set_dirty_context_manager, mkdir, human_size) UPLOAD_POLICY_FORCE = "force-upload" @@ -274,13 +275,26 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): tgz_path = os.path.join(dest_dir, name) if compression_plugin: - compression_plugin.tar_compress( - archive_path=tgz_path, + t1 = time.time() + compressed_path = compression_plugin.tar_compress( + archive_path=os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME), files=files, recursive=recursive, conf=conf, ref=ref, ) + ConanOutput().debug(f"{name} compressed in {time.time() - t1} time in plugin") + ConanOutput(scope=str(ref or "")).info(f"Compressing {compressed_path}") + t1 = time.time() + ConanOutput().debug(f"Wrapping {compressed_path} in {name}") + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) + tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) + tgz.close() + ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") + # Only remove wrapped if it is different from the tgz_path + if compressed_path != os.path.basename(tgz_path): + remove(compressed_path) return tgz_path t1 = time.time() diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index 13ccf6e5efe..9d4d70bf817 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -304,12 +304,13 @@ def uncompress_file(src_path, dest_folder, scope="", config_api=None): hs = human_size(filesize) ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}") - if config_api and config_api.compression_plugin: - config_api.compression_plugin.tar_extract(archive_path=src_path, dest_dir=dest_folder, - conf=config_api.global_conf) - else: - with open(src_path, mode='rb') as file_handler: - tar_extract(file_handler, dest_folder) + with open(src_path, mode='rb') as file_handler: + tar_extract( + fileobj=file_handler, + destination_dir=dest_folder, + compression_plugin=config_api.compression_plugin if config_api and config_api.compression_plugin else None, + conf=config_api.global_conf if config_api else None + ) except Exception as e: error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\ % (src_path, dest_folder, str(e)) diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 2cd89b90cb5..a5c172aae20 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -1,4 +1,6 @@ import errno +from pathlib import Path +import tempfile import gzip import hashlib import os @@ -11,10 +13,13 @@ from contextlib import contextmanager +from conan.api.output import ConanOutput from conan.errors import ConanException _DIRTY_FOLDER = ".dirty" +# Name (without extension) of the tar file to be created by the compression plugin +COMPRESSED_PLUGIN_TAR_NAME = "__conan_plugin_compressed_contents__" def set_dirty(folder): dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER @@ -256,20 +261,43 @@ def mkdir(path): os.makedirs(path) -def tar_extract(fileobj, destination_dir): - try: - the_tar = tarfile.open(fileobj=fileobj) - # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to - # "could not change modification time", with time=0 - # the_tar.errorlevel = 2 # raise exception if any error - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=destination_dir) - the_tar.close() - except tarfile.ReadError: - raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}. The file compression is not recogniced.\n" - "This file could have been compressed using a `compression` plugin.\n" +def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): + if compression_plugin: + _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) + return + the_tar = tarfile.open(fileobj=fileobj) + # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to + # "could not change modification time", with time=0 + # the_tar.errorlevel = 2 # raise exception if any error + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + the_tar.close() + if Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*"): + raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n" + "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment.") +def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf): + """First remove tar.gz wrapper and then call the plugin to extract""" + with tempfile.TemporaryDirectory() as temp_dir: + t1 = time.time() + the_tar = tarfile.open(fileobj=fileobj) + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=temp_dir) + # Check if the tar was compressed with the compression plugin by checking the existence of + # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) + if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): + # Get the only extracted file: the plugin tar + plugin_tar_path = os.path.join(temp_dir, the_tar.getnames()[0]) + the_tar.close() + ConanOutput().debug(f"Unwrapped in {time.time() - t1} time") + t1 = time.time() + compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf) + ConanOutput().debug(f"Extracted in {time.time() - t1} time on plugin") + else: + # The tar was not compressed using the plugin, copy files to destination + from conan.tools.files import copy + copy(None, pattern="*", src=temp_dir, dst=destination_dir) def merge_directories(src, dst): from conan.tools.files import copy diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 43b64b369dd..82900518c22 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,7 +1,7 @@ import os import textwrap -from conan.internal.util.files import tar_extract +from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, tar_extract from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -26,7 +26,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'", assert_error=True) + c.run("cache save 'pkg/*:*'", assert_error=True) assert ( "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions" in c.out @@ -48,6 +48,7 @@ def test_compression_plugin_correctly_load(): # xz compression def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): + archive_path += ".xz" name = os.path.basename(archive_path) ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None @@ -55,6 +56,7 @@ def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **k with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) + return archive_path def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") @@ -75,11 +77,12 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'") - assert "Compressing conan_cache_save.tgz using compression plugin (xz)" in c.out + c.run("cache save 'pkg/*:*'") + print(c.out) + assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") - assert "Decompressing conan_cache_save.tgz using compression plugin (xz)" in c.out + assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out c.run("list pkg/1.0") assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out @@ -87,12 +90,12 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): c.run("remove pkg/* -c") c.run("create .") # Check the plugin is also used on remote interactions - c.run("upload * -r=default -c") - assert "Compressing conan_package.tgz using compression plugin (xz)" in c.out + c.run("upload *:* -r=default -c") + assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out assert "pkg/1.0: Uploading recipe" in c.out c.run("remove pkg/* -c") c.run("download 'pkg/*' -r=default") - assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out + assert f"Decompressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out def test_compression_plugin_tar_not_compatible_with_builtin(): @@ -111,6 +114,7 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): # zip compression def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): # compress files using zipfile library taking into account recursive + archive_path += ".zip" name = os.path.basename(archive_path) compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None ConanOutput().info(f"Compressing {name} using compression plugin (zip)") @@ -121,6 +125,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): zipf.write(abs_path, arcname) else: zipf.write(abs_path, filename) + return archive_path def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): # extract tar using zipfile library @@ -139,13 +144,13 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'") + c.run("cache save 'pkg/*:*'") c.run("remove pkg/* -c") os.unlink(os.path.join(c.cache_folder, "extensions", "plugins", "compression.py")) c.run("cache restore conan_cache_save.tgz", assert_error=True) assert ( - "Error while extracting conan_cache_save.tgz. The file compression is not recogniced.\n" - "This file could have been compressed using a `compression` plugin.\n" + "Error while extracting conan_cache_save.tgz.\n" + "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment." ) in c.out @@ -160,6 +165,7 @@ def test_compress_in_subdirectory(): from conan.api.output import ConanOutput def tar_compress(archive_path, files, recursive, *args, **kwargs): # compress files using tarfile putting all content in a `conan/` subfolder + archive_path += ".tgz" name = os.path.basename(archive_path) ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") with open(archive_path, "wb") as tgz_handle: @@ -167,6 +173,7 @@ def tar_compress(archive_path, files, recursive, *args, **kwargs): for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) tgz.close() + return archive_path def tar_extract(archive_path, dest_dir, *args, **kwargs): ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") @@ -189,12 +196,6 @@ def tar_extract(archive_path, dest_dir, *args, **kwargs): } ) c.run("create .") - c.run("cache save 'pkg/*'") + c.run("cache save 'pkg/*:*'") c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") - with open(os.path.join(c.current_folder, "conan_cache_save.tgz"), 'rb') as file_handler: - dest_dir = os.path.join(c.cache_folder, "extracted") - tar_extract(file_handler, dest_dir) - assert os.listdir(dest_dir) == ["conan"] - assert os.path.exists(os.path.join(dest_dir, "conan", "pkglist.json")) - From f986651d0afe3867afd27d8376b469e43e315db9 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 23 Jun 2025 17:50:41 +0200 Subject: [PATCH 21/24] Fix condition error --- conan/internal/api/uploader.py | 1 - conan/internal/util/files.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index abd1ce3322d..c1f945fcf31 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -4,7 +4,6 @@ import shutil import tarfile import time -from pathlib import Path from conan.internal.conan_app import ConanApp from conan.api.output import ConanOutput diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index a5c172aae20..b7985c733fe 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -272,7 +272,7 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break the_tar.extractall(path=destination_dir) the_tar.close() - if Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*"): + if list(Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n" "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment.") @@ -297,6 +297,7 @@ def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) else: # The tar was not compressed using the plugin, copy files to destination from conan.tools.files import copy + ConanOutput().debug(f"Extracted in {time.time() - t1} time built in") copy(None, pattern="*", src=temp_dir, dst=destination_dir) def merge_directories(src, dst): From 8009bed44ed708357e4b6e5f915e45f6260aed12 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Wed, 25 Jun 2025 10:39:15 +0200 Subject: [PATCH 22/24] Addressed some issues --- conan/internal/api/uploader.py | 53 +++++++++++-------- conan/internal/util/files.py | 6 ++- .../extensions/test_compression_plugin.py | 31 ++++++++++- 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index c1f945fcf31..24a49a6c911 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -272,30 +272,10 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None): def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None): - tgz_path = os.path.join(dest_dir, name) if compression_plugin: - t1 = time.time() - compressed_path = compression_plugin.tar_compress( - archive_path=os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME), - files=files, - recursive=recursive, - conf=conf, - ref=ref, - ) - ConanOutput().debug(f"{name} compressed in {time.time() - t1} time in plugin") - ConanOutput(scope=str(ref or "")).info(f"Compressing {compressed_path}") - t1 = time.time() - ConanOutput().debug(f"Wrapping {compressed_path} in {name}") - with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: - tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) - tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) - tgz.close() - ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") - # Only remove wrapped if it is different from the tgz_path - if compressed_path != os.path.basename(tgz_path): - remove(compressed_path) - return tgz_path + return _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin) + tgz_path = os.path.join(dest_dir, name) t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory ConanOutput(scope=str(ref or "")).info(f"Compressing {name}") @@ -311,6 +291,35 @@ def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, ConanOutput().debug(f"{name} compressed in {duration} time") return tgz_path +def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin): + t1 = time.time() + abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME) + ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") + compressed_path = compression_plugin.tar_compress( + archive_path=abs_path_without_extension, + files=files, + recursive=recursive, + conf=conf, + ref=ref, + ) + ConanOutput().debug(f"Compressed {compressed_path} in {time.time() - t1} time") + # Check if compressed_path == abs_path_without_extension + .* (any extension) + path, extension = os.path.splitext(compressed_path) + if path != abs_path_without_extension or not extension: + raise ConanException("The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension") + + t1 = time.time() + ConanOutput().debug(f"Wrapping {compressed_path} in {name}") + tgz_path = os.path.join(dest_dir, name) + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: + tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) + tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) + tgz.close() + ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") + # Only remove wrapped if it is different from the tgz_path + if compressed_path != os.path.basename(tgz_path): + remove(compressed_path) + return tgz_path def _total_size(cache_files): total_size = 0 diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index b7985c733fe..36bbe80daaa 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -265,6 +265,7 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): if compression_plugin: _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) return + the_tar = tarfile.open(fileobj=fileobj) # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to # "could not change modification time", with time=0 @@ -284,12 +285,13 @@ def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf) the_tar = tarfile.open(fileobj=fileobj) the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break the_tar.extractall(path=temp_dir) + extracted_file = the_tar.getnames()[0] + the_tar.close() # Check if the tar was compressed with the compression plugin by checking the existence of # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): # Get the only extracted file: the plugin tar - plugin_tar_path = os.path.join(temp_dir, the_tar.getnames()[0]) - the_tar.close() + plugin_tar_path = os.path.join(temp_dir, extracted_file) ConanOutput().debug(f"Unwrapped in {time.time() - t1} time") t1 = time.time() compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf) diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 82900518c22..96144d95aa9 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,7 +1,7 @@ import os import textwrap -from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, tar_extract +from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -32,6 +32,34 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): in c.out ) +def test_compression_plugin_returning_invalid_path(): + """Test an error is raised if the compression plugin does not return expected path""" + + c = TestClient() + compression_plugin = textwrap.dedent( + """ + def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): + return archive_path + def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): + pass + """ + ) + + c.save( + { + os.path.join( + c.cache_folder, "extensions", "plugins", "compression.py" + ): compression_plugin, + "conanfile.py": GenConanfile("pkg", "1.0"), + } + ) + c.run("create .") + c.run("cache save 'pkg/*:*'", assert_error=True) + assert ( + "ERROR: The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension" + in c.out + ) + def test_compression_plugin_correctly_load(): """Test that the compression plugin is correctly loaded and used on: @@ -78,7 +106,6 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ) c.run("create .") c.run("cache save 'pkg/*:*'") - print(c.out) assert f"Compressing {COMPRESSED_PLUGIN_TAR_NAME}.xz using compression plugin (xz)" in c.out c.run("remove pkg/* -c") c.run("cache restore conan_cache_save.tgz") From 0db1a7b2e38737ca398ab2f8e530775d35394ed2 Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Wed, 25 Jun 2025 11:24:13 +0200 Subject: [PATCH 23/24] Make plugin return compressed extension --- conan/internal/api/uploader.py | 16 ++++++---------- .../extensions/test_compression_plugin.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 24a49a6c911..04d8c5a6248 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -295,30 +295,26 @@ def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, com t1 = time.time() abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME) ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin") - compressed_path = compression_plugin.tar_compress( + compressed_extension = compression_plugin.tar_compress( archive_path=abs_path_without_extension, files=files, recursive=recursive, conf=conf, ref=ref, ) - ConanOutput().debug(f"Compressed {compressed_path} in {time.time() - t1} time") - # Check if compressed_path == abs_path_without_extension + .* (any extension) - path, extension = os.path.splitext(compressed_path) - if path != abs_path_without_extension or not extension: - raise ConanException("The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension") + ConanOutput().debug(f"Compressed in {time.time() - t1} time") + if not compressed_extension or not compressed_extension.startswith("."): + raise ConanException("The 'compression.py' did not return the compressed extension.") + compressed_path = abs_path_without_extension + compressed_extension t1 = time.time() - ConanOutput().debug(f"Wrapping {compressed_path} in {name}") tgz_path = os.path.join(dest_dir, name) with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0) tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive) tgz.close() ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time") - # Only remove wrapped if it is different from the tgz_path - if compressed_path != os.path.basename(tgz_path): - remove(compressed_path) + remove(compressed_path) return tgz_path def _total_size(cache_files): diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 96144d95aa9..55bbd7257e6 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -39,7 +39,7 @@ def test_compression_plugin_returning_invalid_path(): compression_plugin = textwrap.dedent( """ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): - return archive_path + return def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): pass """ @@ -56,7 +56,7 @@ def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): c.run("create .") c.run("cache save 'pkg/*:*'", assert_error=True) assert ( - "ERROR: The 'compression.py' plugin returned an unexpected path. Plugin should return the 'archive_path' with an extra extension" + "ERROR: The 'compression.py' did not return the compressed extension." in c.out ) @@ -76,7 +76,8 @@ def test_compression_plugin_correctly_load(): # xz compression def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **kwargs): - archive_path += ".xz" + extension = ".xz" + archive_path += extension name = os.path.basename(archive_path) ConanOutput(scope=ref).info(f"Compressing {name} using compression plugin (xz)") compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None @@ -84,7 +85,7 @@ def tar_compress(archive_path, files, recursive, conf=None, ref=None, *args, **k with tarfile.open(archive_path, f"w:xz", **kwargs) as tgz: for filename, abs_path in sorted(files.items()): tgz.add(abs_path, filename, recursive=True) - return archive_path + return extension def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): ConanOutput().info(f"Decompressing {os.path.basename(archive_path)} using compression plugin (xz)") @@ -141,7 +142,8 @@ def test_compression_plugin_tar_not_compatible_with_builtin(): # zip compression def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): # compress files using zipfile library taking into account recursive - archive_path += ".zip" + extension = ".zip" + archive_path += extension name = os.path.basename(archive_path) compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None ConanOutput().info(f"Compressing {name} using compression plugin (zip)") @@ -152,7 +154,7 @@ def tar_compress(archive_path, files, recursive, conf=None, *args, **kwargs): zipf.write(abs_path, arcname) else: zipf.write(abs_path, filename) - return archive_path + return extension def tar_extract(archive_path, dest_dir, conf=None, *args, **kwargs): # extract tar using zipfile library @@ -192,7 +194,8 @@ def test_compress_in_subdirectory(): from conan.api.output import ConanOutput def tar_compress(archive_path, files, recursive, *args, **kwargs): # compress files using tarfile putting all content in a `conan/` subfolder - archive_path += ".tgz" + extension = ".tgz" + archive_path += extension name = os.path.basename(archive_path) ConanOutput().info(f"Compressing {os.path.basename(name)} in conan subfolder") with open(archive_path, "wb") as tgz_handle: @@ -200,7 +203,7 @@ def tar_compress(archive_path, files, recursive, *args, **kwargs): for filename, abs_path in sorted(files.items()): tgz.add(abs_path, os.path.join("conan", filename), recursive=recursive) tgz.close() - return archive_path + return extension def tar_extract(archive_path, dest_dir, *args, **kwargs): ConanOutput().info(f"Decompressing {archive_path} in conan subfolder") From dcbb29e5f71bbab0d1737108173d1403f25bb50e Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Mon, 14 Jul 2025 12:46:45 +0200 Subject: [PATCH 24/24] Adapt changes to support metadata in wrapped files Add new test case to check those files are correctly pruned after extraction Improve performance by extracting in the actual destination folder even when plugin is enabled --- conan/internal/api/uploader.py | 1 + conan/internal/util/files.py | 44 ++++++++++--------- .../extensions/test_compression_plugin.py | 42 +++++++++++++++++- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 04d8c5a6248..7960ebc1226 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -303,6 +303,7 @@ def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, com ref=ref, ) ConanOutput().debug(f"Compressed in {time.time() - t1} time") + ConanOutput().success(f"{time.time() - t1}") if not compressed_extension or not compressed_extension.startswith("."): raise ConanException("The 'compression.py' did not return the compressed extension.") diff --git a/conan/internal/util/files.py b/conan/internal/util/files.py index 36bbe80daaa..40a989cc5e7 100644 --- a/conan/internal/util/files.py +++ b/conan/internal/util/files.py @@ -278,29 +278,33 @@ def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None): "This file has been compressed using a `compression` plugin.\n" "If your organization uses this plugin, ensure it is correctly installed on your environment.") + def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf): """First remove tar.gz wrapper and then call the plugin to extract""" - with tempfile.TemporaryDirectory() as temp_dir: - t1 = time.time() - the_tar = tarfile.open(fileobj=fileobj) - the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break - the_tar.extractall(path=temp_dir) - extracted_file = the_tar.getnames()[0] - the_tar.close() - # Check if the tar was compressed with the compression plugin by checking the existence of - # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) - if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")): - # Get the only extracted file: the plugin tar - plugin_tar_path = os.path.join(temp_dir, extracted_file) - ConanOutput().debug(f"Unwrapped in {time.time() - t1} time") + t1 = time.time() + the_tar = tarfile.open(fileobj=fileobj) + the_tar.extraction_filter = (lambda member, path: member) # fully_trusted, avoid Py3.14 break + the_tar.extractall(path=destination_dir) + extracted_files = the_tar.getnames() + the_tar.close() + # Check if the tar was compressed with the compression plugin by checking the existence of + # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin) + ConanOutput().success(f"{time.time() - t1}") + for path in extracted_files: + if os.path.basename(path).startswith(COMPRESSED_PLUGIN_TAR_NAME): + # Extract the actual contents from the plugin tar (ignore other files present). + ConanOutput().debug(f"Unwrapped in {time.time() - t1}") t1 = time.time() - compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf) - ConanOutput().debug(f"Extracted in {time.time() - t1} time on plugin") - else: - # The tar was not compressed using the plugin, copy files to destination - from conan.tools.files import copy - ConanOutput().debug(f"Extracted in {time.time() - t1} time built in") - copy(None, pattern="*", src=temp_dir, dst=destination_dir) + compression_plugin.tar_extract( + archive_path=os.path.join(destination_dir, path), + dest_dir=destination_dir, + conf=conf, + ) + # Remove extracted files from tar + for f in extracted_files: + remove(os.path.join(destination_dir, f)) + break + ConanOutput().debug(f"Extracted in {time.time() - t1}") def merge_directories(src, dst): from conan.tools.files import copy diff --git a/test/integration/extensions/test_compression_plugin.py b/test/integration/extensions/test_compression_plugin.py index 55bbd7257e6..0090d03fd34 100644 --- a/test/integration/extensions/test_compression_plugin.py +++ b/test/integration/extensions/test_compression_plugin.py @@ -1,7 +1,8 @@ import os import textwrap +import tarfile -from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME +from conan.internal.util.files import COMPRESSED_PLUGIN_TAR_NAME, mkdir from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.tools import TestClient @@ -227,5 +228,42 @@ def tar_extract(archive_path, dest_dir, *args, **kwargs): ) c.run("create .") c.run("cache save 'pkg/*:*'") + tgz = os.path.join(c.current_folder, "conan_cache_save.tgz") + assert os.path.exists(tgz) + + mkdir("extract_folder") + destination_dir = os.path.join(c.current_folder, "extract_folder") + extracted_files = _tar_extract(tgz, destination_dir) + assert extracted_files == [COMPRESSED_PLUGIN_TAR_NAME + ".tgz"] + # Create example files + c.save({os.path.join(destination_dir, "README.md"): "This is a readme file.", + os.path.join(destination_dir, "Cache-contents-graph-report.html"): "Cache contents graph report", + os.path.join(destination_dir, "Cache-contents-graph-report.exe"): "executable file...", + os.path.join(destination_dir, "conan_cache_save.tgz"): "this should also be ignored", + }) + # Recompress the file with metadata + _tar_compress(os.path.join(c.current_folder, "conan_cache_save_rearchived.tgz"), destination_dir) + c.run("remove pkg/* -c") - c.run("cache restore conan_cache_save.tgz") + c.run("cache restore conan_cache_save_rearchived.tgz") + + # Check any of the metadata are present in the cache + assert any(item not in os.listdir(os.path.join(c.cache_folder, "p")) for item in ("README.md", + "Cache-contents-graph-report.html", + "Cache-contents-graph-report.exe")) + + +def _tar_extract(tgz_path, destination_dir): + with open(tgz_path, "rb") as fileobj: + the_tar = tarfile.open(fileobj=fileobj) + the_tar.extraction_filter = (lambda member, path: member) + the_tar.extractall(path=destination_dir) + return the_tar.getnames() + + +def _tar_compress(archive_path, folder): + with open(archive_path, "wb") as tgz_handle: + tgz = tarfile.open(os.path.basename(archive_path), "w", fileobj=tgz_handle) + for filename in os.listdir(folder): + tgz.add(os.path.join(folder, filename), filename, recursive=True) + tgz.close()