Compression plugin

conan/api/subapi/cache.py

@@ -1,7 +1,6 @@
 import json
 import os
 import shutil
-import tarfile
 import tempfile
 
 from conan.api.model import PackagesList
@@ -17,7 +16,7 @@
 from conan.api.model import PkgReference
 from conan.api.model import RecipeReference
 from conan.internal.util.dates import revision_timestamp_now
-from conan.internal.util.files import rmdir, mkdir, remove, save
+from conan.internal.util.files import rmdir, mkdir, remove, save, tar_extract
 
 
 class CacheAPI:
@@ -172,7 +171,8 @@ def save(self, package_list, tgz_path, no_source=False):
         pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json")
         save(pkglist_path, serialized)
         tar_files["pkglist.json"] = pkglist_path
-        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, recursive=True)
+        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel,
+                       recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin)
         remove(pkglist_path)
 
     def restore(self, path):
@@ -182,13 +182,16 @@ def restore(self, path):
         cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf)
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
 
-        with open(path, mode='rb') as file_handler:
-            the_tar = tarfile.open(fileobj=file_handler)
-            fileobj = the_tar.extractfile("pkglist.json")
-            pkglist = fileobj.read()
-            the_tar.extraction_filter = (lambda member, _: member)  # fully_trusted (Py 3.14)
-            the_tar.extractall(path=cache_folder)
-            the_tar.close()
+        compression_plugin = self.conan_api.config.compression_plugin
+        if compression_plugin:
+            compression_plugin.tar_extract(path, cache_folder)
+        else:
+            with open(path, mode='rb') as file_handler:
+                tar_extract(file_handler, cache_folder)
+
+        # Retrieve the package list from the already extracted archive
+        with open(os.path.join(cache_folder, "pkglist.json")) as file_handler:
+            pkglist = file_handler.read()
 
         # After unzipping the files, we need to update the DB that references these files
         out = ConanOutput()

conan/api/subapi/config.py

@@ -3,6 +3,7 @@
 import platform
 import textwrap
 import yaml
+from conan.internal.loader import load_python_file
 from jinja2 import Environment, FileSystemLoader
 
 from conan import conan_version
@@ -31,6 +32,7 @@ def __init__(self, conan_api):
         self._new_config = None
         self._cli_core_confs = None
         self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path)
+        self._compression_plugin = None
 
     def home(self):
         return self.conan_api.cache_folder
@@ -236,4 +238,20 @@ def reinit(self):
         if self._new_config is not None:
             self._new_config.clear()
             self._populate_global_conf()
+            self._compression_plugin = None
         self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path)
+
+    @property
+    def compression_plugin(self):
+        def load_compression_plugin():
+            compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path
+            if not os.path.exists(compression_plugin_path):
+                return None
+            mod, _ = load_python_file(compression_plugin_path)
+            if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"):
+                raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions")
+            return mod
+        if not self._compression_plugin:
+            self._compression_plugin = load_compression_plugin()
+        return self._compression_plugin
+

conan/internal/api/uploader.py

@@ -157,7 +157,8 @@ def add_tgz(tgz_name, tgz_files):
             elif tgz_files:
                 compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
                 tgz = compress_files(tgz_files, tgz_name, download_export_folder,
-                                     compresslevel=compresslevel, ref=ref)
+                                     compresslevel=compresslevel, ref=ref,
+                                     compression_plugin=self._app.conan_api.config.compression_plugin)
                 result[tgz_name] = tgz
 
         add_tgz(EXPORT_TGZ_NAME, files)
@@ -204,7 +205,8 @@ def _compress_package_files(self, layout, pref):
             tgz_files = {f: path for f, path in files.items()}
             compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
             tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel, ref=pref)
+                                      compresslevel=compresslevel, ref=pref,
+                                      compression_plugin=self._app.conan_api.config.compression_plugin)
             assert tgz_path == package_tgz
             assert os.path.exists(package_tgz)
 
@@ -271,12 +273,14 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, compression_plugin=None):
+    if compression_plugin:
+        return compression_plugin.tar_compress(files, name, dest_dir, compresslevel, ref, recursive)
+
     t1 = time.time()
     # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
-    if ref:
-        ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}")
+    ConanOutput(scope=str(ref or "")).info(f"Compressing {name}")
     with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
         tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
         for filename, abs_path in sorted(files.items()):

conan/internal/cache/home_paths.py

@@ -90,3 +90,7 @@ def settings_path_user(self):
     @property
     def config_version_path(self):
         return os.path.join(self._home, "config_version.json")
+
+    @property
+    def compression_plugin_path(self):
+        return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py")

conan/internal/conan_app.py

@@ -52,7 +52,7 @@ def __init__(self, conan_api):
         localdb = LocalDB(cache_folder)
         auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, global_conf)
         # Handle remote connections
-        self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder)
+        self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, self.conan_api.config)
         global_editables = conan_api.local.editable_packages
         ws_editables = conan_api.workspace.editable_packages
         self.editable_packages = global_editables.update_copy(ws_editables)
@@ -81,10 +81,10 @@ class LocalRecipesIndexApp:
     - loader (for the export phase of local-recipes-index)
     The others are internally use by other collaborators
     """
-    def __init__(self, cache_folder):
+    def __init__(self, cache_folder, config_api):
         self.global_conf = ConfDefinition()
         self.cache = PkgCache(cache_folder, self.global_conf)
-        self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder)
+        self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder, config_api=config_api)
         editable_packages = EditablePackages()
         self.proxy = ConanProxy(self, editable_packages)
         self.range_resolver = RangeResolver(self, self.global_conf, editable_packages)

conan/internal/rest/remote_manager.py

@@ -1,3 +1,4 @@
+from gzip import compress
 import os
 import shutil
 from typing import List
@@ -22,15 +23,16 @@
 
 class RemoteManager:
     """ Will handle the remotes to get recipes, packages etc """
-    def __init__(self, cache, auth_manager, home_folder):
+    def __init__(self, cache, auth_manager, home_folder, config_api):
         self._cache = cache
         self._auth_manager = auth_manager
         self._signer = PkgSignaturesPlugin(cache, home_folder)
         self._home_folder = home_folder
+        self._config_api = config_api
 
     def _local_folder_remote(self, remote):
         if remote.remote_type == LOCAL_RECIPES_INDEX:
-            return RestApiClientLocalRecipesIndex(remote, self._home_folder)
+            return RestApiClientLocalRecipesIndex(remote, self._home_folder, self._config_api)
 
     def check_credentials(self, remote, force_auth=False):
         self._call_remote(remote, "check_credentials", force_auth)
@@ -81,7 +83,7 @@ def get_recipe(self, ref, remote, metadata=None):
         tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
 
         if tgz_file:
-            uncompress_file(tgz_file, export_folder, scope=str(ref))
+            uncompress_file(tgz_file, export_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin)
         mkdir(export_folder)
         for file_name, file_path in zipped_files.items():  # copy CONANFILE
             shutil.move(file_path, os.path.join(export_folder, file_name))
@@ -123,7 +125,7 @@ def get_recipe_sources(self, ref, layout, remote):
 
         self._signer.verify(ref, download_folder, files=zipped_files)
         tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
-        uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
+        uncompress_file(tgz_file, export_sources_folder, scope=str(ref), compression_plugin=self._config_api.compression_plugin)
 
     def get_package(self, pref, remote, metadata=None):
         output = ConanOutput(scope=str(pref.ref))
@@ -171,7 +173,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
 
             tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
             package_folder = layout.package()
-            uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
+            uncompress_file(tgz_file, package_folder, scope=str(pref.ref), compression_plugin=self._config_api.compression_plugin)
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
             for file_name, file_path in zipped_files.items():  # copy CONANINFO and CONANMANIFEST
                 shutil.move(file_path, os.path.join(package_folder, file_name))
@@ -281,15 +283,19 @@ def _call_remote(self, remote, method, *args, **kwargs):
             raise ConanException(exc, remote=remote)
 
 
-def uncompress_file(src_path, dest_folder, scope=None):
+def uncompress_file(src_path, dest_folder, scope=None, compression_plugin=None):
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB
         if big_file:
             hs = human_size(filesize)
             ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}")
-        with open(src_path, mode='rb') as file_handler:
-            tar_extract(file_handler, dest_folder)
+
+        if compression_plugin:
+            compression_plugin.tar_extract(src_path, dest_folder)
+        else:
+            with open(src_path, mode='rb') as file_handler:
+                tar_extract(file_handler, dest_folder)
     except Exception as e:
         error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\
                     % (src_path, dest_folder, str(e))

conan/internal/rest/rest_client_local_recipe_index.py

@@ -58,14 +58,14 @@ class RestApiClientLocalRecipesIndex:
     a local folder assuming the conan-center-index repo layout
     """
 
-    def __init__(self, remote, home_folder):
+    def __init__(self, remote, home_folder, config_api):
         self._remote = remote
         local_recipes_index_path = HomePaths(home_folder).local_recipes_index_path
         local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name, ".conan")
         repo_folder = self._remote.url
 
         from conan.internal.conan_app import LocalRecipesIndexApp
-        self._app = LocalRecipesIndexApp(local_recipes_index_path)
+        self._app = LocalRecipesIndexApp(local_recipes_index_path, config_api)
         self._hook_manager = HookManager(HomePaths(local_recipes_index_path).hooks_path)
         self._layout = _LocalRecipesIndexLayout(repo_folder)
 

test/integration/extensions/test_compression_plugin.py

@@ -0,0 +1,106 @@
+import os
+import textwrap
+
+from conan.test.assets.genconanfile import GenConanfile
+from conan.test.utils.tools import TestClient
+
+
+def test_compression_plugin_not_existing():
+    """Test that the compression plugin is not used if it does not exist"""
+    c = TestClient()
+    c.save({"conanfile.py": GenConanfile("pkg", "1.0")})
+    c.run("create .")
+    c.run("cache save 'pkg/*'")
+    assert "Compressing conan_cache_save.tgz\n" in c.out
+    c.run("cache restore conan_cache_save.tgz")
+    # Default decompress does not have any output
+    assert "Decompressing conan_cache_save.tgz" not in c.out
+
+
+def test_compression_plugin_not_valid():
+    """Test an error is raised if the compression plugin is not valid"""
+
+    c = TestClient()
+    compression_plugin = textwrap.dedent(
+        """
+        def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+            return None
+    """
+    )
+
+    c.save(
+        {
+            os.path.join(
+                c.cache_folder, "extensions", "plugins", "compression.py"
+            ): compression_plugin,
+            "conanfile.py": GenConanfile("pkg", "1.0"),
+        }
+    )
+    c.run("create .")
+    c.run("cache save 'pkg/*'", assert_error=True)
+    assert (
+        "ERROR: The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions"
+        in c.out
+    )
+
+
+def test_compression_plugin_correctly_load():
+    """Test that the compression plugin is correctly loaded and used on:
+    - cache save/restore
+    - remote upload/download
+    """
+    c = TestClient(default_server_user=True)
+
+    compression_plugin = textwrap.dedent(
+        """
+        import os
+        import tarfile
+        from conan.api.output import ConanOutput
+
+        # xz compression
+        def tar_compress(files, name, dest_dir, compresslevel=None, ref=None, recursive=False, *args, **kwargs):
+            tgz_path = os.path.join(dest_dir, name)
+            ConanOutput(scope=str(ref) if ref else "").info(f"Compressing {name} using compression plugin (xz)")
+            kwargs = {"preset": compresslevel} if compresslevel else {}
+            with tarfile.open(tgz_path, f"w:xz", **kwargs) as tgz:
+                for filename, abs_path in sorted(files.items()):
+                    tgz.add(abs_path, filename, recursive=True)
+            return tgz_path
+
+        def tar_extract(src_path, destination_dir, *args, **kwargs):
+            ConanOutput().info(f"Decompressing {os.path.basename(src_path)} using compression plugin (xz)")
+            with open(src_path, mode='rb') as file_handler:
+                the_tar = tarfile.open(fileobj=file_handler)
+                the_tar.extraction_filter = (lambda member, path: member)
+                the_tar.extractall(path=destination_dir)
+                the_tar.close()
+    """
+    )
+
+    c.save(
+        {
+            os.path.join(
+                c.cache_folder, "extensions", "plugins", "compression.py"
+            ): compression_plugin,
+            "conanfile.py": GenConanfile("pkg", "1.0"),
+        }
+    )
+    c.run("create .")
+    c.run("cache save 'pkg/*'")
+    assert "Compressing conan_cache_save.tgz using compression plugin (xz)" in c.out
+    c.run("remove pkg/* -c")
+    c.run("cache restore conan_cache_save.tgz")
+    assert "Decompressing conan_cache_save.tgz using compression plugin (xz)" in c.out
+    c.run("list pkg/1.0")
+    assert "Found 1 pkg/version recipes matching pkg/1.0 in local cache" in c.out
+
+    # Remove pre existing tgz to force a recompression
+    c.run("remove pkg/* -c")
+    c.run("create .")
+    # Check the plugin is also used on remote interactions
+    c.run("upload * -r=default -c")
+    assert "Compressing conan_package.tgz using compression plugin (xz)" in c.out
+    assert "pkg/1.0: Uploading recipe" in c.out
+    c.run("remove pkg/* -c")
+    c.run("download 'pkg/*' -r=default")
+    assert "Decompressing conan_package.tgz using compression plugin (xz)" in c.out