From aa84d9c34740387c4ece10575be9624595fecfa5 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Mon, 1 Apr 2024 13:24:15 -0700 Subject: [PATCH 1/3] Support zstd compression in image commit Without this change, specifying `Compression: imagebuildah.Zstd` in `imagebuildah`'s `BuildOptions fails, so it is not possible to push cache to a registry with zstd compression. Note this is only applicable to OCI manifests. Signed-off-by: Aaron Lehmann --- image.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/image.go b/image.go index 9e6c700bb22..fd61890bea1 100644 --- a/image.go +++ b/image.go @@ -621,9 +621,8 @@ func (mb *ociManifestBuilder) computeLayerMIMEType(what string, layerCompression // how to decompress them, we can't try to compress layers with xz. return errors.New("media type for xz-compressed layers is not defined") case archive.Zstd: - // Until the image specs define a media type for zstd-compressed layers, even if we know - // how to decompress them, we can't try to compress layers with zstd. - return errors.New("media type for zstd-compressed layers is not defined") + omediaType = v1.MediaTypeImageLayerZstd + logrus.Debugf("compressing %s with zstd", what) default: logrus.Debugf("compressing %s with unknown compressor(?)", what) } From 873e5458c63ded100d1a61e52cdf37e4282b8632 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Wed, 11 Jun 2025 17:43:30 -0400 Subject: [PATCH 2/3] Add a unit test for compression types in OCI images Add a unit test that commits OCI layouts with various types of compression specified, and verifies that the layers end up written with the desired compression and media type descriptors. Signed-off-by: Nalin Dahyabhai --- commit_test.go | 128 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 117 insertions(+), 11 deletions(-) diff --git a/commit_test.go b/commit_test.go index 41f63bdae21..12f08cf9dc3 100644 --- a/commit_test.go +++ b/commit_test.go @@ -11,8 +11,12 @@ import ( "testing" "time" + "github.com/containers/image/v5/manifest" + ociLayout "github.com/containers/image/v5/oci/layout" imageStorage "github.com/containers/image/v5/storage" + "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" storageTypes "github.com/containers/storage/types" v1 "github.com/opencontainers/image-spec/specs-go/v1" rspec "github.com/opencontainers/runtime-spec/specs-go" @@ -20,6 +24,20 @@ import ( "github.com/stretchr/testify/require" ) +func makeFile(t *testing.T, base string, size int64) string { + t.Helper() + fn := filepath.Join(t.TempDir(), base) + f, err := os.Create(fn) + require.NoError(t, err) + defer f.Close() + if size == 0 { + size = 512 + } + _, err = io.CopyN(f, rand.Reader, size) + require.NoErrorf(t, err, "writing payload file %d", base) + return f.Name() +} + func TestCommitLinkedLayers(t *testing.T) { // This test cannot be parallized as this uses NewBuilder() // which eventually and indirectly accesses a global variable @@ -34,6 +52,7 @@ func TestCommitLinkedLayers(t *testing.T) { if graphDriverName == "" { graphDriverName = "vfs" } + t.Logf("using storage driver %q", graphDriverName) store, err := storage.GetStore(storageTypes.StoreOptions{ RunRoot: t.TempDir(), GraphRoot: t.TempDir(), @@ -44,17 +63,7 @@ func TestCommitLinkedLayers(t *testing.T) { imageName := func(i int) string { return fmt.Sprintf("image%d", i) } makeFile := func(base string, size int64) string { - t.Helper() - fn := filepath.Join(t.TempDir(), base) - f, err := os.Create(fn) - require.NoError(t, err) - defer f.Close() - if size == 0 { - size = 512 - } - _, err = io.CopyN(f, rand.Reader, size) - require.NoErrorf(t, err, "writing payload file %d", base) - return f.Name() + return makeFile(t, base, size) } makeArchive := func(base string, size int64) string { t.Helper() @@ -242,3 +251,100 @@ func TestCommitLinkedLayers(t *testing.T) { }() } } + +func TestCommitCompression(t *testing.T) { + // This test cannot be parallized as this uses NewBuilder() + // which eventually and indirectly accesses a global variable + // defined in `go-selinux`, this must be fixed at `go-selinux` + // or builder must enable sometime of locking mechanism i.e if + // routine is creating Builder other's must wait for it. + // Tracked here: https://github.com/containers/buildah/issues/5967 + ctx := context.TODO() + + graphDriverName := os.Getenv("STORAGE_DRIVER") + if graphDriverName == "" { + graphDriverName = "vfs" + } + t.Logf("using storage driver %q", graphDriverName) + store, err := storage.GetStore(storageTypes.StoreOptions{ + RunRoot: t.TempDir(), + GraphRoot: t.TempDir(), + GraphDriverName: graphDriverName, + }) + require.NoError(t, err, "initializing storage") + t.Cleanup(func() { _, err := store.Shutdown(true); assert.NoError(t, err) }) + + builderOptions := BuilderOptions{ + FromImage: "scratch", + NamespaceOptions: []NamespaceOption{{ + Name: string(rspec.NetworkNamespace), + Host: true, + }}, + SystemContext: &testSystemContext, + } + b, err := NewBuilder(ctx, store, builderOptions) + require.NoError(t, err, "creating builder") + payload := makeFile(t, "file0", 0) + b.SetCreatedBy("ADD file0 in /") + err = b.Add("/", false, AddAndCopyOptions{}, payload) + require.NoError(t, err, "adding", payload) + for _, compressor := range []struct { + compression archive.Compression + name string + expectError bool + layerMediaType string + }{ + {archive.Uncompressed, "uncompressed", false, v1.MediaTypeImageLayer}, + {archive.Gzip, "gzip", false, v1.MediaTypeImageLayerGzip}, + {archive.Bzip2, "bz2", true, ""}, + {archive.Xz, "xz", true, ""}, + {archive.Zstd, "zstd", false, v1.MediaTypeImageLayerZstd}, + } { + t.Run(compressor.name, func(t *testing.T) { + var ref types.ImageReference + commitOptions := CommitOptions{ + PreferredManifestType: v1.MediaTypeImageManifest, + SystemContext: &testSystemContext, + Compression: compressor.compression, + } + imageName := compressor.name + ref, err := imageStorage.Transport.ParseStoreReference(store, imageName) + require.NoErrorf(t, err, "parsing reference for to-be-committed local image %q", imageName) + _, _, _, err = b.Commit(ctx, ref, commitOptions) + if compressor.expectError { + require.Errorf(t, err, "committing local image %q", imageName) + } else { + require.NoErrorf(t, err, "committing local image %q", imageName) + } + imageName = t.TempDir() + ref, err = ociLayout.Transport.ParseReference(imageName) + require.NoErrorf(t, err, "parsing reference for to-be-committed oci layout %q", imageName) + _, _, _, err = b.Commit(ctx, ref, commitOptions) + if compressor.expectError { + require.Errorf(t, err, "committing oci layout %q", imageName) + return + } else { + require.NoErrorf(t, err, "committing oci layout %q", imageName) + } + src, err := ref.NewImageSource(ctx, &testSystemContext) + require.NoErrorf(t, err, "reading oci layout %q", imageName) + defer src.Close() + manifestBytes, manifestType, err := src.GetManifest(ctx, nil) + require.NoErrorf(t, err, "reading manifest from oci layout %q", imageName) + require.Equalf(t, v1.MediaTypeImageManifest, manifestType, "manifest type from oci layout %q looked wrong", imageName) + parsedManifest, err := manifest.OCI1FromManifest(manifestBytes) + require.NoErrorf(t, err, "parsing manifest from oci layout %q", imageName) + require.Lenf(t, parsedManifest.Layers, 1, "expected exactly one layer in oci layout %q", imageName) + require.Equalf(t, compressor.layerMediaType, parsedManifest.Layers[0].MediaType, "expected the layer media type to reflect compression in oci layout %q", imageName) + blobReadCloser, _, err := src.GetBlob(ctx, types.BlobInfo{ + Digest: parsedManifest.Layers[0].Digest, + MediaType: parsedManifest.Layers[0].MediaType, + }, nil) + require.NoErrorf(t, err, "reading the first layer from oci layout %q", imageName) + defer blobReadCloser.Close() + blob, err := io.ReadAll(blobReadCloser) + require.NoErrorf(t, err, "consuming the first layer from oci layout %q", imageName) + require.Equalf(t, compressor.compression, archive.DetectCompression(blob), "detected compression looks wrong for layer in oci layout %q") + }) + } +} From 2eb666c22d3d3f3c2eedde29a8affe9ee689c104 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 11 Jun 2025 22:50:46 +0000 Subject: [PATCH 3/3] Fix lint issue in TestCommitCompression Signed-off-by: Aaron Lehmann --- commit_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/commit_test.go b/commit_test.go index 12f08cf9dc3..ed2f903235d 100644 --- a/commit_test.go +++ b/commit_test.go @@ -323,9 +323,8 @@ func TestCommitCompression(t *testing.T) { if compressor.expectError { require.Errorf(t, err, "committing oci layout %q", imageName) return - } else { - require.NoErrorf(t, err, "committing oci layout %q", imageName) } + require.NoErrorf(t, err, "committing oci layout %q", imageName) src, err := ref.NewImageSource(ctx, &testSystemContext) require.NoErrorf(t, err, "reading oci layout %q", imageName) defer src.Close()