Support zstd compression in image commit

commit_test.go

@@ -11,15 +11,33 @@ import (
 	"testing"
 	"time"
 
+	"github.com/containers/image/v5/manifest"
+	ociLayout "github.com/containers/image/v5/oci/layout"
 	imageStorage "github.com/containers/image/v5/storage"
+	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
+	"github.com/containers/storage/pkg/archive"
 	storageTypes "github.com/containers/storage/types"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func makeFile(t *testing.T, base string, size int64) string {
+	t.Helper()
+	fn := filepath.Join(t.TempDir(), base)
+	f, err := os.Create(fn)
+	require.NoError(t, err)
+	defer f.Close()
+	if size == 0 {
+		size = 512
+	}
+	_, err = io.CopyN(f, rand.Reader, size)
+	require.NoErrorf(t, err, "writing payload file %d", base)
+	return f.Name()
+}
+
 func TestCommitLinkedLayers(t *testing.T) {
 	// This test cannot be parallized as this uses NewBuilder()
 	// which eventually and indirectly accesses a global variable
@@ -34,6 +52,7 @@ func TestCommitLinkedLayers(t *testing.T) {
 	if graphDriverName == "" {
 		graphDriverName = "vfs"
 	}
+	t.Logf("using storage driver %q", graphDriverName)
 	store, err := storage.GetStore(storageTypes.StoreOptions{
 		RunRoot:         t.TempDir(),
 		GraphRoot:       t.TempDir(),
@@ -44,17 +63,7 @@ func TestCommitLinkedLayers(t *testing.T) {
 
 	imageName := func(i int) string { return fmt.Sprintf("image%d", i) }
 	makeFile := func(base string, size int64) string {
-		t.Helper()
-		fn := filepath.Join(t.TempDir(), base)
-		f, err := os.Create(fn)
-		require.NoError(t, err)
-		defer f.Close()
-		if size == 0 {
-			size = 512
-		}
-		_, err = io.CopyN(f, rand.Reader, size)
-		require.NoErrorf(t, err, "writing payload file %d", base)
-		return f.Name()
+		return makeFile(t, base, size)
 	}
 	makeArchive := func(base string, size int64) string {
 		t.Helper()
@@ -242,3 +251,99 @@ func TestCommitLinkedLayers(t *testing.T) {
 		}()
 	}
 }
+
+func TestCommitCompression(t *testing.T) {
+	// This test cannot be parallized as this uses NewBuilder()
+	// which eventually and indirectly accesses a global variable
+	// defined in `go-selinux`, this must be fixed at `go-selinux`
+	// or builder must enable sometime of locking mechanism i.e if
+	// routine is creating Builder other's must wait for it.
+	// Tracked here: https://github.com/containers/buildah/issues/5967
+	ctx := context.TODO()
+
+	graphDriverName := os.Getenv("STORAGE_DRIVER")
+	if graphDriverName == "" {
+		graphDriverName = "vfs"
+	}
+	t.Logf("using storage driver %q", graphDriverName)
+	store, err := storage.GetStore(storageTypes.StoreOptions{
+		RunRoot:         t.TempDir(),
+		GraphRoot:       t.TempDir(),
+		GraphDriverName: graphDriverName,
+	})
+	require.NoError(t, err, "initializing storage")
+	t.Cleanup(func() { _, err := store.Shutdown(true); assert.NoError(t, err) })
+
+	builderOptions := BuilderOptions{
+		FromImage: "scratch",
+		NamespaceOptions: []NamespaceOption{{
+			Name: string(rspec.NetworkNamespace),
+			Host: true,
+		}},
+		SystemContext: &testSystemContext,
+	}
+	b, err := NewBuilder(ctx, store, builderOptions)
+	require.NoError(t, err, "creating builder")
+	payload := makeFile(t, "file0", 0)
+	b.SetCreatedBy("ADD file0 in /")
+	err = b.Add("/", false, AddAndCopyOptions{}, payload)
+	require.NoError(t, err, "adding", payload)
+	for _, compressor := range []struct {
+		compression    archive.Compression
+		name           string
+		expectError    bool
+		layerMediaType string
+	}{
+		{archive.Uncompressed, "uncompressed", false, v1.MediaTypeImageLayer},
+		{archive.Gzip, "gzip", false, v1.MediaTypeImageLayerGzip},
+		{archive.Bzip2, "bz2", true, ""},
+		{archive.Xz, "xz", true, ""},
+		{archive.Zstd, "zstd", false, v1.MediaTypeImageLayerZstd},
+	} {
+		t.Run(compressor.name, func(t *testing.T) {
+			var ref types.ImageReference
+			commitOptions := CommitOptions{
+				PreferredManifestType: v1.MediaTypeImageManifest,
+				SystemContext:         &testSystemContext,
+				Compression:           compressor.compression,
+			}
+			imageName := compressor.name
+			ref, err := imageStorage.Transport.ParseStoreReference(store, imageName)
+			require.NoErrorf(t, err, "parsing reference for to-be-committed local image %q", imageName)
+			_, _, _, err = b.Commit(ctx, ref, commitOptions)
+			if compressor.expectError {
+				require.Errorf(t, err, "committing local image %q", imageName)
+			} else {
+				require.NoErrorf(t, err, "committing local image %q", imageName)
+			}
+			imageName = t.TempDir()
+			ref, err = ociLayout.Transport.ParseReference(imageName)
+			require.NoErrorf(t, err, "parsing reference for to-be-committed oci layout %q", imageName)
+			_, _, _, err = b.Commit(ctx, ref, commitOptions)
+			if compressor.expectError {
+				require.Errorf(t, err, "committing oci layout %q", imageName)
+				return
+			}
+			require.NoErrorf(t, err, "committing oci layout %q", imageName)
+			src, err := ref.NewImageSource(ctx, &testSystemContext)
+			require.NoErrorf(t, err, "reading oci layout %q", imageName)
+			defer src.Close()
+			manifestBytes, manifestType, err := src.GetManifest(ctx, nil)
+			require.NoErrorf(t, err, "reading manifest from oci layout %q", imageName)
+			require.Equalf(t, v1.MediaTypeImageManifest, manifestType, "manifest type from oci layout %q looked wrong", imageName)
+			parsedManifest, err := manifest.OCI1FromManifest(manifestBytes)
+			require.NoErrorf(t, err, "parsing manifest from oci layout %q", imageName)
+			require.Lenf(t, parsedManifest.Layers, 1, "expected exactly one layer in oci layout %q", imageName)
+			require.Equalf(t, compressor.layerMediaType, parsedManifest.Layers[0].MediaType, "expected the layer media type to reflect compression in oci layout %q", imageName)
+			blobReadCloser, _, err := src.GetBlob(ctx, types.BlobInfo{
+				Digest:    parsedManifest.Layers[0].Digest,
+				MediaType: parsedManifest.Layers[0].MediaType,
+			}, nil)
+			require.NoErrorf(t, err, "reading the first layer from oci layout %q", imageName)
+			defer blobReadCloser.Close()
+			blob, err := io.ReadAll(blobReadCloser)
+			require.NoErrorf(t, err, "consuming the first layer from oci layout %q", imageName)
+			require.Equalf(t, compressor.compression, archive.DetectCompression(blob), "detected compression looks wrong for layer in oci layout %q")
+		})
+	}
+}

image.go

@@ -621,9 +621,8 @@ func (mb *ociManifestBuilder) computeLayerMIMEType(what string, layerCompression
 			// how to decompress them, we can't try to compress layers with xz.
 			return errors.New("media type for xz-compressed layers is not defined")
 		case archive.Zstd:
-			// Until the image specs define a media type for zstd-compressed layers, even if we know
-			// how to decompress them, we can't try to compress layers with zstd.
-			return errors.New("media type for zstd-compressed layers is not defined")
+			omediaType = v1.MediaTypeImageLayerZstd
+			logrus.Debugf("compressing %s with zstd", what)
 		default:
 			logrus.Debugf("compressing %s with unknown compressor(?)", what)
 		}