more generic and full zstd compression support

.github/workflows/ci.yml

@@ -149,7 +149,7 @@ jobs:
       matrix:
         image:
           - opensuse/leap:latest
-          - centos:latest
+          - almalinux:latest
           - debian:latest
           - ubuntu:latest
           - fedora:latest

CHANGELOG.md

@@ -21,6 +21,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
   had no real impact on umoci but for safety we implemented the now-recommended
   media-type embedding and verification. CVE-2021-41190
 
+### Added ###
+- `umoci unpack` now supports handling layers compressed with zstd. This is
+  something that was added in image-spec v1.2 (which we do not yet support
+  fully) but at least this will allow users to operate on zstd-compressed
+  images, which are slowly becoming more common.
+- `umoci repack` and `umoci insert` now support creating zstd-compressed
+  layers. The default behaviour (called `auto`) is to try to match the last
+  layer's compression algorithm, with a fallback to `gzip` if none of the layer
+  algorithms were supported.
+  * Users can specify their preferred compression algorithm using the new
+    `--compress` flag. You can also disable compression entirely using
+    `--compress=none` but `--compress=auto` will never automatically choose
+    `none` compression.
+
 ### Changes ###
 - In this release, the primary development branch was renamed to `main`.
 - The runtime-spec version of the `config.json` version we generate is no

cmd/umoci/insert.go

@@ -29,12 +29,13 @@ import (
 	"github.com/opencontainers/umoci/mutate"
 	"github.com/opencontainers/umoci/oci/cas/dir"
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/casext/blobcompress"
 	igen "github.com/opencontainers/umoci/oci/config/generate"
 	"github.com/opencontainers/umoci/oci/layer"
 	"github.com/urfave/cli"
 )
 
-var insertCommand = uxRemap(uxHistory(uxTag(cli.Command{
+var insertCommand = uxCompress(uxRemap(uxHistory(uxTag(cli.Command{
 	Name:  "insert",
 	Usage: "insert content into an OCI image",
 	ArgsUsage: `--image <image-path>[:<tag>] [--opaque] <source> <target>
@@ -107,14 +108,19 @@ Some examples:
 		ctx.App.Metadata["--target-path"] = targetPath
 		return nil
 	},
-})))
+}))))
 
 func insert(ctx *cli.Context) error {
 	imagePath := ctx.App.Metadata["--image-path"].(string)
 	fromName := ctx.App.Metadata["--image-tag"].(string)
 	sourcePath := ctx.App.Metadata["--source-path"].(string)
 	targetPath := ctx.App.Metadata["--target-path"].(string)
 
+	var compressAlgo blobcompress.Algorithm
+	if algo, ok := ctx.App.Metadata["--compress"].(blobcompress.Algorithm); ok {
+		compressAlgo = algo
+	}
+
 	// By default we clobber the old tag.
 	tagName := fromName
 	if val, ok := ctx.App.Metadata["--tag"]; ok {
@@ -188,9 +194,7 @@ func insert(ctx *cli.Context) error {
 		}
 	}
 
-	// TODO: We should add a flag to allow for a new layer to be made
-	//       non-distributable.
-	if _, err := mutator.Add(context.Background(), ispec.MediaTypeImageLayer, reader, history, mutate.GzipCompressor, nil); err != nil {
+	if _, err := mutator.Add(context.Background(), ispec.MediaTypeImageLayer, reader, history, compressAlgo, nil); err != nil {
 		return fmt.Errorf("add diff layer: %w", err)
 	}
 

cmd/umoci/raw-add-layer.go

@@ -30,11 +30,12 @@ import (
 	"github.com/opencontainers/umoci/mutate"
 	"github.com/opencontainers/umoci/oci/cas/dir"
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/casext/blobcompress"
 	igen "github.com/opencontainers/umoci/oci/config/generate"
 	"github.com/urfave/cli"
 )
 
-var rawAddLayerCommand = uxHistory(uxTag(cli.Command{
+var rawAddLayerCommand = uxCompress(uxHistory(uxTag(cli.Command{
 	Name:  "add-layer",
 	Usage: "add a layer archive verbatim to an image",
 	ArgsUsage: `--image <image-path>[:<tag>] <new-layer.tar>
@@ -65,13 +66,18 @@ only supports uncompressed archives.`,
 		ctx.App.Metadata["newlayer"] = ctx.Args().First()
 		return nil
 	},
-}))
+})))
 
 func rawAddLayer(ctx *cli.Context) error {
 	imagePath := ctx.App.Metadata["--image-path"].(string)
 	fromName := ctx.App.Metadata["--image-tag"].(string)
 	newLayerPath := ctx.App.Metadata["newlayer"].(string)
 
+	var compressAlgo blobcompress.Algorithm
+	if algo, ok := ctx.App.Metadata["--compress"].(blobcompress.Algorithm); ok {
+		compressAlgo = algo
+	}
+
 	// Overide the from tag by default, otherwise use the one specified.
 	tagName := fromName
 	if overrideTagName, ok := ctx.App.Metadata["--tag"]; ok {
@@ -154,9 +160,7 @@ func rawAddLayer(ctx *cli.Context) error {
 		}
 	}
 
-	// TODO: We should add a flag to allow for a new layer to be made
-	//       non-distributable.
-	if _, err := mutator.Add(context.Background(), ispec.MediaTypeImageLayer, newLayer, history, mutate.GzipCompressor, nil); err != nil {
+	if _, err := mutator.Add(context.Background(), ispec.MediaTypeImageLayer, newLayer, history, compressAlgo, nil); err != nil {
 		return fmt.Errorf("add diff layer: %w", err)
 	}
 

cmd/umoci/repack.go

@@ -29,12 +29,13 @@ import (
 	"github.com/opencontainers/umoci/mutate"
 	"github.com/opencontainers/umoci/oci/cas/dir"
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/casext/blobcompress"
 	igen "github.com/opencontainers/umoci/oci/config/generate"
 	"github.com/opencontainers/umoci/pkg/mtreefilter"
 	"github.com/urfave/cli"
 )
 
-var repackCommand = uxHistory(cli.Command{
+var repackCommand = uxCompress(uxHistory(cli.Command{
 	Name:  "repack",
 	Usage: "repacks an OCI runtime bundle into a reference",
 	ArgsUsage: `--image <image-path>[:<new-tag>] <bundle>
@@ -88,13 +89,18 @@ manifest and configuration information uses the new diff atop the old manifest.`
 		ctx.App.Metadata["bundle"] = ctx.Args().First()
 		return nil
 	},
-})
+}))
 
 func repack(ctx *cli.Context) error {
 	imagePath := ctx.App.Metadata["--image-path"].(string)
 	tagName := ctx.App.Metadata["--image-tag"].(string)
 	bundlePath := ctx.App.Metadata["bundle"].(string)
 
+	var compressAlgo blobcompress.Algorithm
+	if algo, ok := ctx.App.Metadata["--compress"].(blobcompress.Algorithm); ok {
+		compressAlgo = algo
+	}
+
 	// Read the metadata first.
 	meta, err := umoci.ReadBundleMeta(bundlePath)
 	if err != nil {
@@ -176,5 +182,5 @@ func repack(ctx *cli.Context) error {
 		mtreefilter.MaskFilter(maskedPaths),
 	}
 
-	return umoci.Repack(engineExt, tagName, bundlePath, meta, history, filters, ctx.Bool("refresh-bundle"), mutator)
+	return umoci.Repack(engineExt, tagName, bundlePath, meta, history, filters, ctx.Bool("refresh-bundle"), mutator, compressAlgo)
 }

cmd/umoci/utils_ux.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/casext/blobcompress"
 	"github.com/urfave/cli"
 )
 
@@ -88,6 +89,45 @@ func uxHistory(cmd cli.Command) cli.Command {
 	return cmd
 }
 
+// uxCompress adds the --compress flag to the given cli.Command as well as
+// adding relevant validation logic to the .Before of the command. The value
+// will be stored in ctx.Metadata["--compress"] as a string (or nil if --tag
+// was not specified).
+func uxCompress(cmd cli.Command) cli.Command {
+	cmd.Flags = append(cmd.Flags, cli.StringFlag{
+		Name:  "compress",
+		Usage: "compression algorithm for newly created layer blobs",
+		Value: "auto",
+	})
+
+	oldBefore := cmd.Before
+	cmd.Before = func(ctx *cli.Context) error {
+		// Verify compression algorithm value.
+		var layerCompressor blobcompress.Algorithm
+		if ctx.IsSet("compress") {
+			compressType := ctx.String("compress")
+			if compressType == "none" {
+				compressType = ""
+			}
+			if compressType != "auto" {
+				layerCompressor = blobcompress.GetAlgorithm(compressType)
+				if layerCompressor == nil {
+					return fmt.Errorf("invalid --compress: unknown layer compression type %q", ctx.String("compress"))
+				}
+			}
+		}
+		ctx.App.Metadata["--compress"] = layerCompressor
+
+		// Include any old befores set.
+		if oldBefore != nil {
+			return oldBefore(ctx)
+		}
+		return nil
+	}
+
+	return cmd
+}
+
 // uxTag adds a --tag flag to the given cli.Command as well as adding relevant
 // validation logic to the .Before of the command. The value will be stored in
 // ctx.Metadata["--tag"] as a string (or nil if --tag was not specified).

doc/man/umoci-insert.1.md

@@ -8,6 +8,7 @@ umoci insert - insert content into an OCI image
 **umoci insert**
 **--image**=*image*[:*tag*]
 [**--tag**=*new-tag*]
+[**--compress**=*compression-type*]
 [**--opaque**]
 [**--rootless**]
 [**--uid-map**=*value*]
@@ -60,6 +61,21 @@ The global options are defined in **umoci**(1).
   Tag name for the modified image, if unspecified then the original tag
   provided to **--image** will be clobbered.
 
+**--compress**=*compression-type*
+  Specify the compression type to use when creating a new layer. Supported
+  compression types are *none*, *gzip*, and *zstd*. **umoci-unpack**(1)
+  transparently supports all compression methods you can specify with
+  **--compress**.
+  <!-- paragraph break -->
+  The special value *auto* will cause **umoci**(1) to auto-select the most
+  appropriate compression algorithm based on what previous layers are
+  compressed with (it will try to use the most recent layer's compression
+  algorithm which it supports). Note that *auto* will never select *none*
+  compression automatically, as not compressing **tar**(1) archives is really
+  not advisable.
+  <!-- paragraph break -->
+  If no *compression-type* is provided, it defaults to *auto*.
+
 **--opaque**
   (Assuming *target* is a directory.) Add an opaque whiteout entry for *target*
   so that any child path of *target* in previous layers is masked by the new

doc/man/umoci-raw-add-layer.1.md

@@ -8,6 +8,7 @@ umoci raw add-layer - add a layer archive verbatim to an image
 **umoci raw add-layer**
 **--image**=*image*
 [**--tag**=*tag*]
+[**--compress**=*compression-type*]
 [**--no-history**]
 [**--history.comment**=*comment*]
 [**--history.created_by**=*created_by*]
@@ -39,6 +40,21 @@ The global options are defined in **umoci**(1).
   tag in the image. If *tag* is not provided it defaults to the *tag* specified
   in **--image** (overwriting it).
 
+**--compress**=*compression-type*
+  Specify the compression type to use when creating a new layer. Supported
+  compression types are *none*, *gzip*, and *zstd*. **umoci-unpack**(1)
+  transparently supports all compression methods you can specify with
+  **--compress**.
+  <!-- paragraph break -->
+  The special value *auto* will cause **umoci**(1) to auto-select the most
+  appropriate compression algorithm based on what previous layers are
+  compressed with (it will try to use the most recent layer's compression
+  algorithm which it supports). Note that *auto* will never select *none*
+  compression automatically, as not compressing **tar**(1) archives is really
+  not advisable.
+  <!-- paragraph break -->
+  If no *compression-type* is provided, it defaults to *auto*.
+
 **--no-history**
   Causes no history entry to be added for this operation. **This is not
   recommended for use with umoci-raw-add-layer(1), since it results in the

doc/man/umoci-repack.1.md

@@ -7,6 +7,7 @@ umoci repack - Repacks an OCI runtime bundle into an image tag
 # SYNOPSIS
 **umoci repack**
 **--image**=*image*[:*tag*]
+[**--compress**=*compression-type*]
 [**--no-history**]
 [**--history.comment**=*comment*]
 [**--history.created_by**=*created_by*]
@@ -45,6 +46,21 @@ The global options are defined in **umoci**(1).
   the same name as *tag* it will be overwritten. If *tag* is not provided it
   defaults to "latest".
 
+**--compress**=*compression-type*
+  Specify the compression type to use when creating a new layer. Supported
+  compression types are *none*, *gzip*, and *zstd*. **umoci-unpack**(1)
+  transparently supports all compression methods you can specify with
+  **--compress**.
+  <!-- paragraph break -->
+  The special value *auto* will cause **umoci**(1) to auto-select the most
+  appropriate compression algorithm based on what previous layers are
+  compressed with (it will try to use the most recent layer's compression
+  algorithm which it supports). Note that *auto* will never select *none*
+  compression automatically, as not compressing **tar**(1) archives is really
+  not advisable.
+  <!-- paragraph break -->
+  If no *compression-type* is provided, it defaults to *auto*.
+
 **--no-history**
   Causes no history entry to be added for this operation. **This is not
   recommended for use with umoci-repack(1), since it results in the history not