overlay: add StartStagingDiffToApply()

Add a function to apply the diff into a tmporary directory so we can do
that unlcoked and only rename under the lock.

Signed-off-by: Paul Holzinger <[email protected]>

Author: Luap99

storage/drivers/driver.go

@@ -299,6 +299,15 @@ type DriverWithDiffer interface {
 	DifferTarget(id string) (string, error)
 }
 
+// ApplyDiffStaging is an interface for driver who can apply the diff without holding the main storage lock.
+// This API is experimental and can be changed without bumping the major version number.
+type ApplyDiffStaging interface {
+	// StartStagingDiffToApply applies the layer in a temporary directory. This can be done without holding the storage lock.
+	StartStagingDiffToApply(id, parent string, options ApplyDiffOpts) (tempdir.CleanupTempDirFunc, tempdir.CommitFunc, int64, error)
+	// CommitStagedLayer commits the staged layer from StartStagingDiffToApply(). This must be done while the storage lock.
+	CommitStagedLayer(id string, commit tempdir.CommitFunc) error
+}
+
 // Capabilities defines a list of capabilities a driver may implement.
 // These capabilities are not required; however, they do determine how a
 // graphdriver can be used.

storage/drivers/overlay/overlay.go

@@ -2369,21 +2369,67 @@ func (d *Driver) DifferTarget(id string) (string, error) {
 	return d.getDiffPath(id)
 }
 
-// ApplyDiff applies the new layer into a root
-func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) {
-	idMappings := options.Mappings
-	if idMappings == nil {
-		idMappings = &idtools.IDMappings{}
+// StartStagingDiffToApply applies the new layer into a temporary directory.
+// It returns a CleanupTempDirFunc which can nil or set regardless if the function return an error or not.
+// CommitFunc is only set when there is no error returned and the int64 value returns the size of the layer.
+//
+// This API is experimental and can be changed without bumping the major version number.
+func (d *Driver) StartStagingDiffToApply(id, parent string, options graphdriver.ApplyDiffOpts) (tempdir.CleanupTempDirFunc, tempdir.CommitFunc, int64, error) {
+	tempDirRoot := d.getTempDirRoot(id)
+	t, err := tempdir.NewTempDir(tempDirRoot)
+	if err != nil {
+		return nil, nil, -1, err
+	}
+
+	var size int64
+	sa, err := t.StageDirectoryAddition(func(path string) error {
+		size, err = d.applyDiff(id, path, options)
+		return err
+	})
+	if err != nil {
+		return t.Cleanup, nil, -1, err
+	}
+
+	return t.Cleanup, sa.Commit, size, nil
+}
+
+// CommitStagedLayer that was created with StartStagingDiffToApply().
+//
+// This API is experimental and can be changed without bumping the major version number.
+func (d *Driver) CommitStagedLayer(id string, commit tempdir.CommitFunc) error {
+	applyDir, err := d.getDiffPath(id)
+	if err != nil {
+		return err
 	}
 
+	// The os.Rename() function used by CommitFunc errors when the target directory already
+	// exists, as such delete the dir if it already exists.
+	if err := os.Remove(applyDir); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	return commit(applyDir)
+}
+
+// ApplyDiff applies the new layer into a root
+func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) (size int64, err error) {
 	applyDir, err := d.getDiffPath(id)
 	if err != nil {
 		return 0, err
 	}
+	return d.applyDiff(id, applyDir, options)
+}
+
+// ApplyDiff applies the new layer into a root
+func (d *Driver) applyDiff(id, target string, options graphdriver.ApplyDiffOpts) (size int64, err error) {
+	idMappings := options.Mappings
+	if idMappings == nil {
+		idMappings = &idtools.IDMappings{}
+	}
 
-	logrus.Debugf("Applying tar in %s", applyDir)
+	logrus.Debugf("Applying tar in %s", target)
 	// Overlay doesn't need the parent id to apply the diff
-	if err := untar(options.Diff, applyDir, &archive.TarOptions{
+	if err := untar(options.Diff, target, &archive.TarOptions{
 		UIDMaps:           idMappings.UIDs(),
 		GIDMaps:           idMappings.GIDs(),
 		IgnoreChownErrors: d.options.ignoreChownErrors,
@@ -2394,7 +2440,7 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
 		return 0, err
 	}
 
-	return directory.Size(applyDir)
+	return directory.Size(target)
 }
 
 func (d *Driver) getComposefsData(id string) string {

storage/drivers/overlay/overlay_test.go

@@ -17,6 +17,9 @@ import (
 
 const driverName = "overlay"
 
+// check that Driver correctly implements the ApplyDiffTemporary interface
+var _ graphdriver.ApplyDiffStaging = &Driver{}
+
 func init() {
 	// Do not sure chroot to speed run time and allow archive
 	// errors or hangs to be debugged directly from the test process.