RFC DO NOT MERGE: Store signatures in c/i/docker/daemon/extra

distribution/pull_v2.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	cimagedocker "github.com/containers/image/docker"
+	ciImage "github.com/containers/image/image"
 	"github.com/containers/image/signature"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/digest"
@@ -99,8 +100,13 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e
 	var layersDownloaded bool
 	if !reference.IsNameOnly(ref) {
 		var err error
+		ciImage, closer, err := p.ciImage(ctx, ref)
+		if err != nil {
+			return err
+		}
+		defer closer.Close()
 		if p.config.SignatureCheck {
-			ref, err = p.checkTrusted(ctx, ref)
+			ref, err = p.checkTrusted(ref, ciImage)
 			if err != nil {
 				if err == cimagedocker.ErrV1NotSupported {
 					return fmt.Errorf("unable to pull from V1 Docker registries with image signature verification enabled. If you need to accept this risk and disable signature verification (for ALL images), run the docker daemon with --signature-enabled=false")
@@ -109,7 +115,7 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e
 				return err
 			}
 		}
-		layersDownloaded, err = p.pullV2Tag(ctx, ref)
+		layersDownloaded, err = p.pullV2Tag(ctx, ref, ciImage)
 		if err != nil {
 			return err
 		}
@@ -135,8 +141,13 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e
 			}
 			var ref reference.Named
 			ref = tagRef
+			ciImage, closer, err := p.ciImage(ctx, ref)
+			if err != nil {
+				return err
+			}
+			defer closer.Close()
 			if p.config.SignatureCheck {
-				trustedRef, err := p.checkTrusted(ctx, tagRef)
+				trustedRef, err := p.checkTrusted(tagRef, ciImage)
 				if err != nil {
 					p.originalRef = nil
 					if err == cimagedocker.ErrV1NotSupported {
@@ -146,7 +157,7 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named) (e
 				}
 				ref = trustedRef
 			}
-			pulledNew, err := p.pullV2Tag(ctx, ref)
+			pulledNew, err := p.pullV2Tag(ctx, ref, ciImage)
 			if err != nil {
 				// Since this is the pull-all-tags case, don't
 				// allow an error pulling a particular tag to
@@ -366,7 +377,7 @@ func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
 	ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})
 }
 
-func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdated bool, err error) {
+func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, ciImage *ciImage.UnparsedImage) (tagUpdated bool, err error) {
 	manSvc, err := p.repo.Manifests(ctx)
 	if err != nil {
 		return false, err
@@ -443,6 +454,10 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat
 
 	progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
 
+	if err = p.storeSignatures(ctx, ciImage); err != nil {
+		return false, err
+	}
+
 	oldTagImageID, err := p.config.ReferenceStore.Get(ref)
 	if err == nil {
 		if oldTagImageID == imageID {

distribution/pull_v2_unix.go

@@ -4,10 +4,13 @@ package distribution
 
 import (
 	"fmt"
+	"io"
 	"path/filepath"
 
 	"github.com/containers/image/docker"
+	"github.com/containers/image/docker/daemon/signatures"
 	containersImageRef "github.com/containers/image/docker/reference"
+	ciImage "github.com/containers/image/image"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/signature"
 	"github.com/containers/image/types"
@@ -43,19 +46,19 @@ func configurePolicyContext() (*signature.PolicyContext, error) {
 	return pc, nil
 }
 
-func (p *v2Puller) checkTrusted(c gctx.Context, ref reference.Named) (reference.Named, error) {
-	p.originalRef = ref
+// ciImage returns a *containers/image/image.UnparsedImage and a close callback for ref.
+func (p *v2Puller) ciImage(c gctx.Context, ref reference.Named) (*ciImage.UnparsedImage, io.Closer, error) {
 	// we can't use upstream docker/docker/reference since in projectatomic/docker
 	// we modified docker/docker/reference and it's not doing any normalization.
 	// we instead forked docker/docker/reference in containers/image and we need
 	// this parsing here to make sure signature naming checks are consistent.
 	dockerRef, err := containersImageRef.ParseNormalizedNamed(ref.String())
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	imgRef, err := docker.NewReference(dockerRef)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	isSecure := (p.endpoint.TLSConfig == nil || !p.endpoint.TLSConfig.InsecureSkipVerify)
 	authConfig := registry.ResolveAuthConfig(p.config.AuthConfigs, p.repoInfo.Index)
@@ -72,11 +75,17 @@ func (p *v2Puller) checkTrusted(c gctx.Context, ref reference.Named) (reference.
 	if p.config.RegistryService.SecureIndex(p.repoInfo.Index.Name) {
 		ctx.DockerCertPath = filepath.Join(registry.CertsDir, p.repoInfo.Index.Name)
 	}
-	img, err := imgRef.NewImage(ctx)
+	src, err := imgRef.NewImageSource(ctx)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	allowed, err := p.policyContext.IsRunningImageAllowed(img)
+	unparsed := ciImage.UnparsedInstance(src, nil)
+	return unparsed, src, nil
+}
+
+func (p *v2Puller) checkTrusted(ref reference.Named, unparsed types.UnparsedImage) (reference.Named, error) {
+	p.originalRef = ref
+	allowed, err := p.policyContext.IsRunningImageAllowed(unparsed)
 	if !allowed {
 		if err != nil {
 			return nil, fmt.Errorf("%s isn't allowed: %v", ref.String(), err)
@@ -86,7 +95,7 @@ func (p *v2Puller) checkTrusted(c gctx.Context, ref reference.Named) (reference.
 	if err != nil {
 		return nil, err
 	}
-	mfst, _, err := img.Manifest()
+	mfst, _, err := unparsed.Manifest()
 	if err != nil {
 		return nil, err
 	}
@@ -100,3 +109,13 @@ func (p *v2Puller) checkTrusted(c gctx.Context, ref reference.Named) (reference.
 	}
 	return ref, nil
 }
+
+// storeSignature stores the signatures of ciImage and updates the tag in ciImage.Reference() if necessary.
+func (p *v2Puller) storeSignatures(c gctx.Context, unparsed *ciImage.UnparsedImage) error {
+	img, err := ciImage.FromUnparsedImage(nil, unparsed)
+	if err != nil {
+		return err
+	}
+	store := signatures.NewStore(nil)
+	return store.RecordImage(c, img)
+}

distribution/pull_v2_windows.go

@@ -5,10 +5,13 @@ package distribution
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 
+	ciImage "github.com/containers/image/image"
 	"github.com/containers/image/signature"
+	"github.com/containers/image/types"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/context"
 	"github.com/docker/distribution/manifest/schema1"
@@ -75,6 +78,14 @@ func configurePolicyContext() (*signature.PolicyContext, error) {
 	return nil, nil
 }
 
-func (p *v2Puller) checkTrusted(c gctx.Context, ref reference.Named) (reference.Named, error) {
+func (p *v2Puller) ciImage(c gctx.Context, ref reference.Named) (*ciImage.UnparsedImage, io.Closer, error) {
+	return nil, nil, nil
+}
+
+func (p *v2Puller) checkTrusted(ref reference.Named, unparsed types.UnparsedImage) (reference.Named, error) {
 	return ref, nil
 }
+
+func (p *v2Puller) storeSignatures(c gctx.Context, unparsed *ciImage.UnparsedImage) error {
+	return nil
+}

hack/vendor.sh

@@ -168,10 +168,8 @@ clone git github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42
 clone git github.com/flynn-archive/go-shlex 3f9db97f856818214da2e1057f8ad84803971cff
 
 # signatures
-clone git github.com/containers/image e6ced5e26c9b60ec23f0f7bd49b82a106f1d4db4
+clone git github.com/containers/image dockerd-verification https://github.com/mtrmac/image.git
 clone git github.com/opencontainers/image-spec v1.0.0
-clone git k8s.io/kubernetes 4a3f9c5b19c7ff804cbc1bf37a15c044ca5d2353 https://github.com/openshift/kubernetes
-clone git github.com/golang/glog 44145f04b68cf362d9c4df2182967c2275eaefed
 clone git github.com/ghodss/yaml 73d445a93680fa1a78ae23a5839bad48f32ba1ee
 clone git gopkg.in/yaml.v2 d466437aa4adc35830964cffc5b5f262c63ddcb4
 clone git github.com/mtrmac/gpgme master

vendor/src/github.com/containers/image/docker/daemon/signatures/data.go

@@ -0,0 +1,128 @@
+package signatures
+
+// dataBucket stores the original manifests and signatures for an image.
+
+// dataBucket keys are (config digest, manifest digest), and contains a sub-bucket for each key.
+// This sub-bucket stores a manifest in manifestKey, and individual signatures in
+// (signatureKeyPrefix + zero-based index.)
+
+// Safety WRT concurrent access to data in a dataBucket sub-bucket:
+// The bucket is identified by the manifest digest, so no substantial updates to the
+// manifest are ever expected.
+// Readers expect signatures stored within a sub-bucket to be replaced/updated atomically
+// to ensure the bucket contents are consistent.
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+
+	"github.com/boltdb/bolt"
+	digest "github.com/opencontainers/go-digest"
+)
+
+var (
+	dataBucket         = []byte("config+manifest->bucket")
+	manifestKey        = []byte("manifest")
+	signatureKeyPrefix = []byte("sig")
+)
+
+// dataBucketKey returns a key for use in dataBucket.
+func dataBucketKey(configDigest, manifestDigest digest.Digest) ([]byte, error) {
+	configBytes, err := stringToNonNULBytes(configDigest.String())
+	if err != nil {
+		return nil, err
+	}
+	manifestBytes, err := stringToNonNULBytes(manifestDigest.String())
+	if err != nil {
+		return nil, err
+	}
+	return bytes.Join([][]byte{configBytes, manifestBytes}, []byte{0}), nil
+}
+
+// copyBytes returns a freshly allocated clone of input.
+// This is needed because the data pointers returned by boltdb are invalid after the end of the transaction.
+func copyBytes(input []byte) []byte {
+	res := make([]byte, len(input))
+	copy(res, input)
+	return res
+}
+
+// readManifest returns the original manifest stored in b, or nil if not available.
+func readManifest(b *bolt.Bucket) []byte {
+	m := b.Get(manifestKey)
+	if m == nil {
+		return nil
+	}
+	return copyBytes(m)
+}
+
+// writeManifest stores the original manifest to b.
+func writeManifest(b *bolt.Bucket, manifest []byte) error {
+	return b.Put(manifestKey, manifest)
+}
+
+// readSignatures returns the original signatures stored in bucket, which is dataKey.
+func readSignatures(bucket *bolt.Bucket, dataKey []byte) ([][]byte, error) {
+	// Iterate through all keys in the sub-bucket; we need all of the except for manifestKey, so it seems fastest to read them in the database order and then reorder in memory.
+	signatureMap := map[int][]byte{}
+	if err := bucket.ForEach(func(k, v []byte) error {
+		if !bytes.HasPrefix(k, signatureKeyPrefix) {
+			return nil
+		}
+		i, err := strconv.Atoi(string(bytes.TrimPrefix(k, signatureKeyPrefix)))
+		if err != nil {
+			return err
+		}
+		if _, ok := signatureMap[i]; ok {
+			return fmt.Errorf("Internal error: Duplicate key %q in dataBucket key %q", k, dataKey)
+		}
+		signatureMap[i] = copyBytes(v)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	signatures := [][]byte{}
+	for i := 0; ; i++ {
+		signature, ok := signatureMap[i]
+		if !ok {
+			break
+		}
+		signatures = append(signatures, signature)
+	}
+	if len(signatures) != len(signatureMap) {
+		// The use of transactions to update signatures should prevent this from happening
+		return nil, fmt.Errorf("Internal error: Non-consecutive signatures in dataBucket key %q", dataKey)
+	}
+	return signatures, nil
+}
+
+// writeSignatures stores the original signatures to b, which is dataKey.
+func writeSignatures(b *bolt.Bucket, dataKey []byte, signatures [][]byte) error {
+	if len(signatures) == 0 {
+		return nil // Don't bother reading the old ones.
+	}
+
+	existingSigs, err := readSignatures(b, dataKey)
+	if err != nil {
+		return err
+	}
+
+	nextIndex := len(existingSigs)
+sigExists:
+	for _, sig := range signatures {
+		for _, existingSig := range existingSigs {
+			if bytes.Equal(sig, existingSig) {
+				continue sigExists
+			}
+		}
+
+		key := bytes.Join([][]byte{signatureKeyPrefix, []byte(strconv.Itoa(nextIndex))}, []byte{})
+		if err := b.Put(key, sig); err != nil {
+			return err
+		}
+		nextIndex++
+	}
+	return nil
+}