diff --git a/pkg/imgpkg/cmd/copy_repo_src_test.go b/pkg/imgpkg/cmd/copy_repo_src_test.go index 52667459c..951c84f2a 100644 --- a/pkg/imgpkg/cmd/copy_repo_src_test.go +++ b/pkg/imgpkg/cmd/copy_repo_src_test.go @@ -964,8 +964,8 @@ bundle: func TestToRepoImage(t *testing.T) { imageName := "library/image" fakeRegistry := helpers.NewFakeRegistry(t, &helpers.Logger{LogLevel: helpers.LogDebug}) - image1 := fakeRegistry.WithImageFromPath(imageName, "test_assets/image_with_config", map[string]string{}) defer fakeRegistry.CleanUp() + image1 := fakeRegistry.WithImageFromPath(imageName, "test_assets/image_with_config", map[string]string{}) subject := subject subject.ImageFlags = ImageFlags{ fakeRegistry.ReferenceOnTestServer(imageName), @@ -1054,12 +1054,12 @@ images: }) t.Run("When copying to same registry but have no permission to mount layer", func(t *testing.T) { - t.Skip("Skipping this test because the registry in ggcr does not validate that a blob is part of repository or not, so we will get false positives") - // we will need https://github.com/google/go-containerregistry/pull/1158 or something similar to ensure no false positives happen for this test - // skipping it for now, but we should review in the future, To run this test use the code in the commit e4c2b6acd5adc569c155a2353bb12b06f46f3e78 assets := &helpers.Assets{T: t} defer assets.CleanCreatedFolders() + fakeRegistry := helpers.NewFakeRegistryWithRepoSeparation(t, &helpers.Logger{LogLevel: helpers.LogDebug}) + defer fakeRegistry.CleanUp() + destinationImageName := fakeRegistry.ReferenceOnTestServer("some/other/copied-img") originImageName := "repo/image" @@ -1091,15 +1091,56 @@ images: assert.Equal(t, image2RefDigest, processedImages.All()[0].UnprocessedImageRef.DigestRef) }) - t.Run("When a temporary error happens it retries the configured number of times", func(t *testing.T) { + t.Run("When copying to same registry have permission to mount layer", func(t *testing.T) { assets := &helpers.Assets{T: t} defer assets.CleanCreatedFolders() + fakeRegistry := helpers.NewFakeRegistryWithRepoSeparation(t, &helpers.Logger{LogLevel: helpers.LogDebug}) + defer fakeRegistry.CleanUp() + destinationImageName := fakeRegistry.ReferenceOnTestServer("some/other/copied-img") originImageName := "repo/image" image2RefDigest := fakeRegistry.WithRandomImage(originImageName).RefDigest + subject := subject + subject.ImageFlags.Image = image2RefDigest + subject.registry = fakeRegistry.BuildWithRegistryOpts(registry.Opts{ + EnvironFunc: func() []string { + return []string{ + "IMGPKG_REGISTRY_HOSTNAME_0=" + fakeRegistry.ReferenceOnTestServer("repo"), + "IMGPKG_REGISTRY_USERNAME_0=some-user", + "IMGPKG_REGISTRY_PASSWORD_0=some-password", + "IMGPKG_REGISTRY_HOSTNAME_1=" + fakeRegistry.ReferenceOnTestServer("some/other"), + "IMGPKG_REGISTRY_USERNAME_1=some-user", + "IMGPKG_REGISTRY_PASSWORD_1=some-password", + } + }, + }) + + // Authentication added in this step to ensure the images are created beforehand + // because we are not testing here the authentication of image pushing + fakeRegistry.WithBasicAuthPerRepository("repo", "some-user", "some-password") + fakeRegistry.WithBasicAuthPerRepository("some/other", "some-user", "some-password") + + processedImages, err := subject.CopyToRepo(destinationImageName) + require.NoError(t, err) + require.Len(t, processedImages.All(), 1) + assert.Equal(t, image2RefDigest, processedImages.All()[0].UnprocessedImageRef.DigestRef) + }) + + t.Run("When a temporary error happens it retries the configured number of times", func(t *testing.T) { + assets := &helpers.Assets{T: t} + defer assets.CleanCreatedFolders() + + fakeRegistry := helpers.NewFakeRegistry(t, &helpers.Logger{LogLevel: helpers.LogDebug}) + defer fakeRegistry.CleanUp() + + destinationImageName := fakeRegistry.ReferenceOnTestServer("some/other/copied-img") + originImageName := "temp-repo/image" + + image2RefDigest := fakeRegistry.WithRandomImage(originImageName).RefDigest + subject := subject subject.ImageFlags.Image = image2RefDigest subject.registry = fakeRegistry.BuildWithRegistryOpts(registry.Opts{ diff --git a/pkg/imgpkg/imagedesc/described_compressed_layer.go b/pkg/imgpkg/imagedesc/described_compressed_layer.go index d7b81fd19..2654e4bea 100644 --- a/pkg/imgpkg/imagedesc/described_compressed_layer.go +++ b/pkg/imgpkg/imagedesc/described_compressed_layer.go @@ -44,7 +44,7 @@ func (l DescribedCompressedLayer) Compressed() (io.ReadCloser, error) { return nil, fmt.Errorf("Computing digest: %v", err) } - rc, err = verify.ReadCloser(rc, h) + rc, err = verify.ReadCloser(rc, verify.SizeUnknown, h) if err != nil { return nil, fmt.Errorf("Creating verified reader: %v", err) } diff --git a/pkg/imgpkg/imageutils/verify/verify.go b/pkg/imgpkg/imageutils/verify/verify.go index 0133b3cf2..87f295e7d 100644 --- a/pkg/imgpkg/imageutils/verify/verify.go +++ b/pkg/imgpkg/imageutils/verify/verify.go @@ -1,8 +1,3 @@ -// Copyright 2020 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -// Using this code as is from: https://github.com/google/go-containerregistry/tree/master/pkg/v1/internal - // Copyright 2020 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package verify provides a ReadCloser that verifies content matches the +// expected hash values. package verify import ( + "bytes" "encoding/hex" + "errors" "fmt" "hash" "io" @@ -29,20 +28,43 @@ import ( "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/imageutils/and" ) +// SizeUnknown is a sentinel value to indicate that the expected size is not known. +const SizeUnknown = -1 + type verifyReader struct { - inner io.Reader - hasher hash.Hash - expected v1.Hash + inner io.Reader + hasher hash.Hash + expected v1.Hash + gotSize, wantSize int64 +} + +// Error provides information about the failed hash verification. +type Error struct { + got string + want v1.Hash + gotSize int64 +} + +func (v Error) Error() string { + return fmt.Sprintf("error verifying %s checksum after reading %d bytes; got %q, want %q", + v.want.Algorithm, v.gotSize, v.got, v.want) } // Read implements io.Reader func (vc *verifyReader) Read(b []byte) (int, error) { n, err := vc.inner.Read(b) + vc.gotSize += int64(n) if err == io.EOF { - got := hex.EncodeToString(vc.hasher.Sum(make([]byte, 0, vc.hasher.Size()))) + if vc.wantSize != SizeUnknown && vc.gotSize != vc.wantSize { + return n, fmt.Errorf("error verifying size; got %d, want %d", vc.gotSize, vc.wantSize) + } + got := hex.EncodeToString(vc.hasher.Sum(nil)) if want := vc.expected.Hex; got != want { - return n, fmt.Errorf("error verifying %s checksum; got %q, want %q", - vc.expected.Algorithm, got, want) + return n, Error{ + got: vc.expected.Algorithm + ":" + got, + want: vc.expected, + gotSize: vc.gotSize, + } } } return n, err @@ -50,18 +72,51 @@ func (vc *verifyReader) Read(b []byte) (int, error) { // ReadCloser wraps the given io.ReadCloser to verify that its contents match // the provided v1.Hash before io.EOF is returned. -func ReadCloser(r io.ReadCloser, h v1.Hash) (io.ReadCloser, error) { +// +// The reader will only be read up to size bytes, to prevent resource +// exhaustion. If EOF is returned before size bytes are read, an error is +// returned. +// +// A size of SizeUnknown (-1) indicates disables size verification when the size +// is unknown ahead of time. +func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) { w, err := v1.Hasher(h.Algorithm) if err != nil { return nil, err } - r2 := io.TeeReader(r, w) + r2 := io.TeeReader(r, w) // pass all writes to the hasher. + if size != SizeUnknown { + r2 = io.LimitReader(r2, size) // if we know the size, limit to that size. + } return &and.ReadCloser{ Reader: &verifyReader{ inner: r2, hasher: w, expected: h, + wantSize: size, }, CloseFunc: r.Close, }, nil } + +// Descriptor verifies that the embedded Data field matches the Size and Digest +// fields of the given v1.Descriptor, returning an error if the Data field is +// missing or if it contains incorrect data. +func Descriptor(d v1.Descriptor) error { + if d.Data == nil { + return errors.New("error verifying descriptor; Data == nil") + } + + h, sz, err := v1.SHA256(bytes.NewReader(d.Data)) + if err != nil { + return err + } + if h != d.Digest { + return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest) + } + if sz != d.Size { + return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size) + } + + return nil +} diff --git a/pkg/imgpkg/imageutils/verify/verify_test.go b/pkg/imgpkg/imageutils/verify/verify_test.go index 95ab8f7c0..07484ae1a 100644 --- a/pkg/imgpkg/imageutils/verify/verify_test.go +++ b/pkg/imgpkg/imageutils/verify/verify_test.go @@ -1,7 +1,7 @@ // Copyright 2020 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -// Using this code as is from: https://github.com/google/go-containerregistry/tree/master/pkg/v1/internal +// Using this code as is from: https://github.com/google/go-containerregistry/tree/main/internal // Copyright 2020 Google LLC All Rights Reserved. // @@ -21,6 +21,8 @@ package verify import ( "bytes" + "errors" + "fmt" "io/ioutil" "strings" "testing" @@ -33,6 +35,7 @@ func mustHash(s string, t *testing.T) v1.Hash { if err != nil { t.Fatalf("v1.SHA256(%s) = %v", s, err) } + t.Logf("Hashed: %q -> %q", s, h) return h } @@ -40,7 +43,7 @@ func TestVerificationFailure(t *testing.T) { want := "This is the input string." buf := bytes.NewBufferString(want) - verified, err := ReadCloser(ioutil.NopCloser(buf), mustHash("not the same", t)) + verified, err := ReadCloser(ioutil.NopCloser(buf), int64(len(want)), mustHash("not the same", t)) if err != nil { t.Fatal("ReadCloser() =", err) } @@ -53,7 +56,20 @@ func TestVerification(t *testing.T) { want := "This is the input string." buf := bytes.NewBufferString(want) - verified, err := ReadCloser(ioutil.NopCloser(buf), mustHash(want, t)) + verified, err := ReadCloser(ioutil.NopCloser(buf), int64(len(want)), mustHash(want, t)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + if _, err := ioutil.ReadAll(verified); err != nil { + t.Error("ReadAll() =", err) + } +} + +func TestVerificationSizeUnknown(t *testing.T) { + want := "This is the input string." + buf := bytes.NewBufferString(want) + + verified, err := ReadCloser(ioutil.NopCloser(buf), SizeUnknown, mustHash(want, t)) if err != nil { t.Fatal("ReadCloser() =", err) } @@ -67,8 +83,70 @@ func TestBadHash(t *testing.T) { Algorithm: "fake256", Hex: "whatever", } - _, err := ReadCloser(ioutil.NopCloser(strings.NewReader("hi")), h) + _, err := ReadCloser(ioutil.NopCloser(strings.NewReader("hi")), 0, h) if err == nil { t.Errorf("ReadCloser() = %v, wanted err", err) } } + +func TestBadSize(t *testing.T) { + want := "This is the input string." + + // having too much content or expecting too much content returns an error. + for _, size := range []int64{3, 100} { + t.Run(fmt.Sprintf("expecting size %d", size), func(t *testing.T) { + buf := bytes.NewBufferString(want) + rc, err := ReadCloser(ioutil.NopCloser(buf), size, mustHash(want, t)) + if err != nil { + t.Fatal("ReadCloser() =", err) + } + if b, err := ioutil.ReadAll(rc); err == nil { + t.Errorf("ReadAll() = %q; want verification error", string(b)) + } + }) + } +} + +func TestDescriptor(t *testing.T) { + for _, tc := range []struct { + err error + desc v1.Descriptor + }{{ + err: errors.New("error verifying descriptor; Data == nil"), + }, { + err: errors.New(`error verifying Digest; got "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", want ":"`), + desc: v1.Descriptor{ + Data: []byte("abc"), + }, + }, { + err: errors.New("error verifying Size; got 3, want 0"), + desc: v1.Descriptor{ + Data: []byte("abc"), + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }, + }, + }, { + desc: v1.Descriptor{ + Data: []byte("abc"), + Size: 3, + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }, + }, + }} { + got, want := Descriptor(tc.desc), tc.err + + if got == nil { + if want != nil { + t.Errorf("Descriptor(): got nil, want %v", want) + } + } else if want == nil { + t.Errorf("Descriptor(): got %v, want nil", got) + } else if got, want := got.Error(), want.Error(); got != want { + t.Errorf("Descriptor(): got %q, want %q", got, want) + } + } +} diff --git a/pkg/imgpkg/registry/registry.go b/pkg/imgpkg/registry/registry.go index 7e4dae379..25e031aba 100644 --- a/pkg/imgpkg/registry/registry.go +++ b/pkg/imgpkg/registry/registry.go @@ -83,12 +83,19 @@ type ImagesReaderWriter interface { var _ Registry = &SimpleRegistry{} +// RoundTripperStorage Storage of RoundTripper that will be used to talk to the registry +type RoundTripperStorage interface { + RoundTripper(repo regname.Repository, scope string) http.RoundTripper + CreateRoundTripper(reg regname.Registry, auth regauthn.Authenticator, scope string) (http.RoundTripper, error) + BaseRoundTripper() http.RoundTripper +} + // SimpleRegistry Implements Registry interface type SimpleRegistry struct { remoteOpts []regremote.Option refOpts []regname.Option keychain regauthn.Keychain - roundTrippers *RoundTripperStorage + roundTrippers RoundTripperStorage transportAccess *sync.Mutex } @@ -150,7 +157,7 @@ func NewSimpleRegistry(opts Opts, regOpts ...regremote.Option) (*SimpleRegistry, remoteOpts: regRemoteOptions, refOpts: refOpts, keychain: keychain, - roundTrippers: NewRoundTripperStorage(baseRoundTripper), + roundTrippers: NewMultiRoundTripperStorage(baseRoundTripper), transportAccess: &sync.Mutex{}, }, nil } @@ -165,12 +172,16 @@ func (r SimpleRegistry) CloneWithSingleAuth(imageRef regname.Tag) (Registry, err } keychain := auth.NewSingleAuthKeychain(imgAuth) + rt := r.roundTrippers.RoundTripper(imageRef.Repository, imageRef.Scope(transport.PullScope)) + if rt == nil { + rt = r.roundTrippers.BaseRoundTripper() + } return &SimpleRegistry{ remoteOpts: r.remoteOpts, refOpts: r.refOpts, keychain: keychain, - roundTrippers: r.roundTrippers, + roundTrippers: NewSingleTripperStorage(rt), transportAccess: &sync.Mutex{}, }, nil } diff --git a/pkg/imgpkg/registry/transport.go b/pkg/imgpkg/registry/transport.go index 291a6e6a5..30657bb2a 100644 --- a/pkg/imgpkg/registry/transport.go +++ b/pkg/imgpkg/registry/transport.go @@ -15,24 +15,37 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) -// RoundTripperStorage Maintains a storage of all the available RoundTripper for different registries and repositories -type RoundTripperStorage struct { +// MultiRoundTripperStorage Maintains a storage of all the available RoundTripper for different registries and repositories +type MultiRoundTripperStorage struct { baseRoundTripper http.RoundTripper transports map[string]map[string]map[string]http.RoundTripper readWriteAccess *sync.Mutex } -// NewRoundTripperStorage Creates a struct that holds RoundTripper -func NewRoundTripperStorage(baseRoundTripper http.RoundTripper) *RoundTripperStorage { - return &RoundTripperStorage{ +// NewMultiRoundTripperStorage Creates a struct that holds RoundTripper +func NewMultiRoundTripperStorage(baseRoundTripper http.RoundTripper) *MultiRoundTripperStorage { + return &MultiRoundTripperStorage{ baseRoundTripper: baseRoundTripper, readWriteAccess: &sync.Mutex{}, transports: map[string]map[string]map[string]http.RoundTripper{}, } } +// NewSingleTripperStorage Creates a struct that holds RoundTripper +func NewSingleTripperStorage(baseRoundTripper http.RoundTripper) *SingleTripperStorage { + return &SingleTripperStorage{ + baseRoundTripper: baseRoundTripper, + readWriteAccess: &sync.Mutex{}, + } +} + +// BaseRoundTripper retrieves the base RoundTripper used by the store +func (r MultiRoundTripperStorage) BaseRoundTripper() http.RoundTripper { + return r.baseRoundTripper +} + // RoundTripper Retrieve the RoundTripper to be used for a particular registry and repository or nil if it cannot be found -func (r *RoundTripperStorage) RoundTripper(repo regname.Repository, scope string) http.RoundTripper { +func (r *MultiRoundTripperStorage) RoundTripper(repo regname.Repository, scope string) http.RoundTripper { r.readWriteAccess.Lock() defer r.readWriteAccess.Unlock() @@ -66,7 +79,7 @@ func (r *RoundTripperStorage) RoundTripper(repo regname.Repository, scope string // CreateRoundTripper Creates a new RoundTripper // scope field has the following format "repository:/org/suborg/repo_name:pull,push" // for more information check https://github.com/distribution/distribution/blob/263da70ea6a4e96f61f7a6770273ec6baac38941/docs/spec/auth/token.md#requesting-a-token -func (r *RoundTripperStorage) CreateRoundTripper(reg regname.Registry, auth authn.Authenticator, scope string) (http.RoundTripper, error) { +func (r *MultiRoundTripperStorage) CreateRoundTripper(reg regname.Registry, auth authn.Authenticator, scope string) (http.RoundTripper, error) { r.readWriteAccess.Lock() defer r.readWriteAccess.Unlock() @@ -94,3 +107,40 @@ func (r *RoundTripperStorage) CreateRoundTripper(reg regname.Registry, auth auth return rt, nil } + +// SingleTripperStorage Maintains a storage of all the available RoundTripper for different registries and repositories +type SingleTripperStorage struct { + baseRoundTripper http.RoundTripper + transport http.RoundTripper + readWriteAccess *sync.Mutex +} + +// RoundTripper Retrieve the RoundTripper to be used for a particular registry and repository or nil if it cannot be found +func (r *SingleTripperStorage) RoundTripper(_ regname.Repository, _ string) http.RoundTripper { + r.readWriteAccess.Lock() + defer r.readWriteAccess.Unlock() + + return r.transport +} + +// BaseRoundTripper retrieves the base RoundTripper used by the store +func (r SingleTripperStorage) BaseRoundTripper() http.RoundTripper { + return r.baseRoundTripper +} + +// CreateRoundTripper Creates a new RoundTripper +// scope field has the following format "repository:/org/suborg/repo_name:pull,push" +// for more information check https://github.com/distribution/distribution/blob/263da70ea6a4e96f61f7a6770273ec6baac38941/docs/spec/auth/token.md#requesting-a-token +func (r *SingleTripperStorage) CreateRoundTripper(reg regname.Registry, auth authn.Authenticator, scope string) (http.RoundTripper, error) { + r.readWriteAccess.Lock() + defer r.readWriteAccess.Unlock() + + rt, err := transport.NewWithContext(context.Background(), reg, auth, r.baseRoundTripper, []string{scope}) + if err != nil { + return nil, fmt.Errorf("Unable to create round tripper: %s", err) + } + + r.transport = rt + + return rt, nil +} diff --git a/test/helpers/fake_registry.go b/test/helpers/fake_registry.go index 7e756ee39..f9c53367e 100644 --- a/test/helpers/fake_registry.go +++ b/test/helpers/fake_registry.go @@ -22,7 +22,6 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" regname "github.com/google/go-containerregistry/pkg/name" - regregistry "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/random" @@ -34,6 +33,7 @@ import ( "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/image" "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/registry" + regregistry "github.com/vmware-tanzu/carvel-imgpkg/test/helpers/registry" ) type FakeTestRegistryBuilder struct { @@ -45,6 +45,7 @@ type FakeTestRegistryBuilder struct { originalHandler http.Handler } +// NewFakeRegistry Creates a registry that uses the ggcr version func NewFakeRegistry(t *testing.T, logger *Logger) *FakeTestRegistryBuilder { r := &FakeTestRegistryBuilder{images: map[string]*ImageOrImageIndexWithTarPath{}, t: t, logger: logger} r.server = httptest.NewServer(regregistry.New(regregistry.Logger(log.New(io.Discard, "", 0)))) @@ -52,6 +53,22 @@ func NewFakeRegistry(t *testing.T, logger *Logger) *FakeTestRegistryBuilder { return r } +// NewFakeRegistryWithDiskBackend Creates a registry that saves blobs to disk +func NewFakeRegistryWithDiskBackend(t *testing.T, logger *Logger) *FakeTestRegistryBuilder { + r := &FakeTestRegistryBuilder{images: map[string]*ImageOrImageIndexWithTarPath{}, t: t, logger: logger} + r.server = httptest.NewServer(regregistry.New(regregistry.Logger(log.New(io.Discard, "", 0)), regregistry.DiskBlobStorage())) + + return r +} + +// NewFakeRegistryWithRepoSeparation Creates a registry that saves the blobs based on the repository +func NewFakeRegistryWithRepoSeparation(t *testing.T, logger *Logger) *FakeTestRegistryBuilder { + r := &FakeTestRegistryBuilder{images: map[string]*ImageOrImageIndexWithTarPath{}, t: t, logger: logger} + r.server = httptest.NewServer(regregistry.New(regregistry.Logger(log.New(io.Discard, "", 0)), regregistry.MemStorageWithRepoSeparation())) + + return r +} + func (r *FakeTestRegistryBuilder) Build() registry.Registry { return r.BuildWithRegistryOpts(registry.Opts{ EnvironFunc: os.Environ, diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/README.md b/test/helpers/registry/README.md similarity index 100% rename from vendor/github.com/google/go-containerregistry/pkg/registry/README.md rename to test/helpers/registry/README.md diff --git a/test/helpers/registry/README_imgpkg.md b/test/helpers/registry/README_imgpkg.md new file mode 100644 index 000000000..2472ef376 --- /dev/null +++ b/test/helpers/registry/README_imgpkg.md @@ -0,0 +1 @@ +Copy this package from github.com/google/go-containerregistry version 0.7.0 diff --git a/test/helpers/registry/blobs.go b/test/helpers/registry/blobs.go new file mode 100644 index 000000000..be7574b4f --- /dev/null +++ b/test/helpers/registry/blobs.go @@ -0,0 +1,487 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Copied from https://github.com/google/go-containerregistry/tree/v0.8.0/pkg/registry +// Updated to ensure that blobs are mounted instead of re-uploaded + +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net/http" + "path" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/imageutils/verify" +) + +// Returns whether this url should be handled by the blob handler +// This is complicated because blob is indicated by the trailing path, not the leading path. +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-a-layer +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-a-layer +func isBlob(req *http.Request) bool { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + if len(elem) < 3 { + return false + } + return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" && + elem[len(elem)-2] == "uploads") +} + +// blobHandler represents a minimal blob storage backend, capable of serving +// blob contents. +type blobHandler interface { + // Get gets the blob contents, or errNotFound if the blob wasn't found. + Get(ctx context.Context, repo string, h v1.Hash) (io.ReadCloser, error) +} + +// blobStatHandler is an extension interface representing a blob storage +// backend that can serve metadata about blobs. +type blobStatHandler interface { + // Stat returns the size of the blob, or errNotFound if the blob wasn't + // found, or redirectError if the blob can be found elsewhere. + Stat(ctx context.Context, repo string, h v1.Hash) (int64, error) +} + +// blobPutHandler is an extension interface representing a blob storage backend +// that can write blob contents. +type blobPutHandler interface { + // Put puts the blob contents. + // + // The contents will be verified against the expected size and digest + // as the contents are read, and an error will be returned if these + // don't match. Implementations should return that error, or a wrapper + // around that error, to return the correct error when these don't match. + Put(ctx context.Context, repo string, h v1.Hash, rc io.ReadCloser) error +} + +// blobMountHandler is an extension interface representing a blob storage backend +// that can write blob contents. +type blobMountHandler interface { + // Mount puts the blob contents. + Mount(ctx context.Context, repo, from string, h v1.Hash) error +} + +// redirectError represents a signal that the blob handler doesn't have the blob +// contents, but that those contents are at another location which registry +// clients should redirect to. +type redirectError struct { + // Location is the location to find the contents. + Location string + + // Code is the HTTP redirect status code to return to clients. + Code int +} + +func (e redirectError) Error() string { return fmt.Sprintf("redirecting (%d): %s", e.Code, e.Location) } + +// errNotFound represents an error locating the blob. +var errNotFound = errors.New("not found") + +type memHandler struct { + m map[string][]byte + lock sync.Mutex +} + +func (m *memHandler) Stat(_ context.Context, _ string, h v1.Hash) (int64, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[h.String()] + if !found { + return 0, errNotFound + } + return int64(len(b)), nil +} +func (m *memHandler) Get(_ context.Context, _ string, h v1.Hash) (io.ReadCloser, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[h.String()] + if !found { + return nil, errNotFound + } + return ioutil.NopCloser(bytes.NewReader(b)), nil +} +func (m *memHandler) Put(_ context.Context, _ string, h v1.Hash, rc io.ReadCloser) error { + m.lock.Lock() + defer m.lock.Unlock() + + defer rc.Close() + all, err := ioutil.ReadAll(rc) + if err != nil { + return err + } + m.m[h.String()] = all + return nil +} + +// Mount is a no-op since all the blobs are store indexed by sha +func (m *memHandler) Mount(_ context.Context, _, _ string, _ v1.Hash) error { + m.lock.Lock() + defer m.lock.Unlock() + + return nil +} + +// blobs +type blobs struct { + blobHandler blobHandler + + // Each upload gets a unique id that writes occur to until finalized. + uploads map[string][]byte + lock sync.Mutex +} + +func (b *blobs) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + // Must have a path of form /v2/{name}/blobs/{upload,sha256:} + if len(elem) < 4 { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "blobs must be attached to a repo", + } + } + target := elem[len(elem)-1] + service := elem[len(elem)-2] + digest := req.URL.Query().Get("digest") + contentRange := req.Header.Get("Content-Range") + + repo := req.URL.Host + path.Join(elem[1:len(elem)-2]...) + + switch req.Method { + case http.MethodHead: + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + var size int64 + if bsh, ok := b.blobHandler.(blobStatHandler); ok { + size, err = bsh.Stat(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + } else { + rc, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + defer rc.Close() + size, err = io.Copy(ioutil.Discard, rc) + if err != nil { + return regErrInternal(err) + } + } + + resp.Header().Set("Content-Length", fmt.Sprint(size)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusOK) + return nil + + case http.MethodGet: + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + var size int64 + var r io.Reader + if bsh, ok := b.blobHandler.(blobStatHandler); ok { + size, err = bsh.Stat(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + + rc, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + + return regErrInternal(err) + } + defer rc.Close() + r = rc + } else { + tmp, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, errNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr redirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + + return regErrInternal(err) + } + defer tmp.Close() + var buf bytes.Buffer + io.Copy(&buf, tmp) + size = int64(buf.Len()) + r = &buf + } + + resp.Header().Set("Content-Length", fmt.Sprint(size)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, r) + return nil + + case http.MethodPost: + bph, ok := b.blobHandler.(blobPutHandler) + if !ok { + return regErrUnsupported + } + bmh, ok := b.blobHandler.(blobMountHandler) + if !ok { + return regErrUnsupported + } + + // It is weird that this is "target" instead of "service", but + // that's how the index math works out above. + if target != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("POST to /blobs must be followed by /uploads, got %s", target), + } + } + + if digest != "" { + h, err := v1.NewHash(digest) + if err != nil { + return regErrDigestInvalid + } + + vrc, err := verify.ReadCloser(req.Body, req.ContentLength, h) + if err != nil { + return regErrInternal(err) + } + defer vrc.Close() + + if err = bph.Put(req.Context(), repo, h, vrc); err != nil { + if errors.As(err, &verify.Error{}) { + log.Printf("Digest mismatch: %v", err) + return regErrDigestMismatch + } + return regErrInternal(err) + } + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + } + + from := req.URL.Query().Get("from") + mount := req.URL.Query().Get("mount") + if from != "" && mount != "" { + h, err := v1.NewHash(mount) + if err != nil { + return regErrDigestInvalid + } + + err = bmh.Mount(req.Context(), repo, from, h) + if err != nil { + return regErrInternal(err) + } + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + } + + id := fmt.Sprint(rand.Int63()) + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id)) + resp.Header().Set("Range", "0-0") + resp.WriteHeader(http.StatusAccepted) + return nil + + case http.MethodPatch: + if service != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("PATCH to /blobs must be followed by /uploads, got %s", service), + } + } + + if contentRange != "" { + start, end := 0, 0 + if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "We don't understand your Content-Range", + } + } + b.lock.Lock() + defer b.lock.Unlock() + if start != len(b.uploads[target]) { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "Your content range doesn't match what we have", + } + } + l := bytes.NewBuffer(b.uploads[target]) + io.Copy(l, req.Body) + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + resp.WriteHeader(http.StatusNoContent) + return nil + } + + b.lock.Lock() + defer b.lock.Unlock() + if _, ok := b.uploads[target]; ok { + return ®Error{ + Status: http.StatusBadRequest, + Code: "BLOB_UPLOAD_INVALID", + Message: "Stream uploads after first write are not allowed", + } + } + + l := &bytes.Buffer{} + io.Copy(l, req.Body) + + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + resp.WriteHeader(http.StatusNoContent) + return nil + + case http.MethodPut: + bph, ok := b.blobHandler.(blobPutHandler) + if !ok { + return regErrUnsupported + } + + if service != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("PUT to /blobs must be followed by /uploads, got %s", service), + } + } + + if digest == "" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest not specified", + } + } + + b.lock.Lock() + defer b.lock.Unlock() + + h, err := v1.NewHash(digest) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + defer req.Body.Close() + in := ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(b.uploads[target]), req.Body)) + + size := int64(verify.SizeUnknown) + if req.ContentLength > 0 { + size = int64(len(b.uploads[target])) + req.ContentLength + } + + vrc, err := verify.ReadCloser(in, size, h) + if err != nil { + return regErrInternal(err) + } + defer vrc.Close() + + if err := bph.Put(req.Context(), repo, h, vrc); err != nil { + if errors.As(err, &verify.Error{}) { + log.Printf("Digest mismatch: %v", err) + return regErrDigestMismatch + } + return regErrInternal(err) + } + + delete(b.uploads, target) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + + default: + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } +} diff --git a/test/helpers/registry/blobs_access_controll.go b/test/helpers/registry/blobs_access_controll.go new file mode 100644 index 000000000..7ac6224ce --- /dev/null +++ b/test/helpers/registry/blobs_access_controll.go @@ -0,0 +1,184 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package registry + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +const blobFolder = "blobs" + +func newBlobDiskHandler() *diskHandler { + tmpFolder, err := os.MkdirTemp("", blobFolder) + if err != nil { + panic(fmt.Errorf("unable to create temporary folder: %s", err)) + } + + return &diskHandler{ + m: map[string]blobLocation{}, + access: map[string]string{}, + tmpDir: tmpFolder, + lock: sync.Mutex{}, + } +} + +type blobLocation struct { + size int64 + location string +} + +type diskHandler struct { + m map[string]blobLocation + access map[string]string + tmpDir string + lock sync.Mutex +} + +func (m *diskHandler) Stat(_ context.Context, repo string, h v1.Hash) (int64, error) { + m.lock.Lock() + defer m.lock.Unlock() + + if !m.exists(repo, h) { + return 0, errNotFound + } + + return m.m[h.String()].size, nil +} + +func (m *diskHandler) exists(repo string, h v1.Hash) bool { + _, found := m.access[m.accessKey(repo, h)] + return found +} + +func (m *diskHandler) Get(_ context.Context, repo string, h v1.Hash) (io.ReadCloser, error) { + m.lock.Lock() + defer m.lock.Unlock() + + if !m.exists(repo, h) { + return nil, errNotFound + } + + blobFile, err := os.Open(m.m[h.String()].location) + if err != nil { + return nil, err + } + + return blobFile, nil +} + +func (m *diskHandler) accessKey(repo string, h v1.Hash) string { + if strings.HasSuffix(repo, "/blobs") { + return repo[:len(repo)-len("/blobs")] + "@" + h.String() + } + return repo + "@" + h.String() +} + +func (m *diskHandler) Put(_ context.Context, repo string, h v1.Hash, rc io.ReadCloser) error { + m.lock.Lock() + defer m.lock.Unlock() + + defer rc.Close() + + // if the blob already exists there is no need to copy it + if m.exists(repo, h) { + return nil + } + + if _, found := m.m[h.String()]; found { + m.access[m.accessKey(repo, h)] = h.String() + return nil + } + + blobFile, err := os.CreateTemp(m.tmpDir, h.String()) + if err != nil { + return err + } + defer blobFile.Close() + + s, err := io.Copy(blobFile, rc) + if err != nil { + return err + } + + m.m[h.String()] = blobLocation{ + size: s, + location: blobFile.Name(), + } + m.access[m.accessKey(repo, h)] = h.String() + return nil +} +func (m *diskHandler) Mount(_ context.Context, repo, from string, h v1.Hash) error { + m.lock.Lock() + defer m.lock.Unlock() + + m.access[m.accessKey(repo, h)] = h.String() + return nil +} + +func newBlobWithAccessControlHandler() *memWithAccessControlHandler { + return &memWithAccessControlHandler{ + m: map[string][]byte{}, + lock: sync.Mutex{}, + } +} + +type memWithAccessControlHandler struct { + m map[string][]byte + lock sync.Mutex +} + +func accessKey(repo string, h v1.Hash) string { + if strings.HasSuffix(repo, "/blobs") { + return repo[:len(repo)-len("/blobs")] + "@" + h.String() + } + return repo + "@" + h.String() +} +func (m *memWithAccessControlHandler) Stat(_ context.Context, repo string, h v1.Hash) (int64, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[accessKey(repo, h)] + if !found { + return 0, errNotFound + } + return int64(len(b)), nil +} +func (m *memWithAccessControlHandler) Get(_ context.Context, repo string, h v1.Hash) (io.ReadCloser, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[accessKey(repo, h)] + if !found { + return nil, errNotFound + } + return ioutil.NopCloser(bytes.NewReader(b)), nil +} +func (m *memWithAccessControlHandler) Put(_ context.Context, repo string, h v1.Hash, rc io.ReadCloser) error { + m.lock.Lock() + defer m.lock.Unlock() + + defer rc.Close() + all, err := ioutil.ReadAll(rc) + if err != nil { + return err + } + m.m[accessKey(repo, h)] = all + return nil +} +func (m *memWithAccessControlHandler) Mount(_ context.Context, repo, from string, h v1.Hash) error { + m.lock.Lock() + defer m.lock.Unlock() + + m.m[accessKey(repo, h)] = m.m[accessKey(from, h)] + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/error.go b/test/helpers/registry/error.go similarity index 52% rename from vendor/github.com/google/go-containerregistry/pkg/registry/error.go rename to test/helpers/registry/error.go index 64e98671c..c660231e9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/error.go +++ b/test/helpers/registry/error.go @@ -1,3 +1,9 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Copied from https://github.com/google/go-containerregistry/tree/v0.8.0/pkg/registry +// Updated to ensure that blobs are mounted instead of re-uploaded + // Copyright 2018 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,3 +50,36 @@ func (r *regError) Write(resp http.ResponseWriter) error { }, }) } + +// regErrInternal returns an internal server error. +func regErrInternal(err error) *regError { + return ®Error{ + Status: http.StatusInternalServerError, + Code: "INTERNAL_SERVER_ERROR", + Message: err.Error(), + } +} + +var regErrBlobUnknown = ®Error{ + Status: http.StatusNotFound, + Code: "BLOB_UNKNOWN", + Message: "Unknown blob", +} + +var regErrUnsupported = ®Error{ + Status: http.StatusMethodNotAllowed, + Code: "UNSUPPORTED", + Message: "Unsupported operation", +} + +var regErrDigestMismatch = ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest does not match contents", +} + +var regErrDigestInvalid = ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", +} diff --git a/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go b/test/helpers/registry/httptest/httptest.go similarity index 94% rename from vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go rename to test/helpers/registry/httptest/httptest.go index 85b171907..8cc5b8861 100644 --- a/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go +++ b/test/helpers/registry/httptest/httptest.go @@ -1,3 +1,8 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Copied from https://github.com/google/go-containerregistry/tree/v0.8.0/internal + // Copyright 2020 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go b/test/helpers/registry/manifest.go similarity index 97% rename from vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go rename to test/helpers/registry/manifest.go index 2a9723916..b45edeb5a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go +++ b/test/helpers/registry/manifest.go @@ -1,3 +1,9 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Copied from https://github.com/google/go-containerregistry/tree/v0.8.0/pkg/registry +// Updated to ensure that blobs are mounted instead of re-uploaded + // Copyright 2018 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go b/test/helpers/registry/registry.go similarity index 81% rename from vendor/github.com/google/go-containerregistry/pkg/registry/registry.go rename to test/helpers/registry/registry.go index c56dae26d..a5c11b02c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go +++ b/test/helpers/registry/registry.go @@ -1,3 +1,9 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Copied from https://github.com/google/go-containerregistry/tree/v0.8.0/pkg/registry +// Updated to ensure that blobs are mounted instead of re-uploaded + // Copyright 2018 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -77,8 +83,8 @@ func New(opts ...Option) http.Handler { r := ®istry{ log: log.New(os.Stderr, "", log.LstdFlags), blobs: blobs{ - contents: map[string][]byte{}, - uploads: map[string][]byte{}, + blobHandler: &memHandler{m: map[string][]byte{}}, + uploads: map[string][]byte{}, }, manifests: manifests{ manifests: map[string]map[string]manifest{}, @@ -102,3 +108,17 @@ func Logger(l *log.Logger) Option { r.manifests.log = l } } + +// DiskBlobStorage Save blobs to disk +func DiskBlobStorage() Option { + return func(r *registry) { + r.blobs.blobHandler = newBlobDiskHandler() + } +} + +// MemStorageWithRepoSeparation Save the blobs in memory but separated per repository +func MemStorageWithRepoSeparation() Option { + return func(r *registry) { + r.blobs.blobHandler = newBlobWithAccessControlHandler() + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go b/test/helpers/registry/tls.go similarity index 77% rename from vendor/github.com/google/go-containerregistry/pkg/registry/tls.go rename to test/helpers/registry/tls.go index cb2644e61..dfa860c88 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go +++ b/test/helpers/registry/tls.go @@ -1,3 +1,9 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Copied from https://github.com/google/go-containerregistry/tree/v0.8.0/pkg/registry +// Updated to ensure that blobs are mounted instead of re-uploaded + // Copyright 2018 Google LLC All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +23,7 @@ package registry import ( "net/http/httptest" - ggcrtest "github.com/google/go-containerregistry/internal/httptest" + ggcrtest "github.com/vmware-tanzu/carvel-imgpkg/test/helpers/registry/httptest" ) // TLS returns an httptest server, with an http client that has been configured to diff --git a/test/perf/perf_test.go b/test/perf/perf_test.go index e5c30f654..b38641d38 100644 --- a/test/perf/perf_test.go +++ b/test/perf/perf_test.go @@ -55,18 +55,19 @@ func TestBenchmarkCopyingLargeBundleThatContainsImagesMostlyOnDockerHub(t *testi defer env.Cleanup() imgpkg := helpers.Imgpkg{T: t, L: logger, ImgpkgPath: env.ImgpkgPath} + perfTestingRepo := startRegistryForPerfTesting(t, env) - imgpkg.Run([]string{"push", "-f", "./assets/cf-for-k8s-bundle", "-b", env.RelocationRepo}) + imgpkg.Run([]string{"push", "-f", "./assets/cf-for-k8s-bundle", "-b", perfTestingRepo}) benchmarkResultCopyLargeBundle := testing.Benchmark(func(b *testing.B) { - imgpkg.Run([]string{"copy", "-b", env.RelocationRepo, "--to-repo", env.RelocationRepo + "copy"}) + imgpkg.Run([]string{"copy", "-b", perfTestingRepo, "--to-repo", perfTestingRepo + "copy"}) }) logger.Debugf("imgpkg copy took: %v\n", benchmarkResultCopyLargeBundle.T) actualTimeTaken := benchmarkResultCopyLargeBundle.T.Nanoseconds() - reference, err := regname.ParseReference(env.RelocationRepo) + reference, err := regname.ParseReference(perfTestingRepo) require.NoError(t, err) maxTimeCopyShouldTake := time.Minute.Nanoseconds() @@ -78,7 +79,7 @@ func TestBenchmarkCopyingLargeBundleThatContainsImagesMostlyOnDockerHub(t *testi } func startRegistryForPerfTesting(t *testing.T, env *helpers.Env) string { - fakeRegistry := helpers.NewFakeRegistry(t, env.Logger) + fakeRegistry := helpers.NewFakeRegistryWithDiskBackend(t, env.Logger) env.AddCleanup(func() { fakeRegistry.CleanUp() diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go deleted file mode 100644 index 067227271..000000000 --- a/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package registry - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "math/rand" - "net/http" - "path" - "strings" - "sync" -) - -// Returns whether this url should be handled by the blob handler -// This is complicated because blob is indicated by the trailing path, not the leading path. -// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-a-layer -// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-a-layer -func isBlob(req *http.Request) bool { - elem := strings.Split(req.URL.Path, "/") - elem = elem[1:] - if elem[len(elem)-1] == "" { - elem = elem[:len(elem)-1] - } - if len(elem) < 3 { - return false - } - return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" && - elem[len(elem)-2] == "uploads") -} - -// blobs -type blobs struct { - // Blobs are content addresses. we store them globally underneath their sha and make no distinctions per image. - contents map[string][]byte - // Each upload gets a unique id that writes occur to until finalized. - uploads map[string][]byte - lock sync.Mutex -} - -func (b *blobs) handle(resp http.ResponseWriter, req *http.Request) *regError { - elem := strings.Split(req.URL.Path, "/") - elem = elem[1:] - if elem[len(elem)-1] == "" { - elem = elem[:len(elem)-1] - } - // Must have a path of form /v2/{name}/blobs/{upload,sha256:} - if len(elem) < 4 { - return ®Error{ - Status: http.StatusBadRequest, - Code: "NAME_INVALID", - Message: "blobs must be attached to a repo", - } - } - target := elem[len(elem)-1] - service := elem[len(elem)-2] - digest := req.URL.Query().Get("digest") - contentRange := req.Header.Get("Content-Range") - - switch req.Method { - case http.MethodHead: - b.lock.Lock() - defer b.lock.Unlock() - b, ok := b.contents[target] - if !ok { - return ®Error{ - Status: http.StatusNotFound, - Code: "BLOB_UNKNOWN", - Message: "Unknown blob", - } - } - - resp.Header().Set("Content-Length", fmt.Sprint(len(b))) - resp.Header().Set("Docker-Content-Digest", target) - resp.WriteHeader(http.StatusOK) - return nil - - case http.MethodGet: - b.lock.Lock() - defer b.lock.Unlock() - b, ok := b.contents[target] - if !ok { - return ®Error{ - Status: http.StatusNotFound, - Code: "BLOB_UNKNOWN", - Message: "Unknown blob", - } - } - - resp.Header().Set("Content-Length", fmt.Sprint(len(b))) - resp.Header().Set("Docker-Content-Digest", target) - resp.WriteHeader(http.StatusOK) - io.Copy(resp, bytes.NewReader(b)) - return nil - - case http.MethodPost: - // It is weird that this is "target" instead of "service", but - // that's how the index math works out above. - if target != "uploads" { - return ®Error{ - Status: http.StatusBadRequest, - Code: "METHOD_UNKNOWN", - Message: fmt.Sprintf("POST to /blobs must be followed by /uploads, got %s", target), - } - } - - if digest != "" { - l := &bytes.Buffer{} - io.Copy(l, req.Body) - rd := sha256.Sum256(l.Bytes()) - d := "sha256:" + hex.EncodeToString(rd[:]) - if d != digest { - return ®Error{ - Status: http.StatusBadRequest, - Code: "DIGEST_INVALID", - Message: "digest does not match contents", - } - } - - b.lock.Lock() - defer b.lock.Unlock() - b.contents[d] = l.Bytes() - resp.Header().Set("Docker-Content-Digest", d) - resp.WriteHeader(http.StatusCreated) - return nil - } - - id := fmt.Sprint(rand.Int63()) - resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id)) - resp.Header().Set("Range", "0-0") - resp.WriteHeader(http.StatusAccepted) - return nil - - case http.MethodPatch: - if service != "uploads" { - return ®Error{ - Status: http.StatusBadRequest, - Code: "METHOD_UNKNOWN", - Message: fmt.Sprintf("PATCH to /blobs must be followed by /uploads, got %s", service), - } - } - - if contentRange != "" { - start, end := 0, 0 - if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { - return ®Error{ - Status: http.StatusRequestedRangeNotSatisfiable, - Code: "BLOB_UPLOAD_UNKNOWN", - Message: "We don't understand your Content-Range", - } - } - b.lock.Lock() - defer b.lock.Unlock() - if start != len(b.uploads[target]) { - return ®Error{ - Status: http.StatusRequestedRangeNotSatisfiable, - Code: "BLOB_UPLOAD_UNKNOWN", - Message: "Your content range doesn't match what we have", - } - } - l := bytes.NewBuffer(b.uploads[target]) - io.Copy(l, req.Body) - b.uploads[target] = l.Bytes() - resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) - resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) - resp.WriteHeader(http.StatusNoContent) - return nil - } - - b.lock.Lock() - defer b.lock.Unlock() - if _, ok := b.uploads[target]; ok { - return ®Error{ - Status: http.StatusBadRequest, - Code: "BLOB_UPLOAD_INVALID", - Message: "Stream uploads after first write are not allowed", - } - } - - l := &bytes.Buffer{} - io.Copy(l, req.Body) - - b.uploads[target] = l.Bytes() - resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) - resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) - resp.WriteHeader(http.StatusNoContent) - return nil - - case http.MethodPut: - if service != "uploads" { - return ®Error{ - Status: http.StatusBadRequest, - Code: "METHOD_UNKNOWN", - Message: fmt.Sprintf("PUT to /blobs must be followed by /uploads, got %s", service), - } - } - - if digest == "" { - return ®Error{ - Status: http.StatusBadRequest, - Code: "DIGEST_INVALID", - Message: "digest not specified", - } - } - - b.lock.Lock() - defer b.lock.Unlock() - l := bytes.NewBuffer(b.uploads[target]) - io.Copy(l, req.Body) - rd := sha256.Sum256(l.Bytes()) - d := "sha256:" + hex.EncodeToString(rd[:]) - if d != digest { - return ®Error{ - Status: http.StatusBadRequest, - Code: "DIGEST_INVALID", - Message: "digest does not match contents", - } - } - - b.contents[d] = l.Bytes() - delete(b.uploads, target) - resp.Header().Set("Docker-Content-Digest", d) - resp.WriteHeader(http.StatusCreated) - return nil - - default: - return ®Error{ - Status: http.StatusBadRequest, - Code: "METHOD_UNKNOWN", - Message: "We don't understand your method + url", - } - } -} diff --git a/vendor/modules.txt b/vendor/modules.txt index e790a3fe3..5b1ada1ce 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -137,7 +137,6 @@ github.com/google/go-cmp/cmp/internal/value github.com/google/go-containerregistry/internal/and github.com/google/go-containerregistry/internal/estargz github.com/google/go-containerregistry/internal/gzip -github.com/google/go-containerregistry/internal/httptest github.com/google/go-containerregistry/internal/redact github.com/google/go-containerregistry/internal/retry github.com/google/go-containerregistry/internal/retry/wait @@ -145,7 +144,6 @@ github.com/google/go-containerregistry/internal/verify github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/logs github.com/google/go-containerregistry/pkg/name -github.com/google/go-containerregistry/pkg/registry github.com/google/go-containerregistry/pkg/v1 github.com/google/go-containerregistry/pkg/v1/empty github.com/google/go-containerregistry/pkg/v1/fake