From 05944b51937695b30598b52f3ac472acf9053d84 Mon Sep 17 00:00:00 2001 From: Andrew Farries Date: Wed, 27 Jul 2022 10:54:23 +0000 Subject: [PATCH 1/3] Make usage report bucket name configurable The name of the object storage bucket to which usage reports are uploaded needs to be configurable for Gitpod SaaS. GCloud bucket names are globally unique, so the bucket name must vary between staging and production. --- .../content-service-api/go/config/config.go | 10 ++- .../components/content-service/configmap.go | 12 ++++ .../content-service/configmap_test.go | 62 +++++++++++++++++++ .../config/v1/experimental/experimental.go | 4 ++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 install/installer/pkg/components/content-service/configmap_test.go diff --git a/components/content-service-api/go/config/config.go b/components/content-service-api/go/config/config.go index 68eac477d9ce39..65ec7e7feebef4 100644 --- a/components/content-service-api/go/config/config.go +++ b/components/content-service-api/go/config/config.go @@ -114,7 +114,13 @@ type PProf struct { Addr string `json:"address"` } +// UsageReportConfig configures the upload of workspace instance usage reports +type UsageReportConfig struct { + BucketName string `json:"bucketName"` +} + type ServiceConfig struct { - Service baseserver.ServerConfiguration `json:"service"` - Storage StorageConfig `json:"storage"` + Service baseserver.ServerConfiguration `json:"service"` + Storage StorageConfig `json:"storage"` + UsageReports UsageReportConfig `json:"usageReport"` } diff --git a/install/installer/pkg/components/content-service/configmap.go b/install/installer/pkg/components/content-service/configmap.go index 622ef0e6ce5329..e1c984d7a323c9 100644 --- a/install/installer/pkg/components/content-service/configmap.go +++ b/install/installer/pkg/components/content-service/configmap.go @@ -11,6 +11,7 @@ import ( "github.com/gitpod-io/gitpod/content-service/api/config" "github.com/gitpod-io/gitpod/installer/pkg/common" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,11 +19,22 @@ import ( ) func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { + usageReportBucketName := "gitpod-usage-reports" + _ = ctx.WithExperimental(func(cfg *experimental.Config) error { + if cfg.Workspace != nil && cfg.Workspace.ContentService.UsageReportBucketName != "" { + usageReportBucketName = cfg.Workspace.ContentService.UsageReportBucketName + } + return nil + }) + cscfg := config.ServiceConfig{ Service: baseserver.ServerConfiguration{ Address: fmt.Sprintf(":%d", RPCPort), }, Storage: common.StorageConfig(ctx), + UsageReports: config.UsageReportConfig{ + BucketName: usageReportBucketName, + }, } fc, err := common.ToJSONString(cscfg) diff --git a/install/installer/pkg/components/content-service/configmap_test.go b/install/installer/pkg/components/content-service/configmap_test.go new file mode 100644 index 00000000000000..da234cd262c0ea --- /dev/null +++ b/install/installer/pkg/components/content-service/configmap_test.go @@ -0,0 +1,62 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +package content_service + +import ( + "encoding/json" + "testing" + + csconfig "github.com/gitpod-io/gitpod/content-service/api/config" + "github.com/gitpod-io/gitpod/installer/pkg/common" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental" + "github.com/gitpod-io/gitpod/installer/pkg/config/versions" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +func TestConfigMap_CanConfigureUsageReportBucketName(t *testing.T) { + ctx := newRenderContext(t) + objs, err := configmap(ctx) + require.NoError(t, err) + require.Len(t, objs, 1, "must only render one configmap") + + expectedUsageReportConfig := csconfig.UsageReportConfig{BucketName: "some-bucket-name"} + expectedJSON, err := common.ToJSONString(expectedUsageReportConfig) + require.NoError(t, err) + + cm, ok := objs[0].(*corev1.ConfigMap) + require.True(t, ok) + + var fullConfig csconfig.ServiceConfig + err = json.Unmarshal([]byte(cm.Data["config.json"]), &fullConfig) + require.NoError(t, err) + + actualUsageReportConfig := fullConfig.UsageReports + actualJSON, err := common.ToJSONString(actualUsageReportConfig) + require.NoError(t, err) + + require.JSONEq(t, string(expectedJSON), string(actualJSON)) +} + +func newRenderContext(t *testing.T) *common.RenderContext { + t.Helper() + + ctx, err := common.NewRenderContext(config.Config{ + Domain: "test.domain.everything.awesome.is", + ObjectStorage: config.ObjectStorage{InCluster: pointer.Bool(true)}, + Experimental: &experimental.Config{ + Workspace: &experimental.WorkspaceConfig{ + ContentService: struct { + UsageReportBucketName string "json:\"usageReportBucketName\"" + }{UsageReportBucketName: "some-bucket-name"}, + }, + }, + }, versions.Manifest{}, "test-namespace") + + require.NoError(t, err) + + return ctx +} diff --git a/install/installer/pkg/config/v1/experimental/experimental.go b/install/installer/pkg/config/v1/experimental/experimental.go index 2c6aef3ad652f5..9fc365ea6498fc 100644 --- a/install/installer/pkg/config/v1/experimental/experimental.go +++ b/install/installer/pkg/config/v1/experimental/experimental.go @@ -103,6 +103,10 @@ type WorkspaceConfig struct { GitpodInstallationWorkspaceHostSuffix string `json:"gitpodInstallationWorkspaceHostSuffix"` GitpodInstallationWorkspaceHostSuffixRegex string `json:"gitpodInstallationWorkspaceHostSuffixRegex"` } `json:"wsProxy"` + + ContentService struct { + UsageReportBucketName string `json:"usageReportBucketName"` + } `json:"contentService"` } type PersistentVolumeClaim struct { From f6b541707a8a7e64cb869067840b0a51a30737d1 Mon Sep 17 00:00:00 2001 From: Andrew Farries Date: Wed, 27 Jul 2022 11:14:51 +0000 Subject: [PATCH 2/3] Make content-service take bucket name from config Rather than hardcoding the name of the bucket. --- components/content-service/cmd/run.go | 2 +- .../pkg/service/usage-report-service_test.go | 12 ++++++++---- .../pkg/service/usagereport-service.go | 19 ++++++++----------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/components/content-service/cmd/run.go b/components/content-service/cmd/run.go index 94e85c74037924..7ee580462c6bb0 100644 --- a/components/content-service/cmd/run.go +++ b/components/content-service/cmd/run.go @@ -57,7 +57,7 @@ var runCmd = &cobra.Command{ } api.RegisterIDEPluginServiceServer(srv.GRPC(), idePluginService) - usageReportService, err := service.NewUsageReportService(cfg.Storage) + usageReportService, err := service.NewUsageReportService(cfg.Storage, cfg.UsageReports.BucketName) if err != nil { log.WithError(err).Fatalf("Cannot create usage report service") } diff --git a/components/content-service/pkg/service/usage-report-service_test.go b/components/content-service/pkg/service/usage-report-service_test.go index 35b6f22f9cde62..4a87758366383c 100644 --- a/components/content-service/pkg/service/usage-report-service_test.go +++ b/components/content-service/pkg/service/usage-report-service_test.go @@ -19,16 +19,20 @@ import ( // TestUploadURL tests that usageReportService.UploadURL interacts with PresignedAccess // correctly to produce an upload URL for the correct bucket and filename. func TestUploadURL(t *testing.T) { + const ( + fileName = "some-report-filename" + bucketName = "gitpod-usage-reports" + ) + ctrl := gomock.NewController(t) s := storagemock.NewMockPresignedAccess(ctrl) - const fileName = "some-report-filename" - s.EXPECT().EnsureExists(gomock.Any(), usageReportBucketName). + s.EXPECT().EnsureExists(gomock.Any(), bucketName). Return(nil) - s.EXPECT().SignUpload(gomock.Any(), usageReportBucketName, fileName, gomock.Any()). + s.EXPECT().SignUpload(gomock.Any(), bucketName, fileName, gomock.Any()). Return(&storage.UploadInfo{URL: "http://example.com/some-path"}, nil) - svc := &UsageReportService{cfg: config.StorageConfig{}, s: s} + svc := &UsageReportService{cfg: config.StorageConfig{}, s: s, bucketName: bucketName} resp, err := svc.UploadURL(context.Background(), &api.UsageReportUploadURLRequest{Name: fileName}) require.NoError(t, err) diff --git a/components/content-service/pkg/service/usagereport-service.go b/components/content-service/pkg/service/usagereport-service.go index 3c5dee74d83f87..f4437dedb49168 100644 --- a/components/content-service/pkg/service/usagereport-service.go +++ b/components/content-service/pkg/service/usagereport-service.go @@ -18,25 +18,22 @@ import ( "github.com/gitpod-io/gitpod/content-service/pkg/storage" ) -const ( - usageReportBucketName = "usage-reports" -) - // UsageReportService implements UsageReportServiceServer type UsageReportService struct { - cfg config.StorageConfig - s storage.PresignedAccess + cfg config.StorageConfig + s storage.PresignedAccess + bucketName string api.UnimplementedUsageReportServiceServer } // NewUsageReportService create a new usagereport service -func NewUsageReportService(cfg config.StorageConfig) (res *UsageReportService, err error) { +func NewUsageReportService(cfg config.StorageConfig, bucketName string) (res *UsageReportService, err error) { s, err := storage.NewPresignedAccess(&cfg) if err != nil { return nil, err } - return &UsageReportService{cfg: cfg, s: s}, nil + return &UsageReportService{cfg: cfg, s: s, bucketName: bucketName}, nil } // UploadURL provides a URL to which clients can upload the content via HTTP PUT. @@ -45,17 +42,17 @@ func (us *UsageReportService) UploadURL(ctx context.Context, req *api.UsageRepor span.SetTag("name", req.Name) defer tracing.FinishSpan(span, &err) - err = us.s.EnsureExists(ctx, usageReportBucketName) + err = us.s.EnsureExists(ctx, us.bucketName) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) } - info, err := us.s.SignUpload(ctx, usageReportBucketName, req.Name, &storage.SignedURLOptions{ + info, err := us.s.SignUpload(ctx, us.bucketName, req.Name, &storage.SignedURLOptions{ ContentType: "*/*", }) if err != nil { log.WithField("name", req.Name). - WithField("bucket", usageReportBucketName). + WithField("bucket", us.bucketName). WithError(err). Error("Error getting UsageReport SignUpload URL") return nil, status.Error(codes.Unknown, err.Error()) From 17e7c40370685d5c517bd08fde880acd6c493bdf Mon Sep 17 00:00:00 2001 From: Andrew Farries Date: Tue, 2 Aug 2022 07:20:16 +0000 Subject: [PATCH 3/3] Rename file --- .../service/{usagereport-service.go => usage-report-service.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename components/content-service/pkg/service/{usagereport-service.go => usage-report-service.go} (100%) diff --git a/components/content-service/pkg/service/usagereport-service.go b/components/content-service/pkg/service/usage-report-service.go similarity index 100% rename from components/content-service/pkg/service/usagereport-service.go rename to components/content-service/pkg/service/usage-report-service.go