feat(test reports): test_report key functionality

cmd/vela-worker/exec.go

@@ -14,9 +14,10 @@
 
 	"github.com/go-vela/sdk-go/vela"
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/queue/models"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 	"github.com/go-vela/worker/version"
@@ -29,6 +30,10 @@
 func (w *Worker) exec(ctx context.Context, index int, config *api.Worker) error {
 	var err error
 
+	var execStorage storage.Storage
+
+	var _executor executor.Engine
+
 	// setup the version
 	v := version.New()
 
@@ -158,6 +163,14 @@
 	execOutputCtn := *w.Config.Executor.OutputCtn
 	execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID)
 
+	if w.Storage != nil {
+		execStorage = w.Storage
+
+		logrus.Debugf("executor storage is available, setting up storage")
+	} else {
+		logrus.Debugf("executor storage is nil, skipping storage setup")
+	}
+
 	// create logger with extra metadata
 	//
 	// https://pkg.go.dev/github.com/sirupsen/logrus#WithFields
@@ -235,7 +248,7 @@
 	// setup the executor
 	//
 	// https://pkg.go.dev/github.com/go-vela/worker/executor#New
-	_executor, err := executor.New(&executor.Setup{
+	setup := &executor.Setup{
 		Logger:              logger,
 		Mock:                w.Config.Mock,
 		Driver:              w.Config.Executor.Driver,
@@ -250,8 +263,19 @@
 		Pipeline:            p.Sanitize(w.Config.Runtime.Driver),
 		Version:             v.Semantic(),
 		OutputCtn:           &execOutputCtn,
-	})
+	}
 
+	if execStorage != nil {
+		fmt.Printf("setting up executor storage\n")
+
+		setup.Storage = execStorage
+	}
+
+	_executor, err = executor.New(setup)
+	if err != nil {
+		logger.Errorf("unable to setup executor: %v", err)
+		return err
+	}
 	// add the executor to the worker
 	w.Executors[index] = _executor
 

cmd/vela-worker/flags.go

@@ -10,7 +10,8 @@
 
 	"github.com/urfave/cli/v3"
 
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 )
@@ -120,5 +121,8 @@
 
 	f = append(f, runtime.Flags...)
 
+	// Storage Flags
+	f = append(f, storage.Flags...)
+
 	return f
 }

cmd/vela-worker/operate.go

@@ -12,13 +12,14 @@ import (
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/server/storage"
 )
 
 // operate is a helper function to initiate all
 // subprocesses for the operator to poll the
 // queue and execute Vela pipelines.
 //
-//nolint:funlen // refactor candidate
+//nolint:funlen,gocyclo // refactor candidate
 func (w *Worker) operate(ctx context.Context) error {
 	var err error
 	// create the errgroup for managing operator subprocesses
@@ -77,6 +78,58 @@ func (w *Worker) operate(ctx context.Context) error {
 		w.updateWorkerStatus(ctx, registryWorker, constants.WorkerStatusError)
 	}
 
+	// getting storage creds
+	logrus.Trace("getting storage s3 creds..")
+	// fetching queue credentials using registration token
+	stCreds, _, err := w.VelaClient.Storage.GetInfo(ctx)
+	if err != nil {
+		logrus.Tracef("error getting storage creds: %v", err)
+		return err
+	}
+
+	if stCreds.GetEnabled() {
+		logrus.Trace("storage enabled")
+		// if an address was given at start up, use that — else use what is returned from server
+		if len(w.Config.Storage.Endpoint) == 0 {
+			w.Config.Storage.Endpoint = stCreds.GetStorageAddress()
+		}
+
+		// set access key in storage config
+		w.Config.Storage.AccessKey = stCreds.GetAccessKey()
+
+		// set secret key in storage config
+		w.Config.Storage.SecretKey = stCreds.GetSecretKey()
+
+		// set bucket name in storage config
+		w.Config.Storage.Bucket = stCreds.GetStorageBucket()
+
+		// set storage enabled to true
+		w.Config.Storage.Enable = stCreds.GetEnabled()
+
+		s, err := storage.New(w.Config.Storage)
+		if err != nil {
+			logrus.Error("storage setup failed")
+			// set to error as storage setup fails
+			w.updateWorkerStatus(ctx, registryWorker, constants.WorkerStatusError)
+
+			return err
+		}
+
+		w.Storage = s
+		logrus.WithFields(logrus.Fields{
+			"driver":   w.Config.Storage.Driver,
+			"bucket":   w.Config.Storage.Bucket,
+			"endpoint": w.Config.Storage.Endpoint,
+		}).Debug("storage initialized")
+	} else {
+		logrus.Trace("storage not enabled")
+		// storage disabled; nothing to validate
+		w.Config.Storage.Enable = false
+		w.Storage = nil
+
+		logrus.Debug("storage disabled: worker storage unset")
+	}
+
 	// spawn goroutine for phoning home
 	executors.Go(func() error {
 		// five second ticker for signal handling

cmd/vela-worker/run.go

@@ -17,6 +17,7 @@ import (
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 )
@@ -138,6 +139,13 @@ func run(ctx context.Context, c *cli.Command) error {
 				Address: c.String("server.addr"),
 				Secret:  c.String("server.secret"),
 			},
+			Storage: &storage.Setup{
+				Driver:    c.String("storage.driver"),
+				Endpoint:  c.String("storage.endpoint.name"),
+				AccessKey: c.String("storage.access.key"),
+				SecretKey: c.String("storage.secret.key"),
+				Bucket:    c.String("storage.bucket.name"),
+			},
 			// Certificate configuration
 			Certificate: &Certificate{
 				Cert: c.String("server.cert"),

cmd/vela-worker/worker.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-vela/sdk-go/vela"
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 )
@@ -57,6 +58,7 @@ type (
 		Server        *Server
 		Certificate   *Certificate
 		TLSMinVersion string
+		Storage       *storage.Setup
 	}
 
 	// Worker represents all configuration and
@@ -72,5 +74,6 @@ type (
 		RunningBuilds      []*api.Build
 		QueueCheckedIn     bool
 		RunningBuildsMutex sync.Mutex
+		Storage            storage.Storage
 	}
 )

executor/executor_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/constants"
 	"github.com/go-vela/server/mock/server"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/executor/linux"
 	"github.com/go-vela/worker/executor/local"
 	"github.com/go-vela/worker/runtime/docker"
@@ -37,6 +38,38 @@ func TestExecutor_New(t *testing.T) {
 		t.Errorf("unable to create runtime engine: %v", err)
 	}
 
+	_storageT := &storage.Setup{
+		Enable:    true,
+		Driver:    "minio",
+		Endpoint:  "http://localhost:9000",
+		AccessKey: "ad",
+		SecretKey: "asd",
+		Bucket:    "vela",
+		Region:    "",
+		Secure:    false,
+	}
+
+	_sT, err := storage.New(_storageT)
+	if err != nil {
+		t.Errorf("unable to create storage engine: %v", err)
+	}
+
+	_storageF := &storage.Setup{
+		Enable:    false,
+		Driver:    "",
+		Endpoint:  "",
+		AccessKey: "",
+		SecretKey: "",
+		Bucket:    "",
+		Region:    "",
+		Secure:    false,
+	}
+
+	_sF, err := storage.New(_storageF)
+	if err != nil {
+		t.Errorf("unable to create storage engine: %v", err)
+	}
+
 	_linux, err := linux.New(
 		linux.WithBuild(_build),
 		linux.WithHostname("localhost"),
@@ -45,6 +78,7 @@ func TestExecutor_New(t *testing.T) {
 		linux.WithRuntime(_runtime),
 		linux.WithVelaClient(_client),
 		linux.WithVersion("v1.0.0"),
+		linux.WithStorage(_sT),
 	)
 	if err != nil {
 		t.Errorf("unable to create linux engine: %v", err)
@@ -57,6 +91,7 @@ func TestExecutor_New(t *testing.T) {
 		local.WithRuntime(_runtime),
 		local.WithVelaClient(_client),
 		local.WithVersion("v1.0.0"),
+		local.WithStorage(_sT),
 	)
 	if err != nil {
 		t.Errorf("unable to create local engine: %v", err)
@@ -80,6 +115,7 @@ func TestExecutor_New(t *testing.T) {
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sF,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,
@@ -95,6 +131,7 @@ func TestExecutor_New(t *testing.T) {
 				Pipeline:   _pipeline,
 				Runtime:    _runtime,
 				Version:    "v1.0.0",
+				Storage:    _sT,
 			},
 			want:  _linux,
 			equal: linux.Equal,
@@ -109,6 +146,7 @@ func TestExecutor_New(t *testing.T) {
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  _local,
 			equal: local.Equal,
@@ -123,6 +161,7 @@ func TestExecutor_New(t *testing.T) {
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,
@@ -137,6 +176,7 @@ func TestExecutor_New(t *testing.T) {
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,
@@ -151,6 +191,7 @@ func TestExecutor_New(t *testing.T) {
 				Pipeline: _pipeline,
 				Runtime:  _runtime,
 				Version:  "v1.0.0",
+				Storage:  _sT,
 			},
 			want:  nil,
 			equal: reflect.DeepEqual,