feat(test reports): test_report key functionality

cmd/vela-worker/exec.go

@@ -25,8 +25,8 @@
 // exec is a helper function to poll the queue
 // and execute Vela pipelines for the Worker.
 //
 //nolint:nilerr,funlen // ignore returning nil - don't want to crash worker
 func (w *Worker) exec(index int, config *api.Worker) error {
 	var err error
 
 	// setup the version
@@ -150,6 +150,9 @@
 	execOutputCtn := *w.Config.Executor.OutputCtn
 	execOutputCtn.ID = fmt.Sprintf("outputs_%s", p.ID)
 
+	// dereference configured storage config and set the storage config for the executor
+	execStorage := *w.Config.Executor.Storage
+
 	// create logger with extra metadata
 	//
 	// https://pkg.go.dev/github.com/sirupsen/logrus#WithFields
@@ -242,6 +245,7 @@
 		Pipeline:            p.Sanitize(w.Config.Runtime.Driver),
 		Version:             v.Semantic(),
 		OutputCtn:           &execOutputCtn,
+		Storage:             &execStorage,
 	})
 
 	// add the executor to the worker

cmd/vela-worker/flags.go

@@ -5,6 +5,7 @@
 import (
 	"context"
 	"fmt"
+	"github.com/go-vela/server/storage"
 	"strings"
 	"time"
 
@@ -120,5 +121,8 @@
 
 	f = append(f, runtime.Flags...)
 
+	// Storage Flags
+	f = append(f, storage.Flags...)
+
 	return f
 }

cmd/vela-worker/operate.go

@@ -31,7 +31,7 @@
 	registryWorker.SetHostname(w.Config.API.Address.Hostname())
 	registryWorker.SetAddress(w.Config.API.Address.String())
 	registryWorker.SetActive(true)
 	registryWorker.SetBuildLimit(int32(w.Config.Build.Limit))
 
 	// set routes from config if set or defaulted to `vela`
 	if (len(w.Config.Queue.Routes) > 0) && (w.Config.Queue.Routes[0] != "NONE" && w.Config.Queue.Routes[0] != "") {
@@ -76,7 +76,31 @@
 		// set to error as queue setup fails
 		w.updateWorkerStatus(registryWorker, constants.WorkerStatusError)
 	}
+	// getting storage creds
+	logrus.Trace("getting storage s3 creds..")
+	// fetching queue credentials using registration token
+	stCreds, _, err := w.VelaClient.Storage.GetInfo()
+	if err != nil {
+		logrus.Trace("error getting storage creds")
+		return err
+	}
+
+	// if an address was given at start up, use that — else use what is returned from server
+	if len(w.Config.Executor.Storage.Endpoint) == 0 {
+		w.Config.Executor.Storage.Endpoint = stCreds.GetStorageAddress()
+		logrus.Trace("storage address: ", w.Config.Executor.Storage.Driver)
+	}
+
+	// set access key in storage config
+	w.Config.Executor.Storage.AccessKey = stCreds.GetAccessKey()
+	logrus.Trace("access key: ", w.Config.Executor.Storage.AccessKey)
+	// set secret key in storage config
+	w.Config.Executor.Storage.SecretKey = stCreds.GetSecretKey()
+	logrus.Trace("secret key: ", w.Config.Executor.Storage.SecretKey)
 
+	// set bucket name in storage config
+	w.Config.Executor.Storage.Bucket = stCreds.GetStorageBucket()
+	logrus.Trace("bucket name: ", w.Config.Executor.Storage.Bucket)
 	// spawn goroutine for phoning home
 	executors.Go(func() error {
 		// five second ticker for signal handling

cmd/vela-worker/run.go

@@ -5,6 +5,7 @@
 import (
 	"context"
 	"fmt"
+	"github.com/go-vela/server/storage"
 	"net/url"
 
 	"github.com/gin-gonic/gin"
@@ -96,7 +97,7 @@
 			},
 			// build configuration
 			Build: &Build{
 				Limit:   int(c.Int("build.limit")),
 				Timeout: c.Duration("build.timeout"),
 			},
 			// build configuration
@@ -104,10 +105,17 @@
 			// executor configuration
 			Executor: &executor.Setup{
 				Driver:              c.String("executor.driver"),
 				MaxLogSize:          uint(c.Uint("executor.max_log_size")),
 				LogStreamingTimeout: c.Duration("executor.log_streaming_timeout"),
 				EnforceTrustedRepos: c.Bool("executor.enforce-trusted-repos"),
 				OutputCtn:           outputsCtn,
+				Storage: &storage.Setup{
+					Driver:    c.String("storage.driver"),
+					Endpoint:  c.String("storage.endpoint.name"),
+					AccessKey: c.String("storage.access.key"),
+					SecretKey: c.String("storage.secret.key"),
+					Bucket:    c.String("storage.bucket.name"),
+				},
 			},
 			// logger configuration
 			Logger: &Logger{
@@ -138,6 +146,13 @@
 				Address: c.String("server.addr"),
 				Secret:  c.String("server.secret"),
 			},
+			Storage: &storage.Setup{
+				Driver:    c.String("storage.driver"),
+				Endpoint:  c.String("storage.endpoint.name"),
+				AccessKey: c.String("storage.access.key"),
+				SecretKey: c.String("storage.secret.key"),
+				Bucket:    c.String("storage.bucket.name"),
+			},
 			// Certificate configuration
 			Certificate: &Certificate{
 				Cert: c.String("server.cert"),
@@ -172,5 +187,5 @@
 	}
 
 	// start the worker
 	return w.Start()
 }

cmd/vela-worker/worker.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"github.com/go-vela/server/storage"
 	"net/url"
 	"sync"
 	"time"
@@ -57,6 +58,7 @@
 		Server        *Server
 		Certificate   *Certificate
 		TLSMinVersion string
+		Storage       *storage.Setup
 	}
 
 	// Worker represents all configuration and
@@ -72,5 +74,6 @@
 		RunningBuilds      []*api.Build
 		QueueCheckedIn     bool
 		RunningBuildsMutex sync.Mutex
+		Storage            storage.Storage
 	}
 )

executor/linux/build.go

@@ -29,6 +29,11 @@
 	//
 	// https://pkg.go.dev/github.com/go-vela/worker/internal/build#Snapshot
 	defer func() { build.Snapshot(c.build, c.Vela, c.err, c.Logger) }()
+	// Check if storage client is initialized
+	// and if storage is enable
+	if c.Storage == nil {
+		return fmt.Errorf("storage client is not initialized")
+	}
 
 	// update the build fields
 	c.build.SetStatus(constants.StatusRunning)
@@ -290,7 +295,7 @@
 	for _, s := range c.pipeline.Stages {
 		// TODO: remove hardcoded reference
 		//
 		//nolint:goconst // ignore making a constant for now
 		if s.Name == "init" {
 			continue
 		}
@@ -445,6 +450,12 @@
 	// output maps for dynamic environment variables captured from volume
 	var opEnv, maskEnv map[string]string
 
+	// test report object for storing the test report information
+	var tr *api.TestReport
+
+	// Flag to track if we've already created the test report record
+	testReportCreated := false
+
 	// fire up output container to run with the build
 	c.Logger.Infof("creating outputs container %s", c.OutputCtn.ID)
 
@@ -543,6 +554,38 @@
 			_step.Secrets = append(_step.Secrets, sec)
 		}
 
+		// logic for polling files only if the test-report step is present
+		// iterate through the steps in the build
+
+		if !_step.TestReport.Empty() {
+			c.Logger.Debug("creating test report record in database")
+			// send API call to update the test report
+			//
+			// https://pkg.go.dev/github.com/go-vela/sdk-go/vela#TestReportService.Add
+			// TODO: .Add should be .Update
+			// TODO: handle somewhere if multiple test report keys exist in pipeline
+			if !testReportCreated {
+				tr, c.err = c.CreateTestReport()
+				if c.err != nil {
+					return fmt.Errorf("unable to create test report: %w", c.err)
+				}
+
+				testReportCreated = true
+			}
+			if len(_step.TestReport.Results) != 0 {
+				err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Results, c.build, tr)
+				if err != nil {
+					c.Logger.Errorf("unable to poll files for results: %v", err)
+				}
+			}
+			if len(_step.TestReport.Attachments) != 0 {
+				err := c.outputs.pollFiles(ctx, c.OutputCtn, _step.TestReport.Attachments, c.build, tr)
+				if err != nil {
+					c.Logger.Errorf("unable to poll files for attachments: %v", err)
+				}
+			}
+		}
+
 		// perform any substitution on dynamic variables
 		err = _step.Substitute()
 		if err != nil {
@@ -691,7 +734,7 @@
 // into the container right before execution, rather than
 // during build planning. It is only available for the Docker runtime.
 //
 //nolint:funlen // explanation takes up a lot of lines
 func loadLazySecrets(c *client, _step *pipeline.Container) error {
 	_log := new(api.Log)
 

executor/linux/linux.go

@@ -8,13 +8,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/go-vela/sdk-go/vela"
 	api "github.com/go-vela/server/api/types"
 	"github.com/go-vela/server/compiler/types/pipeline"
+	"github.com/go-vela/server/storage"
 	"github.com/go-vela/worker/internal/message"
 	"github.com/go-vela/worker/runtime"
+	"github.com/sirupsen/logrus"
 )
 
 type (
@@ -29,6 +29,7 @@ type (
 		Hostname     string
 		Version      string
 		OutputCtn    *pipeline.Container
+		Storage      storage.Storage
 
 		// clients for build actions
 		secret  *secretSvc

executor/linux/opts.go

@@ -4,6 +4,7 @@
 
 import (
 	"fmt"
+	"github.com/go-vela/server/storage"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -209,3 +210,28 @@
 		return nil
 	}
 }
+
+// WithStorage sets the storage in the executor client for Linux.
+func WithStorage(s *storage.Setup) Opt {
+	return func(c *client) error {
+		c.Logger.Trace("configuring storage in linux executor client")
+
+		// check if the storage provided is empty
+		if &s == nil {
+			return fmt.Errorf("empty storage setup provided")
+		}
+
+		// set the storage in the client
+		var err error
+		c.Storage, err = storage.New(s)
+		if err != nil {
+			return fmt.Errorf("unable to create storage: %v", err)
+
+		}
+		if c.Storage == nil {
+			return fmt.Errorf("empty storage setup: %v", err)
+		}
+
+		return nil
+	}
+}

executor/linux/outputs.go

@@ -8,11 +8,13 @@
 	"encoding/base64"
 	"fmt"
 	"maps"
+	"path/filepath"
+	"strconv"
 
+	api "github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/compiler/types/pipeline"
 	envparse "github.com/hashicorp/go-envparse"
 	"github.com/sirupsen/logrus"
-
-	"github.com/go-vela/server/compiler/types/pipeline"
 )
 
 // outputSvc handles communication with the outputs container during the build.
@@ -204,3 +206,66 @@
 
 	return outputMap, maskMap, nil
 }
+
+// pollFiles polls the output for files from the sidecar container.
+func (o *outputSvc) pollFiles(ctx context.Context, ctn *pipeline.Container, fileList []string, b *api.Build, tr *api.TestReport) error {
+	// exit if outputs container has not been configured
+	if len(ctn.Image) == 0 {
+		return fmt.Errorf("no outputs container configured")
+	}
+
+	// update engine logger with outputs metadata
+	//
+	// https://pkg.go.dev/github.com/sirupsen/logrus#Entry.WithField
+	logger := o.client.Logger.WithField("test-outputs", ctn.Name)
+
+	// grab file paths from the container
+	filesPath, err := o.client.Runtime.PollFileNames(ctx, ctn, fileList)
+	if err != nil {
+		return fmt.Errorf("unable to poll file names: %w", err)
+	}
+
+	if len(filesPath) == 0 {
+		return fmt.Errorf("no files found for file list: %v", fileList)
+	}
+
+	// process each file found
+	for _, filePath := range filesPath {
+		fileName := filepath.Base(filePath)
+		logger.Debugf("processing file: %s (path: %s)", fileName, filePath)
+
+		// get file content from container
+		reader, size, err := o.client.Runtime.PollFileContent(ctx, ctn, filePath)
+		if err != nil {
+			return fmt.Errorf("unable to poll file content for %s: %w", filePath, err)
+		}
+
+		// create storage object path
+		objectName := fmt.Sprintf("%s/%s/%s/%s",
+			b.GetRepo().GetOrg(),
+			b.GetRepo().GetName(),
+			strconv.FormatInt(b.GetNumber(), 10),
+			fileName)
+
+		// upload file to storage
+		err = o.client.Storage.UploadObject(ctx, &api.Object{
+			ObjectName: objectName,
+			Bucket:     api.Bucket{BucketName: o.client.Storage.GetBucket(ctx)},
+			FilePath:   filePath,
+		}, reader, size)
+		if err != nil {
+			return fmt.Errorf("unable to upload object %s: %w", fileName, err)
+		}
+
+		logger.Debugf("successfully uploaded file %s (%d bytes)", fileName, size)
+
+		// create test attachment record in database after successful upload
+		err = o.client.CreateTestAttachment(fileName, filePath, size, tr)
+		if err != nil {
+			logger.Errorf("unable to create test attachment record for %s: %v", fileName, err)
+			// don't return error here to avoid blocking the upload process
+		}
+	}
+
+	return nil
+}

executor/linux/report.go

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package linux
+
+import (
+	"fmt"
+
+	api "github.com/go-vela/server/api/types"
+)
+
+// CreateTestReport creates a test report record in the database for the current build.
+func (c *client) CreateTestReport() (*api.TestReport, error) {
+	// create empty test report for the build
+	testReport := &api.TestReport{}
+
+	// update test report in database
+	tr, resp, err := c.Vela.TestReport.Update(
+		c.build.GetRepo().GetOrg(),
+		c.build.GetRepo().GetName(),
+		c.build.GetNumber(),
+		testReport,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create test report record: build=%d, status=%d, error=%w",
+			c.build.GetNumber(), resp.StatusCode, err)
+	}
+
+	c.Logger.Debugf("created test report record: id=%d, build=%d", tr.GetID(), c.build.GetNumber())
+
+	return tr, nil
+}