Instantiate runners directly on test commands (#1888)

Trigger each test runner from the command with its own
constructor and options. Cleaning up all references to runner
registry in testrunner package.

Author: mrodm

cmd/testrunner.go

@@ -25,7 +25,11 @@ import (
 	"github.com/elastic/elastic-package/internal/testrunner"
 	"github.com/elastic/elastic-package/internal/testrunner/reporters/formats"
 	"github.com/elastic/elastic-package/internal/testrunner/reporters/outputs"
-	_ "github.com/elastic/elastic-package/internal/testrunner/runners" // register all test runners
+	"github.com/elastic/elastic-package/internal/testrunner/runners/asset"
+	"github.com/elastic/elastic-package/internal/testrunner/runners/pipeline"
+	"github.com/elastic/elastic-package/internal/testrunner/runners/policy"
+	"github.com/elastic/elastic-package/internal/testrunner/runners/static"
+	"github.com/elastic/elastic-package/internal/testrunner/runners/system"
 )
 
 const testLongDescription = `Use this command to run tests on a package. Currently, the following types of tests are available:
@@ -164,16 +168,13 @@ func testRunnerAssetCommandAction(cmd *cobra.Command, args []string) error {
 
 	_, pkg := filepath.Split(packageRootPath)
 
-	results, err := testrunner.Run(ctx, testType, testrunner.TestOptions{
-		Profile:                    profile,
-		TestFolder:                 testrunner.TestFolder{Package: pkg},
-		PackageRootPath:            packageRootPath,
-		WithCoverage:               testCoverage,
-		CoverageType:               testCoverageFormat,
-		KibanaClient:               kibanaClient,
-		RunIndependentElasticAgent: false,
+	runner := asset.NewAssetRunner(asset.AssetRunnerOptions{
+		TestFolder:      testrunner.TestFolder{Package: pkg},
+		PackageRootPath: packageRootPath,
+		KibanaClient:    kibanaClient,
 	})
 
+	results, err := testrunner.Run(ctx, runner)
 	if err != nil {
 		return fmt.Errorf("error running package %s tests: %w", testType, err)
 	}
@@ -200,11 +201,6 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error {
 	cmd.Printf("Run static tests for the package\n")
 	testType := testrunner.TestType("static")
 
-	profile, err := cobraext.GetProfileFlag(cmd)
-	if err != nil {
-		return err
-	}
-
 	failOnMissing, err := cmd.Flags().GetBool(cobraext.FailOnMissingFlagName)
 	if err != nil {
 		return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName)
@@ -295,20 +291,15 @@ func testRunnerStaticCommandAction(cmd *cobra.Command, args []string) error {
 
 	var results []testrunner.TestResult
 	for _, folder := range testFolders {
-		r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{
-			Profile:                    profile,
-			TestFolder:                 folder,
-			PackageRootPath:            packageRootPath,
-			RunIndependentElasticAgent: false,
+		runner := static.NewStaticRunner(static.StaticRunnerOptions{
+			TestFolder:      folder,
+			PackageRootPath: packageRootPath,
 		})
-
-		// Results must be appended even if there is an error, since there could be
-		// tests (e.g. system tests) that return both error and results.
-		results = append(results, r...)
-
+		r, err := testrunner.Run(ctx, runner)
 		if err != nil {
 			return fmt.Errorf("error running package %s tests: %w", testType, err)
 		}
+		results = append(results, r...)
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
@@ -449,25 +440,25 @@ func testRunnerPipelineCommandAction(cmd *cobra.Command, args []string) error {
 
 	var results []testrunner.TestResult
 	for _, folder := range testFolders {
-		r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{
-			Profile:                    profile,
-			TestFolder:                 folder,
-			PackageRootPath:            packageRootPath,
-			GenerateTestResult:         generateTestResult,
-			API:                        esClient.API,
-			WithCoverage:               testCoverage,
-			CoverageType:               testCoverageFormat,
-			DeferCleanup:               deferCleanup,
-			RunIndependentElasticAgent: false,
+		runner, err := pipeline.NewPipelineRunner(pipeline.PipelineRunnerOptions{
+			Profile:            profile,
+			TestFolder:         folder,
+			PackageRootPath:    packageRootPath,
+			GenerateTestResult: generateTestResult,
+			API:                esClient.API,
+			WithCoverage:       testCoverage,
+			CoverageType:       testCoverageFormat,
+			DeferCleanup:       deferCleanup,
 		})
+		if err != nil {
+			return fmt.Errorf("failed to create pipeline runner: %w", err)
+		}
 
-		// Results must be appended even if there is an error, since there could be
-		// tests (e.g. system tests) that return both error and results.
-		results = append(results, r...)
-
+		r, err := testrunner.Run(ctx, runner)
 		if err != nil {
 			return fmt.Errorf("error running package %s tests: %w", testType, err)
 		}
+		results = append(results, r...)
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
@@ -691,7 +682,7 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 
 	var results []testrunner.TestResult
 	for _, folder := range testFolders {
-		r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{
+		runner := system.NewSystemRunner(system.SystemRunnerOptions{
 			Profile:                    profile,
 			TestFolder:                 folder,
 			PackageRootPath:            packageRootPath,
@@ -707,13 +698,11 @@ func testRunnerSystemCommandAction(cmd *cobra.Command, args []string) error {
 			RunIndependentElasticAgent: false,
 		})
 
-		// Results must be appended even if there is an error, since there could be
-		// tests (e.g. system tests) that return both error and results.
-		results = append(results, r...)
-
+		r, err := testrunner.Run(ctx, runner)
 		if err != nil {
 			return fmt.Errorf("error running package %s tests: %w", testType, err)
 		}
+		results = append(results, r...)
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)
@@ -845,22 +834,17 @@ func testRunnerPolicyCommandAction(cmd *cobra.Command, args []string) error {
 
 	var results []testrunner.TestResult
 	for _, folder := range testFolders {
-		r, err := testrunner.Run(ctx, testType, testrunner.TestOptions{
-			Profile:                    profile,
-			TestFolder:                 folder,
-			PackageRootPath:            packageRootPath,
-			GenerateTestResult:         generateTestResult,
-			KibanaClient:               kibanaClient,
-			RunIndependentElasticAgent: false,
+		runner := policy.NewPolicyRunner(policy.PolicyRunnerOptions{
+			TestFolder:         folder,
+			PackageRootPath:    packageRootPath,
+			GenerateTestResult: generateTestResult,
+			KibanaClient:       kibanaClient,
 		})
-
-		// Results must be appended even if there is an error, since there could be
-		// tests (e.g. system tests) that return both error and results.
-		results = append(results, r...)
-
+		r, err := testrunner.Run(ctx, runner)
 		if err != nil {
 			return fmt.Errorf("error running package %s tests: %w", testType, err)
 		}
+		results = append(results, r...)
 	}
 
 	return processResults(results, testType, reportFormat, reportOutput, packageRootPath, manifest.Name, manifest.Type, testCoverageFormat, testCoverage)

internal/testrunner/runners/asset/runner.go

@@ -17,10 +17,6 @@ import (
 	"github.com/elastic/elastic-package/internal/testrunner"
 )
 
-func init() {
-	testrunner.RegisterRunner(&runner{})
-}
-
 const (
 	// TestType defining asset loading tests
 	TestType testrunner.TestType = "asset"
@@ -33,6 +29,26 @@ type runner struct {
 	resourcesManager *resources.Manager
 }
 
+type AssetRunnerOptions struct {
+	TestFolder      testrunner.TestFolder
+	PackageRootPath string
+	KibanaClient    *kibana.Client
+}
+
+func NewAssetRunner(options AssetRunnerOptions) *runner {
+	runner := runner{
+		testFolder:      options.TestFolder,
+		packageRootPath: options.PackageRootPath,
+		kibanaClient:    options.KibanaClient,
+	}
+
+	manager := resources.NewManager()
+	manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: options.KibanaClient})
+	runner.resourcesManager = manager
+
+	return &runner
+}
+
 // Ensures that runner implements testrunner.TestRunner interface
 var _ testrunner.TestRunner = new(runner)
 
@@ -47,15 +63,7 @@ func (r runner) String() string {
 }
 
 // Run runs the asset loading tests
-func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]testrunner.TestResult, error) {
-	r.testFolder = options.TestFolder
-	r.packageRootPath = options.PackageRootPath
-	r.kibanaClient = options.KibanaClient
-
-	manager := resources.NewManager()
-	manager.RegisterProvider(resources.DefaultKibanaProviderName, &resources.KibanaProvider{Client: r.kibanaClient})
-	r.resourcesManager = manager
-
+func (r *runner) Run(ctx context.Context) ([]testrunner.TestResult, error) {
 	return r.run(ctx)
 }
 

internal/testrunner/runners/pipeline/coverage.go

@@ -17,8 +17,8 @@ import (
 	"github.com/elastic/elastic-package/internal/testrunner"
 )
 
-// GetPipelineCoverage returns a coverage report for the provided set of ingest pipelines.
-func GetPipelineCoverage(options testrunner.TestOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) {
+// getPipelineCoverage returns a coverage report for the provided set of ingest pipelines.
+func getPipelineCoverage(options PipelineRunnerOptions, pipelines []ingest.Pipeline) (testrunner.CoverageReport, error) {
 	dataStreamPath, found, err := packages.FindDataStreamRootForPath(options.TestFolder.Path)
 	if err != nil {
 		return nil, fmt.Errorf("locating data_stream root failed: %w", err)