From da987333bff889d21e4178b639b24d14ec320d5a Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 23 May 2024 18:35:01 +0200
Subject: [PATCH 01/30] Create policy for no-provision - missing to re-assign
 policy

---
 internal/testrunner/runners/system/runner.go | 30 ++++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index acd1236d1..f3ad1eaca 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -971,13 +971,33 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		}
 	}
 
+	testTime := time.Now().Format("20060102T15:04:05Z")
+
+	policyTesting := kibana.Policy{
+		Name:        fmt.Sprintf("ep-one-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime),
+		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
+		Namespace:   createTestRunID(),
+	}
+	var policyForTesting *kibana.Policy
+	if r.options.RunTestsOnly {
+		policyForTesting, err = r.options.KibanaClient.CreatePolicy(ctx, policyTesting)
+		if err != nil {
+			return nil, fmt.Errorf("could not create test policy: %w", err)
+		}
+		policyToTest = policyForTesting
+	}
+
 	// store the time just before adding the Test Policy, this time will be used to check
 	// the agent logs from that time onwards to avoid possible previous errors present in logs
 	scenario.startTestTime = time.Now()
 
 	logger.Debug("adding package data stream to test policy...")
-	ds := createPackageDatastream(*policyToTest, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, svcInfo.Test.RunID)
-	if r.options.RunTearDown || r.options.RunTestsOnly {
+	suffixDatastream := svcInfo.Test.RunID
+	if r.options.RunTestsOnly {
+		suffixDatastream = policyTesting.Namespace
+	}
+	ds := createPackageDatastream(*policyToTest, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, suffixDatastream)
+	if r.options.RunTearDown {
 		logger.Debug("Skip adding data stream config to policy")
 	} else {
 		if err := r.options.KibanaClient.AddPackageDataStreamToPolicy(ctx, ds); err != nil {
@@ -1100,7 +1120,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil
 	}
 
-	if r.options.RunTearDown || r.options.RunTestsOnly {
+	if r.options.RunTearDown {
 		logger.Debug("Skip assiging package data stream to agent")
 	} else {
 		policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policyToTest.ID)
@@ -1579,7 +1599,7 @@ func createIntegrationPackageDatastream(
 ) kibana.PackageDataStream {
 	r := kibana.PackageDataStream{
 		Name:      fmt.Sprintf("%s-%s-%s", pkg.Name, ds.Name, suffix),
-		Namespace: "ep",
+		Namespace: kibanaPolicy.Namespace,
 		PolicyID:  kibanaPolicy.ID,
 		Enabled:   true,
 		Inputs: []kibana.Input{
@@ -1633,7 +1653,7 @@ func createInputPackageDatastream(
 ) kibana.PackageDataStream {
 	r := kibana.PackageDataStream{
 		Name:      fmt.Sprintf("%s-%s-%s", pkg.Name, policyTemplate.Name, suffix),
-		Namespace: "ep",
+		Namespace: kibanaPolicy.Namespace,
 		PolicyID:  kibanaPolicy.ID,
 		Enabled:   true,
 	}

From dd9bf60c0542245400a77fc12370f9b79b8260f6 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 23 May 2024 18:54:50 +0200
Subject: [PATCH 02/30] Ensure test policy is re-assigned

---
 internal/testrunner/runners/system/runner.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index f3ad1eaca..dd97d0e9e 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -278,7 +278,14 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes
 		if err != nil {
 			return result.WithError(fmt.Errorf("failed to prepare scenario: %w", err))
 		}
-		return r.validateTestScenario(ctx, result, scenario, testConfig)
+		results, err := r.validateTestScenario(ctx, result, scenario, testConfig)
+		// run re-assign policy
+		tdErr := r.resetAgentPolicyHandler(ctx)
+		if tdErr != nil {
+			logger.Errorf("failed to reassign policy: %s", tdErr)
+		}
+		return results, err
+
 	}
 
 	if r.options.RunTearDown {
@@ -1076,7 +1083,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil
 	}
 
-	if r.options.RunTearDown || r.options.RunTestsOnly {
+	if r.options.RunTearDown {
 		origPolicy = serviceStateData.OrigPolicy
 		logger.Debugf("Got orig policy from file: %q - %q", origPolicy.Name, origPolicy.ID)
 	} else {
@@ -1088,7 +1095,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 	// Assign policy to agent
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
-		if r.options.RunIndependentElasticAgent {
+		if r.options.RunIndependentElasticAgent && !r.options.RunTestsOnly {
 			return nil
 		}
 		logger.Debug("reassigning original policy back to agent...")

From df6268a96c17bfb10c94f3d7cd5dfa16b87e97cf Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 23 May 2024 18:59:07 +0200
Subject: [PATCH 03/30] Delete agent policies created for each test

---
 internal/testrunner/runners/system/runner.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index dd97d0e9e..5bb6177bf 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1102,6 +1102,10 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
 			return fmt.Errorf("error reassigning original policy to agent: %w", err)
 		}
+		if r.options.RunTestsOnly {
+			// Clean up policies created
+			return r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID)
+		}
 		return nil
 	}
 

From ed1411e8e0bd9097360c40705bb1a3a0e4ba4018 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 23 May 2024 19:38:14 +0200
Subject: [PATCH 04/30] delete datastream created for testing

---
 internal/testrunner/runners/system/runner.go | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 5bb6177bf..a0dc0db51 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1102,9 +1102,21 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
 			return fmt.Errorf("error reassigning original policy to agent: %w", err)
 		}
-		if r.options.RunTestsOnly {
-			// Clean up policies created
-			return r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID)
+		if !r.options.RunTestsOnly {
+			return nil
+		}
+
+		logger.Debug("Deleting test policy...")
+		err = r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID)
+		if err != nil {
+			return fmt.Errorf("failed to delete policy %s: %w", policyToTest.Name, err)
+		}
+		logger.Debug("Deleting data stream for testing")
+		_, err := r.options.API.Indices.DeleteDataStream([]string{scenario.dataStream},
+			r.options.API.Indices.DeleteDataStream.WithContext(ctx),
+		)
+		if err != nil {
+			return fmt.Errorf("failed to delete data stream %s: %w", scenario.dataStream, err)
 		}
 		return nil
 	}

From 1c55b1cb607759bd6148d45896c1740b5ec2fbc4 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 24 May 2024 13:37:42 +0200
Subject: [PATCH 05/30] Apply changes to all scenarios

---
 internal/testrunner/runners/system/runner.go | 82 +++++++++++++-------
 1 file changed, 53 insertions(+), 29 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index a0dc0db51..d50461224 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -828,6 +828,37 @@ type scenarioTest struct {
 	startTestTime      time.Time
 }
 
+func (r *runner) shouldCreateNewAgentPolicyForTest() bool {
+	if r.options.RunTestsOnly {
+		// always that --no-provision is set, it should create new Agent Policies.
+		return true
+	}
+	if !r.options.RunIndependentElasticAgent {
+		// keep same behaviour as previously when Elastic Agent of the stack is used.
+		return false
+
+	}
+	// No need to create new Agent Policies for these stages
+	if r.options.RunSetup || r.options.RunTearDown {
+		return false
+	}
+	return true
+}
+
+func (r *runner) deleteDataStream(ctx context.Context, dataStream string) error {
+	resp, err := r.options.API.Indices.DeleteDataStream([]string{dataStream},
+		r.options.API.Indices.DeleteDataStream.WithContext(ctx),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to delete data stream %s: %w", dataStream, err)
+	}
+	defer resp.Body.Close()
+	if resp.IsError() {
+		return fmt.Errorf("could not get delete data stream %s: %s", dataStream, resp.String())
+	}
+	return nil
+}
+
 func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInfo servicedeployer.ServiceInfo) (*scenarioTest, error) {
 	serviceOptions := r.createServiceOptions(config.ServiceVariantName)
 
@@ -978,32 +1009,26 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		}
 	}
 
-	testTime := time.Now().Format("20060102T15:04:05Z")
+	// store the time just before adding the Test Policy, this time will be used to check
+	// the agent logs from that time onwards to avoid possible previous errors present in logs
+	scenario.startTestTime = time.Now()
+	suffixDatastream := svcInfo.Test.RunID
 
+	logger.Debug("adding package data stream to test policy...")
 	policyTesting := kibana.Policy{
-		Name:        fmt.Sprintf("ep-one-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime),
+		Name:        fmt.Sprintf("ep-one-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, scenario.startTestTime.Format("20060102T15:04:05Z")),
 		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
 		Namespace:   createTestRunID(),
 	}
-	var policyForTesting *kibana.Policy
-	if r.options.RunTestsOnly {
-		policyForTesting, err = r.options.KibanaClient.CreatePolicy(ctx, policyTesting)
+	policyToAssignDatastreamTests := policyToTest
+	if r.shouldCreateNewAgentPolicyForTest() {
+		policyToAssignDatastreamTests, err = r.options.KibanaClient.CreatePolicy(ctx, policyTesting)
 		if err != nil {
 			return nil, fmt.Errorf("could not create test policy: %w", err)
 		}
-		policyToTest = policyForTesting
-	}
-
-	// store the time just before adding the Test Policy, this time will be used to check
-	// the agent logs from that time onwards to avoid possible previous errors present in logs
-	scenario.startTestTime = time.Now()
-
-	logger.Debug("adding package data stream to test policy...")
-	suffixDatastream := svcInfo.Test.RunID
-	if r.options.RunTestsOnly {
 		suffixDatastream = policyTesting.Namespace
 	}
-	ds := createPackageDatastream(*policyToTest, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, suffixDatastream)
+	ds := createPackageDatastream(*policyToAssignDatastreamTests, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, suffixDatastream)
 	if r.options.RunTearDown {
 		logger.Debug("Skip adding data stream config to policy")
 	} else {
@@ -1095,26 +1120,25 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 	// Assign policy to agent
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
-		if r.options.RunIndependentElasticAgent && !r.options.RunTestsOnly {
-			return nil
-		}
-		logger.Debug("reassigning original policy back to agent...")
-		if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
-			return fmt.Errorf("error reassigning original policy to agent: %w", err)
+		if !r.options.RunSetup || !r.options.RunTearDown {
+			// it should be kept the same policy just when system tests are
+			// triggered with the flags for just setup or just tear-down
+			logger.Debug("reassigning original policy back to agent...")
+			if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
+				return fmt.Errorf("error reassigning original policy to agent: %w", err)
+			}
 		}
-		if !r.options.RunTestsOnly {
+		if !r.shouldCreateNewAgentPolicyForTest() {
 			return nil
 		}
 
 		logger.Debug("Deleting test policy...")
-		err = r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID)
+		err = r.options.KibanaClient.DeletePolicy(ctx, policyToAssignDatastreamTests.ID)
 		if err != nil {
-			return fmt.Errorf("failed to delete policy %s: %w", policyToTest.Name, err)
+			return fmt.Errorf("failed to delete policy %s: %w", policyToAssignDatastreamTests.Name, err)
 		}
 		logger.Debug("Deleting data stream for testing")
-		_, err := r.options.API.Indices.DeleteDataStream([]string{scenario.dataStream},
-			r.options.API.Indices.DeleteDataStream.WithContext(ctx),
-		)
+		r.deleteDataStream(ctx, scenario.dataStream)
 		if err != nil {
 			return fmt.Errorf("failed to delete data stream %s: %w", scenario.dataStream, err)
 		}
@@ -1146,7 +1170,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	if r.options.RunTearDown {
 		logger.Debug("Skip assiging package data stream to agent")
 	} else {
-		policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policyToTest.ID)
+		policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policyToAssignDatastreamTests.ID)
 		if err != nil {
 			return nil, fmt.Errorf("could not read the policy with data stream: %w", err)
 		}

From 2bca8d80a46158281e9743b57b2daa28e832d1f3 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 24 May 2024 13:40:38 +0200
Subject: [PATCH 06/30] Skip tests with terraform

---
 .buildkite/pipeline.trigger.integration.tests.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index dc1e4d2ad..4214370ea 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -96,6 +96,10 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
         label_suffix=" (independent agent)"
     fi
     package_name=$(basename "${package}")
+    if [[ "${package_name}" == "gcp" ||  "${package_name}" == "aws" || "${package_name}" == "aws_logs" ]] ; then
+        echoerr "Skip package temporarily ${packager_name}"
+        continue
+    fi
 
     if [[ "$independent_agent" == "false" && "$package_name" == "oracle" ]]; then
         echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required software)."

From 60c34da89fcf7233271129a0d9d35f5a5f784633 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Fri, 24 May 2024 16:37:27 +0200
Subject: [PATCH 07/30] Fix env. variable name

---
 .buildkite/pipeline.trigger.integration.tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 4214370ea..09c6dfe32 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -97,7 +97,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     fi
     package_name=$(basename "${package}")
     if [[ "${package_name}" == "gcp" ||  "${package_name}" == "aws" || "${package_name}" == "aws_logs" ]] ; then
-        echoerr "Skip package temporarily ${packager_name}"
+        echoerr "Skip package temporarily ${package_name}"
         continue
     fi
 

From 402b631e52f720585091134b99d8fa76262ca597 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 09:02:36 +0200
Subject: [PATCH 08/30] Change order tear down handlers

---
 internal/testrunner/runners/system/runner.go | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index d50461224..e31225151 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -428,6 +428,15 @@ func (r *runner) tearDownTest(ctx context.Context) error {
 	// Avoid cancellations during cleanup.
 	cleanupCtx := context.WithoutCancel(ctx)
 
+	// Run service shutdown first to ensure that resources created
+	// by terraform are deleted avoiding errors in other handlers
+	if r.shutdownServiceHandler != nil {
+		if err := r.shutdownServiceHandler(cleanupCtx); err != nil {
+			return err
+		}
+		r.shutdownServiceHandler = nil
+	}
+
 	if r.resetAgentPolicyHandler != nil {
 		if err := r.resetAgentPolicyHandler(cleanupCtx); err != nil {
 			return err
@@ -465,13 +474,6 @@ func (r *runner) tearDownTest(ctx context.Context) error {
 		return err
 	}
 
-	if r.shutdownServiceHandler != nil {
-		if err := r.shutdownServiceHandler(cleanupCtx); err != nil {
-			return err
-		}
-		r.shutdownServiceHandler = nil
-	}
-
 	if r.shutdownAgentHandler != nil {
 		if err := r.shutdownAgentHandler(cleanupCtx); err != nil {
 			return err

From e7ea9ff4a9fb591a0cc1a3026ec6871711dcbe48 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 09:07:04 +0200
Subject: [PATCH 09/30] Exit loop if getDocs returns zero documents

---
 internal/testrunner/runners/system/runner.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index e31225151..8df7c158e 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1447,6 +1447,8 @@ func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataSt
 	if err := deleteDataStreamDocs(ctx, r.options.API, dataStream); err != nil {
 		return fmt.Errorf("error deleting old data in data stream: %s: %w", dataStream, err)
 	}
+	// TODO: Review if this getDocs call can be deleted after creating
+	// a new Agent Policy in each test
 	startHits, err := r.getDocs(ctx, dataStream)
 	if err != nil {
 		return err
@@ -1464,7 +1466,7 @@ func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataSt
 		if mustBeZero {
 			return hits.size() == 0, nil
 		}
-		return startHits.size() > hits.size(), nil
+		return hits.size() == 0 || startHits.size() > hits.size(), nil
 	}, 1*time.Second, 2*time.Minute)
 	if err != nil || !cleared {
 		if err == nil {

From 8e95b4db5fbe87e5950a81d086fb63eb1e88d79c Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 11:04:21 +0200
Subject: [PATCH 10/30] Reorder handlers and change condition

---
 internal/testrunner/runners/system/runner.go | 22 ++++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 8df7c158e..1ebed5465 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -428,20 +428,24 @@ func (r *runner) tearDownTest(ctx context.Context) error {
 	// Avoid cancellations during cleanup.
 	cleanupCtx := context.WithoutCancel(ctx)
 
-	// Run service shutdown first to ensure that resources created
-	// by terraform are deleted avoiding errors in other handlers
-	if r.shutdownServiceHandler != nil {
-		if err := r.shutdownServiceHandler(cleanupCtx); err != nil {
+	// This handler should be run before shutting down Elastic Agents (agent deployer)
+	// or services that could run agents like Custom Agents (service deployer)
+	// or Kind deployer.
+	if r.resetAgentPolicyHandler != nil {
+		if err := r.resetAgentPolicyHandler(cleanupCtx); err != nil {
 			return err
 		}
-		r.shutdownServiceHandler = nil
+		r.resetAgentPolicyHandler = nil
 	}
 
-	if r.resetAgentPolicyHandler != nil {
-		if err := r.resetAgentPolicyHandler(cleanupCtx); err != nil {
+	// Shutting down the service should be run one of the first actions
+	// to ensure that resources created by terraform are deleted even if other
+	// errors fail.
+	if r.shutdownServiceHandler != nil {
+		if err := r.shutdownServiceHandler(cleanupCtx); err != nil {
 			return err
 		}
-		r.resetAgentPolicyHandler = nil
+		r.shutdownServiceHandler = nil
 	}
 
 	if r.resetAgentLogLevelHandler != nil {
@@ -1122,7 +1126,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 	// Assign policy to agent
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
-		if !r.options.RunSetup || !r.options.RunTearDown {
+		if !r.options.RunSetup {
 			// it should be kept the same policy just when system tests are
 			// triggered with the flags for just setup or just tear-down
 			logger.Debug("reassigning original policy back to agent...")

From a421e83ab248dff36d17f7c48670e35d2523d54e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 11:14:37 +0200
Subject: [PATCH 11/30] Add new tear down handler

---
 internal/testrunner/runners/system/runner.go | 42 +++++++++++++-------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 1ebed5465..1b58b513a 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -150,6 +150,7 @@ type runner struct {
 	// Execution order of following handlers is defined in runner.TearDown() method.
 	removeAgentHandler        func(context.Context) error
 	deleteTestPolicyHandler   func(context.Context) error
+	cleanTestScenarioHandler  func(context.Context) error
 	resetAgentPolicyHandler   func(context.Context) error
 	resetAgentLogLevelHandler func(context.Context) error
 	shutdownServiceHandler    func(context.Context) error
@@ -448,6 +449,13 @@ func (r *runner) tearDownTest(ctx context.Context) error {
 		r.shutdownServiceHandler = nil
 	}
 
+	if r.cleanTestScenarioHandler != nil {
+		if err := r.cleanTestScenarioHandler(cleanupCtx); err != nil {
+			return err
+		}
+		r.cleanTestScenarioHandler = nil
+	}
+
 	if r.resetAgentLogLevelHandler != nil {
 		if err := r.resetAgentLogLevelHandler(cleanupCtx); err != nil {
 			return err
@@ -1075,6 +1083,24 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil
 	}
 
+	r.cleanTestScenarioHandler = func(ctx context.Context) error {
+		if !r.shouldCreateNewAgentPolicyForTest() {
+			return nil
+		}
+
+		logger.Debug("Deleting test policy...")
+		err = r.options.KibanaClient.DeletePolicy(ctx, policyToAssignDatastreamTests.ID)
+		if err != nil {
+			return fmt.Errorf("failed to delete policy %s: %w", policyToAssignDatastreamTests.Name, err)
+		}
+		logger.Debug("Deleting data stream for testing")
+		r.deleteDataStream(ctx, scenario.dataStream)
+		if err != nil {
+			return fmt.Errorf("failed to delete data stream %s: %w", scenario.dataStream, err)
+		}
+		return nil
+	}
+
 	switch {
 	case r.options.RunTearDown:
 		logger.Debugf("Skipped deleting old data in data stream %q", scenario.dataStream)
@@ -1124,7 +1150,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 			Revision: agent.PolicyRevision,
 		}
 	}
-	// Assign policy to agent
+
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
 		if !r.options.RunSetup {
 			// it should be kept the same policy just when system tests are
@@ -1134,20 +1160,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 				return fmt.Errorf("error reassigning original policy to agent: %w", err)
 			}
 		}
-		if !r.shouldCreateNewAgentPolicyForTest() {
-			return nil
-		}
-
-		logger.Debug("Deleting test policy...")
-		err = r.options.KibanaClient.DeletePolicy(ctx, policyToAssignDatastreamTests.ID)
-		if err != nil {
-			return fmt.Errorf("failed to delete policy %s: %w", policyToAssignDatastreamTests.Name, err)
-		}
-		logger.Debug("Deleting data stream for testing")
-		r.deleteDataStream(ctx, scenario.dataStream)
-		if err != nil {
-			return fmt.Errorf("failed to delete data stream %s: %w", scenario.dataStream, err)
-		}
 		return nil
 	}
 

From 8c9c5d8b927aab8bb8d82af6c48329359f3050ef Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 11:57:18 +0200
Subject: [PATCH 12/30] Update comment

---
 internal/testrunner/runners/system/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 1b58b513a..ac0f2f6ab 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1154,7 +1154,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
 		if !r.options.RunSetup {
 			// it should be kept the same policy just when system tests are
-			// triggered with the flags for just setup or just tear-down
+			// triggered with the flags for running setup stage (--setup)
 			logger.Debug("reassigning original policy back to agent...")
 			if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
 				return fmt.Errorf("error reassigning original policy to agent: %w", err)

From c11361f7275284a7a96653aef5b3c35697eac80e Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 13:53:29 +0200
Subject: [PATCH 13/30] Add one more test package

---
 .buildkite/pipeline.trigger.integration.tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 09c6dfe32..1eaa2d7d2 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -96,7 +96,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
         label_suffix=" (independent agent)"
     fi
     package_name=$(basename "${package}")
-    if [[ "${package_name}" == "gcp" ||  "${package_name}" == "aws" || "${package_name}" == "aws_logs" ]] ; then
+    if [[ "${package_name}" == "aws" || "${package_name}" == "aws_logs" ]] ; then
         echoerr "Skip package temporarily ${package_name}"
         continue
     fi

From 7d715621decc06af8585822298bde558d1146e97 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 16:24:52 +0200
Subject: [PATCH 14/30] Remove exceptions in wait loop to delete docs

---
 internal/testrunner/runners/system/runner.go | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index ac0f2f6ab..5d590deba 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1463,16 +1463,6 @@ func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataSt
 	if err := deleteDataStreamDocs(ctx, r.options.API, dataStream); err != nil {
 		return fmt.Errorf("error deleting old data in data stream: %s: %w", dataStream, err)
 	}
-	// TODO: Review if this getDocs call can be deleted after creating
-	// a new Agent Policy in each test
-	startHits, err := r.getDocs(ctx, dataStream)
-	if err != nil {
-		return err
-	}
-	// First call already reports zero documents
-	if startHits.size() == 0 {
-		return nil
-	}
 	cleared, err := wait.UntilTrue(ctx, func(ctx context.Context) (bool, error) {
 		hits, err := r.getDocs(ctx, dataStream)
 		if err != nil {
@@ -1482,7 +1472,7 @@ func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataSt
 		if mustBeZero {
 			return hits.size() == 0, nil
 		}
-		return hits.size() == 0 || startHits.size() > hits.size(), nil
+		return hits.size() == 0, nil
 	}, 1*time.Second, 2*time.Minute)
 	if err != nil || !cleared {
 		if err == nil {

From e4e8751b330330232e75bb2d8e54555ce804297b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 16:28:30 +0200
Subject: [PATCH 15/30] Update some logger calls to use the right formats

---
 internal/agentdeployer/agent.go          | 4 ++--
 internal/servicedeployer/compose.go      | 4 ++--
 internal/servicedeployer/custom_agent.go | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go
index 246348255..ce1ac8534 100644
--- a/internal/agentdeployer/agent.go
+++ b/internal/agentdeployer/agent.go
@@ -391,12 +391,12 @@ func (s *dockerComposeDeployedAgent) TearDown(ctx context.Context) error {
 	defer func() {
 		// Remove the service logs dir for this agent
 		if err := os.RemoveAll(s.agentInfo.Logs.Folder.Local); err != nil {
-			logger.Errorf("could not remove the agent logs (path: %s): %w", s.agentInfo.Logs.Folder.Local, err)
+			logger.Errorf("could not remove the agent logs (path: %s): %v", s.agentInfo.Logs.Folder.Local, err)
 		}
 
 		// Remove the configuration dir for this agent (e.g. compose scenario files)
 		if err := os.RemoveAll(s.agentInfo.ConfigDir); err != nil {
-			logger.Errorf("could not remove the agent configuration directory (path: %s) %w", s.agentInfo.ConfigDir, err)
+			logger.Errorf("could not remove the agent configuration directory (path: %s) %v", s.agentInfo.ConfigDir, err)
 		}
 	}()
 
diff --git a/internal/servicedeployer/compose.go b/internal/servicedeployer/compose.go
index 9dc274b36..989e15d81 100644
--- a/internal/servicedeployer/compose.go
+++ b/internal/servicedeployer/compose.go
@@ -98,7 +98,7 @@ func (d *DockerComposeServiceDeployer) SetUp(ctx context.Context, svcInfo Servic
 		// service logs folder must no be deleted to avoid breaking log files written
 		// by the service. If this is required, those files should be rotated or truncated
 		// so the service can still write to them.
-		logger.Debug("Skipping removing service logs folder folder %s", svcInfo.Logs.Folder.Local)
+		logger.Debugf("Skipping removing service logs folder folder %s", svcInfo.Logs.Folder.Local)
 	} else {
 		err = files.RemoveContent(svcInfo.Logs.Folder.Local)
 		if err != nil {
@@ -242,7 +242,7 @@ func (s *dockerComposeDeployedService) TearDown(ctx context.Context) error {
 		}
 		// Remove the outputs generated by the service container
 		if err = os.RemoveAll(s.svcInfo.OutputDir); err != nil {
-			logger.Errorf("could not remove the temporary output files %w", err)
+			logger.Errorf("could not remove the temporary output files %s", err)
 		}
 	}()
 
diff --git a/internal/servicedeployer/custom_agent.go b/internal/servicedeployer/custom_agent.go
index 36cc111e8..2c8e01ab8 100644
--- a/internal/servicedeployer/custom_agent.go
+++ b/internal/servicedeployer/custom_agent.go
@@ -129,7 +129,7 @@ func (d *CustomAgentDeployer) SetUp(ctx context.Context, svcInfo ServiceInfo) (D
 		// service logs folder must no be deleted to avoid breaking log files written
 		// by the service. If this is required, those files should be rotated or truncated
 		// so the service can still write to them.
-		logger.Debug("Skipping removing service logs folder folder %s", svcInfo.Logs.Folder.Local)
+		logger.Debugf("Skipping removing service logs folder folder %s", svcInfo.Logs.Folder.Local)
 	} else {
 		err = files.RemoveContent(svcInfo.Logs.Folder.Local)
 		if err != nil {

From f43dc7778acb73e6c8ef938237af8c825ab73891 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 17:56:25 +0200
Subject: [PATCH 16/30] Restore packages skipped

---
 .buildkite/pipeline.trigger.integration.tests.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 1eaa2d7d2..3e99ddf42 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -96,11 +96,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
         label_suffix=" (independent agent)"
     fi
     package_name=$(basename "${package}")
-    if [[ "${package_name}" == "aws" || "${package_name}" == "aws_logs" ]] ; then
-        echoerr "Skip package temporarily ${package_name}"
-        continue
-    fi
-
     if [[ "$independent_agent" == "false" && "$package_name" == "oracle" ]]; then
         echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required software)."
         continue

From df1835c62ec25711b36387d098cd97885150b7df Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 21:11:14 +0200
Subject: [PATCH 17/30] Remove empty line

---
 internal/testrunner/runners/system/runner.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 5d590deba..f337b7a44 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -850,7 +850,6 @@ func (r *runner) shouldCreateNewAgentPolicyForTest() bool {
 	if !r.options.RunIndependentElasticAgent {
 		// keep same behaviour as previously when Elastic Agent of the stack is used.
 		return false
-
 	}
 	// No need to create new Agent Policies for these stages
 	if r.options.RunSetup || r.options.RunTearDown {

From d9e800089eb641cc12a254d104bc2a3459e399b2 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 21:16:47 +0200
Subject: [PATCH 18/30] Remove mustBeZero parameter from delete docs function

---
 internal/testrunner/runners/system/runner.go | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index f337b7a44..0034a640b 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1100,18 +1100,10 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil
 	}
 
-	switch {
-	case r.options.RunTearDown:
+	if r.options.RunTearDown {
 		logger.Debugf("Skipped deleting old data in data stream %q", scenario.dataStream)
-	case r.options.RunTestsOnly:
-		// In this mode, service is still running and the agent is sending documents, so sometimes
-		// cannot be guaranteed to be zero documents
-		err := r.deleteOldDocumentsDataStreamAndWait(ctx, scenario.dataStream, false)
-		if err != nil {
-			return nil, err
-		}
-	default:
-		err := r.deleteOldDocumentsDataStreamAndWait(ctx, scenario.dataStream, true)
+	} else {
+		err := r.deleteOldDocumentsDataStreamAndWait(ctx, scenario.dataStream)
 		if err != nil {
 			return nil, err
 		}
@@ -1457,7 +1449,7 @@ func (r *runner) writeScenarioState(opts scenarioStateOpts) error {
 	return nil
 }
 
-func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataStream string, mustBeZero bool) error {
+func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataStream string) error {
 	logger.Debugf("Delete previous documents in data stream %q", dataStream)
 	if err := deleteDataStreamDocs(ctx, r.options.API, dataStream); err != nil {
 		return fmt.Errorf("error deleting old data in data stream: %s: %w", dataStream, err)
@@ -1468,9 +1460,6 @@ func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataSt
 			return false, err
 		}
 
-		if mustBeZero {
-			return hits.size() == 0, nil
-		}
 		return hits.size() == 0, nil
 	}, 1*time.Second, 2*time.Minute)
 	if err != nil || !cleared {

From 7769e3cb119ae2984d5db912da4a8a4074da4254 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 21:23:27 +0200
Subject: [PATCH 19/30] Add handler to clean test scenario - test only stage

---
 internal/testrunner/runners/system/runner.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 0034a640b..33832d8c4 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -280,11 +280,14 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes
 			return result.WithError(fmt.Errorf("failed to prepare scenario: %w", err))
 		}
 		results, err := r.validateTestScenario(ctx, result, scenario, testConfig)
-		// run re-assign policy
 		tdErr := r.resetAgentPolicyHandler(ctx)
 		if tdErr != nil {
 			logger.Errorf("failed to reassign policy: %s", tdErr)
 		}
+		tdErr = r.cleanTestScenarioHandler(ctx)
+		if tdErr != nil {
+			logger.Errorf("failed to clean test scenario: %s", tdErr)
+		}
 		return results, err
 
 	}

From 91676d766424c8445694e124c00e1cd4cd8b5f68 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 28 May 2024 21:31:55 +0200
Subject: [PATCH 20/30] Skip handlers that should not be executed with
 --no-provision

---
 internal/testrunner/runners/system/runner.go | 26 +++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 33832d8c4..129058cfb 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -280,13 +280,9 @@ func (r *runner) Run(ctx context.Context, options testrunner.TestOptions) ([]tes
 			return result.WithError(fmt.Errorf("failed to prepare scenario: %w", err))
 		}
 		results, err := r.validateTestScenario(ctx, result, scenario, testConfig)
-		tdErr := r.resetAgentPolicyHandler(ctx)
-		if tdErr != nil {
-			logger.Errorf("failed to reassign policy: %s", tdErr)
-		}
-		tdErr = r.cleanTestScenarioHandler(ctx)
+		tdErr := r.tearDownTest(ctx)
 		if tdErr != nil {
-			logger.Errorf("failed to clean test scenario: %s", tdErr)
+			logger.Errorf("failed to tear down runner: %s", tdErr.Error())
 		}
 		return results, err
 
@@ -971,6 +967,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		}
 	}
 	r.deleteTestPolicyHandler = func(ctx context.Context) error {
+		if r.options.RunTestsOnly {
+			return nil
+		}
 		logger.Debug("deleting test policies...")
 		if err := r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID); err != nil {
 			return fmt.Errorf("error cleaning up test policy: %w", err)
@@ -1078,6 +1077,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	)
 
 	r.wipeDataStreamHandler = func(ctx context.Context) error {
+		if r.options.RunTestsOnly {
+			return nil
+		}
 		logger.Debugf("deleting data in data stream...")
 		if err := deleteDataStreamDocs(ctx, r.options.API, scenario.dataStream); err != nil {
 			return fmt.Errorf("error deleting data in data stream: %w", err)
@@ -1122,6 +1124,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	logger.Debugf("Selected enrolled agent %q", agent.ID)
 
 	r.removeAgentHandler = func(ctx context.Context) error {
+		if r.options.RunTestsOnly {
+			return nil
+		}
 		// When not using independent agents, service deployers like kubernetes or custom agents create new Elastic Agent
 		if !r.options.RunIndependentElasticAgent && !svcInfo.Agent.Independent {
 			return nil
@@ -1171,6 +1176,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		}
 	}
 	r.resetAgentLogLevelHandler = func(ctx context.Context) error {
+		if r.options.RunTestsOnly {
+			return nil
+		}
 		logger.Debugf("reassigning original log level %q back to agent...", origLogLevel)
 
 		if err := r.options.KibanaClient.SetAgentLogLevel(ctx, agent.ID, origLogLevel); err != nil {
@@ -1323,6 +1331,9 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp
 	}
 
 	r.shutdownServiceHandler = func(ctx context.Context) error {
+		if r.options.RunTestsOnly {
+			return nil
+		}
 		logger.Debug("tearing down service...")
 		if err := service.TearDown(ctx); err != nil {
 			return fmt.Errorf("error tearing down service: %w", err)
@@ -1362,6 +1373,9 @@ func (r *runner) setupAgent(ctx context.Context, config *testConfig, state Servi
 		return nil, agentInfo, fmt.Errorf("could not setup agent: %w", err)
 	}
 	r.shutdownAgentHandler = func(ctx context.Context) error {
+		if r.options.RunTestsOnly {
+			return nil
+		}
 		if agentDeployer == nil {
 			return nil
 		}

From 46b3ff89924ac7bc3886a98c1b0045985c9c1664 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jun 2024 14:08:25 +0200
Subject: [PATCH 21/30] Change condition in handler

---
 internal/testrunner/runners/system/runner.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 129058cfb..d7dfad09e 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1151,13 +1151,14 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
-		if !r.options.RunSetup {
+		if r.options.RunSetup {
 			// it should be kept the same policy just when system tests are
 			// triggered with the flags for running setup stage (--setup)
-			logger.Debug("reassigning original policy back to agent...")
-			if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
-				return fmt.Errorf("error reassigning original policy to agent: %w", err)
-			}
+			return nil
+		}
+		logger.Debug("reassigning original policy back to agent...")
+		if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
+			return fmt.Errorf("error reassigning original policy to agent: %w", err)
 		}
 		return nil
 	}

From 2379f8128782df52099f026f2923ec19e65fd804 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jun 2024 14:11:24 +0200
Subject: [PATCH 22/30] Use helper from common

---
 internal/testrunner/runners/system/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index c295b13ef..359c23f49 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1007,7 +1007,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	policyTesting := kibana.Policy{
 		Name:        fmt.Sprintf("ep-one-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, scenario.startTestTime.Format("20060102T15:04:05Z")),
 		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
-		Namespace:   createTestRunID(),
+		Namespace:   common.CreateTestRunID(),
 	}
 	policyToAssignDatastreamTests := policyToTest
 	if r.shouldCreateNewAgentPolicyForTest() {

From 8372d732cfdb490a5f90fefd0e9edc4423033d2b Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jun 2024 14:13:50 +0200
Subject: [PATCH 23/30] Add comment

---
 internal/testrunner/runners/system/runner.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 359c23f49..3c7d93ed9 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1130,6 +1130,11 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 			// triggered with the flags for running setup stage (--setup)
 			return nil
 		}
+		if r.options.RunIndependentElasticAgent {
+			// no need to reassign policy back, since this agent
+			// is going to be unenrolled and removed
+			return nil
+		}
 		logger.Debug("reassigning original policy back to agent...")
 		if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
 			return fmt.Errorf("error reassigning original policy to agent: %w", err)

From 84883442e1a43d894014f82da2c7f47bbb28e0e6 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jun 2024 14:47:40 +0200
Subject: [PATCH 24/30] Reassign policy back to agent for both independent and
 stack agents

---
 internal/testrunner/runners/system/runner.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 3c7d93ed9..359c23f49 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -1130,11 +1130,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 			// triggered with the flags for running setup stage (--setup)
 			return nil
 		}
-		if r.options.RunIndependentElasticAgent {
-			// no need to reassign policy back, since this agent
-			// is going to be unenrolled and removed
-			return nil
-		}
 		logger.Debug("reassigning original policy back to agent...")
 		if err := r.options.KibanaClient.AssignPolicyToAgent(ctx, agent, origPolicy); err != nil {
 			return fmt.Errorf("error reassigning original policy to agent: %w", err)

From b3b7a1d9c6f970b886b6bcc52e12cd8b3c30238f Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 6 Jun 2024 10:52:44 +0200
Subject: [PATCH 25/30] Update namespace in policyEnroll

---
 internal/testrunner/runners/system/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 359c23f49..3a61b2b48 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -919,7 +919,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		policyEnroll := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime),
 			Description: fmt.Sprintf("test policy created by elastic-package to enroll agent for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
-			Namespace:   "ep",
+			Namespace:   "enrollep",
 		}
 
 		policyTest := kibana.Policy{

From f39c935c3d8769e018967d9ae30ccdc55976c171 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 6 Jun 2024 12:54:46 +0200
Subject: [PATCH 26/30] Avoid creating a new policy if no stages are used

---
 internal/testrunner/runners/system/runner.go | 102 +++++++------------
 1 file changed, 39 insertions(+), 63 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 3a61b2b48..8d1cf483f 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -815,22 +815,6 @@ type scenarioTest struct {
 	startTestTime      time.Time
 }
 
-func (r *runner) shouldCreateNewAgentPolicyForTest() bool {
-	if r.options.RunTestsOnly {
-		// always that --no-provision is set, it should create new Agent Policies.
-		return true
-	}
-	if !r.options.RunIndependentElasticAgent {
-		// keep same behaviour as previously when Elastic Agent of the stack is used.
-		return false
-	}
-	// No need to create new Agent Policies for these stages
-	if r.options.RunSetup || r.options.RunTearDown {
-		return false
-	}
-	return true
-}
-
 func (r *runner) deleteDataStream(ctx context.Context, dataStream string) error {
 	resp, err := r.options.API.Indices.DeleteDataStream([]string{dataStream},
 		r.options.API.Indices.DeleteDataStream.WithContext(ctx),
@@ -904,50 +888,66 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 
 	// Configure package (single data stream) via Fleet APIs.
-	var policyToTest, policyToEnroll *kibana.Policy
+	testTime := time.Now().Format("20060102T15:04:05Z")
+	var policyToTest, policyCurrent, policyToEnroll *kibana.Policy
 	if r.options.RunTearDown || r.options.RunTestsOnly {
-		policyToTest = &serviceStateData.CurrentPolicy
+		policyCurrent = &serviceStateData.CurrentPolicy
 		policyToEnroll = &serviceStateData.EnrollPolicy
-		logger.Debugf("Got policy from file: %q - %q", policyToTest.Name, policyToTest.ID)
+		logger.Debugf("Got current policy from file: %q - %q", policyCurrent.Name, policyCurrent.ID)
 	} else {
 		// Create two different policies, one for enrolling the agent and the other for testing.
 		// This allows us to ensure that the Agent Policy used for testing is
 		// assigned to the agent with all the required changes (e.g. Package DataStream)
-		logger.Debug("creating test policies...")
-		testTime := time.Now().Format("20060102T15:04:05Z")
-
+		logger.Debug("creating enroll policy...")
 		policyEnroll := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime),
 			Description: fmt.Sprintf("test policy created by elastic-package to enroll agent for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
 			Namespace:   "enrollep",
 		}
 
-		policyTest := kibana.Policy{
+		policyToEnroll, err = r.options.KibanaClient.CreatePolicy(ctx, policyEnroll)
+		if err != nil {
+			return nil, fmt.Errorf("could not create test policy: %w", err)
+		}
+	}
+
+	if r.options.RunTearDown {
+		// required to assign the policy stored in the service state file
+		// so data stream related to this Agent Policy can be obtained (and deleted)
+		// in the cleanTestScenarioHandler handler
+		policyToTest = policyCurrent
+	} else {
+		logger.Debug("creating test policy...")
+		policyToAssignDatastreamTests := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime),
 			Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
-			Namespace:   "ep",
+			Namespace:   common.CreateTestRunID(),
 		}
 		// Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl
 		if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" {
-			policyTest.DataOutputID = "fleet-logstash-output"
+			policyToAssignDatastreamTests.DataOutputID = "fleet-logstash-output"
 		}
-		policyToTest, err = r.options.KibanaClient.CreatePolicy(ctx, policyTest)
-		if err != nil {
-			return nil, fmt.Errorf("could not create test policy: %w", err)
-		}
-		policyToEnroll, err = r.options.KibanaClient.CreatePolicy(ctx, policyEnroll)
+		policyToTest, err = r.options.KibanaClient.CreatePolicy(ctx, policyToAssignDatastreamTests)
 		if err != nil {
 			return nil, fmt.Errorf("could not create test policy: %w", err)
 		}
 	}
+
 	r.deleteTestPolicyHandler = func(ctx context.Context) error {
-		if r.options.RunTestsOnly {
-			return nil
-		}
 		logger.Debug("deleting test policies...")
 		if err := r.options.KibanaClient.DeletePolicy(ctx, policyToTest.ID); err != nil {
 			return fmt.Errorf("error cleaning up test policy: %w", err)
 		}
+		if r.options.RunTestsOnly {
+			return nil
+		}
+		// if !r.options.RunTearDown {
+		// 	// when RunTearDown is true, this Agent Policy is already removed with the previous
+		// 	// call, removing `policyToTest`
+		// 	if err := r.options.KibanaClient.DeletePolicy(ctx, policyCurrent.ID); err != nil {
+		// 		return fmt.Errorf("error cleaning up test policy: %w", err)
+		// 	}
+		// }
 		if err := r.options.KibanaClient.DeletePolicy(ctx, policyToEnroll.ID); err != nil {
 			return fmt.Errorf("error cleaning up test policy: %w", err)
 		}
@@ -959,7 +959,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	if r.options.RunTearDown || r.options.RunTestsOnly {
 		// required in order to be able select the right agent in `checkEnrolledAgents` when
 		// using independent agents or custom/kubernetes agents since policy data is set into `agentInfo` variable`
-		policy = policyToTest
+		policy = policyCurrent
 	}
 
 	agentDeployed, agentInfo, err := r.setupAgent(ctx, config, serviceStateData, policy, scenario.pkgManifest.Agent)
@@ -1001,23 +1001,8 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	// store the time just before adding the Test Policy, this time will be used to check
 	// the agent logs from that time onwards to avoid possible previous errors present in logs
 	scenario.startTestTime = time.Now()
-	suffixDatastream := svcInfo.Test.RunID
 
-	logger.Debug("adding package data stream to test policy...")
-	policyTesting := kibana.Policy{
-		Name:        fmt.Sprintf("ep-one-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, scenario.startTestTime.Format("20060102T15:04:05Z")),
-		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
-		Namespace:   common.CreateTestRunID(),
-	}
-	policyToAssignDatastreamTests := policyToTest
-	if r.shouldCreateNewAgentPolicyForTest() {
-		policyToAssignDatastreamTests, err = r.options.KibanaClient.CreatePolicy(ctx, policyTesting)
-		if err != nil {
-			return nil, fmt.Errorf("could not create test policy: %w", err)
-		}
-		suffixDatastream = policyTesting.Namespace
-	}
-	ds := createPackageDatastream(*policyToAssignDatastreamTests, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, suffixDatastream)
+	ds := createPackageDatastream(*policyToTest, *scenario.pkgManifest, policyTemplate, *scenario.dataStreamManifest, *config, policyToTest.Namespace)
 	if r.options.RunTearDown {
 		logger.Debug("Skip adding data stream config to policy")
 	} else {
@@ -1062,16 +1047,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 
 	r.cleanTestScenarioHandler = func(ctx context.Context) error {
-		if !r.shouldCreateNewAgentPolicyForTest() {
-			return nil
-		}
-
-		logger.Debug("Deleting test policy...")
-		err = r.options.KibanaClient.DeletePolicy(ctx, policyToAssignDatastreamTests.ID)
-		if err != nil {
-			return fmt.Errorf("failed to delete policy %s: %w", policyToAssignDatastreamTests.Name, err)
-		}
-		logger.Debug("Deleting data stream for testing")
+		logger.Debugf("Deleting data stream for testing %s", scenario.dataStream)
 		r.deleteDataStream(ctx, scenario.dataStream)
 		if err != nil {
 			return fmt.Errorf("failed to delete data stream %s: %w", scenario.dataStream, err)
@@ -1127,7 +1103,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	r.resetAgentPolicyHandler = func(ctx context.Context) error {
 		if r.options.RunSetup {
 			// it should be kept the same policy just when system tests are
-			// triggered with the flags for running setup stage (--setup)
+			// triggered with the flags for running spolicyToAssignDatastreamTestsetup stage (--setup)
 			return nil
 		}
 		logger.Debug("reassigning original policy back to agent...")
@@ -1163,9 +1139,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 	}
 
 	if r.options.RunTearDown {
-		logger.Debug("Skip assiging package data stream to agent")
+		logger.Debug("Skip assigning package data stream to agent")
 	} else {
-		policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policyToAssignDatastreamTests.ID)
+		policyWithDataStream, err := r.options.KibanaClient.GetPolicy(ctx, policyToTest.ID)
 		if err != nil {
 			return nil, fmt.Errorf("could not read the policy with data stream: %w", err)
 		}

From 3e94058e167bc752ad8c3612dce1ce2b2287e7f7 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 6 Jun 2024 13:18:14 +0200
Subject: [PATCH 27/30] Remove code to wait until docs are removed from
 datastream (and wipe handler)

---
 internal/testrunner/runners/system/runner.go | 89 ++------------------
 1 file changed, 7 insertions(+), 82 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 50cf63346..caf8029e2 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -166,7 +166,6 @@ type runner struct {
 	resetAgentLogLevelHandler func(context.Context) error
 	shutdownServiceHandler    func(context.Context) error
 	shutdownAgentHandler      func(context.Context) error
-	wipeDataStreamHandler     func(context.Context) error
 }
 
 type SystemRunnerOptions struct {
@@ -527,13 +526,6 @@ func (r *runner) tearDownTest(ctx context.Context) error {
 		r.shutdownAgentHandler = nil
 	}
 
-	if r.wipeDataStreamHandler != nil {
-		if err := r.wipeDataStreamHandler(cleanupCtx); err != nil {
-			return err
-		}
-		r.wipeDataStreamHandler = nil
-	}
-
 	return nil
 }
 
@@ -951,10 +943,10 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		policyCurrent = &serviceStateData.CurrentPolicy
 		policyToEnroll = &serviceStateData.EnrollPolicy
 		logger.Debugf("Got current policy from file: %q - %q", policyCurrent.Name, policyCurrent.ID)
-	} else {
-		// Create two different policies, one for enrolling the agent and the other for testing.
-		// This allows us to ensure that the Agent Policy used for testing is
-		// assigned to the agent with all the required changes (e.g. Package DataStream)
+	} else if r.runIndependentElasticAgent {
+		// Created a specific Agent Policy to enrolling purposes
+		// There are some issues when the stack is running for some time,
+		// agents cannot enroll with the default policy
 		logger.Debug("creating enroll policy...")
 		policyEnroll := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime),
@@ -974,6 +966,9 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		// in the cleanTestScenarioHandler handler
 		policyToTest = policyCurrent
 	} else {
+		// Create a specific Agent Policy just for testing this test.
+		// This allows us to ensure that the Agent Policy used for testing is
+		// assigned to the agent with all the required changes (e.g. Package DataStream)
 		logger.Debug("creating test policy...")
 		policyToAssignDatastreamTests := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime),
@@ -998,13 +993,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		if r.runTestsOnly {
 			return nil
 		}
-		// if !r.runTearDown {
-		// 	// when RunTearDown is true, this Agent Policy is already removed with the previous
-		// 	// call, removing `policyToTest`
-		// 	if err := r.kibanaClient.DeletePolicy(ctx, policyCurrent.ID); err != nil {
-		// 		return fmt.Errorf("error cleaning up test policy: %w", err)
-		// 	}
-		// }
 		if err := r.kibanaClient.DeletePolicy(ctx, policyToEnroll.ID); err != nil {
 			return fmt.Errorf("error cleaning up test policy: %w", err)
 		}
@@ -1093,17 +1081,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		dataStreamDataset,
 	)
 
-	r.wipeDataStreamHandler = func(ctx context.Context) error {
-		if r.runTestsOnly {
-			return nil
-		}
-		logger.Debugf("deleting data in data stream...")
-		if err := deleteDataStreamDocs(ctx, r.esAPI, scenario.dataStream); err != nil {
-			return fmt.Errorf("error deleting data in data stream: %w", err)
-		}
-		return nil
-	}
-
 	r.cleanTestScenarioHandler = func(ctx context.Context) error {
 		logger.Debugf("Deleting data stream for testing %s", scenario.dataStream)
 		r.deleteDataStream(ctx, scenario.dataStream)
@@ -1113,15 +1090,6 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		return nil
 	}
 
-	if r.runTearDown {
-		logger.Debugf("Skipped deleting old data in data stream %q", scenario.dataStream)
-	} else {
-		err := r.deleteOldDocumentsDataStreamAndWait(ctx, scenario.dataStream)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	// FIXME: running per stages does not work when multiple agents are created
 	var origPolicy kibana.Policy
 	agents, err := checkEnrolledAgents(ctx, r.kibanaClient, agentInfo, svcInfo, r.runIndependentElasticAgent)
@@ -1475,28 +1443,6 @@ func (r *runner) writeScenarioState(opts scenarioStateOpts) error {
 	return nil
 }
 
-func (r *runner) deleteOldDocumentsDataStreamAndWait(ctx context.Context, dataStream string) error {
-	logger.Debugf("Delete previous documents in data stream %q", dataStream)
-	if err := deleteDataStreamDocs(ctx, r.esAPI, dataStream); err != nil {
-		return fmt.Errorf("error deleting old data in data stream: %s: %w", dataStream, err)
-	}
-	cleared, err := wait.UntilTrue(ctx, func(ctx context.Context) (bool, error) {
-		hits, err := r.getDocs(ctx, dataStream)
-		if err != nil {
-			return false, err
-		}
-
-		return hits.size() == 0, nil
-	}, 1*time.Second, 2*time.Minute)
-	if err != nil || !cleared {
-		if err == nil {
-			err = errors.New("unable to clear previous data")
-		}
-		return err
-	}
-	return nil
-}
-
 func (r *runner) validateTestScenario(ctx context.Context, result *testrunner.ResultComposer, scenario *scenarioTest, config *testConfig) ([]testrunner.TestResult, error) {
 	// Validate fields in docs
 	// when reroute processors are used, expectedDatasets should be set depends on the processor config
@@ -2021,27 +1967,6 @@ func (r *runner) previewTransform(ctx context.Context, transformId string) ([]co
 	return preview.Documents, nil
 }
 
-func deleteDataStreamDocs(ctx context.Context, api *elasticsearch.API, dataStream string) error {
-	body := strings.NewReader(`{ "query": { "match_all": {} } }`)
-	resp, err := api.DeleteByQuery([]string{dataStream}, body,
-		api.DeleteByQuery.WithContext(ctx),
-	)
-	if err != nil {
-		return fmt.Errorf("failed to delete data stream docs: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusNotFound {
-		// Unavailable index is ok, this means that data is already not there.
-		return nil
-	}
-	if resp.IsError() {
-		return fmt.Errorf("failed to delete data stream docs for data stream %s: %s", dataStream, resp.String())
-	}
-
-	return nil
-}
-
 func filterAgents(allAgents []kibana.Agent, svcInfo servicedeployer.ServiceInfo) []kibana.Agent {
 	if svcInfo.Agent.Host.NamePrefix != "" {
 		logger.Debugf("filter agents using criteria: NamePrefix=%s", svcInfo.Agent.Host.NamePrefix)

From 7dd578fedae1516d185dac4470234726dda37daf Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 6 Jun 2024 14:08:11 +0200
Subject: [PATCH 28/30] Randomized namespace in enroll Agent Policy

---
 internal/testrunner/runners/system/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index caf8029e2..9f5b7c75e 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -951,7 +951,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		policyEnroll := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime),
 			Description: fmt.Sprintf("test policy created by elastic-package to enroll agent for data stream %s/%s", r.testFolder.Package, r.testFolder.DataStream),
-			Namespace:   "enrollep",
+			Namespace:   common.CreateTestRunID(),
 		}
 
 		policyToEnroll, err = r.kibanaClient.CreatePolicy(ctx, policyEnroll)

From f6ee2bdd8e23ed9bad2db3604b41663f0e998281 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 6 Jun 2024 14:57:33 +0200
Subject: [PATCH 29/30] Create enroll policy if not stages are used

---
 internal/testrunner/runners/system/runner.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 9f5b7c75e..330c97eaa 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -943,10 +943,12 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		policyCurrent = &serviceStateData.CurrentPolicy
 		policyToEnroll = &serviceStateData.EnrollPolicy
 		logger.Debugf("Got current policy from file: %q - %q", policyCurrent.Name, policyCurrent.ID)
-	} else if r.runIndependentElasticAgent {
+	} else {
 		// Created a specific Agent Policy to enrolling purposes
 		// There are some issues when the stack is running for some time,
 		// agents cannot enroll with the default policy
+		// This enroll policy must be created even if independent Elastic Agents are not used. Agents created
+		// in Kubernetes or Custom Agents require this enroll policy too (service deployer).
 		logger.Debug("creating enroll policy...")
 		policyEnroll := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-enroll-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime),

From 0837447cd344229d8725a4f1183f43e9b801e0d4 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Thu, 6 Jun 2024 16:39:29 +0200
Subject: [PATCH 30/30] Rename variable

---
 internal/testrunner/runners/system/runner.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 330c97eaa..3ba839b73 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -972,16 +972,16 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf
 		// This allows us to ensure that the Agent Policy used for testing is
 		// assigned to the agent with all the required changes (e.g. Package DataStream)
 		logger.Debug("creating test policy...")
-		policyToAssignDatastreamTests := kibana.Policy{
+		policy := kibana.Policy{
 			Name:        fmt.Sprintf("ep-test-system-%s-%s-%s", r.testFolder.Package, r.testFolder.DataStream, testTime),
 			Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.testFolder.Package, r.testFolder.DataStream),
 			Namespace:   common.CreateTestRunID(),
 		}
 		// Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl
 		if r.profile.Config("stack.logstash_enabled", "false") == "true" {
-			policyToAssignDatastreamTests.DataOutputID = "fleet-logstash-output"
+			policy.DataOutputID = "fleet-logstash-output"
 		}
-		policyToTest, err = r.kibanaClient.CreatePolicy(ctx, policyToAssignDatastreamTests)
+		policyToTest, err = r.kibanaClient.CreatePolicy(ctx, policy)
 		if err != nil {
 			return nil, fmt.Errorf("could not create test policy: %w", err)
 		}