Merge remote-tracking branch 'upstream/main' into update-documentation-llm

Author: mjwolf

.buildkite/pipeline.cloud-cleanup.yml

@@ -19,15 +19,6 @@ steps:
         - label: "False"
           value: "false"
       default: "true"
-    - select: "DRY_RUN (Deprecated step)"
-      key: "DRY_RUN_DEPRECATED"
-      options:
-        - label: "True"
-          value: "true"
-        - label: "False"
-          value: "false"
-      default: "true"
-    if: "build.source == 'ui'"
 
   - wait: ~
     if: "build.source == 'ui'"

.buildkite/pipeline.serverless.yml

@@ -1,6 +1,6 @@
 env:
   NOTIFY_TO: "[email protected]"
-  SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  SETUP_GVM_VERSION: 'v0.6.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.27.0'

.buildkite/pipeline.test-with-integrations-repo.yml

@@ -1,5 +1,5 @@
 env:
-  SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  SETUP_GVM_VERSION: 'v0.6.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   GH_CLI_VERSION: "2.29.0"
   JQ_VERSION: "1.7"
   # Agent images used in pipeline steps

.buildkite/pipeline.yml

@@ -1,5 +1,5 @@
 env:
-  SETUP_GVM_VERSION: 'v0.5.2' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
+  SETUP_GVM_VERSION: 'v0.6.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.27.0'

.github/workflows/bump-elastic-stack-version.yml

@@ -35,7 +35,7 @@ jobs:
           echo "UPDATECLI_ACTION=apply" >> $GITHUB_ENV
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@57aa8966d4d775cb1420b90c270ba97a4b5abe47 #v2.93.0
+        uses: updatecli/updatecli-action@9c04c99c508b3ac7851020724a912adde546f64b #v2.95.0
 
       - name: Update default stack version
         # --experimental needed for commitusingapi option.

.go-version

@@ -1 +1 @@
-1.24.2
+1.25.3

Makefile

@@ -73,10 +73,10 @@ test-stack-command-86:
 	./scripts/test-stack-command.sh 8.6.2
 
 test-stack-command-8x:
-	./scripts/test-stack-command.sh 8.19.5-12d7b56d-SNAPSHOT
+	./scripts/test-stack-command.sh 8.19.7-d893c2bd-SNAPSHOT
 
 test-stack-command-9x:
-	./scripts/test-stack-command.sh 9.2.0-dd4147ff-SNAPSHOT
+	./scripts/test-stack-command.sh 9.3.0-6f40f4f1-SNAPSHOT
 
 test-stack-command-with-apm-server:
 	APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh

README.md

@@ -39,6 +39,8 @@ Run the `help` command and see available commands:
 elastic-package help
 ```
 
+Some sub-commands are Docker-based, check you also have Docker installed. In case you are using Podman Desktop, check [this guide](./docs/howto/use_podman.md) to make it compatible.
+
 ## Development
 
 Even though the project is "go-gettable", there is the [`Makefile`](./Makefile) present, which can be used to build,
@@ -490,6 +492,8 @@ You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, s
 
 Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold.
 
+You can use Podman Desktop instead of Docker, see [this document](./docs/howto/use_podman.md)
+
 For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).
 
 ### `elastic-package stack down`
@@ -542,6 +546,8 @@ You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, s
 
 Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold.
 
+You can use Podman Desktop instead of Docker, see [this document](./docs/howto/use_podman.md)
+
 To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. Starting with Elastic stack version >= 8.7.0, it is not mandatory to be available local packages in the Package Registry to run the tests.
 
 For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).
@@ -850,7 +856,7 @@ There are available some environment variables that could be used to change some
     - `ELASTIC_PACKAGE_DATA_HOME`: Custom path to be used for `elastic-package` data directory. By default this is `~/.elastic-package`.
 
 - Related to the build process:
-    - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license.
+    - `ELASTIC_PACKAGE_REPOSITORY_LICENSE`: Path to the default repository license. This path should be relative to the repository root.
     - `ELASTIC_PACKAGE_LINKS_FILE_PATH`: Path to the links table file (e.g. `links_table.yml`) with the link definitions to be used in the build process of a package.
 
 - Related to signing packages:

cmd/benchmark.go

@@ -25,6 +25,7 @@ import (
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/common"
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
@@ -136,10 +137,12 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error {
 		return cobraext.FlagParsingError(err, cobraext.BenchNumTopProcsFlagName)
 	}
 
-	packageRootPath, found, err := packages.FindPackageRoot()
-	if !found {
-		return errors.New("package root not found")
+	repositoryRoot, err := files.FindRepositoryRoot()
+	if err != nil {
+		return fmt.Errorf("locating repository root failed: %w", err)
 	}
+
+	packageRootPath, err := packages.FindPackageRoot()
 	if err != nil {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}
@@ -203,6 +206,7 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error {
 			pipeline.WithESAPI(esClient.API),
 			pipeline.WithNumTopProcs(numTopProcs),
 			pipeline.WithFormat(reportFormat),
+			pipeline.WithRepositoryRoot(repositoryRoot),
 		)
 		runner := pipeline.NewPipelineBenchmark(opts)
 
@@ -291,17 +295,18 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 	}
 
 	var packageRootPath string
-	var found bool
 	if len(packageName) == 0 {
-		packageRootPath, found, err = packages.FindPackageRoot()
-		if !found {
-			return errors.New("package root not found")
-		}
+		packageRootPath, err = packages.FindPackageRoot()
 		if err != nil {
 			return fmt.Errorf("locating package root failed: %w", err)
 		}
 	}
 
+	repositoryRoot, err := files.FindRepositoryRoot()
+	if err != nil {
+		return fmt.Errorf("locating repository root failed: %w", err)
+	}
+
 	profile, err := cobraext.GetProfileFlag(cmd)
 	if err != nil {
 		return err
@@ -336,6 +341,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error {
 		rally.WithRallyDryRun(rallyDryRun),
 		rally.WithRallyPackageFromRegistry(packageName, packageVersion),
 		rally.WithRallyCorpusAtPath(corpusAtPath),
+		rally.WithRepositoryRoot(repositoryRoot),
 	}
 
 	esMetricsClient, err := initializeESMetricsClient(ctx)
@@ -465,14 +471,16 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		return cobraext.FlagParsingError(err, cobraext.BenchStreamTimestampFieldFlagName)
 	}
 
-	packageRootPath, found, err := packages.FindPackageRoot()
-	if !found {
-		return errors.New("package root not found")
-	}
+	packageRootPath, err := packages.FindPackageRoot()
 	if err != nil {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}
 
+	repositoryRoot, err := files.FindRepositoryRoot()
+	if err != nil {
+		return fmt.Errorf("locating repository root failed: %w", err)
+	}
+
 	profile, err := cobraext.GetProfileFlag(cmd)
 	if err != nil {
 		return err
@@ -507,6 +515,7 @@ func streamCommandAction(cmd *cobra.Command, args []string) error {
 		stream.WithESAPI(esClient.API),
 		stream.WithKibanaClient(kc),
 		stream.WithProfile(profile),
+		stream.WithRepositoryRoot(repositoryRoot),
 	}
 
 	runner := stream.NewStreamBenchmark(stream.NewOptions(withOpts...))
@@ -572,10 +581,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 		return cobraext.FlagParsingError(err, cobraext.BenchReindexToMetricstoreFlagName)
 	}
 
-	packageRootPath, found, err := packages.FindPackageRoot()
-	if !found {
-		return errors.New("package root not found")
-	}
+	packageRootPath, err := packages.FindPackageRoot()
 	if err != nil {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}

cmd/build.go

@@ -61,18 +61,38 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	repositoryRoot, err := files.FindRepositoryRoot()
+	if err != nil {
+		return fmt.Errorf("locating repository root failed: %w", err)
+	}
+	defer repositoryRoot.Close()
+
 	packageRoot, err := packages.MustFindPackageRoot()
 	if err != nil {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}
 
+	// Currently the build directory is placed inside the repository build/ folder.
+	// In the future we might want to make this configurable.
 	buildDir, err := builder.BuildDirectory()
 	if err != nil {
 		return fmt.Errorf("can't prepare build directory: %w", err)
 	}
 	logger.Debugf("Use build directory: %s", buildDir)
 
-	targets, err := docs.UpdateReadmes(packageRoot, buildDir)
+	target, err := builder.BuildPackage(cmd.Context(), builder.BuildOptions{
+		PackageRootPath: packageRoot,
+		BuildDir:        buildDir,
+		CreateZip:       createZip,
+		SignPackage:     signPackage,
+		SkipValidation:  skipValidation,
+		RepositoryRoot:  repositoryRoot,
+	})
+	if err != nil {
+		return fmt.Errorf("building package failed: %w", err)
+	}
+
+	targets, err := docs.UpdateReadmes(repositoryRoot, packageRoot, buildDir)
 	if err != nil {
 		return fmt.Errorf("updating files failed: %w", err)
 	}
@@ -82,16 +102,6 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 		cmd.Printf("%s file rendered: %s\n", fileName, target)
 	}
 
-	target, err := builder.BuildPackage(cmd.Context(), builder.BuildOptions{
-		PackageRoot:    packageRoot,
-		BuildDir:       buildDir,
-		CreateZip:      createZip,
-		SignPackage:    signPackage,
-		SkipValidation: skipValidation,
-	})
-	if err != nil {
-		return fmt.Errorf("building package failed: %w", err)
-	}
 	cmd.Printf("Package built: %s\n", target)
 
 	cmd.Println("Done")