From 04fb45da0ed284746d528eb42997c71bc1fa85ca Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 18 Mar 2025 11:34:40 +0100 Subject: [PATCH 01/20] Simple includes implementation --- cmd/build.go | 9 ++ cmd/lint.go | 14 ++- go.mod | 4 +- go.sum | 8 +- internal/includes/includes.go | 166 ++++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 internal/includes/includes.go diff --git a/cmd/build.go b/cmd/build.go index 00d86d329f..c5d063ce9c 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/docs" "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/includes" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" ) @@ -72,6 +73,14 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { } logger.Debugf("Use build directory: %s", buildDir) + included, err := includes.IncludeSharedFiles() + if err != nil { + return fmt.Errorf("updating included files failed: %w", err) + } + for _, i := range included { + cmd.Printf("%s file copied to: %s\n", filepath.Join(i.Package, i.From), i.To) + } + targets, err := docs.UpdateReadmes(packageRoot) if err != nil { return fmt.Errorf("updating files failed: %w", err) diff --git a/cmd/lint.go b/cmd/lint.go index a2a26025fe..93a06492f4 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/docs" + "github.com/elastic/elastic-package/internal/includes" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/validation" @@ -45,7 +46,6 @@ func setupLintCommand() *cobraext.Command { func lintCommandAction(cmd *cobra.Command, args []string) error { cmd.Println("Lint the package") - readmeFiles, err := docs.AreReadmesUpToDate() if err != nil { for _, f := range readmeFiles { @@ -58,6 +58,18 @@ func lintCommandAction(cmd *cobra.Command, args []string) error { } return fmt.Errorf("checking readme files are up-to-date failed: %w", err) } + includedFiles, err := includes.AreFilesUpToDate() + if err != nil { + for _, f := range includedFiles { + if !f.UpToDate { + cmd.Printf("%s is outdated. Rebuild the package with 'elastic-package build'\n%s", f.To, f.Diff) + } + if f.Error != nil { + cmd.Printf("check if %s is up-to-date failed: %s\n", f.To, f.Error) + } + } + return fmt.Errorf("checking included files are up-to-date failed: %w", err) + } return nil } diff --git a/go.mod b/go.mod index 403424a728..b55bcdef13 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.23.0 toolchain go1.23.1 +replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28 + require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Masterminds/semver/v3 v3.3.1 @@ -73,7 +75,7 @@ require ( github.com/elastic/kbncontent v0.1.4 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect diff --git a/go.sum b/go.sum index d81b21d4e6..65b521f6a1 100644 --- a/go.sum +++ b/go.sum @@ -84,14 +84,12 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU= github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= -github.com/elastic/package-spec/v3 v3.3.2 h1:hUoGvVZU19akdx7gyOtSNGVHjWXeN50NDSqtcGxXtJk= -github.com/elastic/package-spec/v3 v3.3.2/go.mod h1:worvQ1JmqxT8/SaW3Tijspa5nKR/X7jS6u/hmAxa93w= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -219,6 +217,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28 h1:abIqzB3NBb971NNZPR8cWa+zNItK4/Z5a76DzCIhwdg= +github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= diff --git a/internal/includes/includes.go b/internal/includes/includes.go new file mode 100644 index 0000000000..e10205dcb3 --- /dev/null +++ b/internal/includes/includes.go @@ -0,0 +1,166 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package includes + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/go-ucfg/yaml" + "github.com/pmezard/go-difflib/difflib" +) + +// IncludesFileEntry contains a file reference to include. +type IncludesFileEntry struct { + Package string `config:"package"` + From string `config:"from"` + To string `config:"to"` + UpToDate bool `config:"-"` + Diff string `config:"-"` + Error error `config:"-"` +} + +type IncludesFile []IncludesFileEntry + +// IncludeSharedFiles function collects any necessary files to include in the package. +func IncludeSharedFiles() (IncludesFile, error) { + packageRoot, err := packages.MustFindPackageRoot() + if err != nil { + return nil, fmt.Errorf("package root not found: %w", err) + } + + includesFile, err := readIncludes(packageRoot) + if err != nil { + return nil, fmt.Errorf("could not read includes.yml: %w", err) + } + + for _, f := range includesFile { + b, err := collectFile(packageRoot, f) + if err != nil { + return nil, fmt.Errorf("could not collect file %q: %w", filepath.Join(f.Package, f.From), err) + } + if err := writeFile(packageRoot, f.To, b); err != nil { + return nil, fmt.Errorf("could not write destination file %q: %w", filepath.Join(packageRoot, f.To), err) + } + } + return includesFile, nil +} + +// AreFilesUpToDate function checks if all the included files are up-to-date. +func AreFilesUpToDate() (IncludesFile, error) { + packageRoot, err := packages.MustFindPackageRoot() + if err != nil { + return nil, fmt.Errorf("package root not found: %w", err) + } + + includesFile, err := readIncludes(packageRoot) + if err != nil { + return nil, fmt.Errorf("could not read includes.yml: %w", err) + } + + var outdated bool + for i := 0; i < len(includesFile); i++ { + f := includesFile[i] + uptodate, diff, err := isFileUpToDate(packageRoot, f) + if !uptodate || err != nil { + includesFile[i].UpToDate = uptodate + includesFile[i].Diff = diff + includesFile[i].Error = err + outdated = true + } + } + + if outdated { + return includesFile, fmt.Errorf("files do not match") + } + return includesFile, nil +} + +func isFileUpToDate(packageRoot string, includedFile IncludesFileEntry) (bool, string, error) { + logger.Debugf("Check if %s is up-to-date", includedFile.To) + + newFile, err := collectFile(packageRoot, includedFile) + if err != nil { + return false, "", err + } + + existing, found, err := readFile(packageRoot, includedFile.To) + if err != nil { + return false, "", fmt.Errorf("reading file failed: %w", err) + } + if !found { + return false, "", nil + } + if bytes.Equal(existing, newFile) { + return true, "", nil + } + var buf bytes.Buffer + err = difflib.WriteUnifiedDiff(&buf, difflib.UnifiedDiff{ + A: difflib.SplitLines(string(existing)), + B: difflib.SplitLines(string(newFile)), + FromFile: "want", + ToFile: "got", + Context: 1, + }) + return false, buf.String(), err +} + +func collectFile(packageRoot string, includedFile IncludesFileEntry) ([]byte, error) { + var filePath string + if includedFile.Package != "" { + filePath = filepath.Join("..", includedFile.Package, includedFile.From) + } else { + filePath = filepath.Join(packageRoot, includedFile.From) + } + b, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + return b, nil +} + +func writeFile(packageRoot, to string, b []byte) error { + filePath := filepath.Join(packageRoot, to) + return os.WriteFile(filePath, b, 0644) +} + +func readIncludes(packageRoot string) (IncludesFile, error) { + includesPath := filepath.Join(packageRoot, "_dev", "shared", "includes.yml") + + b, err := os.ReadFile(includesPath) + if err != nil && errors.Is(err, os.ErrNotExist) { + return nil, nil + } + + cfg, err := yaml.NewConfig(b) + if err != nil { + return nil, fmt.Errorf("could not load includes config: %w", err) + } + + var includesFile IncludesFile + if err := cfg.Unpack(&includesFile); err != nil { + return nil, fmt.Errorf("could not parse includes config: %w", err) + } + return includesFile, nil +} + +func readFile(packageRoot, filePath string) ([]byte, bool, error) { + logger.Debugf("Read existing %s file (package: %s)", filePath, packageRoot) + + includesFilepath := filepath.Join(packageRoot, filePath) + b, err := os.ReadFile(includesFilepath) + if err != nil && errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("readfile failed (path: %s): %w", includesFilepath, err) + } + return b, true, err +} From f778462e701cea04ce93b0ebb370478c048a164a Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 18 Mar 2025 15:17:19 +0100 Subject: [PATCH 02/20] Second iteration --- cmd/build.go | 2 +- go.mod | 6 +-- go.sum | 4 +- internal/includes/includes.go | 70 +++++++++++++++++++---------------- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/cmd/build.go b/cmd/build.go index c5d063ce9c..bb4a5c05ba 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -78,7 +78,7 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("updating included files failed: %w", err) } for _, i := range included { - cmd.Printf("%s file copied to: %s\n", filepath.Join(i.Package, i.From), i.To) + cmd.Printf("%s file copied to: %s\n", filepath.FromSlash(i.From), i.To) } targets, err := docs.UpdateReadmes(packageRoot) diff --git a/go.mod b/go.mod index b55bcdef13..17b5700947 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/elastic/elastic-package -go 1.23.0 +go 1.24.0 -toolchain go1.23.1 +toolchain go1.24.1 -replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28 +replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250318133830-8c7c5b9aebde require ( github.com/AlecAivazis/survey/v2 v2.3.7 diff --git a/go.sum b/go.sum index 65b521f6a1..3d7a90f09c 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28 h1:abIqzB3NBb971NNZPR8cWa+zNItK4/Z5a76DzCIhwdg= -github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= +github.com/marc-gr/package-spec/v3 v3.0.0-20250318133830-8c7c5b9aebde h1:CR67Va0mMnQHq16yp6f5kL3nvETSITzy3CujjvmFTsw= +github.com/marc-gr/package-spec/v3 v3.0.0-20250318133830-8c7c5b9aebde/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= diff --git a/internal/includes/includes.go b/internal/includes/includes.go index e10205dcb3..b8db8f42e4 100644 --- a/internal/includes/includes.go +++ b/internal/includes/includes.go @@ -8,6 +8,7 @@ import ( "bytes" "errors" "fmt" + "io/fs" "os" "path/filepath" @@ -19,7 +20,6 @@ import ( // IncludesFileEntry contains a file reference to include. type IncludesFileEntry struct { - Package string `config:"package"` From string `config:"from"` To string `config:"to"` UpToDate bool `config:"-"` @@ -27,66 +27,80 @@ type IncludesFileEntry struct { Error error `config:"-"` } -type IncludesFile []IncludesFileEntry +type IncludesFile struct { + Include []IncludesFileEntry `config:"include"` +} // IncludeSharedFiles function collects any necessary files to include in the package. -func IncludeSharedFiles() (IncludesFile, error) { +func IncludeSharedFiles() ([]IncludesFileEntry, error) { packageRoot, err := packages.MustFindPackageRoot() if err != nil { return nil, fmt.Errorf("package root not found: %w", err) } - includesFile, err := readIncludes(packageRoot) + // scope any possible operation in the packages/ folder + dirRoot, err := os.OpenRoot(filepath.Join(packageRoot, "..")) + if err != nil { + return nil, fmt.Errorf("could not open root: %w", err) + } + + includes, err := readIncludes(packageRoot) if err != nil { return nil, fmt.Errorf("could not read includes.yml: %w", err) } - for _, f := range includesFile { - b, err := collectFile(packageRoot, f) + for _, f := range includes { + b, err := collectFile(dirRoot.FS().(fs.ReadFileFS), f) if err != nil { - return nil, fmt.Errorf("could not collect file %q: %w", filepath.Join(f.Package, f.From), err) + return nil, fmt.Errorf("could not collect file %q: %w", filepath.FromSlash(f.From), err) } if err := writeFile(packageRoot, f.To, b); err != nil { - return nil, fmt.Errorf("could not write destination file %q: %w", filepath.Join(packageRoot, f.To), err) + return nil, fmt.Errorf("could not write destination file %q: %w", filepath.Join(packageRoot, filepath.FromSlash(f.To)), err) } } - return includesFile, nil + return includes, nil } // AreFilesUpToDate function checks if all the included files are up-to-date. -func AreFilesUpToDate() (IncludesFile, error) { +func AreFilesUpToDate() ([]IncludesFileEntry, error) { packageRoot, err := packages.MustFindPackageRoot() if err != nil { return nil, fmt.Errorf("package root not found: %w", err) } - includesFile, err := readIncludes(packageRoot) + includes, err := readIncludes(packageRoot) if err != nil { return nil, fmt.Errorf("could not read includes.yml: %w", err) } var outdated bool - for i := 0; i < len(includesFile); i++ { - f := includesFile[i] + for i := 0; i < len(includes); i++ { + f := includes[i] uptodate, diff, err := isFileUpToDate(packageRoot, f) if !uptodate || err != nil { - includesFile[i].UpToDate = uptodate - includesFile[i].Diff = diff - includesFile[i].Error = err + includes[i].UpToDate = uptodate + includes[i].Diff = diff + includes[i].Error = err outdated = true } } if outdated { - return includesFile, fmt.Errorf("files do not match") + return includes, fmt.Errorf("files do not match") } - return includesFile, nil + return includes, nil } func isFileUpToDate(packageRoot string, includedFile IncludesFileEntry) (bool, string, error) { logger.Debugf("Check if %s is up-to-date", includedFile.To) - newFile, err := collectFile(packageRoot, includedFile) + // scope any possible operation in the packages/ folder + dirRoot, err := os.OpenRoot(filepath.Join(packageRoot, "..")) + if err != nil { + return false, "", fmt.Errorf("could not open root: %w", err) + } + + newFile, err := collectFile(dirRoot.FS().(fs.ReadFileFS), includedFile) if err != nil { return false, "", err } @@ -112,14 +126,8 @@ func isFileUpToDate(packageRoot string, includedFile IncludesFileEntry) (bool, s return false, buf.String(), err } -func collectFile(packageRoot string, includedFile IncludesFileEntry) ([]byte, error) { - var filePath string - if includedFile.Package != "" { - filePath = filepath.Join("..", includedFile.Package, includedFile.From) - } else { - filePath = filepath.Join(packageRoot, includedFile.From) - } - b, err := os.ReadFile(filePath) +func collectFile(root fs.ReadFileFS, includedFile IncludesFileEntry) ([]byte, error) { + b, err := root.ReadFile(filepath.FromSlash(includedFile.From)) if err != nil { return nil, err } @@ -127,12 +135,12 @@ func collectFile(packageRoot string, includedFile IncludesFileEntry) ([]byte, er } func writeFile(packageRoot, to string, b []byte) error { - filePath := filepath.Join(packageRoot, to) + filePath := filepath.Join(packageRoot, filepath.FromSlash(to)) return os.WriteFile(filePath, b, 0644) } -func readIncludes(packageRoot string) (IncludesFile, error) { - includesPath := filepath.Join(packageRoot, "_dev", "shared", "includes.yml") +func readIncludes(packageRoot string) ([]IncludesFileEntry, error) { + includesPath := filepath.Join(packageRoot, "_dev", "build", "build.yml") b, err := os.ReadFile(includesPath) if err != nil && errors.Is(err, os.ErrNotExist) { @@ -148,7 +156,7 @@ func readIncludes(packageRoot string) (IncludesFile, error) { if err := cfg.Unpack(&includesFile); err != nil { return nil, fmt.Errorf("could not parse includes config: %w", err) } - return includesFile, nil + return includesFile.Include, nil } func readFile(packageRoot, filePath string) ([]byte, bool, error) { From f24f9486791956b623959ce3ae11ce70587f45c8 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 19 Mar 2025 13:36:54 +0100 Subject: [PATCH 03/20] Use link files instead of includes file --- cmd/build.go | 9 -- cmd/lint.go | 13 --- go.mod | 2 +- go.sum | 4 +- internal/builder/linked_files.go | 120 +++++++++++++++++++++ internal/builder/packages.go | 6 ++ internal/files/copy.go | 21 +++- internal/includes/includes.go | 174 ------------------------------- internal/stack/boot.go | 2 +- 9 files changed, 148 insertions(+), 203 deletions(-) create mode 100644 internal/builder/linked_files.go delete mode 100644 internal/includes/includes.go diff --git a/cmd/build.go b/cmd/build.go index bb4a5c05ba..00d86d329f 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -15,7 +15,6 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/docs" "github.com/elastic/elastic-package/internal/files" - "github.com/elastic/elastic-package/internal/includes" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" ) @@ -73,14 +72,6 @@ func buildCommandAction(cmd *cobra.Command, args []string) error { } logger.Debugf("Use build directory: %s", buildDir) - included, err := includes.IncludeSharedFiles() - if err != nil { - return fmt.Errorf("updating included files failed: %w", err) - } - for _, i := range included { - cmd.Printf("%s file copied to: %s\n", filepath.FromSlash(i.From), i.To) - } - targets, err := docs.UpdateReadmes(packageRoot) if err != nil { return fmt.Errorf("updating files failed: %w", err) diff --git a/cmd/lint.go b/cmd/lint.go index 93a06492f4..bba2188f9d 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -12,7 +12,6 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/docs" - "github.com/elastic/elastic-package/internal/includes" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/validation" @@ -58,18 +57,6 @@ func lintCommandAction(cmd *cobra.Command, args []string) error { } return fmt.Errorf("checking readme files are up-to-date failed: %w", err) } - includedFiles, err := includes.AreFilesUpToDate() - if err != nil { - for _, f := range includedFiles { - if !f.UpToDate { - cmd.Printf("%s is outdated. Rebuild the package with 'elastic-package build'\n%s", f.To, f.Diff) - } - if f.Error != nil { - cmd.Printf("check if %s is up-to-date failed: %s\n", f.To, f.Error) - } - } - return fmt.Errorf("checking included files are up-to-date failed: %w", err) - } return nil } diff --git a/go.mod b/go.mod index 17b5700947..81d80f0aaa 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.24.0 toolchain go1.24.1 -replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250318133830-8c7c5b9aebde +replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250319094545-d89aaf66115e require ( github.com/AlecAivazis/survey/v2 v2.3.7 diff --git a/go.sum b/go.sum index 3d7a90f09c..2c31a16699 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marc-gr/package-spec/v3 v3.0.0-20250318133830-8c7c5b9aebde h1:CR67Va0mMnQHq16yp6f5kL3nvETSITzy3CujjvmFTsw= -github.com/marc-gr/package-spec/v3 v3.0.0-20250318133830-8c7c5b9aebde/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= +github.com/marc-gr/package-spec/v3 v3.0.0-20250319094545-d89aaf66115e h1:wmqI2aDNnzLQ+uIB4xU6VTwNApkAOUJ3DIn2xhKL/kw= +github.com/marc-gr/package-spec/v3 v3.0.0-20250319094545-d89aaf66115e/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go new file mode 100644 index 0000000000..5479dc2fe7 --- /dev/null +++ b/internal/builder/linked_files.go @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package builder + +import ( + "bufio" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/elastic/elastic-package/internal/logger" +) + +type link struct { + filePath string + includedFilePath string +} + +func includeSharedFiles(packageRoot, destinationDir string) error { + links, err := getLinks(packageRoot) + if err != nil { + return fmt.Errorf("could not list link files: %w", err) + } + + if len(links) == 0 { + return nil + } + + logger.Debugf("Package has linked files defined") + + // scope any possible operation in the packages/ folder + dirRoot, err := os.OpenRoot(filepath.Join(packageRoot, "..")) + if err != nil { + return fmt.Errorf("could not open root: %w", err) + } + + for _, l := range links { + b, err := collectFile(dirRoot.FS().(fs.ReadFileFS), l.includedFilePath) + if err != nil { + return fmt.Errorf("could not collect file %v: %w", l.includedFilePath, err) + } + cleanPath := strings.TrimSuffix(l.filePath, ".link") + toFilePath := strings.Replace( + cleanPath, + packageRoot, + destinationDir, + 1, + ) + if err := writeFile(toFilePath, b); err != nil { + return fmt.Errorf("could not write file %v: %w", toFilePath, err) + } + logger.Debugf("%v included in package", cleanPath) + } + + return nil +} + +func collectFile(root fs.ReadFileFS, path string) ([]byte, error) { + b, err := root.ReadFile(filepath.FromSlash(path)) + if err != nil { + return nil, err + } + return b, nil +} + +func writeFile(to string, b []byte) error { + return os.WriteFile(filepath.FromSlash(to), b, 0644) +} + +func getLinks(packageRoot string) ([]link, error) { + var linkFiles []string + if err := filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() && strings.HasSuffix(info.Name(), ".link") { + linkFiles = append(linkFiles, path) + } + return nil + }); err != nil { + return nil, err + } + + links := make([]link, len(linkFiles)) + + for i, f := range linkFiles { + firstLine, err := readFirstLine(f) + if err != nil { + return nil, err + } + links[i].filePath = f + links[i].includedFilePath = firstLine + } + + return links, nil +} + +func readFirstLine(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + if scanner.Scan() { + return scanner.Text(), nil + } + + if err := scanner.Err(); err != nil { + return "", err + } + + return "", fmt.Errorf("file is empty or first line is missing") +} diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 94c1b89b77..1a8310025e 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -184,6 +184,12 @@ func BuildPackage(options BuildOptions) (string, error) { return "", fmt.Errorf("adding dynamic mappings: %w", err) } + logger.Debug("Include linked files") + err = includeSharedFiles(options.PackageRoot, destinationDir) + if err != nil { + return "", fmt.Errorf("including linked files failed: %w", err) + } + if options.CreateZip { return buildZippedPackage(options, destinationDir) } diff --git a/internal/files/copy.go b/internal/files/copy.go index 207790791e..32e90195f1 100644 --- a/internal/files/copy.go +++ b/internal/files/copy.go @@ -13,16 +13,16 @@ import ( // CopyAll method copies files from the source to the destination skipping empty directories. func CopyAll(sourcePath, destinationPath string) error { - return CopyWithSkipped(sourcePath, destinationPath, []string{}) + return CopyWithSkipped(sourcePath, destinationPath, nil, nil) } // CopyWithoutDev method copies files from the source to the destination, but skips _dev directories and empty folders. func CopyWithoutDev(sourcePath, destinationPath string) error { - return CopyWithSkipped(sourcePath, destinationPath, []string{"_dev", "build", ".git"}) + return CopyWithSkipped(sourcePath, destinationPath, []string{"_dev", "build", ".git"}, []string{"*.link"}) } // CopyWithSkipped method copies files from the source to the destination, but skips selected directories and empty folders. -func CopyWithSkipped(sourcePath, destinationPath string, skippedDirs []string) error { +func CopyWithSkipped(sourcePath, destinationPath string, skippedDirs []string, skippedFilesGlobs []string) error { return filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -45,6 +45,10 @@ func CopyWithSkipped(sourcePath, destinationPath string, skippedDirs []string) e return nil // don't create empty directories inside packages, if the directory is empty, skip it. } + if shouldFileBeSkipped(relativePath, skippedFilesGlobs) { + return nil + } + err = os.MkdirAll(filepath.Join(destinationPath, filepath.Dir(relativePath)), 0755) if err != nil { return err @@ -65,3 +69,14 @@ func shouldDirectoryBeSkipped(path string, skippedDirs []string) bool { } return false } + +// shouldFileBeSkipped function checks if absolute path should be skipped. +func shouldFileBeSkipped(path string, skippedFilesGlobs []string) bool { + for _, g := range skippedFilesGlobs { + m, _ := filepath.Match(g, filepath.Base(path)) + if m { + return true + } + } + return false +} diff --git a/internal/includes/includes.go b/internal/includes/includes.go deleted file mode 100644 index b8db8f42e4..0000000000 --- a/internal/includes/includes.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package includes - -import ( - "bytes" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - - "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/packages" - "github.com/elastic/go-ucfg/yaml" - "github.com/pmezard/go-difflib/difflib" -) - -// IncludesFileEntry contains a file reference to include. -type IncludesFileEntry struct { - From string `config:"from"` - To string `config:"to"` - UpToDate bool `config:"-"` - Diff string `config:"-"` - Error error `config:"-"` -} - -type IncludesFile struct { - Include []IncludesFileEntry `config:"include"` -} - -// IncludeSharedFiles function collects any necessary files to include in the package. -func IncludeSharedFiles() ([]IncludesFileEntry, error) { - packageRoot, err := packages.MustFindPackageRoot() - if err != nil { - return nil, fmt.Errorf("package root not found: %w", err) - } - - // scope any possible operation in the packages/ folder - dirRoot, err := os.OpenRoot(filepath.Join(packageRoot, "..")) - if err != nil { - return nil, fmt.Errorf("could not open root: %w", err) - } - - includes, err := readIncludes(packageRoot) - if err != nil { - return nil, fmt.Errorf("could not read includes.yml: %w", err) - } - - for _, f := range includes { - b, err := collectFile(dirRoot.FS().(fs.ReadFileFS), f) - if err != nil { - return nil, fmt.Errorf("could not collect file %q: %w", filepath.FromSlash(f.From), err) - } - if err := writeFile(packageRoot, f.To, b); err != nil { - return nil, fmt.Errorf("could not write destination file %q: %w", filepath.Join(packageRoot, filepath.FromSlash(f.To)), err) - } - } - return includes, nil -} - -// AreFilesUpToDate function checks if all the included files are up-to-date. -func AreFilesUpToDate() ([]IncludesFileEntry, error) { - packageRoot, err := packages.MustFindPackageRoot() - if err != nil { - return nil, fmt.Errorf("package root not found: %w", err) - } - - includes, err := readIncludes(packageRoot) - if err != nil { - return nil, fmt.Errorf("could not read includes.yml: %w", err) - } - - var outdated bool - for i := 0; i < len(includes); i++ { - f := includes[i] - uptodate, diff, err := isFileUpToDate(packageRoot, f) - if !uptodate || err != nil { - includes[i].UpToDate = uptodate - includes[i].Diff = diff - includes[i].Error = err - outdated = true - } - } - - if outdated { - return includes, fmt.Errorf("files do not match") - } - return includes, nil -} - -func isFileUpToDate(packageRoot string, includedFile IncludesFileEntry) (bool, string, error) { - logger.Debugf("Check if %s is up-to-date", includedFile.To) - - // scope any possible operation in the packages/ folder - dirRoot, err := os.OpenRoot(filepath.Join(packageRoot, "..")) - if err != nil { - return false, "", fmt.Errorf("could not open root: %w", err) - } - - newFile, err := collectFile(dirRoot.FS().(fs.ReadFileFS), includedFile) - if err != nil { - return false, "", err - } - - existing, found, err := readFile(packageRoot, includedFile.To) - if err != nil { - return false, "", fmt.Errorf("reading file failed: %w", err) - } - if !found { - return false, "", nil - } - if bytes.Equal(existing, newFile) { - return true, "", nil - } - var buf bytes.Buffer - err = difflib.WriteUnifiedDiff(&buf, difflib.UnifiedDiff{ - A: difflib.SplitLines(string(existing)), - B: difflib.SplitLines(string(newFile)), - FromFile: "want", - ToFile: "got", - Context: 1, - }) - return false, buf.String(), err -} - -func collectFile(root fs.ReadFileFS, includedFile IncludesFileEntry) ([]byte, error) { - b, err := root.ReadFile(filepath.FromSlash(includedFile.From)) - if err != nil { - return nil, err - } - return b, nil -} - -func writeFile(packageRoot, to string, b []byte) error { - filePath := filepath.Join(packageRoot, filepath.FromSlash(to)) - return os.WriteFile(filePath, b, 0644) -} - -func readIncludes(packageRoot string) ([]IncludesFileEntry, error) { - includesPath := filepath.Join(packageRoot, "_dev", "build", "build.yml") - - b, err := os.ReadFile(includesPath) - if err != nil && errors.Is(err, os.ErrNotExist) { - return nil, nil - } - - cfg, err := yaml.NewConfig(b) - if err != nil { - return nil, fmt.Errorf("could not load includes config: %w", err) - } - - var includesFile IncludesFile - if err := cfg.Unpack(&includesFile); err != nil { - return nil, fmt.Errorf("could not parse includes config: %w", err) - } - return includesFile.Include, nil -} - -func readFile(packageRoot, filePath string) ([]byte, bool, error) { - logger.Debugf("Read existing %s file (package: %s)", filePath, packageRoot) - - includesFilepath := filepath.Join(packageRoot, filePath) - b, err := os.ReadFile(includesFilepath) - if err != nil && errors.Is(err, os.ErrNotExist) { - return nil, false, nil - } - if err != nil { - return nil, false, fmt.Errorf("readfile failed (path: %s): %w", includesFilepath, err) - } - return b, true, err -} diff --git a/internal/stack/boot.go b/internal/stack/boot.go index 0210132b0f..0187bf785e 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -170,7 +170,7 @@ func copyUniquePackages(sourcePath, destinationPath string) error { } skippedDirs = append(skippedDirs, filepath.Join(name, version)) } - return files.CopyWithSkipped(sourcePath, destinationPath, skippedDirs) + return files.CopyWithSkipped(sourcePath, destinationPath, skippedDirs, nil) } // stringsCut has been imported from Go source code. From 52132f5ae1ef94318e647e4afc2c8664dcffdf5d Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 20 Mar 2025 11:48:13 +0100 Subject: [PATCH 04/20] Collect included pipelines in tests and benchmarks --- internal/builder/linked_files.go | 73 +++++++++++---------- internal/builder/packages.go | 2 +- internal/elasticsearch/ingest/datastream.go | 16 +++++ 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go index 5479dc2fe7..1b48724fc4 100644 --- a/internal/builder/linked_files.go +++ b/internal/builder/linked_files.go @@ -13,67 +13,58 @@ import ( "strings" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" ) -type link struct { - filePath string - includedFilePath string +type Link struct { + FilePath string + IncludedFilePath string } -func includeSharedFiles(packageRoot, destinationDir string) error { - links, err := getLinks(packageRoot) +func IncludeSharedFiles(fromPath, destinationDir string) ([]Link, error) { + links, err := GetLinksFromPath(fromPath) if err != nil { - return fmt.Errorf("could not list link files: %w", err) + return nil, fmt.Errorf("could not list link files: %w", err) } if len(links) == 0 { - return nil + return nil, nil } + packageRootPath, found, err := packages.FindPackageRoot() + if !found { + return nil, fmt.Errorf("package root not found: %w", err) + } logger.Debugf("Package has linked files defined") - // scope any possible operation in the packages/ folder - dirRoot, err := os.OpenRoot(filepath.Join(packageRoot, "..")) + dirRoot, err := os.OpenRoot(filepath.Join(packageRootPath, "..")) if err != nil { - return fmt.Errorf("could not open root: %w", err) + return nil, fmt.Errorf("could not open root: %w", err) } for _, l := range links { - b, err := collectFile(dirRoot.FS().(fs.ReadFileFS), l.includedFilePath) + b, err := collectFile(dirRoot.FS().(fs.ReadFileFS), l.IncludedFilePath) if err != nil { - return fmt.Errorf("could not collect file %v: %w", l.includedFilePath, err) + return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) } - cleanPath := strings.TrimSuffix(l.filePath, ".link") toFilePath := strings.Replace( - cleanPath, - packageRoot, + l.FilePath, + fromPath, destinationDir, 1, ) if err := writeFile(toFilePath, b); err != nil { - return fmt.Errorf("could not write file %v: %w", toFilePath, err) + return nil, fmt.Errorf("could not write file %v: %w", toFilePath, err) } - logger.Debugf("%v included in package", cleanPath) - } - - return nil -} - -func collectFile(root fs.ReadFileFS, path string) ([]byte, error) { - b, err := root.ReadFile(filepath.FromSlash(path)) - if err != nil { - return nil, err + logger.Debugf("%v included in package", l.FilePath) } - return b, nil -} -func writeFile(to string, b []byte) error { - return os.WriteFile(filepath.FromSlash(to), b, 0644) + return links, nil } -func getLinks(packageRoot string) ([]link, error) { +func GetLinksFromPath(dirPath string) ([]Link, error) { var linkFiles []string - if err := filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error { + if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -86,20 +77,32 @@ func getLinks(packageRoot string) ([]link, error) { return nil, err } - links := make([]link, len(linkFiles)) + links := make([]Link, len(linkFiles)) for i, f := range linkFiles { firstLine, err := readFirstLine(f) if err != nil { return nil, err } - links[i].filePath = f - links[i].includedFilePath = firstLine + links[i].FilePath = strings.TrimSuffix(f, ".link") + links[i].IncludedFilePath = firstLine } return links, nil } +func collectFile(root fs.ReadFileFS, path string) ([]byte, error) { + b, err := root.ReadFile(filepath.FromSlash(path)) + if err != nil { + return nil, err + } + return b, nil +} + +func writeFile(to string, b []byte) error { + return os.WriteFile(filepath.FromSlash(to), b, 0644) +} + func readFirstLine(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 1a8310025e..9ab8592451 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -185,7 +185,7 @@ func BuildPackage(options BuildOptions) (string, error) { } logger.Debug("Include linked files") - err = includeSharedFiles(options.PackageRoot, destinationDir) + _, err = IncludeSharedFiles(options.PackageRoot, destinationDir) if err != nil { return "", fmt.Errorf("including linked files failed: %w", err) } diff --git a/internal/elasticsearch/ingest/datastream.go b/internal/elasticsearch/ingest/datastream.go index 6d4848446c..e388909280 100644 --- a/internal/elasticsearch/ingest/datastream.go +++ b/internal/elasticsearch/ingest/datastream.go @@ -18,7 +18,9 @@ import ( "gopkg.in/yaml.v3" + "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" ) @@ -70,6 +72,20 @@ func InstallDataStreamPipelines(ctx context.Context, api *elasticsearch.API, dat func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, error) { elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline") + // Include shared pipelines before installing them + links, err := builder.IncludeSharedFiles(elasticsearchPath, elasticsearchPath) + if err != nil { + return nil, fmt.Errorf("including shared files failed: %w", err) + } + defer func() { + // Remove linked files after installing them + for _, link := range links { + if err := os.Remove(link.FilePath); err != nil { + logger.Errorf("Failed to remove linked file %s: %v\n", link.FilePath, err) + } + } + }() + var pipelineFiles []string for _, pattern := range []string{"*.json", "*.yml"} { files, err := filepath.Glob(filepath.Join(elasticsearchPath, pattern)) From fc8f0549a81f368a04ad9b6ac55ca78a34e36880 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 25 Mar 2025 12:39:04 +0100 Subject: [PATCH 05/20] Create complete path if not exist --- internal/builder/linked_files.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go index 1b48724fc4..659b43f9fa 100644 --- a/internal/builder/linked_files.go +++ b/internal/builder/linked_files.go @@ -100,6 +100,11 @@ func collectFile(root fs.ReadFileFS, path string) ([]byte, error) { } func writeFile(to string, b []byte) error { + if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { + return err + } + } return os.WriteFile(filepath.FromSlash(to), b, 0644) } From 268e23e339b4c35eb9296b1124213b125680d43a Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 25 Mar 2025 12:39:10 +0100 Subject: [PATCH 06/20] Add test package --- .../other/with_includes/_dev/build/build.yml | 4 + .../with_includes/_dev/build/docs/README.md | 9 ++ .../_dev/build/shared/stream.yml.hbs | 7 ++ .../other/with_includes/changelog.yml | 6 ++ .../first/agent/stream/stream.yml.hbs.link | 1 + .../elasticsearch/ingest_pipeline/default.yml | 10 +++ .../data_stream/first/fields/base-fields.yml | 12 +++ .../first/fields/histogram-fields.yml | 3 + .../data_stream/first/manifest.yml | 13 +++ .../data_stream/first/sample_event.json | 26 ++++++ .../second/agent/stream/stream.yml.hbs.link | 1 + .../elasticsearch/ingest_pipeline/default.yml | 10 +++ .../data_stream/second/fields/base-fields.yml | 12 +++ .../data_stream/second/fields/geo-fields.yml | 2 + .../second/fields/histogram-fields.yml | 3 + .../data_stream/second/manifest.yml | 13 +++ .../data_stream/second/sample_event.json | 26 ++++++ .../other/with_includes/docs/README.md | 85 +++++++++++++++++++ .../packages/other/with_includes/manifest.yml | 20 +++++ 19 files changed, 263 insertions(+) create mode 100644 test/packages/other/with_includes/_dev/build/build.yml create mode 100644 test/packages/other/with_includes/_dev/build/docs/README.md create mode 100644 test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs create mode 100644 test/packages/other/with_includes/changelog.yml create mode 100644 test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link create mode 100644 test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/other/with_includes/data_stream/first/fields/base-fields.yml create mode 100644 test/packages/other/with_includes/data_stream/first/fields/histogram-fields.yml create mode 100644 test/packages/other/with_includes/data_stream/first/manifest.yml create mode 100644 test/packages/other/with_includes/data_stream/first/sample_event.json create mode 100644 test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link create mode 100644 test/packages/other/with_includes/data_stream/second/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/other/with_includes/data_stream/second/fields/base-fields.yml create mode 100644 test/packages/other/with_includes/data_stream/second/fields/geo-fields.yml create mode 100644 test/packages/other/with_includes/data_stream/second/fields/histogram-fields.yml create mode 100644 test/packages/other/with_includes/data_stream/second/manifest.yml create mode 100644 test/packages/other/with_includes/data_stream/second/sample_event.json create mode 100644 test/packages/other/with_includes/docs/README.md create mode 100644 test/packages/other/with_includes/manifest.yml diff --git a/test/packages/other/with_includes/_dev/build/build.yml b/test/packages/other/with_includes/_dev/build/build.yml new file mode 100644 index 0000000000..8a08f65dea --- /dev/null +++ b/test/packages/other/with_includes/_dev/build/build.yml @@ -0,0 +1,4 @@ +dependencies: + ecs: + reference: git@v8.5.2 + import_mappings: true diff --git a/test/packages/other/with_includes/_dev/build/docs/README.md b/test/packages/other/with_includes/_dev/build/docs/README.md new file mode 100644 index 0000000000..591d5fa57c --- /dev/null +++ b/test/packages/other/with_includes/_dev/build/docs/README.md @@ -0,0 +1,9 @@ +# Imported Mappings Tests + +{{event "first"}} + +{{fields "first"}} + +{{event "second"}} + +{{fields "second"}} \ No newline at end of file diff --git a/test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs b/test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs new file mode 100644 index 0000000000..5845510de8 --- /dev/null +++ b/test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/other/with_includes/changelog.yml b/test/packages/other/with_includes/changelog.yml new file mode 100644 index 0000000000..bb0320a524 --- /dev/null +++ b/test/packages/other/with_includes/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link b/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link new file mode 100644 index 0000000000..d705970720 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link @@ -0,0 +1 @@ +with_includes/_dev/build/shared/stream.yml.hbs \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..81221adf3f --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/first/fields/base-fields.yml b/test/packages/other/with_includes/data_stream/first/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/other/with_includes/data_stream/first/fields/histogram-fields.yml b/test/packages/other/with_includes/data_stream/first/fields/histogram-fields.yml new file mode 100644 index 0000000000..128a0cb1ca --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/fields/histogram-fields.yml @@ -0,0 +1,3 @@ +- name: service.status.*.histogram + type: object + object_type: histogram diff --git a/test/packages/other/with_includes/data_stream/first/manifest.yml b/test/packages/other/with_includes/data_stream/first/manifest.yml new file mode 100644 index 0000000000..979ef29d64 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/manifest.yml @@ -0,0 +1,13 @@ +title: "First" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/other/with_includes/data_stream/first/sample_event.json b/test/packages/other/with_includes/data_stream/first/sample_event.json new file mode 100644 index 0000000000..a242024f51 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/sample_event.json @@ -0,0 +1,26 @@ +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link b/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link new file mode 100644 index 0000000000..d705970720 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link @@ -0,0 +1 @@ +with_includes/_dev/build/shared/stream.yml.hbs \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/second/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/with_includes/data_stream/second/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..81221adf3f --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/second/fields/base-fields.yml b/test/packages/other/with_includes/data_stream/second/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/other/with_includes/data_stream/second/fields/geo-fields.yml b/test/packages/other/with_includes/data_stream/second/fields/geo-fields.yml new file mode 100644 index 0000000000..a618607d34 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/fields/geo-fields.yml @@ -0,0 +1,2 @@ +- name: destination.geo.location + external: ecs diff --git a/test/packages/other/with_includes/data_stream/second/fields/histogram-fields.yml b/test/packages/other/with_includes/data_stream/second/fields/histogram-fields.yml new file mode 100644 index 0000000000..128a0cb1ca --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/fields/histogram-fields.yml @@ -0,0 +1,3 @@ +- name: service.status.*.histogram + type: object + object_type: histogram diff --git a/test/packages/other/with_includes/data_stream/second/manifest.yml b/test/packages/other/with_includes/data_stream/second/manifest.yml new file mode 100644 index 0000000000..979ef29d64 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/manifest.yml @@ -0,0 +1,13 @@ +title: "First" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/other/with_includes/data_stream/second/sample_event.json b/test/packages/other/with_includes/data_stream/second/sample_event.json new file mode 100644 index 0000000000..a242024f51 --- /dev/null +++ b/test/packages/other/with_includes/data_stream/second/sample_event.json @@ -0,0 +1,26 @@ +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} \ No newline at end of file diff --git a/test/packages/other/with_includes/docs/README.md b/test/packages/other/with_includes/docs/README.md new file mode 100644 index 0000000000..6f99a892f2 --- /dev/null +++ b/test/packages/other/with_includes/docs/README.md @@ -0,0 +1,85 @@ +# Imported Mappings Tests + +An example event for `first` looks as following: + +```json +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| service.status.\*.histogram | | object | + + +An example event for `second` looks as following: + +```json +{ + "source.geo.location": { + "lat": 1.0, + "lon": "2.0" + }, + "destination.geo.location.lat": 3.0, + "destination.geo.location.lon": 4.0, + "service.status.duration.histogram": { + "counts": [ + 8, + 17, + 8, + 7, + 6, + 2 + ], + "values": [ + 0.1, + 0.25, + 0.35, + 0.4, + 0.45, + 0.5 + ] + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| service.status.\*.histogram | | object | diff --git a/test/packages/other/with_includes/manifest.yml b/test/packages/other/with_includes/manifest.yml new file mode 100644 index 0000000000..838496d40b --- /dev/null +++ b/test/packages/other/with_includes/manifest.yml @@ -0,0 +1,20 @@ +format_version: 2.3.0 +name: with_includes +title: "With Includes Tests" +version: 0.0.1 +description: "These are tests of field validation with includes." +type: integration +categories: + - custom +conditions: + kibana.version: "^8.0.0" +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations From 4b6dcb0e3f052c139d6ea5eccb1c85ae68187391 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 25 Mar 2025 17:33:49 +0100 Subject: [PATCH 07/20] Add links commands --- cmd/links.go | 126 +++++++++++ cmd/root.go | 1 + internal/builder/linked_files.go | 214 +++++++++++++++--- internal/builder/packages.go | 2 +- internal/cobraext/command.go | 3 + internal/elasticsearch/ingest/datastream.go | 6 +- .../first/agent/stream/stream.yml.hbs.link | 2 +- .../second/agent/stream/stream.yml.hbs.link | 2 +- 8 files changed, 320 insertions(+), 36 deletions(-) create mode 100644 cmd/links.go diff --git a/cmd/links.go b/cmd/links.go new file mode 100644 index 0000000000..a007d9782d --- /dev/null +++ b/cmd/links.go @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/builder" + "github.com/elastic/elastic-package/internal/cobraext" +) + +const linksLongDescription = `` + +func setupLinksCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "links", + Short: "Manage linked files", + Long: linksLongDescription, + RunE: func(parent *cobra.Command, args []string) error { + return cobraext.ComposeCommandsParentContext(parent, args, parent.Commands()...) + }, + } + + cmd.AddCommand(getLinksCheckCommand()) + cmd.AddCommand(getLinksUpdateCommand()) + cmd.AddCommand(getLinksListCommand()) + + return cobraext.NewCommand(cmd, cobraext.ContextBoth) +} + +func getLinksCheckCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "check", + Short: "Check for linked files changes", + Args: cobra.NoArgs, + RunE: linksCheckCommandAction, + } + return cmd +} + +func linksCheckCommandAction(cmd *cobra.Command, args []string) error { + cmd.Printf("Check for linked files changes\n") + pwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("reading current working directory failed: %w", err) + } + + linkedFiles, err := builder.AreLinkedFilesUpToDate(pwd) + if err != nil { + return fmt.Errorf("checking linked files are up-to-date failed: %w", err) + } + for _, f := range linkedFiles { + if !f.UpToDate { + cmd.Printf("%s is outdated.\n", f.Path) + } + } + if len(linkedFiles) > 0 { + return fmt.Errorf("linked files are outdated") + } + return nil +} + +func getLinksUpdateCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "update", + Short: "Update linked files checksums if needed.", + Args: cobra.NoArgs, + RunE: linksUpdateCommandAction, + } + return cmd +} + +func linksUpdateCommandAction(cmd *cobra.Command, args []string) error { + cmd.Printf("Update linked files checksums if needed.\n") + pwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("reading current working directory failed: %w", err) + } + + linkedFiles, err := builder.UpdateLinkedFilesChecksums(pwd) + if err != nil { + return fmt.Errorf("updating linked files checksums failed: %w", err) + } + + for _, f := range linkedFiles { + if !f.UpToDate { + cmd.Printf("%s is outdated.\n", f.Path) + } + } + + return nil +} + +func getLinksListCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List packages linking files from this path", + Args: cobra.NoArgs, + RunE: linksListCommandAction, + } + return cmd +} + +func linksListCommandAction(cmd *cobra.Command, args []string) error { + cmd.Printf("List packages linking files from this path.\n") + pwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("reading current working directory failed: %w", err) + } + + packages, err := builder.ListPackagesWithLinkedFilesFrom(pwd) + if err != nil { + return fmt.Errorf("listing linked packages failed: %w", err) + } + + for _, p := range packages { + cmd.Printf("%s\n", p) + } + + return nil +} diff --git a/cmd/root.go b/cmd/root.go index 52478b8b2b..e449ff5169 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -28,6 +28,7 @@ var commands = []*cobraext.Command{ setupExportCommand(), setupFormatCommand(), setupInstallCommand(), + setupLinksCommand(), setupLintCommand(), setupProfilesCommand(), setupReportsCommand(), diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go index 659b43f9fa..f853b36bcc 100644 --- a/internal/builder/linked_files.go +++ b/internal/builder/linked_files.go @@ -6,65 +6,167 @@ package builder import ( "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" "fmt" + "io" "io/fs" "os" "path/filepath" + "slices" "strings" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" ) type Link struct { - FilePath string - IncludedFilePath string + Path string + Checksum string + + TargetFilePath string + + IncludedFilePath string + IncludedFileContents []byte + IncludedFileContentsChecksum string + + UpToDate bool } -func IncludeSharedFiles(fromPath, destinationDir string) ([]Link, error) { - links, err := GetLinksFromPath(fromPath) +// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. +func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { + links, err := collectLinkedFiles(fromDir, "") if err != nil { - return nil, fmt.Errorf("could not list link files: %w", err) + return nil, fmt.Errorf("including linked files failed: %w", err) } - if len(links) == 0 { - return nil, nil + var outdated []Link + for _, l := range links { + logger.Debugf("Check if %s is up-to-date", l.Path) + if !l.UpToDate { + outdated = append(outdated, l) + } } - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return nil, fmt.Errorf("package root not found: %w", err) + return outdated, nil +} + +func IncludeLinkedFiles(fromDir, toDir string) ([]Link, error) { + links, err := collectLinkedFiles(fromDir, toDir) + if err != nil { + return nil, fmt.Errorf("including linked files failed: %w", err) } - logger.Debugf("Package has linked files defined") - // scope any possible operation in the packages/ folder - dirRoot, err := os.OpenRoot(filepath.Join(packageRootPath, "..")) + + for _, l := range links { + if err := writeFile(l.TargetFilePath, l.IncludedFileContents); err != nil { + return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) + } + if !l.UpToDate { + newContent := fmt.Sprintf("%v %v", l.IncludedFilePath, l.IncludedFileContentsChecksum) + if err := writeFile(l.Path, []byte(newContent)); err != nil { + return nil, fmt.Errorf("could not update checksum for file %v: %w", l.Path, err) + } + } + logger.Debugf("%v included in package", l.TargetFilePath) + } + + return links, nil +} + +func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { + links, err := collectLinkedFiles(fromDir, "") + if err != nil { + return nil, fmt.Errorf("updating linked files failed: %w", err) + } + + for _, l := range links { + if !l.UpToDate { + newContent := fmt.Sprintf("%v %v", l.IncludedFilePath, l.IncludedFileContentsChecksum) + if err := writeFile(l.Path, []byte(newContent)); err != nil { + return nil, fmt.Errorf("could not update checksum for file %v: %w", l.Path, err) + } + logger.Debugf("%v updated", l.Path) + } + } + + return links, nil +} + +func ListPackagesWithLinkedFilesFrom(includedPath string) ([]string, error) { + defer func() { + if err := os.Chdir(filepath.Dir(includedPath)); err != nil { + logger.Errorf("could not change directory: %w", err) + } + }() + + rootPath, err := files.FindRepositoryRootDirectory() + if err != nil { + return nil, fmt.Errorf("root not found: %w", err) + } + + links, err := collectLinkedFiles(rootPath, "") + if err != nil { + return nil, fmt.Errorf("updating linked files failed: %w", err) + } + + dirRoot, err := os.OpenRoot(rootPath) if err != nil { return nil, fmt.Errorf("could not open root: %w", err) } + m := map[string]struct{}{} for _, l := range links { - b, err := collectFile(dirRoot.FS().(fs.ReadFileFS), l.IncludedFilePath) + if _, err := dirRoot.Stat(l.IncludedFilePath); os.IsNotExist(err) { + continue + } + if err := os.Chdir(filepath.Dir(l.Path)); err != nil { + return nil, fmt.Errorf("could not change directory: %w", err) + } + p, found, err := packages.FindPackageRoot() + if !found || err != nil { + if err != nil { + logger.Errorf("could not find package root directory: %w", err) + } + continue + } + m[filepath.Base(p)] = struct{}{} + } + + packages := make([]string, 0, len(m)) + for p := range m { + packages = append(packages, p) + } + slices.Sort(packages) + return packages, nil +} + +func collectLinkedFiles(fromDir, toDir string) ([]Link, error) { + links, root, err := getLinksAndRoot(fromDir, toDir) + if err != nil { + return nil, err + } + + for i := range links { + l := links[i] + b, cs, err := collectFile(root, l.IncludedFilePath) if err != nil { return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) } - toFilePath := strings.Replace( - l.FilePath, - fromPath, - destinationDir, - 1, - ) - if err := writeFile(toFilePath, b); err != nil { - return nil, fmt.Errorf("could not write file %v: %w", toFilePath, err) + if l.Checksum == cs { + links[i].UpToDate = true } - logger.Debugf("%v included in package", l.FilePath) + links[i].IncludedFileContents = b + links[i].IncludedFileContentsChecksum = cs } return links, nil } -func GetLinksFromPath(dirPath string) ([]Link, error) { +func getLinksFrom(fromDir, toDir string) ([]Link, error) { var linkFiles []string - if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err := filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -84,19 +186,63 @@ func GetLinksFromPath(dirPath string) ([]Link, error) { if err != nil { return nil, err } - links[i].FilePath = strings.TrimSuffix(f, ".link") - links[i].IncludedFilePath = firstLine + links[i].Path = f + links[i].TargetFilePath = strings.TrimSuffix(f, ".link") + // if a destination dir is set we replace the source dir with the destination dir + if toDir != "" { + links[i].TargetFilePath = strings.Replace( + links[i].TargetFilePath, + fromDir, + toDir, + 1, + ) + } + fields := strings.Fields(firstLine) + links[i].IncludedFilePath = fields[0] + if len(fields) == 2 { + links[i].Checksum = fields[1] + } } return links, nil } -func collectFile(root fs.ReadFileFS, path string) ([]byte, error) { +func getLinksAndRoot(fromDir, toDir string) ([]Link, fs.ReadFileFS, error) { + links, err := getLinksFrom(fromDir, toDir) + if err != nil { + return nil, nil, fmt.Errorf("could not list link files: %w", err) + } + + if len(links) == 0 { + return nil, nil, nil + } + + logger.Debugf("Package has linked files defined") + + rootPath, err := files.FindRepositoryRootDirectory() + if err != nil { + return nil, nil, fmt.Errorf("root not found: %w", err) + } + + // scope any possible operation to the repository folder + dirRoot, err := os.OpenRoot(rootPath) + if err != nil { + return nil, nil, fmt.Errorf("could not open root: %w", err) + } + + return links, dirRoot.FS().(fs.ReadFileFS), nil +} + +func collectFile(root fs.ReadFileFS, path string) ([]byte, string, error) { b, err := root.ReadFile(filepath.FromSlash(path)) if err != nil { - return nil, err + return nil, "", err + } + cs, err := checksum(b) + if err != nil { + return nil, "", err } - return b, nil + return b, cs, nil } func writeFile(to string, b []byte) error { @@ -126,3 +272,11 @@ func readFirstLine(filePath string) (string, error) { return "", fmt.Errorf("file is empty or first line is missing") } + +func checksum(b []byte) (string, error) { + hash := sha256.New() + if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 9ab8592451..6f5bdc4f76 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -185,7 +185,7 @@ func BuildPackage(options BuildOptions) (string, error) { } logger.Debug("Include linked files") - _, err = IncludeSharedFiles(options.PackageRoot, destinationDir) + _, err = IncludeLinkedFiles(options.PackageRoot, destinationDir) if err != nil { return "", fmt.Errorf("including linked files failed: %w", err) } diff --git a/internal/cobraext/command.go b/internal/cobraext/command.go index d2a0e7ff45..563787c5e1 100644 --- a/internal/cobraext/command.go +++ b/internal/cobraext/command.go @@ -20,6 +20,9 @@ const ( // ContextPackage means the command runs in the contexts of a specific package. ContextPackage CommandContext = "package" + + // ContextBoth means the command can run in global and package contexts. + ContextBoth CommandContext = "both" ) // Command wraps a cobra.Command and adds some additional information relevant diff --git a/internal/elasticsearch/ingest/datastream.go b/internal/elasticsearch/ingest/datastream.go index e388909280..94fe8945a1 100644 --- a/internal/elasticsearch/ingest/datastream.go +++ b/internal/elasticsearch/ingest/datastream.go @@ -73,15 +73,15 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline") // Include shared pipelines before installing them - links, err := builder.IncludeSharedFiles(elasticsearchPath, elasticsearchPath) + links, err := builder.IncludeLinkedFiles(elasticsearchPath, elasticsearchPath) if err != nil { return nil, fmt.Errorf("including shared files failed: %w", err) } defer func() { // Remove linked files after installing them for _, link := range links { - if err := os.Remove(link.FilePath); err != nil { - logger.Errorf("Failed to remove linked file %s: %v\n", link.FilePath, err) + if err := os.Remove(link.TargetFilePath); err != nil { + logger.Errorf("failed to remove linked file %s: %v\n", link.TargetFilePath, err) } } }() diff --git a/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link b/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link index d705970720..537ff60f76 100644 --- a/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link +++ b/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link @@ -1 +1 @@ -with_includes/_dev/build/shared/stream.yml.hbs \ No newline at end of file +test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs 069381d45bffbd532a4af8953766a053e75a2aceebdafdffc2264e800fcd1363 \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link b/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link index d705970720..537ff60f76 100644 --- a/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link +++ b/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link @@ -1 +1 @@ -with_includes/_dev/build/shared/stream.yml.hbs \ No newline at end of file +test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs 069381d45bffbd532a4af8953766a053e75a2aceebdafdffc2264e800fcd1363 \ No newline at end of file From 0fb950c1a6d343bdf29488ebb0f124e333ec612c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 26 Mar 2025 11:22:49 +0100 Subject: [PATCH 08/20] Improve links commands --- cmd/links.go | 14 ++- internal/builder/linked_files.go | 97 +++++++++++-------- .../elasticsearch/ingest_pipeline/default.yml | 10 -- .../ingest_pipeline/default.yml.link | 1 + 4 files changed, 69 insertions(+), 53 deletions(-) delete mode 100644 test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link diff --git a/cmd/links.go b/cmd/links.go index a007d9782d..14524f32c2 100644 --- a/cmd/links.go +++ b/cmd/links.go @@ -14,7 +14,12 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" ) -const linksLongDescription = `` +const ( + linksLongDescription = `Use this command to manage linked files in the repository.` + linksCheckLongDescription = `Use this command to check if linked files references inside the current directory are up to date.` + linksUpdateLongDescription = `Use this command to update all linked files references inside the current directory.` + linksListLongDescription = `Use this command to list all packages that have linked file references that include the current directory.` +) func setupLinksCommand() *cobraext.Command { cmd := &cobra.Command{ @@ -37,6 +42,7 @@ func getLinksCheckCommand() *cobra.Command { cmd := &cobra.Command{ Use: "check", Short: "Check for linked files changes", + Long: linksCheckLongDescription, Args: cobra.NoArgs, RunE: linksCheckCommandAction, } @@ -56,7 +62,7 @@ func linksCheckCommandAction(cmd *cobra.Command, args []string) error { } for _, f := range linkedFiles { if !f.UpToDate { - cmd.Printf("%s is outdated.\n", f.Path) + cmd.Printf("%s is outdated.\n", f.LinkFilePath) } } if len(linkedFiles) > 0 { @@ -69,6 +75,7 @@ func getLinksUpdateCommand() *cobra.Command { cmd := &cobra.Command{ Use: "update", Short: "Update linked files checksums if needed.", + Long: linksUpdateLongDescription, Args: cobra.NoArgs, RunE: linksUpdateCommandAction, } @@ -89,7 +96,7 @@ func linksUpdateCommandAction(cmd *cobra.Command, args []string) error { for _, f := range linkedFiles { if !f.UpToDate { - cmd.Printf("%s is outdated.\n", f.Path) + cmd.Printf("%s is outdated.\n", f.LinkFilePath) } } @@ -107,7 +114,6 @@ func getLinksListCommand() *cobra.Command { } func linksListCommandAction(cmd *cobra.Command, args []string) error { - cmd.Printf("List packages linking files from this path.\n") pwd, err := os.Getwd() if err != nil { return fmt.Errorf("reading current working directory failed: %w", err) diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go index f853b36bcc..09bc2aff86 100644 --- a/internal/builder/linked_files.go +++ b/internal/builder/linked_files.go @@ -23,11 +23,15 @@ import ( ) type Link struct { - Path string - Checksum string + LinkPackageName string + LinkPackagePath string + LinkFilePath string + LinkChecksum string TargetFilePath string + IncludedPackageName string + IncludedPackagePath string IncludedFilePath string IncludedFileContents []byte IncludedFileContentsChecksum string @@ -44,7 +48,7 @@ func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { var outdated []Link for _, l := range links { - logger.Debugf("Check if %s is up-to-date", l.Path) + logger.Debugf("Check if %s is up-to-date", l.LinkFilePath) if !l.UpToDate { outdated = append(outdated, l) } @@ -65,9 +69,10 @@ func IncludeLinkedFiles(fromDir, toDir string) ([]Link, error) { } if !l.UpToDate { newContent := fmt.Sprintf("%v %v", l.IncludedFilePath, l.IncludedFileContentsChecksum) - if err := writeFile(l.Path, []byte(newContent)); err != nil { - return nil, fmt.Errorf("could not update checksum for file %v: %w", l.Path, err) + if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { + return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) } + logger.Debugf("%v updated", l.LinkFilePath) } logger.Debugf("%v included in package", l.TargetFilePath) } @@ -84,19 +89,19 @@ func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { for _, l := range links { if !l.UpToDate { newContent := fmt.Sprintf("%v %v", l.IncludedFilePath, l.IncludedFileContentsChecksum) - if err := writeFile(l.Path, []byte(newContent)); err != nil { - return nil, fmt.Errorf("could not update checksum for file %v: %w", l.Path, err) + if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { + return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) } - logger.Debugf("%v updated", l.Path) + logger.Debugf("%v updated", l.LinkFilePath) } } return links, nil } -func ListPackagesWithLinkedFilesFrom(includedPath string) ([]string, error) { +func ListPackagesWithLinkedFilesFrom(fromPath string) ([]string, error) { defer func() { - if err := os.Chdir(filepath.Dir(includedPath)); err != nil { + if err := os.Chdir(filepath.Dir(fromPath)); err != nil { logger.Errorf("could not change directory: %w", err) } }() @@ -108,30 +113,18 @@ func ListPackagesWithLinkedFilesFrom(includedPath string) ([]string, error) { links, err := collectLinkedFiles(rootPath, "") if err != nil { - return nil, fmt.Errorf("updating linked files failed: %w", err) - } - - dirRoot, err := os.OpenRoot(rootPath) - if err != nil { - return nil, fmt.Errorf("could not open root: %w", err) + return nil, fmt.Errorf("collect linked files failed: %w", err) } - + packagePath, _, _ := packages.FindPackageRoot() + packageName := filepath.Base(packagePath) m := map[string]struct{}{} for _, l := range links { - if _, err := dirRoot.Stat(l.IncludedFilePath); os.IsNotExist(err) { - continue - } - if err := os.Chdir(filepath.Dir(l.Path)); err != nil { - return nil, fmt.Errorf("could not change directory: %w", err) - } - p, found, err := packages.FindPackageRoot() - if !found || err != nil { - if err != nil { - logger.Errorf("could not find package root directory: %w", err) - } + if l.LinkPackageName == "" || + l.LinkPackageName == l.IncludedPackageName || + l.IncludedPackageName != packageName { continue } - m[filepath.Base(p)] = struct{}{} + m[l.LinkPackageName] = struct{}{} } packages := make([]string, 0, len(m)) @@ -154,7 +147,7 @@ func collectLinkedFiles(fromDir, toDir string) ([]Link, error) { if err != nil { return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) } - if l.Checksum == cs { + if l.LinkChecksum == cs { links[i].UpToDate = true } links[i].IncludedFileContents = b @@ -164,7 +157,17 @@ func collectLinkedFiles(fromDir, toDir string) ([]Link, error) { return links, nil } -func getLinksFrom(fromDir, toDir string) ([]Link, error) { +func getLinksFrom(fromDir, toDir, rootPath string) ([]Link, error) { + pwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("reading current working directory failed: %w", err) + } + defer func() { + if err := os.Chdir(pwd); err != nil { + logger.Errorf("could not change directory: %w", err) + } + }() + var linkFiles []string if err := filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -186,7 +189,7 @@ func getLinksFrom(fromDir, toDir string) ([]Link, error) { if err != nil { return nil, err } - links[i].Path = f + links[i].LinkFilePath = f links[i].TargetFilePath = strings.TrimSuffix(f, ".link") // if a destination dir is set we replace the source dir with the destination dir if toDir != "" { @@ -200,15 +203,36 @@ func getLinksFrom(fromDir, toDir string) ([]Link, error) { fields := strings.Fields(firstLine) links[i].IncludedFilePath = fields[0] if len(fields) == 2 { - links[i].Checksum = fields[1] + links[i].LinkChecksum = fields[1] } + + if err := os.Chdir(filepath.Dir(filepath.Join(rootPath, links[i].IncludedFilePath))); err != nil { + return nil, fmt.Errorf("could not change directory: %w", err) + } + + p, _, _ := packages.FindPackageRoot() + links[i].IncludedPackageName = filepath.Base(p) + links[i].IncludedPackagePath = p + + if err := os.Chdir(filepath.Dir(links[i].LinkFilePath)); err != nil { + return nil, fmt.Errorf("could not change directory: %w", err) + } + + p, _, _ = packages.FindPackageRoot() + links[i].LinkPackageName = filepath.Base(p) + links[i].LinkPackagePath = p } return links, nil } func getLinksAndRoot(fromDir, toDir string) ([]Link, fs.ReadFileFS, error) { - links, err := getLinksFrom(fromDir, toDir) + rootPath, err := files.FindRepositoryRootDirectory() + if err != nil { + return nil, nil, fmt.Errorf("root not found: %w", err) + } + + links, err := getLinksFrom(fromDir, toDir, rootPath) if err != nil { return nil, nil, fmt.Errorf("could not list link files: %w", err) } @@ -219,11 +243,6 @@ func getLinksAndRoot(fromDir, toDir string) ([]Link, fs.ReadFileFS, error) { logger.Debugf("Package has linked files defined") - rootPath, err := files.FindRepositoryRootDirectory() - if err != nil { - return nil, nil, fmt.Errorf("root not found: %w", err) - } - // scope any possible operation to the repository folder dirRoot, err := os.OpenRoot(rootPath) if err != nil { diff --git a/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml deleted file mode 100644 index 81221adf3f..0000000000 --- a/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -description: Pipeline for processing sample logs -processors: -- set: - field: sample_field - value: "1" -on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link b/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link new file mode 100644 index 0000000000..99b4d232ab --- /dev/null +++ b/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link @@ -0,0 +1 @@ +test/packages/other/pipeline_tests/data_stream/test/elasticsearch/ingest_pipeline/default.yml f7c5f0c03aca8ef68c379a62447bdafbf0dcf32b1ff2de143fd6878ee01a91ad \ No newline at end of file From d4f4c367ca30dbb31f597461be034cadb693b33d Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 26 Mar 2025 11:32:29 +0100 Subject: [PATCH 09/20] List also if not in package --- internal/builder/linked_files.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go index 09bc2aff86..003d00192d 100644 --- a/internal/builder/linked_files.go +++ b/internal/builder/linked_files.go @@ -119,8 +119,7 @@ func ListPackagesWithLinkedFilesFrom(fromPath string) ([]string, error) { packageName := filepath.Base(packagePath) m := map[string]struct{}{} for _, l := range links { - if l.LinkPackageName == "" || - l.LinkPackageName == l.IncludedPackageName || + if l.LinkPackageName == l.IncludedPackageName || l.IncludedPackageName != packageName { continue } From f044059bcb41097e5f792960acb83749f0c67989 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 27 Mar 2025 17:00:40 +0100 Subject: [PATCH 10/20] Reorganize code --- cmd/links.go | 34 ++-- internal/builder/linked_files.go | 300 ------------------------------- internal/builder/linkedfiles.go | 38 ++++ internal/builder/packages.go | 3 +- internal/cobraext/command.go | 3 - internal/cobraext/flags.go | 3 + internal/files/linkedfiles.go | 264 +++++++++++++++++++++++++++ internal/files/repository.go | 15 ++ internal/packages/packages.go | 10 +- 9 files changed, 351 insertions(+), 319 deletions(-) delete mode 100644 internal/builder/linked_files.go create mode 100644 internal/builder/linkedfiles.go create mode 100644 internal/files/linkedfiles.go diff --git a/cmd/links.go b/cmd/links.go index 14524f32c2..ab83b1571f 100644 --- a/cmd/links.go +++ b/cmd/links.go @@ -10,8 +10,8 @@ import ( "github.com/spf13/cobra" - "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/files" ) const ( @@ -35,7 +35,7 @@ func setupLinksCommand() *cobraext.Command { cmd.AddCommand(getLinksUpdateCommand()) cmd.AddCommand(getLinksListCommand()) - return cobraext.NewCommand(cmd, cobraext.ContextBoth) + return cobraext.NewCommand(cmd, cobraext.ContextGlobal) } func getLinksCheckCommand() *cobra.Command { @@ -56,7 +56,7 @@ func linksCheckCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("reading current working directory failed: %w", err) } - linkedFiles, err := builder.AreLinkedFilesUpToDate(pwd) + linkedFiles, err := files.AreLinkedFilesUpToDate(pwd) if err != nil { return fmt.Errorf("checking linked files are up-to-date failed: %w", err) } @@ -89,15 +89,13 @@ func linksUpdateCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("reading current working directory failed: %w", err) } - linkedFiles, err := builder.UpdateLinkedFilesChecksums(pwd) + updatedLinks, err := files.UpdateLinkedFilesChecksums(pwd) if err != nil { return fmt.Errorf("updating linked files checksums failed: %w", err) } - for _, f := range linkedFiles { - if !f.UpToDate { - cmd.Printf("%s is outdated.\n", f.LinkFilePath) - } + for _, l := range updatedLinks { + cmd.Printf("%s was updated.\n", l.LinkFilePath) } return nil @@ -110,22 +108,36 @@ func getLinksListCommand() *cobra.Command { Args: cobra.NoArgs, RunE: linksListCommandAction, } + cmd.Flags().BoolP(cobraext.PackagesFlagName, "", false, cobraext.PackagesFlagDescription) return cmd } func linksListCommandAction(cmd *cobra.Command, args []string) error { + onlyPackages, err := cmd.Flags().GetBool(cobraext.PackagesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.PackagesFlagName) + } + pwd, err := os.Getwd() if err != nil { return fmt.Errorf("reading current working directory failed: %w", err) } - packages, err := builder.ListPackagesWithLinkedFilesFrom(pwd) + byPackage, err := files.LinkedFilesByPackageFrom(pwd) if err != nil { return fmt.Errorf("listing linked packages failed: %w", err) } - for _, p := range packages { - cmd.Printf("%s\n", p) + for i := range byPackage { + for p, links := range byPackage[i] { + if onlyPackages { + cmd.Println(p) + continue + } + for _, l := range links { + cmd.Println(l) + } + } } return nil diff --git a/internal/builder/linked_files.go b/internal/builder/linked_files.go deleted file mode 100644 index 003d00192d..0000000000 --- a/internal/builder/linked_files.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package builder - -import ( - "bufio" - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "slices" - "strings" - - "github.com/elastic/elastic-package/internal/files" - "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/packages" -) - -type Link struct { - LinkPackageName string - LinkPackagePath string - LinkFilePath string - LinkChecksum string - - TargetFilePath string - - IncludedPackageName string - IncludedPackagePath string - IncludedFilePath string - IncludedFileContents []byte - IncludedFileContentsChecksum string - - UpToDate bool -} - -// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. -func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { - links, err := collectLinkedFiles(fromDir, "") - if err != nil { - return nil, fmt.Errorf("including linked files failed: %w", err) - } - - var outdated []Link - for _, l := range links { - logger.Debugf("Check if %s is up-to-date", l.LinkFilePath) - if !l.UpToDate { - outdated = append(outdated, l) - } - } - - return outdated, nil -} - -func IncludeLinkedFiles(fromDir, toDir string) ([]Link, error) { - links, err := collectLinkedFiles(fromDir, toDir) - if err != nil { - return nil, fmt.Errorf("including linked files failed: %w", err) - } - - for _, l := range links { - if err := writeFile(l.TargetFilePath, l.IncludedFileContents); err != nil { - return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) - } - if !l.UpToDate { - newContent := fmt.Sprintf("%v %v", l.IncludedFilePath, l.IncludedFileContentsChecksum) - if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { - return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) - } - logger.Debugf("%v updated", l.LinkFilePath) - } - logger.Debugf("%v included in package", l.TargetFilePath) - } - - return links, nil -} - -func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { - links, err := collectLinkedFiles(fromDir, "") - if err != nil { - return nil, fmt.Errorf("updating linked files failed: %w", err) - } - - for _, l := range links { - if !l.UpToDate { - newContent := fmt.Sprintf("%v %v", l.IncludedFilePath, l.IncludedFileContentsChecksum) - if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { - return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) - } - logger.Debugf("%v updated", l.LinkFilePath) - } - } - - return links, nil -} - -func ListPackagesWithLinkedFilesFrom(fromPath string) ([]string, error) { - defer func() { - if err := os.Chdir(filepath.Dir(fromPath)); err != nil { - logger.Errorf("could not change directory: %w", err) - } - }() - - rootPath, err := files.FindRepositoryRootDirectory() - if err != nil { - return nil, fmt.Errorf("root not found: %w", err) - } - - links, err := collectLinkedFiles(rootPath, "") - if err != nil { - return nil, fmt.Errorf("collect linked files failed: %w", err) - } - packagePath, _, _ := packages.FindPackageRoot() - packageName := filepath.Base(packagePath) - m := map[string]struct{}{} - for _, l := range links { - if l.LinkPackageName == l.IncludedPackageName || - l.IncludedPackageName != packageName { - continue - } - m[l.LinkPackageName] = struct{}{} - } - - packages := make([]string, 0, len(m)) - for p := range m { - packages = append(packages, p) - } - slices.Sort(packages) - return packages, nil -} - -func collectLinkedFiles(fromDir, toDir string) ([]Link, error) { - links, root, err := getLinksAndRoot(fromDir, toDir) - if err != nil { - return nil, err - } - - for i := range links { - l := links[i] - b, cs, err := collectFile(root, l.IncludedFilePath) - if err != nil { - return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) - } - if l.LinkChecksum == cs { - links[i].UpToDate = true - } - links[i].IncludedFileContents = b - links[i].IncludedFileContentsChecksum = cs - } - - return links, nil -} - -func getLinksFrom(fromDir, toDir, rootPath string) ([]Link, error) { - pwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("reading current working directory failed: %w", err) - } - defer func() { - if err := os.Chdir(pwd); err != nil { - logger.Errorf("could not change directory: %w", err) - } - }() - - var linkFiles []string - if err := filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() && strings.HasSuffix(info.Name(), ".link") { - linkFiles = append(linkFiles, path) - } - return nil - }); err != nil { - return nil, err - } - - links := make([]Link, len(linkFiles)) - - for i, f := range linkFiles { - firstLine, err := readFirstLine(f) - if err != nil { - return nil, err - } - links[i].LinkFilePath = f - links[i].TargetFilePath = strings.TrimSuffix(f, ".link") - // if a destination dir is set we replace the source dir with the destination dir - if toDir != "" { - links[i].TargetFilePath = strings.Replace( - links[i].TargetFilePath, - fromDir, - toDir, - 1, - ) - } - fields := strings.Fields(firstLine) - links[i].IncludedFilePath = fields[0] - if len(fields) == 2 { - links[i].LinkChecksum = fields[1] - } - - if err := os.Chdir(filepath.Dir(filepath.Join(rootPath, links[i].IncludedFilePath))); err != nil { - return nil, fmt.Errorf("could not change directory: %w", err) - } - - p, _, _ := packages.FindPackageRoot() - links[i].IncludedPackageName = filepath.Base(p) - links[i].IncludedPackagePath = p - - if err := os.Chdir(filepath.Dir(links[i].LinkFilePath)); err != nil { - return nil, fmt.Errorf("could not change directory: %w", err) - } - - p, _, _ = packages.FindPackageRoot() - links[i].LinkPackageName = filepath.Base(p) - links[i].LinkPackagePath = p - } - - return links, nil -} - -func getLinksAndRoot(fromDir, toDir string) ([]Link, fs.ReadFileFS, error) { - rootPath, err := files.FindRepositoryRootDirectory() - if err != nil { - return nil, nil, fmt.Errorf("root not found: %w", err) - } - - links, err := getLinksFrom(fromDir, toDir, rootPath) - if err != nil { - return nil, nil, fmt.Errorf("could not list link files: %w", err) - } - - if len(links) == 0 { - return nil, nil, nil - } - - logger.Debugf("Package has linked files defined") - - // scope any possible operation to the repository folder - dirRoot, err := os.OpenRoot(rootPath) - if err != nil { - return nil, nil, fmt.Errorf("could not open root: %w", err) - } - - return links, dirRoot.FS().(fs.ReadFileFS), nil -} - -func collectFile(root fs.ReadFileFS, path string) ([]byte, string, error) { - b, err := root.ReadFile(filepath.FromSlash(path)) - if err != nil { - return nil, "", err - } - cs, err := checksum(b) - if err != nil { - return nil, "", err - } - return b, cs, nil -} - -func writeFile(to string, b []byte) error { - if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { - return err - } - } - return os.WriteFile(filepath.FromSlash(to), b, 0644) -} - -func readFirstLine(filePath string) (string, error) { - file, err := os.Open(filePath) - if err != nil { - return "", err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - if scanner.Scan() { - return scanner.Text(), nil - } - - if err := scanner.Err(); err != nil { - return "", err - } - - return "", fmt.Errorf("file is empty or first line is missing") -} - -func checksum(b []byte) (string, error) { - hash := sha256.New() - if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil { - return "", err - } - return hex.EncodeToString(hash.Sum(nil)), nil -} diff --git a/internal/builder/linkedfiles.go b/internal/builder/linkedfiles.go new file mode 100644 index 0000000000..0d4b745250 --- /dev/null +++ b/internal/builder/linkedfiles.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package builder + +import ( + "fmt" + + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/logger" +) + +func IncludeLinkedFiles(fromDir, toDir string) ([]files.Link, error) { + links, err := files.ListLinkedFiles(fromDir) + if err != nil { + return nil, fmt.Errorf("including linked files failed: %w", err) + } + + for _, l := range links { + l.ReplaceTargetFilePathDirectory(fromDir, toDir) + + if err := files.WriteFile(l.TargetFilePath, l.IncludedFileContents); err != nil { + return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) + } + + updated, err := l.UpdateChecksum() + if err != nil { + return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) + } + + if updated { + logger.Debugf("%v included in package", l.TargetFilePath) + } + } + + return links, nil +} diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 6f5bdc4f76..6889822695 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -185,8 +185,7 @@ func BuildPackage(options BuildOptions) (string, error) { } logger.Debug("Include linked files") - _, err = IncludeLinkedFiles(options.PackageRoot, destinationDir) - if err != nil { + if _, err := IncludeLinkedFiles(options.PackageRoot, destinationDir); err != nil { return "", fmt.Errorf("including linked files failed: %w", err) } diff --git a/internal/cobraext/command.go b/internal/cobraext/command.go index 563787c5e1..d2a0e7ff45 100644 --- a/internal/cobraext/command.go +++ b/internal/cobraext/command.go @@ -20,9 +20,6 @@ const ( // ContextPackage means the command runs in the contexts of a specific package. ContextPackage CommandContext = "package" - - // ContextBoth means the command can run in global and package contexts. - ContextBoth CommandContext = "both" ) // Command wraps a cobra.Command and adds some additional information relevant diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index c8f19eff22..fda51d0de0 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -138,6 +138,9 @@ const ( GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" + PackagesFlagName = "packages" + PackagesFlagDescription = "whether to return packages names or complete paths for the linked files found" + ProfileFlagName = "profile" ProfileFlagDescription = "select a profile to use for the stack configuration. Can also be set with %s" diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go new file mode 100644 index 0000000000..5372ae6265 --- /dev/null +++ b/internal/files/linkedfiles.go @@ -0,0 +1,264 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package files + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" +) + +const linkExtension = ".link" + +type Link struct { + LinkFilePath string + LinkChecksum string + + TargetFilePath string + + IncludedFilePath string + IncludedFileContents []byte + IncludedFileContentsChecksum string + + UpToDate bool +} + +func NewLinkedFile(linkFilePath string) (Link, error) { + var l Link + firstLine, err := readFirstLine(linkFilePath) + if err != nil { + return l, err + } + l.LinkFilePath = linkFilePath + l.TargetFilePath = strings.TrimSuffix(linkFilePath, linkExtension) + fields := strings.Fields(firstLine) + l.IncludedFilePath = fields[0] + if len(fields) == 2 { + l.LinkChecksum = fields[1] + } + return l, nil +} + +func (l *Link) UpdateChecksum() (bool, error) { + if l.UpToDate { + return false, nil + } + if l.IncludedFilePath == "" { + return false, fmt.Errorf("file path is empty for file %v", l.IncludedFilePath) + } + if l.IncludedFileContentsChecksum == "" { + return false, fmt.Errorf("checksum is empty for file %v", l.IncludedFilePath) + } + newContent := fmt.Sprintf("%v %v", filepath.ToSlash(l.IncludedFilePath), l.IncludedFileContentsChecksum) + if err := WriteFile(l.LinkFilePath, []byte(newContent)); err != nil { + return false, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) + } + return true, nil +} + +func (l *Link) ReplaceTargetFilePathDirectory(fromDir, toDir string) { + // if a destination dir is set we replace the source dir with the destination dir + if toDir == "" { + return + } + l.TargetFilePath = strings.Replace( + l.TargetFilePath, + fromDir, + toDir, + 1, + ) +} + +// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. +func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { + links, err := ListLinkedFiles(fromDir) + if err != nil { + return nil, fmt.Errorf("including linked files failed: %w", err) + } + + var outdated []Link + for _, l := range links { + logger.Debugf("Check if %s is up-to-date", l.LinkFilePath) + if !l.UpToDate { + outdated = append(outdated, l) + } + } + + return outdated, nil +} + +func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { + links, err := ListLinkedFiles(fromDir) + if err != nil { + return nil, fmt.Errorf("updating linked files checksums failed: %w", err) + } + + var updatedLinks []Link + for _, l := range links { + updated, err := l.UpdateChecksum() + if err != nil { + return nil, fmt.Errorf("updating linked files checksums failed: %w", err) + } + if updated { + updatedLinks = append(updatedLinks, l) + } + } + + return updatedLinks, nil +} + +func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { + rootPath, err := FindRepositoryRootDirectory() + if err != nil { + return nil, fmt.Errorf("locating repository root failed: %w", err) + } + links, err := ListLinkedFiles(rootPath) + if err != nil { + return nil, fmt.Errorf("including linked files failed: %w", err) + } + + packageRoot, _, _ := packages.FindPackageRootFrom(fromDir) + packageName := filepath.Base(packageRoot) + byPackageMap := map[string][]string{} + for _, l := range links { + linkPackageRoot, _, _ := packages.FindPackageRootFrom(l.LinkFilePath) + linkPackageName := filepath.Base(linkPackageRoot) + includedPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Join(rootPath, l.IncludedFilePath)) + includedPackageName := filepath.Base(includedPackageRoot) + if linkPackageName == includedPackageName || + packageName != includedPackageName { + continue + } + byPackageMap[linkPackageName] = append(byPackageMap[linkPackageName], l.LinkFilePath) + } + + var packages []string + for p := range byPackageMap { + packages = append(packages, p) + } + slices.Sort(packages) + + var byPackage []map[string][]string + for _, p := range packages { + m := map[string][]string{p: byPackageMap[p]} + byPackage = append(byPackage, m) + } + return byPackage, nil +} + +func ListLinkedFiles(fromDir string) ([]Link, error) { + links, err := getLinksFrom(fromDir) + if err != nil { + return nil, err + } + + root, err := FindRepositoryRoot() + if err != nil { + return nil, fmt.Errorf("could not get root: %w", err) + } + + for i := range links { + l := links[i] + b, cs, err := collectFile(root.FS().(fs.ReadFileFS), l.IncludedFilePath) + if err != nil { + return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) + } + if l.LinkChecksum == cs { + links[i].UpToDate = true + } + links[i].IncludedFileContents = b + links[i].IncludedFileContentsChecksum = cs + } + + return links, nil +} + +func WriteFile(to string, b []byte) error { + to = filepath.FromSlash(to) + if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { + return err + } + } + return os.WriteFile(to, b, 0644) +} + +func getLinksFrom(fromDir string) ([]Link, error) { + var linkFiles []string + if err := filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() && strings.HasSuffix(info.Name(), linkExtension) { + linkFiles = append(linkFiles, path) + } + return nil + }); err != nil { + return nil, err + } + + links := make([]Link, len(linkFiles)) + + for i, f := range linkFiles { + l, err := NewLinkedFile(f) + if err != nil { + return nil, fmt.Errorf("could not create linked file %v: %w", f, err) + } + links[i] = l + } + + return links, nil +} + +func collectFile(root fs.ReadFileFS, path string) ([]byte, string, error) { + b, err := root.ReadFile(filepath.FromSlash(path)) + if err != nil { + return nil, "", err + } + cs, err := checksum(b) + if err != nil { + return nil, "", err + } + return b, cs, nil +} + +func readFirstLine(filePath string) (string, error) { + file, err := os.Open(filepath.FromSlash(filePath)) + if err != nil { + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + if scanner.Scan() { + return scanner.Text(), nil + } + + if err := scanner.Err(); err != nil { + return "", err + } + + return "", fmt.Errorf("file is empty or first line is missing") +} + +func checksum(b []byte) (string, error) { + hash := sha256.New() + if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/internal/files/repository.go b/internal/files/repository.go index 9519e08757..d7ae0bf68d 100644 --- a/internal/files/repository.go +++ b/internal/files/repository.go @@ -13,6 +13,21 @@ import ( "gopkg.in/yaml.v3" ) +func FindRepositoryRoot() (*os.Root, error) { + rootPath, err := FindRepositoryRootDirectory() + if err != nil { + return nil, fmt.Errorf("root not found: %w", err) + } + + // scope any possible operation to the repository folder + dirRoot, err := os.OpenRoot(rootPath) + if err != nil { + return nil, fmt.Errorf("could not open root: %w", err) + } + + return dirRoot, nil +} + func FindRepositoryRootDirectory() (string, error) { workDir, err := os.Getwd() if err != nil { diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 30d505f949..7000d4382b 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -239,18 +239,22 @@ func MustFindPackageRoot() (string, error) { return root, nil } -// FindPackageRoot finds and returns the path to the root folder of a package. +// FindPackageRoot finds and returns the path to the root folder of a package from the working directory. func FindPackageRoot() (string, bool, error) { workDir, err := os.Getwd() if err != nil { return "", false, fmt.Errorf("locating working directory failed: %w", err) } + return FindPackageRootFrom(workDir) +} +// FindPackageRootFrom finds and returns the path to the root folder of a package from a given directory. +func FindPackageRootFrom(fromDir string) (string, bool, error) { // VolumeName() will return something like "C:" in Windows, and "" in other OSs // rootDir will be something like "C:\" in Windows, and "/" everywhere else. - rootDir := filepath.VolumeName(workDir) + string(filepath.Separator) + rootDir := filepath.VolumeName(fromDir) + string(filepath.Separator) - dir := workDir + dir := fromDir for dir != "." { path := filepath.Join(dir, PackageManifestFile) fileInfo, err := os.Stat(path) From 44a1d1ac5654421a6890fe460dade4755af854b7 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 27 Mar 2025 17:02:59 +0100 Subject: [PATCH 11/20] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 2f168891a1..2c31a16699 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,6 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU= github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= -github.com/elastic/package-spec/v3 v3.3.4 h1:lfeNGHRzJauOu342bxXQ9hHMO+D118ijUz0bYwMq4O0= -github.com/elastic/package-spec/v3 v3.3.4/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= From afc4b26621199c13da659323d256db20ab301a0b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 28 Mar 2025 08:42:04 +0100 Subject: [PATCH 12/20] Only read entire file when copying --- internal/builder/linkedfiles.go | 7 ++--- internal/files/linkedfiles.go | 53 ++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/internal/builder/linkedfiles.go b/internal/builder/linkedfiles.go index 0d4b745250..0a73272acf 100644 --- a/internal/builder/linkedfiles.go +++ b/internal/builder/linkedfiles.go @@ -20,16 +20,15 @@ func IncludeLinkedFiles(fromDir, toDir string) ([]files.Link, error) { for _, l := range links { l.ReplaceTargetFilePathDirectory(fromDir, toDir) - if err := files.WriteFile(l.TargetFilePath, l.IncludedFileContents); err != nil { - return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) - } - updated, err := l.UpdateChecksum() if err != nil { return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) } if updated { + if err := files.CopyFile(l.IncludedFilePath, l.TargetFilePath); err != nil { + return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) + } logger.Debugf("%v included in package", l.TargetFilePath) } } diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go index 5372ae6265..b2b7a18dba 100644 --- a/internal/files/linkedfiles.go +++ b/internal/files/linkedfiles.go @@ -30,13 +30,12 @@ type Link struct { TargetFilePath string IncludedFilePath string - IncludedFileContents []byte IncludedFileContentsChecksum string UpToDate bool } -func NewLinkedFile(linkFilePath string) (Link, error) { +func newLinkedFile(linkFilePath string) (Link, error) { var l Link firstLine, err := readFirstLine(linkFilePath) if err != nil { @@ -52,6 +51,8 @@ func NewLinkedFile(linkFilePath string) (Link, error) { return l, nil } +// UpdateChecksum function updates the checksum of the linked file. +// It returns true if the checksum was updated, false if it was already up-to-date. func (l *Link) UpdateChecksum() (bool, error) { if l.UpToDate { return false, nil @@ -63,12 +64,13 @@ func (l *Link) UpdateChecksum() (bool, error) { return false, fmt.Errorf("checksum is empty for file %v", l.IncludedFilePath) } newContent := fmt.Sprintf("%v %v", filepath.ToSlash(l.IncludedFilePath), l.IncludedFileContentsChecksum) - if err := WriteFile(l.LinkFilePath, []byte(newContent)); err != nil { + if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { return false, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) } return true, nil } +// ReplaceTargetFilePathDirectory function replaces the target file path directory. func (l *Link) ReplaceTargetFilePathDirectory(fromDir, toDir string) { // if a destination dir is set we replace the source dir with the destination dir if toDir == "" { @@ -100,6 +102,9 @@ func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { return outdated, nil } +// UpdateLinkedFilesChecksums function updates the checksums of the linked files. +// It returns a slice of updated links. +// If no links were updated, it returns an empty slice. func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { links, err := ListLinkedFiles(fromDir) if err != nil { @@ -120,6 +125,8 @@ func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { return updatedLinks, nil } +// LinkedFilesByPackageFrom function returns a slice of maps containing linked files grouped by package. +// Each map contains the package name as the key and a slice of linked file paths as the value. func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { rootPath, err := FindRepositoryRootDirectory() if err != nil { @@ -159,6 +166,7 @@ func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { return byPackage, nil } +// ListLinkedFiles function returns a slice of Link structs representing linked files. func ListLinkedFiles(fromDir string) ([]Link, error) { links, err := getLinksFrom(fromDir) if err != nil { @@ -172,21 +180,44 @@ func ListLinkedFiles(fromDir string) ([]Link, error) { for i := range links { l := links[i] - b, cs, err := collectFile(root.FS().(fs.ReadFileFS), l.IncludedFilePath) + cs, err := getLinkedFileChecksum(root.FS().(fs.ReadFileFS), l.IncludedFilePath) if err != nil { return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) } if l.LinkChecksum == cs { links[i].UpToDate = true } - links[i].IncludedFileContents = b links[i].IncludedFileContentsChecksum = cs } return links, nil } -func WriteFile(to string, b []byte) error { +func CopyFile(from, to string) error { + from = filepath.FromSlash(from) + source, err := os.Open(from) + if err != nil { + return err + } + defer source.Close() + + to = filepath.FromSlash(to) + if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { + return err + } + } + destination, err := os.Create(to) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +func writeFile(to string, b []byte) error { to = filepath.FromSlash(to) if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { @@ -214,7 +245,7 @@ func getLinksFrom(fromDir string) ([]Link, error) { links := make([]Link, len(linkFiles)) for i, f := range linkFiles { - l, err := NewLinkedFile(f) + l, err := newLinkedFile(f) if err != nil { return nil, fmt.Errorf("could not create linked file %v: %w", f, err) } @@ -224,16 +255,16 @@ func getLinksFrom(fromDir string) ([]Link, error) { return links, nil } -func collectFile(root fs.ReadFileFS, path string) ([]byte, string, error) { +func getLinkedFileChecksum(root fs.ReadFileFS, path string) (string, error) { b, err := root.ReadFile(filepath.FromSlash(path)) if err != nil { - return nil, "", err + return "", err } cs, err := checksum(b) if err != nil { - return nil, "", err + return "", err } - return b, cs, nil + return cs, nil } func readFirstLine(filePath string) (string, error) { From 53eead0ab06980a4e09cea864986af8954fc3263 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 31 Mar 2025 13:02:24 +0200 Subject: [PATCH 13/20] Add unit tests --- internal/files/linkedfiles.go | 38 ++--- internal/files/linkedfiles_test.go | 150 ++++++++++++++++++ internal/files/testdata/links/included.yml | 3 + .../files/testdata/links/outdated.yml.link | 1 + .../files/testdata/links/uptodate.yml.link | 1 + .../testdata/testpackage/included.yml.link | 1 + .../files/testdata/testpackage/manifest.yml | 20 +++ 7 files changed, 195 insertions(+), 19 deletions(-) create mode 100644 internal/files/linkedfiles_test.go create mode 100644 internal/files/testdata/links/included.yml create mode 100644 internal/files/testdata/links/outdated.yml.link create mode 100644 internal/files/testdata/links/uptodate.yml.link create mode 100644 internal/files/testdata/testpackage/included.yml.link create mode 100644 internal/files/testdata/testpackage/manifest.yml diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go index b2b7a18dba..fdb9bc3866 100644 --- a/internal/files/linkedfiles.go +++ b/internal/files/linkedfiles.go @@ -35,11 +35,11 @@ type Link struct { UpToDate bool } -func newLinkedFile(linkFilePath string) (Link, error) { +func newLinkedFile(root *os.Root, linkFilePath string) (Link, error) { var l Link firstLine, err := readFirstLine(linkFilePath) if err != nil { - return l, err + return Link{}, err } l.LinkFilePath = linkFilePath l.TargetFilePath = strings.TrimSuffix(linkFilePath, linkExtension) @@ -48,6 +48,16 @@ func newLinkedFile(linkFilePath string) (Link, error) { if len(fields) == 2 { l.LinkChecksum = fields[1] } + + cs, err := getLinkedFileChecksum(root.FS().(fs.ReadFileFS), l.IncludedFilePath) + if err != nil { + return Link{}, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) + } + if l.LinkChecksum == cs { + l.UpToDate = true + } + l.IncludedFileContentsChecksum = cs + return l, nil } @@ -67,6 +77,8 @@ func (l *Link) UpdateChecksum() (bool, error) { if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { return false, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) } + l.LinkChecksum = l.IncludedFileContentsChecksum + l.UpToDate = true return true, nil } @@ -168,26 +180,14 @@ func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { // ListLinkedFiles function returns a slice of Link structs representing linked files. func ListLinkedFiles(fromDir string) ([]Link, error) { - links, err := getLinksFrom(fromDir) - if err != nil { - return nil, err - } - root, err := FindRepositoryRoot() if err != nil { return nil, fmt.Errorf("could not get root: %w", err) } - for i := range links { - l := links[i] - cs, err := getLinkedFileChecksum(root.FS().(fs.ReadFileFS), l.IncludedFilePath) - if err != nil { - return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) - } - if l.LinkChecksum == cs { - links[i].UpToDate = true - } - links[i].IncludedFileContentsChecksum = cs + links, err := getLinksFrom(root, fromDir) + if err != nil { + return nil, err } return links, nil @@ -227,7 +227,7 @@ func writeFile(to string, b []byte) error { return os.WriteFile(to, b, 0644) } -func getLinksFrom(fromDir string) ([]Link, error) { +func getLinksFrom(root *os.Root, fromDir string) ([]Link, error) { var linkFiles []string if err := filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -245,7 +245,7 @@ func getLinksFrom(fromDir string) ([]Link, error) { links := make([]Link, len(linkFiles)) for i, f := range linkFiles { - l, err := newLinkedFile(f) + l, err := newLinkedFile(root, f) if err != nil { return nil, fmt.Errorf("could not create linked file %v: %w", f, err) } diff --git a/internal/files/linkedfiles_test.go b/internal/files/linkedfiles_test.go new file mode 100644 index 0000000000..c3a85c911a --- /dev/null +++ b/internal/files/linkedfiles_test.go @@ -0,0 +1,150 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package files + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLinkUpdateChecksum(t *testing.T) { + root, err := FindRepositoryRoot() + assert.NoError(t, err) + + outdatedFile, err := newLinkedFile(root, "testdata/links/outdated.yml.link") + t.Cleanup(func() { + _ = writeFile(outdatedFile.LinkFilePath, []byte(outdatedFile.IncludedFilePath)) + }) + assert.NoError(t, err) + assert.False(t, outdatedFile.UpToDate) + assert.Empty(t, outdatedFile.LinkChecksum) + updated, err := outdatedFile.UpdateChecksum() + assert.NoError(t, err) + assert.True(t, updated) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", outdatedFile.LinkChecksum) + assert.True(t, outdatedFile.UpToDate) + + uptodateFile, err := newLinkedFile(root, "testdata/links/uptodate.yml.link") + assert.NoError(t, err) + assert.True(t, uptodateFile.UpToDate) + updated, err = uptodateFile.UpdateChecksum() + assert.NoError(t, err) + assert.False(t, updated) +} + +func TestLinkReplaceTargetFilePathDirectory(t *testing.T) { + root, err := FindRepositoryRoot() + assert.NoError(t, err) + + linkedFile, err := newLinkedFile(root, "testdata/links/uptodate.yml.link") + assert.NoError(t, err) + assert.Equal(t, "testdata/links/uptodate.yml", linkedFile.TargetFilePath) + + linkedFile.ReplaceTargetFilePathDirectory("testdata/links", "build/testdata/links") + assert.Equal(t, "build/testdata/links/uptodate.yml", linkedFile.TargetFilePath) +} + +func TestAreLinkedFilesUpToDate(t *testing.T) { + linkedFiles, err := AreLinkedFilesUpToDate("testdata/links") + assert.NoError(t, err) + assert.NotEmpty(t, linkedFiles) + assert.Len(t, linkedFiles, 1) + assert.Equal(t, "testdata/links/outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.Empty(t, linkedFiles[0].LinkChecksum) + assert.Equal(t, "testdata/links/outdated.yml", linkedFiles[0].TargetFilePath) + assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[0].IncludedFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) + assert.False(t, linkedFiles[0].UpToDate) +} + +func TestUpdateLinkedFilesChecksums(t *testing.T) { + updated, err := UpdateLinkedFilesChecksums("testdata/links") + t.Cleanup(func() { + _ = writeFile(updated[0].LinkFilePath, []byte(updated[0].IncludedFilePath)) + }) + assert.NoError(t, err) + assert.NotEmpty(t, updated) + assert.Len(t, updated, 1) + assert.True(t, updated[0].UpToDate) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", updated[0].LinkChecksum) + +} + +func TestLinkedFilesByPackageFrom(t *testing.T) { + m, err := LinkedFilesByPackageFrom("testdata/links") + assert.NoError(t, err) + assert.NotEmpty(t, m) + assert.Len(t, m, 1) + assert.NotEmpty(t, m[0]) + assert.Len(t, m[0], 1) + assert.NotEmpty(t, m[0]["testpackage"]) + assert.Len(t, m[0]["testpackage"], 1) + match := strings.HasSuffix( + filepath.ToSlash(m[0]["testpackage"][0]), + "/testdata/testpackage/included.yml.link", + ) + assert.True(t, match) +} + +func TestListLinkedFiles(t *testing.T) { + linkedFiles, err := ListLinkedFiles("testdata/links") + assert.NoError(t, err) + assert.NotEmpty(t, linkedFiles) + assert.Len(t, linkedFiles, 2) + assert.Equal(t, "testdata/links/outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.Empty(t, linkedFiles[0].LinkChecksum) + assert.Equal(t, "testdata/links/outdated.yml", linkedFiles[0].TargetFilePath) + assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[0].IncludedFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) + assert.False(t, linkedFiles[0].UpToDate) + assert.Equal(t, "testdata/links/uptodate.yml.link", linkedFiles[1].LinkFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].LinkChecksum) + assert.Equal(t, "testdata/links/uptodate.yml", linkedFiles[1].TargetFilePath) + assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[1].IncludedFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].IncludedFileContentsChecksum) + assert.True(t, linkedFiles[1].UpToDate) +} + +func TestCopyFile(t *testing.T) { + fileA := filepath.Join(t.TempDir(), "fileA.txt") + fileB := filepath.Join(t.TempDir(), "fileB.txt") + t.Cleanup(func() { _ = os.Remove(fileA) }) + t.Cleanup(func() { _ = os.Remove(fileB) }) + + createDummyFile(t, fileA, "This is the content of the file.") + + assert.NoError(t, CopyFile(fileA, fileB)) + + equal, err := filesEqual(fileA, fileB) + assert.NoError(t, err) + assert.True(t, equal, "files should be equal after copying") +} + +func createDummyFile(t *testing.T, filename, content string) { + file, err := os.Create(filename) + assert.NoError(t, err) + defer file.Close() + _, err = file.WriteString(content) + assert.NoError(t, err) +} + +func filesEqual(file1, file2 string) (bool, error) { + f1, err := os.ReadFile(file1) + if err != nil { + return false, err + } + + f2, err := os.ReadFile(file2) + if err != nil { + return false, err + } + + return bytes.Equal(f1, f2), nil +} diff --git a/internal/files/testdata/links/included.yml b/internal/files/testdata/links/included.yml new file mode 100644 index 0000000000..923ec99b96 --- /dev/null +++ b/internal/files/testdata/links/included.yml @@ -0,0 +1,3 @@ +processors: + - test: + foo: bar \ No newline at end of file diff --git a/internal/files/testdata/links/outdated.yml.link b/internal/files/testdata/links/outdated.yml.link new file mode 100644 index 0000000000..8d5de3bbd4 --- /dev/null +++ b/internal/files/testdata/links/outdated.yml.link @@ -0,0 +1 @@ +internal/files/testdata/links/included.yml \ No newline at end of file diff --git a/internal/files/testdata/links/uptodate.yml.link b/internal/files/testdata/links/uptodate.yml.link new file mode 100644 index 0000000000..3f0925ac7f --- /dev/null +++ b/internal/files/testdata/links/uptodate.yml.link @@ -0,0 +1 @@ +internal/files/testdata/links/included.yml d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e \ No newline at end of file diff --git a/internal/files/testdata/testpackage/included.yml.link b/internal/files/testdata/testpackage/included.yml.link new file mode 100644 index 0000000000..3f0925ac7f --- /dev/null +++ b/internal/files/testdata/testpackage/included.yml.link @@ -0,0 +1 @@ +internal/files/testdata/links/included.yml d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e \ No newline at end of file diff --git a/internal/files/testdata/testpackage/manifest.yml b/internal/files/testdata/testpackage/manifest.yml new file mode 100644 index 0000000000..cef06e0e0f --- /dev/null +++ b/internal/files/testdata/testpackage/manifest.yml @@ -0,0 +1,20 @@ +format_version: 2.3.0 +name: testpackage +title: "With Includes Tests" +version: 0.0.1 +description: "These are tests of field validation with includes." +type: integration +categories: + - custom +conditions: + kibana.version: "^8.0.0" +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations From c2cdf7b318611212a73b2cee13594e3912b33f3e Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 31 Mar 2025 13:29:33 +0200 Subject: [PATCH 14/20] Always copy file on build --- internal/builder/linkedfiles.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/builder/linkedfiles.go b/internal/builder/linkedfiles.go index 0a73272acf..12259bfaf1 100644 --- a/internal/builder/linkedfiles.go +++ b/internal/builder/linkedfiles.go @@ -26,11 +26,12 @@ func IncludeLinkedFiles(fromDir, toDir string) ([]files.Link, error) { } if updated { - if err := files.CopyFile(l.IncludedFilePath, l.TargetFilePath); err != nil { - return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) - } logger.Debugf("%v included in package", l.TargetFilePath) } + + if err := files.CopyFile(l.IncludedFilePath, l.TargetFilePath); err != nil { + return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) + } } return links, nil From eb8b05c801b1e0a4b1fc99830fff8d5988efa6fe Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 31 Mar 2025 13:46:06 +0200 Subject: [PATCH 15/20] remove unused function --- internal/files/copy.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/files/copy.go b/internal/files/copy.go index a551fbe56e..9e0733f0c4 100644 --- a/internal/files/copy.go +++ b/internal/files/copy.go @@ -80,14 +80,3 @@ func shouldDirectoryBeSkipped(path string, skippedDirs []string) bool { } return false } - -// shouldFileBeSkipped function checks if absolute path should be skipped. -func shouldFileBeSkipped(path string, skippedFilesGlobs []string) bool { - for _, g := range skippedFilesGlobs { - m, _ := filepath.Match(g, filepath.Base(path)) - if m { - return true - } - } - return false -} From 0fdd2df998b8bf483a499a15120641883c3bb3cb Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 2 Apr 2025 14:50:04 +0200 Subject: [PATCH 16/20] Use package in spec --- go.mod | 6 +- go.sum | 4 +- internal/builder/linkedfiles.go | 35 ++--- internal/files/linkedfiles.go | 242 +++++------------------------ internal/files/linkedfiles_test.go | 110 ++----------- 5 files changed, 63 insertions(+), 334 deletions(-) diff --git a/go.mod b/go.mod index 6f26de7d26..c89a01968b 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,8 @@ module github.com/elastic/elastic-package -go 1.24.0 +go 1.24.2 -toolchain go1.24.1 - -replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250319094545-d89aaf66115e +replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250402123950-e2d4cc314a62 require ( github.com/AlecAivazis/survey/v2 v2.3.7 diff --git a/go.sum b/go.sum index 2c31a16699..44c9f89368 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marc-gr/package-spec/v3 v3.0.0-20250319094545-d89aaf66115e h1:wmqI2aDNnzLQ+uIB4xU6VTwNApkAOUJ3DIn2xhKL/kw= -github.com/marc-gr/package-spec/v3 v3.0.0-20250319094545-d89aaf66115e/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= +github.com/marc-gr/package-spec/v3 v3.0.0-20250402123950-e2d4cc314a62 h1:D8cgXB26uhUcybSkdz3dYK4YoNKuRdtD5OUnSKoOhKg= +github.com/marc-gr/package-spec/v3 v3.0.0-20250402123950-e2d4cc314a62/go.mod h1:GHWGHGvWE6OtH0ZC8YWUQT/G3Cuufhmx9MKHVUcgtK8= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= diff --git a/internal/builder/linkedfiles.go b/internal/builder/linkedfiles.go index 12259bfaf1..9d2fa5a230 100644 --- a/internal/builder/linkedfiles.go +++ b/internal/builder/linkedfiles.go @@ -5,34 +5,25 @@ package builder import ( - "fmt" + "path/filepath" - "github.com/elastic/elastic-package/internal/files" - "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles" ) -func IncludeLinkedFiles(fromDir, toDir string) ([]files.Link, error) { - links, err := files.ListLinkedFiles(fromDir) +func IncludeLinkedFiles(fromDir, toDir string) ([]linkedfiles.Link, error) { + root, err := linkedfiles.FindRepositoryRoot() if err != nil { - return nil, fmt.Errorf("including linked files failed: %w", err) + return nil, err } - for _, l := range links { - l.ReplaceTargetFilePathDirectory(fromDir, toDir) - - updated, err := l.UpdateChecksum() - if err != nil { - return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) - } - - if updated { - logger.Debugf("%v included in package", l.TargetFilePath) - } - - if err := files.CopyFile(l.IncludedFilePath, l.TargetFilePath); err != nil { - return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err) - } + fromRel, err := filepath.Rel(root.Name(), fromDir) + if err != nil { + return nil, err + } + toRel, err := filepath.Rel(root.Name(), toDir) + if err != nil { + return nil, err } - return links, nil + return linkedfiles.IncludeLinkedFiles(root, fromRel, toRel) } diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go index fdb9bc3866..ac3c252e92 100644 --- a/internal/files/linkedfiles.go +++ b/internal/files/linkedfiles.go @@ -5,105 +5,38 @@ package files import ( - "bufio" - "bytes" - "crypto/sha256" - "encoding/hex" "fmt" - "io" - "io/fs" - "os" "path/filepath" "slices" - "strings" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles" ) -const linkExtension = ".link" - -type Link struct { - LinkFilePath string - LinkChecksum string - - TargetFilePath string - - IncludedFilePath string - IncludedFileContentsChecksum string - - UpToDate bool -} - -func newLinkedFile(root *os.Root, linkFilePath string) (Link, error) { - var l Link - firstLine, err := readFirstLine(linkFilePath) +// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. +func AreLinkedFilesUpToDate(fromDir string) ([]linkedfiles.Link, error) { + root, err := linkedfiles.FindRepositoryRoot() if err != nil { - return Link{}, err - } - l.LinkFilePath = linkFilePath - l.TargetFilePath = strings.TrimSuffix(linkFilePath, linkExtension) - fields := strings.Fields(firstLine) - l.IncludedFilePath = fields[0] - if len(fields) == 2 { - l.LinkChecksum = fields[1] + return nil, err } - cs, err := getLinkedFileChecksum(root.FS().(fs.ReadFileFS), l.IncludedFilePath) + fromRel, err := func() (string, error) { + if filepath.IsAbs(fromDir) { + return filepath.Rel(root.Name(), fromDir) + } + return fromDir, nil + }() if err != nil { - return Link{}, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) - } - if l.LinkChecksum == cs { - l.UpToDate = true - } - l.IncludedFileContentsChecksum = cs - - return l, nil -} - -// UpdateChecksum function updates the checksum of the linked file. -// It returns true if the checksum was updated, false if it was already up-to-date. -func (l *Link) UpdateChecksum() (bool, error) { - if l.UpToDate { - return false, nil - } - if l.IncludedFilePath == "" { - return false, fmt.Errorf("file path is empty for file %v", l.IncludedFilePath) - } - if l.IncludedFileContentsChecksum == "" { - return false, fmt.Errorf("checksum is empty for file %v", l.IncludedFilePath) - } - newContent := fmt.Sprintf("%v %v", filepath.ToSlash(l.IncludedFilePath), l.IncludedFileContentsChecksum) - if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil { - return false, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) - } - l.LinkChecksum = l.IncludedFileContentsChecksum - l.UpToDate = true - return true, nil -} - -// ReplaceTargetFilePathDirectory function replaces the target file path directory. -func (l *Link) ReplaceTargetFilePathDirectory(fromDir, toDir string) { - // if a destination dir is set we replace the source dir with the destination dir - if toDir == "" { - return + return nil, err } - l.TargetFilePath = strings.Replace( - l.TargetFilePath, - fromDir, - toDir, - 1, - ) -} -// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. -func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { - links, err := ListLinkedFiles(fromDir) + links, err := linkedfiles.ListLinkedFilesInRoot(root, fromRel) if err != nil { return nil, fmt.Errorf("including linked files failed: %w", err) } - var outdated []Link + var outdated []linkedfiles.Link for _, l := range links { logger.Debugf("Check if %s is up-to-date", l.LinkFilePath) if !l.UpToDate { @@ -117,13 +50,28 @@ func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { // UpdateLinkedFilesChecksums function updates the checksums of the linked files. // It returns a slice of updated links. // If no links were updated, it returns an empty slice. -func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { - links, err := ListLinkedFiles(fromDir) +func UpdateLinkedFilesChecksums(fromDir string) ([]linkedfiles.Link, error) { + root, err := linkedfiles.FindRepositoryRoot() + if err != nil { + return nil, err + } + + fromRel, err := func() (string, error) { + if filepath.IsAbs(fromDir) { + return filepath.Rel(root.Name(), fromDir) + } + return fromDir, nil + }() + if err != nil { + return nil, err + } + + links, err := linkedfiles.ListLinkedFilesInRoot(root, fromRel) if err != nil { return nil, fmt.Errorf("updating linked files checksums failed: %w", err) } - var updatedLinks []Link + var updatedLinks []linkedfiles.Link for _, l := range links { updated, err := l.UpdateChecksum() if err != nil { @@ -140,11 +88,11 @@ func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { // LinkedFilesByPackageFrom function returns a slice of maps containing linked files grouped by package. // Each map contains the package name as the key and a slice of linked file paths as the value. func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { - rootPath, err := FindRepositoryRootDirectory() + root, err := linkedfiles.FindRepositoryRoot() if err != nil { - return nil, fmt.Errorf("locating repository root failed: %w", err) + return nil, err } - links, err := ListLinkedFiles(rootPath) + links, err := linkedfiles.ListLinkedFilesInRoot(root, ".") if err != nil { return nil, fmt.Errorf("including linked files failed: %w", err) } @@ -153,9 +101,9 @@ func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { packageName := filepath.Base(packageRoot) byPackageMap := map[string][]string{} for _, l := range links { - linkPackageRoot, _, _ := packages.FindPackageRootFrom(l.LinkFilePath) + linkPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Join(root.Name(), l.LinkFilePath)) linkPackageName := filepath.Base(linkPackageRoot) - includedPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Join(rootPath, l.IncludedFilePath)) + includedPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Join(root.Name(), l.IncludedFilePath)) includedPackageName := filepath.Base(includedPackageRoot) if linkPackageName == includedPackageName || packageName != includedPackageName { @@ -177,119 +125,3 @@ func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { } return byPackage, nil } - -// ListLinkedFiles function returns a slice of Link structs representing linked files. -func ListLinkedFiles(fromDir string) ([]Link, error) { - root, err := FindRepositoryRoot() - if err != nil { - return nil, fmt.Errorf("could not get root: %w", err) - } - - links, err := getLinksFrom(root, fromDir) - if err != nil { - return nil, err - } - - return links, nil -} - -func CopyFile(from, to string) error { - from = filepath.FromSlash(from) - source, err := os.Open(from) - if err != nil { - return err - } - defer source.Close() - - to = filepath.FromSlash(to) - if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { - return err - } - } - destination, err := os.Create(to) - if err != nil { - return err - } - defer destination.Close() - - _, err = io.Copy(destination, source) - return err -} - -func writeFile(to string, b []byte) error { - to = filepath.FromSlash(to) - if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { - return err - } - } - return os.WriteFile(to, b, 0644) -} - -func getLinksFrom(root *os.Root, fromDir string) ([]Link, error) { - var linkFiles []string - if err := filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() && strings.HasSuffix(info.Name(), linkExtension) { - linkFiles = append(linkFiles, path) - } - return nil - }); err != nil { - return nil, err - } - - links := make([]Link, len(linkFiles)) - - for i, f := range linkFiles { - l, err := newLinkedFile(root, f) - if err != nil { - return nil, fmt.Errorf("could not create linked file %v: %w", f, err) - } - links[i] = l - } - - return links, nil -} - -func getLinkedFileChecksum(root fs.ReadFileFS, path string) (string, error) { - b, err := root.ReadFile(filepath.FromSlash(path)) - if err != nil { - return "", err - } - cs, err := checksum(b) - if err != nil { - return "", err - } - return cs, nil -} - -func readFirstLine(filePath string) (string, error) { - file, err := os.Open(filepath.FromSlash(filePath)) - if err != nil { - return "", err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - if scanner.Scan() { - return scanner.Text(), nil - } - - if err := scanner.Err(); err != nil { - return "", err - } - - return "", fmt.Errorf("file is empty or first line is missing") -} - -func checksum(b []byte) (string, error) { - hash := sha256.New() - if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil { - return "", err - } - return hex.EncodeToString(hash.Sum(nil)), nil -} diff --git a/internal/files/linkedfiles_test.go b/internal/files/linkedfiles_test.go index c3a85c911a..1842f1a931 100644 --- a/internal/files/linkedfiles_test.go +++ b/internal/files/linkedfiles_test.go @@ -5,69 +5,33 @@ package files import ( - "bytes" - "os" "path/filepath" "strings" "testing" + "github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles" "github.com/stretchr/testify/assert" ) -func TestLinkUpdateChecksum(t *testing.T) { - root, err := FindRepositoryRoot() - assert.NoError(t, err) - - outdatedFile, err := newLinkedFile(root, "testdata/links/outdated.yml.link") - t.Cleanup(func() { - _ = writeFile(outdatedFile.LinkFilePath, []byte(outdatedFile.IncludedFilePath)) - }) - assert.NoError(t, err) - assert.False(t, outdatedFile.UpToDate) - assert.Empty(t, outdatedFile.LinkChecksum) - updated, err := outdatedFile.UpdateChecksum() - assert.NoError(t, err) - assert.True(t, updated) - assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", outdatedFile.LinkChecksum) - assert.True(t, outdatedFile.UpToDate) - - uptodateFile, err := newLinkedFile(root, "testdata/links/uptodate.yml.link") - assert.NoError(t, err) - assert.True(t, uptodateFile.UpToDate) - updated, err = uptodateFile.UpdateChecksum() - assert.NoError(t, err) - assert.False(t, updated) -} - -func TestLinkReplaceTargetFilePathDirectory(t *testing.T) { - root, err := FindRepositoryRoot() - assert.NoError(t, err) - - linkedFile, err := newLinkedFile(root, "testdata/links/uptodate.yml.link") - assert.NoError(t, err) - assert.Equal(t, "testdata/links/uptodate.yml", linkedFile.TargetFilePath) - - linkedFile.ReplaceTargetFilePathDirectory("testdata/links", "build/testdata/links") - assert.Equal(t, "build/testdata/links/uptodate.yml", linkedFile.TargetFilePath) -} - func TestAreLinkedFilesUpToDate(t *testing.T) { - linkedFiles, err := AreLinkedFilesUpToDate("testdata/links") + linkedFiles, err := AreLinkedFilesUpToDate("internal/files/testdata/links") assert.NoError(t, err) assert.NotEmpty(t, linkedFiles) assert.Len(t, linkedFiles, 1) - assert.Equal(t, "testdata/links/outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.Equal(t, "internal/files/testdata/links/outdated.yml.link", linkedFiles[0].LinkFilePath) assert.Empty(t, linkedFiles[0].LinkChecksum) - assert.Equal(t, "testdata/links/outdated.yml", linkedFiles[0].TargetFilePath) + assert.Equal(t, "internal/files/testdata/links/outdated.yml", linkedFiles[0].TargetFilePath) assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[0].IncludedFilePath) assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) assert.False(t, linkedFiles[0].UpToDate) } func TestUpdateLinkedFilesChecksums(t *testing.T) { - updated, err := UpdateLinkedFilesChecksums("testdata/links") + root, err := linkedfiles.FindRepositoryRoot() + assert.NoError(t, err) + updated, err := UpdateLinkedFilesChecksums("internal/files/testdata/links") t.Cleanup(func() { - _ = writeFile(updated[0].LinkFilePath, []byte(updated[0].IncludedFilePath)) + _ = linkedfiles.WriteFileToRoot(root, updated[0].LinkFilePath, []byte(updated[0].IncludedFilePath)) }) assert.NoError(t, err) assert.NotEmpty(t, updated) @@ -78,7 +42,7 @@ func TestUpdateLinkedFilesChecksums(t *testing.T) { } func TestLinkedFilesByPackageFrom(t *testing.T) { - m, err := LinkedFilesByPackageFrom("testdata/links") + m, err := LinkedFilesByPackageFrom("internal/files/testdata/links") assert.NoError(t, err) assert.NotEmpty(t, m) assert.Len(t, m, 1) @@ -92,59 +56,3 @@ func TestLinkedFilesByPackageFrom(t *testing.T) { ) assert.True(t, match) } - -func TestListLinkedFiles(t *testing.T) { - linkedFiles, err := ListLinkedFiles("testdata/links") - assert.NoError(t, err) - assert.NotEmpty(t, linkedFiles) - assert.Len(t, linkedFiles, 2) - assert.Equal(t, "testdata/links/outdated.yml.link", linkedFiles[0].LinkFilePath) - assert.Empty(t, linkedFiles[0].LinkChecksum) - assert.Equal(t, "testdata/links/outdated.yml", linkedFiles[0].TargetFilePath) - assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[0].IncludedFilePath) - assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) - assert.False(t, linkedFiles[0].UpToDate) - assert.Equal(t, "testdata/links/uptodate.yml.link", linkedFiles[1].LinkFilePath) - assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].LinkChecksum) - assert.Equal(t, "testdata/links/uptodate.yml", linkedFiles[1].TargetFilePath) - assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[1].IncludedFilePath) - assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].IncludedFileContentsChecksum) - assert.True(t, linkedFiles[1].UpToDate) -} - -func TestCopyFile(t *testing.T) { - fileA := filepath.Join(t.TempDir(), "fileA.txt") - fileB := filepath.Join(t.TempDir(), "fileB.txt") - t.Cleanup(func() { _ = os.Remove(fileA) }) - t.Cleanup(func() { _ = os.Remove(fileB) }) - - createDummyFile(t, fileA, "This is the content of the file.") - - assert.NoError(t, CopyFile(fileA, fileB)) - - equal, err := filesEqual(fileA, fileB) - assert.NoError(t, err) - assert.True(t, equal, "files should be equal after copying") -} - -func createDummyFile(t *testing.T, filename, content string) { - file, err := os.Create(filename) - assert.NoError(t, err) - defer file.Close() - _, err = file.WriteString(content) - assert.NoError(t, err) -} - -func filesEqual(file1, file2 string) (bool, error) { - f1, err := os.ReadFile(file1) - if err != nil { - return false, err - } - - f2, err := os.ReadFile(file2) - if err != nil { - return false, err - } - - return bytes.Equal(f1, f2), nil -} From a4322e50284850e4d240d365ac20e196ff7265bb Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 2 Apr 2025 14:58:40 +0200 Subject: [PATCH 17/20] Use package spec --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c89a01968b..d7543275b4 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/elastic/elastic-package go 1.24.2 -replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250402123950-e2d4cc314a62 +replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250402125715-45c058345f1e require ( github.com/AlecAivazis/survey/v2 v2.3.7 diff --git a/go.sum b/go.sum index 44c9f89368..009930407b 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marc-gr/package-spec/v3 v3.0.0-20250402123950-e2d4cc314a62 h1:D8cgXB26uhUcybSkdz3dYK4YoNKuRdtD5OUnSKoOhKg= -github.com/marc-gr/package-spec/v3 v3.0.0-20250402123950-e2d4cc314a62/go.mod h1:GHWGHGvWE6OtH0ZC8YWUQT/G3Cuufhmx9MKHVUcgtK8= +github.com/marc-gr/package-spec/v3 v3.0.0-20250402125715-45c058345f1e h1:lhEtKVEjYwKDhsC/CJ2CrQOEtq+YeUEeNXXiL9i0NJs= +github.com/marc-gr/package-spec/v3 v3.0.0-20250402125715-45c058345f1e/go.mod h1:GHWGHGvWE6OtH0ZC8YWUQT/G3Cuufhmx9MKHVUcgtK8= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= From 663ec29834b3cb996071fd65ac3ec079452e001b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 10 Apr 2025 11:30:40 +0200 Subject: [PATCH 18/20] Update links usage --- README.md | 24 ++ cmd/links.go | 3 +- docs/howto/links.md | 34 ++ go.mod | 2 - go.sum | 4 +- internal/builder/linkedfiles.go | 29 -- internal/builder/packages.go | 6 +- internal/elasticsearch/ingest/datastream.go | 35 +- internal/files/linkedfiles.go | 363 ++++++++++++++++-- internal/files/linkedfiles_test.go | 128 +++++- .../files/testdata/links/outdated.yml.link | 2 +- .../files/testdata/links/uptodate.yml.link | 2 +- .../testdata/testpackage/included.yml.link | 2 +- .../first/agent/stream/stream.yml.hbs.link | 1 - .../ingest_pipeline/default.yml.link | 1 - .../second/agent/stream/stream.yml.hbs.link | 1 - .../_dev/build/build.yml | 0 .../_dev/build/docs/README.md | 0 .../_dev/build/shared/stream.yml.hbs | 0 .../changelog.yml | 0 .../_dev/test/pipeline/test-access-raw.log | 1 + .../pipeline/test-access-raw.log-config.yml | 4 + .../test-access-raw.log-expected.json | 5 + .../first/agent/stream/stream.yml.hbs.link | 1 + .../ingest_pipeline/default.yml.link | 1 + .../data_stream/first/fields/base-fields.yml | 0 .../first/fields/histogram-fields.yml | 0 .../data_stream/first/manifest.yml | 0 .../data_stream/first/sample_event.json | 0 .../second/agent/stream/stream.yml.hbs.link | 1 + .../elasticsearch/ingest_pipeline/default.yml | 0 .../data_stream/second/fields/base-fields.yml | 0 .../data_stream/second/fields/geo-fields.yml | 0 .../second/fields/histogram-fields.yml | 0 .../data_stream/second/manifest.yml | 0 .../data_stream/second/sample_event.json | 0 .../docs/README.md | 0 .../manifest.yml | 6 +- 38 files changed, 539 insertions(+), 117 deletions(-) create mode 100644 docs/howto/links.md delete mode 100644 internal/builder/linkedfiles.go delete mode 100644 test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link delete mode 100644 test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link delete mode 100644 test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link rename test/packages/other/{with_includes => with_links}/_dev/build/build.yml (100%) rename test/packages/other/{with_includes => with_links}/_dev/build/docs/README.md (100%) rename test/packages/other/{with_includes => with_links}/_dev/build/shared/stream.yml.hbs (100%) rename test/packages/other/{with_includes => with_links}/changelog.yml (100%) create mode 100644 test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log create mode 100644 test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-config.yml create mode 100644 test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-expected.json create mode 100644 test/packages/other/with_links/data_stream/first/agent/stream/stream.yml.hbs.link create mode 100644 test/packages/other/with_links/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link rename test/packages/other/{with_includes => with_links}/data_stream/first/fields/base-fields.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/first/fields/histogram-fields.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/first/manifest.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/first/sample_event.json (100%) create mode 100644 test/packages/other/with_links/data_stream/second/agent/stream/stream.yml.hbs.link rename test/packages/other/{with_includes => with_links}/data_stream/second/elasticsearch/ingest_pipeline/default.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/second/fields/base-fields.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/second/fields/geo-fields.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/second/fields/histogram-fields.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/second/manifest.yml (100%) rename test/packages/other/{with_includes => with_links}/data_stream/second/sample_event.json (100%) rename test/packages/other/{with_includes => with_links}/docs/README.md (100%) rename test/packages/other/{with_includes => with_links}/manifest.yml (76%) diff --git a/README.md b/README.md index 1a6c103a0c..f691d9c0d3 100644 --- a/README.md +++ b/README.md @@ -370,6 +370,30 @@ Use this command to install the package in Kibana. The command uses Kibana API to install the package in Kibana. The package must be exposed via the Package Registry or built locally in zip format so they can be installed using --zip parameter. Zip packages can be installed directly in Kibana >= 8.7.0. More details in this [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/install_package.md). +### `elastic-package links` + +_Context: global_ + +Use this command to manage linked files in the repository. + +### `elastic-package links check` + +_Context: global_ + +Use this command to check if linked files references inside the current directory are up to date. + +### `elastic-package links list` + +_Context: global_ + +List packages linking files from this path. + +### `elastic-package links update` + +_Context: global_ + +Use this command to update all linked files references inside the current directory. + ### `elastic-package lint` _Context: package_ diff --git a/cmd/links.go b/cmd/links.go index ab83b1571f..e3f03d7373 100644 --- a/cmd/links.go +++ b/cmd/links.go @@ -7,6 +7,7 @@ package cmd import ( "fmt" "os" + "path/filepath" "github.com/spf13/cobra" @@ -62,7 +63,7 @@ func linksCheckCommandAction(cmd *cobra.Command, args []string) error { } for _, f := range linkedFiles { if !f.UpToDate { - cmd.Printf("%s is outdated.\n", f.LinkFilePath) + cmd.Printf("%s is outdated.\n", filepath.Join(f.WorkDir, f.LinkFilePath)) } } if len(linkedFiles) > 0 { diff --git a/docs/howto/links.md b/docs/howto/links.md new file mode 100644 index 0000000000..35a9ad6cda --- /dev/null +++ b/docs/howto/links.md @@ -0,0 +1,34 @@ +# HOWTO: Use links to reuse common files. + +## Introduction + +Many packages have files that are equal between them. This is more common in pipelines, +input configurations, and field definitions. + +In order to help developers, there is the ability to define links, so a file that might be reused needs to only be defined once, and can be reused from any other packages. + + +# Links + +Currently, there are some specific places where links can be defined: + +- `elasticsearch/ingest_pipeline` +- `data_stream/**/elasticsearch/ingest_pipeline` +- `agent/input` +- `data_stream/**/agent/stream` +- `data_stream/**/fields` + +A link consists of a file with a `.link` extension that contains a path, relative to its location, to the file that it will be replaced with. It also consists of a checksum to validate the linked file is up to date with the package expectations. + +`data_stream/foo/elasticsearch/ingest_pipeline/default.yml.link` + +``` +../../../../../testpackage/data_stream/test/elasticsearch/ingest_pipeline/default.yml f7c5f0c03aca8ef68c379a62447bdafbf0dcf32b1ff2de143fd6878ee01a91ad +``` + +This will use the contents of the linked file during validation, tests, and building of the package, so functionally nothing changes from the package point of view. + +## The `_dev/shared` folder + +As a convenience, shared files can be placed under `_dev/shared` if they are going to be +reused from several places. They can even be added outside of any package, in any place in the repository. diff --git a/go.mod b/go.mod index d7543275b4..ee6c775214 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/elastic/elastic-package go 1.24.2 -replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250402125715-45c058345f1e - require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Masterminds/semver/v3 v3.3.1 diff --git a/go.sum b/go.sum index 009930407b..96047ae177 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU= github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= +github.com/elastic/package-spec/v3 v3.3.4 h1:lfeNGHRzJauOu342bxXQ9hHMO+D118ijUz0bYwMq4O0= +github.com/elastic/package-spec/v3 v3.3.4/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= @@ -217,8 +219,6 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marc-gr/package-spec/v3 v3.0.0-20250402125715-45c058345f1e h1:lhEtKVEjYwKDhsC/CJ2CrQOEtq+YeUEeNXXiL9i0NJs= -github.com/marc-gr/package-spec/v3 v3.0.0-20250402125715-45c058345f1e/go.mod h1:GHWGHGvWE6OtH0ZC8YWUQT/G3Cuufhmx9MKHVUcgtK8= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= diff --git a/internal/builder/linkedfiles.go b/internal/builder/linkedfiles.go deleted file mode 100644 index 9d2fa5a230..0000000000 --- a/internal/builder/linkedfiles.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package builder - -import ( - "path/filepath" - - "github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles" -) - -func IncludeLinkedFiles(fromDir, toDir string) ([]linkedfiles.Link, error) { - root, err := linkedfiles.FindRepositoryRoot() - if err != nil { - return nil, err - } - - fromRel, err := filepath.Rel(root.Name(), fromDir) - if err != nil { - return nil, err - } - toRel, err := filepath.Rel(root.Name(), toDir) - if err != nil { - return nil, err - } - - return linkedfiles.IncludeLinkedFiles(root, fromRel, toRel) -} diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 6889822695..6293740628 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -185,9 +185,13 @@ func BuildPackage(options BuildOptions) (string, error) { } logger.Debug("Include linked files") - if _, err := IncludeLinkedFiles(options.PackageRoot, destinationDir); err != nil { + links, err := files.IncludeLinkedFiles(options.PackageRoot, destinationDir) + if err != nil { return "", fmt.Errorf("including linked files failed: %w", err) } + for _, l := range links { + logger.Debugf("Linked file included (path: %s)", l.TargetFilePath(destinationDir)) + } if options.CreateZip { return buildZippedPackage(options, destinationDir) diff --git a/internal/elasticsearch/ingest/datastream.go b/internal/elasticsearch/ingest/datastream.go index 94fe8945a1..acab9590cc 100644 --- a/internal/elasticsearch/ingest/datastream.go +++ b/internal/elasticsearch/ingest/datastream.go @@ -18,16 +18,17 @@ import ( "gopkg.in/yaml.v3" - "github.com/elastic/elastic-package/internal/builder" "github.com/elastic/elastic-package/internal/elasticsearch" - "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/packages" ) var ( - ingestPipelineTag = regexp.MustCompile(`{{\s*IngestPipeline.+}}`) - defaultPipelineJSON = "default.json" - defaultPipelineYML = "default.yml" + ingestPipelineTag = regexp.MustCompile(`{{\s*IngestPipeline.+}}`) + defaultPipelineJSON = "default.json" + defaultPipelineJSONLink = "default.json" + defaultPipelineYML = "default.yml.link" + defaultPipelineYMLLink = "default.yml.link" ) type Rule struct { @@ -72,22 +73,8 @@ func InstallDataStreamPipelines(ctx context.Context, api *elasticsearch.API, dat func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, error) { elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline") - // Include shared pipelines before installing them - links, err := builder.IncludeLinkedFiles(elasticsearchPath, elasticsearchPath) - if err != nil { - return nil, fmt.Errorf("including shared files failed: %w", err) - } - defer func() { - // Remove linked files after installing them - for _, link := range links { - if err := os.Remove(link.TargetFilePath); err != nil { - logger.Errorf("failed to remove linked file %s: %v\n", link.TargetFilePath, err) - } - } - }() - var pipelineFiles []string - for _, pattern := range []string{"*.json", "*.yml"} { + for _, pattern := range []string{"*.json", "*.yml", "*.link"} { files, err := filepath.Glob(filepath.Join(elasticsearchPath, pattern)) if err != nil { return nil, fmt.Errorf("listing '%s' in '%s': %w", pattern, elasticsearchPath, err) @@ -95,9 +82,10 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er pipelineFiles = append(pipelineFiles, files...) } + linksFS := files.NewLinksFS(elasticsearchPath) var pipelines []Pipeline for _, path := range pipelineFiles { - c, err := os.ReadFile(path) + c, err := linksFS.ReadFile(path) if err != nil { return nil, fmt.Errorf("reading ingest pipeline failed (path: %s): %w", path, err) } @@ -124,7 +112,7 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er pipelines = append(pipelines, Pipeline{ Path: path, Name: getPipelineNameWithNonce(name[:strings.Index(name, ".")], nonce), - Format: filepath.Ext(path)[1:], + Format: filepath.Ext(strings.TrimSuffix(path, ".link"))[1:], Content: cWithRerouteProcessors, ContentOriginal: c, }) @@ -135,7 +123,8 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er func addRerouteProcessors(pipeline []byte, dataStreamPath, path string) ([]byte, error) { // Only attach routing_rules.yml reroute processors after the default pipeline filename := filepath.Base(path) - if filename != defaultPipelineJSON && filename != defaultPipelineYML { + if filename != defaultPipelineJSON && filename != defaultPipelineYML && + filename != defaultPipelineJSONLink && filename != defaultPipelineYMLLink { return pipeline, nil } diff --git a/internal/files/linkedfiles.go b/internal/files/linkedfiles.go index ac3c252e92..d384b1a352 100644 --- a/internal/files/linkedfiles.go +++ b/internal/files/linkedfiles.go @@ -5,38 +5,278 @@ package files import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" "fmt" + "io" + "io/fs" + "os" "path/filepath" "slices" + "strings" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" - "github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles" ) -// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. -func AreLinkedFilesUpToDate(fromDir string) ([]linkedfiles.Link, error) { - root, err := linkedfiles.FindRepositoryRoot() +const linkExtension = ".link" + +var _ fs.FS = (*LinksFS)(nil) + +// LinksFS is a filesystem that handles linked files. +// It wraps another filesystem and checks for linked files with the ".link" extension. +// If a linked file is found, it reads the link file to determine the target file +// and its checksum. If the target file is up to date, it returns the target file. +// Otherwise, it returns an error. +type LinksFS struct { + workDir string + inner fs.FS +} + +// NewLinksFS creates a new LinksFS. +func NewLinksFS(workDir string) *LinksFS { + return &LinksFS{workDir: workDir, inner: os.DirFS(workDir)} +} + +// Open opens a file in the filesystem. +func (lfs *LinksFS) Open(name string) (fs.File, error) { + name, err := filepath.Rel(lfs.workDir, name) + if err != nil { + return nil, fmt.Errorf("could not get relative path: %w", err) + } + fmt.Println(name) + if filepath.Ext(name) != linkExtension { + return lfs.inner.Open(name) + } + pathName := filepath.Join(lfs.workDir, name) + l, err := NewLinkedFile(pathName) if err != nil { return nil, err } + if !l.UpToDate { + return nil, fmt.Errorf("linked file %s is not up to date", name) + } + includedPath := filepath.Join(lfs.workDir, filepath.Dir(name), l.IncludedFilePath) + return os.Open(includedPath) +} - fromRel, err := func() (string, error) { - if filepath.IsAbs(fromDir) { - return filepath.Rel(root.Name(), fromDir) - } - return fromDir, nil - }() +// ReadFile reads a file from the filesystem. +func (lfs *LinksFS) ReadFile(name string) ([]byte, error) { + f, err := lfs.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + b, err := io.ReadAll(f) if err != nil { return nil, err } + return b, nil +} + +// A Link represents a linked file. +// It contains the path to the link file, the checksum of the linked file, +// the path to the target file, and the checksum of the included file contents. +// It also contains a boolean indicating whether the link is up to date. +type Link struct { + WorkDir string + + LinkFilePath string + LinkChecksum string + LinkPackageName string + + IncludedFilePath string + IncludedFileContentsChecksum string + IncludedPackageName string + + UpToDate bool +} + +// NewLinkedFile creates a new Link from the given link file path. +func NewLinkedFile(linkFilePath string) (Link, error) { + var l Link + l.WorkDir = filepath.Dir(linkFilePath) + if linkPackageRoot, _, _ := packages.FindPackageRootFrom(l.WorkDir); linkPackageRoot != "" { + l.LinkPackageName = filepath.Base(linkPackageRoot) + } + + firstLine, err := readFirstLine(linkFilePath) + if err != nil { + return Link{}, err + } + l.LinkFilePath, err = filepath.Rel(l.WorkDir, linkFilePath) + if err != nil { + return Link{}, fmt.Errorf("could not get relative path: %w", err) + } + + fields := strings.Fields(firstLine) + l.IncludedFilePath = fields[0] + if len(fields) == 2 { + l.LinkChecksum = fields[1] + } + + pathName := filepath.Join(l.WorkDir, filepath.FromSlash(l.IncludedFilePath)) + if _, err := os.Stat(pathName); err != nil { + return Link{}, err + } + + notInRoot, err := pathIsInRepositoryRoot(pathName) + if err != nil { + return Link{}, fmt.Errorf("could not check if path %v is in repository root: %w", pathName, err) + } + if !notInRoot { + return Link{}, fmt.Errorf("path %v escapes the repository root", pathName) + } + + cs, err := getLinkedFileChecksum(pathName) + if err != nil { + return Link{}, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err) + } + if l.LinkChecksum == cs { + l.UpToDate = true + } + l.IncludedFileContentsChecksum = cs + + if includedPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Dir(pathName)); includedPackageRoot != "" { + l.IncludedPackageName = filepath.Base(includedPackageRoot) + } + + return l, nil +} + +// UpdateChecksum function updates the checksum of the linked file. +// It returns true if the checksum was updated, false if it was already up-to-date. +func (l *Link) UpdateChecksum() (bool, error) { + if l.UpToDate { + return false, nil + } + if l.IncludedFilePath == "" { + return false, fmt.Errorf("file path is empty for file %v", l.IncludedFilePath) + } + if l.IncludedFileContentsChecksum == "" { + return false, fmt.Errorf("checksum is empty for file %v", l.IncludedFilePath) + } + newContent := fmt.Sprintf("%v %v", filepath.ToSlash(l.IncludedFilePath), l.IncludedFileContentsChecksum) + if err := WriteFile(filepath.Join(l.WorkDir, l.LinkFilePath), []byte(newContent)); err != nil { + return false, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) + } + l.LinkChecksum = l.IncludedFileContentsChecksum + l.UpToDate = true + return true, nil +} + +func (l *Link) TargetFilePath(workDir ...string) string { + targetFilePath := filepath.FromSlash(strings.TrimSuffix(l.LinkFilePath, linkExtension)) + wd := l.WorkDir + if len(workDir) > 0 { + wd = workDir[0] + } + return filepath.Join(wd, targetFilePath) +} + +// IncludeLinkedFiles function includes linked files from the source +// directory to the target directory. +// It returns a slice of Link structs representing the included files. +// It also updates the checksum of the linked files. +// Both directories must be relative to the root. +func IncludeLinkedFiles(fromDir, toDir string) ([]Link, error) { + links, err := ListLinkedFiles(fromDir) + if err != nil { + return nil, fmt.Errorf("including linked files failed: %w", err) + } + for _, l := range links { + if _, err := l.UpdateChecksum(); err != nil { + return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err) + } + targetFilePath := l.TargetFilePath(toDir) + if err := CopyFile( + filepath.Join(l.WorkDir, filepath.FromSlash(l.IncludedFilePath)), + targetFilePath, + ); err != nil { + return nil, fmt.Errorf("could not write file %v: %w", targetFilePath, err) + } + } + + return links, nil +} + +// ListLinkedFiles function returns a slice of Link structs representing linked files. +func ListLinkedFiles(fromDir string) ([]Link, error) { + var linkFiles []string + if err := filepath.Walk( + filepath.FromSlash(fromDir), + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), linkExtension) { + linkFiles = append(linkFiles, path) + } + return nil + }); err != nil { + return nil, err + } + + links := make([]Link, len(linkFiles)) + + for i, f := range linkFiles { + l, err := NewLinkedFile(filepath.FromSlash(f)) + if err != nil { + return nil, fmt.Errorf("could not initialize linked file %v: %w", f, err) + } + links[i] = l + } + + return links, nil +} + +// CopyFile function copies a file from to to inside the root. +func CopyFile(from, to string) error { + from = filepath.FromSlash(from) + source, err := os.Open(from) + if err != nil { + return err + } + defer source.Close() + + to = filepath.FromSlash(to) + dir := filepath.Dir(to) + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + } + destination, err := os.Create(to) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +// WriteFile function writes a byte slice to a file inside the root. +func WriteFile(to string, b []byte) error { + to = filepath.FromSlash(to) + if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil { + return err + } + } + return os.WriteFile(to, b, 0644) +} - links, err := linkedfiles.ListLinkedFilesInRoot(root, fromRel) +// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date. +func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) { + links, err := ListLinkedFiles(fromDir) if err != nil { return nil, fmt.Errorf("including linked files failed: %w", err) } - var outdated []linkedfiles.Link + var outdated []Link for _, l := range links { logger.Debugf("Check if %s is up-to-date", l.LinkFilePath) if !l.UpToDate { @@ -50,28 +290,13 @@ func AreLinkedFilesUpToDate(fromDir string) ([]linkedfiles.Link, error) { // UpdateLinkedFilesChecksums function updates the checksums of the linked files. // It returns a slice of updated links. // If no links were updated, it returns an empty slice. -func UpdateLinkedFilesChecksums(fromDir string) ([]linkedfiles.Link, error) { - root, err := linkedfiles.FindRepositoryRoot() - if err != nil { - return nil, err - } - - fromRel, err := func() (string, error) { - if filepath.IsAbs(fromDir) { - return filepath.Rel(root.Name(), fromDir) - } - return fromDir, nil - }() - if err != nil { - return nil, err - } - - links, err := linkedfiles.ListLinkedFilesInRoot(root, fromRel) +func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) { + links, err := ListLinkedFiles(fromDir) if err != nil { return nil, fmt.Errorf("updating linked files checksums failed: %w", err) } - var updatedLinks []linkedfiles.Link + var updatedLinks []Link for _, l := range links { updated, err := l.UpdateChecksum() if err != nil { @@ -88,28 +313,26 @@ func UpdateLinkedFilesChecksums(fromDir string) ([]linkedfiles.Link, error) { // LinkedFilesByPackageFrom function returns a slice of maps containing linked files grouped by package. // Each map contains the package name as the key and a slice of linked file paths as the value. func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { - root, err := linkedfiles.FindRepositoryRoot() + root, err := FindRepositoryRoot() if err != nil { return nil, err } - links, err := linkedfiles.ListLinkedFilesInRoot(root, ".") + links, err := ListLinkedFiles(root.Name()) if err != nil { return nil, fmt.Errorf("including linked files failed: %w", err) } - packageRoot, _, _ := packages.FindPackageRootFrom(fromDir) - packageName := filepath.Base(packageRoot) + var packageName string + if packageRoot, _, _ := packages.FindPackageRootFrom(fromDir); packageRoot != "" { + packageName = filepath.Base(packageRoot) + } byPackageMap := map[string][]string{} for _, l := range links { - linkPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Join(root.Name(), l.LinkFilePath)) - linkPackageName := filepath.Base(linkPackageRoot) - includedPackageRoot, _, _ := packages.FindPackageRootFrom(filepath.Join(root.Name(), l.IncludedFilePath)) - includedPackageName := filepath.Base(includedPackageRoot) - if linkPackageName == includedPackageName || - packageName != includedPackageName { + if l.LinkPackageName == l.IncludedPackageName || + packageName != l.IncludedPackageName { continue } - byPackageMap[linkPackageName] = append(byPackageMap[linkPackageName], l.LinkFilePath) + byPackageMap[l.LinkPackageName] = append(byPackageMap[l.LinkPackageName], filepath.Join(l.WorkDir, l.LinkFilePath)) } var packages []string @@ -125,3 +348,61 @@ func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) { } return byPackage, nil } + +func getLinkedFileChecksum(path string) (string, error) { + b, err := os.ReadFile(filepath.FromSlash(path)) + if err != nil { + return "", err + } + cs, err := checksum(b) + if err != nil { + return "", err + } + return cs, nil +} + +func readFirstLine(filePath string) (string, error) { + file, err := os.Open(filepath.FromSlash(filePath)) + if err != nil { + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + if scanner.Scan() { + return scanner.Text(), nil + } + + if err := scanner.Err(); err != nil { + return "", err + } + + return "", fmt.Errorf("file is empty or first line is missing") +} + +func checksum(b []byte) (string, error) { + hash := sha256.New() + if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func pathIsInRepositoryRoot(path string) (bool, error) { + path = filepath.FromSlash(path) + root, err := FindRepositoryRoot() + if err != nil { + return false, err + } + if filepath.IsAbs(path) { + path, err = filepath.Rel(root.Name(), path) + if err != nil { + return false, fmt.Errorf("could not get relative path: %w", err) + } + } + + if _, err := root.Stat(path); err != nil { + return false, nil + } + return true, nil +} diff --git a/internal/files/linkedfiles_test.go b/internal/files/linkedfiles_test.go index 1842f1a931..05befac9be 100644 --- a/internal/files/linkedfiles_test.go +++ b/internal/files/linkedfiles_test.go @@ -5,33 +5,101 @@ package files import ( + "bytes" + "os" "path/filepath" "strings" "testing" - "github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestLinkUpdateChecksum(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + basePath := filepath.Join(wd, filepath.FromSlash("testdata/links")) + outdatedFile, err := NewLinkedFile(filepath.Join(basePath, "outdated.yml.link")) + t.Cleanup(func() { + _ = WriteFile(filepath.Join(outdatedFile.WorkDir, outdatedFile.LinkFilePath), []byte(outdatedFile.IncludedFilePath)) + }) + assert.NoError(t, err) + assert.False(t, outdatedFile.UpToDate) + assert.Empty(t, outdatedFile.LinkChecksum) + updated, err := outdatedFile.UpdateChecksum() + assert.NoError(t, err) + assert.True(t, updated) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", outdatedFile.LinkChecksum) + assert.True(t, outdatedFile.UpToDate) + + uptodateFile, err := NewLinkedFile(filepath.Join(basePath, "uptodate.yml.link")) + assert.NoError(t, err) + assert.True(t, uptodateFile.UpToDate) + updated, err = uptodateFile.UpdateChecksum() + assert.NoError(t, err) + assert.False(t, updated) +} + +func TestListLinkedFiles(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + basePath := filepath.Join(wd, filepath.FromSlash("testdata/links")) + linkedFiles, err := ListLinkedFiles(basePath) + require.NoError(t, err) + require.NotEmpty(t, linkedFiles) + require.Len(t, linkedFiles, 2) + assert.Equal(t, "outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.Empty(t, linkedFiles[0].LinkChecksum) + assert.Equal(t, "outdated.yml", linkedFiles[0].TargetFilePath("")) + assert.Equal(t, "./included.yml", linkedFiles[0].IncludedFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) + assert.False(t, linkedFiles[0].UpToDate) + assert.Equal(t, "uptodate.yml.link", linkedFiles[1].LinkFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].LinkChecksum) + assert.Equal(t, "uptodate.yml", linkedFiles[1].TargetFilePath("")) + assert.Equal(t, "./included.yml", linkedFiles[1].IncludedFilePath) + assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[1].IncludedFileContentsChecksum) + assert.True(t, linkedFiles[1].UpToDate) +} + +func TestCopyFile(t *testing.T) { + fileA := "fileA.txt" + fileB := "fileB.txt" + t.Cleanup(func() { _ = os.Remove(fileA) }) + t.Cleanup(func() { _ = os.Remove(fileB) }) + + createDummyFile(t, fileA, "This is the content of the file.") + + assert.NoError(t, CopyFile(fileA, fileB)) + + equal, err := filesEqual(fileA, fileB) + assert.NoError(t, err) + assert.True(t, equal, "files should be equal after copying") +} + func TestAreLinkedFilesUpToDate(t *testing.T) { - linkedFiles, err := AreLinkedFilesUpToDate("internal/files/testdata/links") + wd, err := os.Getwd() + assert.NoError(t, err) + basePath := filepath.Join(wd, filepath.FromSlash("testdata/links")) + linkedFiles, err := AreLinkedFilesUpToDate(basePath) assert.NoError(t, err) assert.NotEmpty(t, linkedFiles) assert.Len(t, linkedFiles, 1) - assert.Equal(t, "internal/files/testdata/links/outdated.yml.link", linkedFiles[0].LinkFilePath) + assert.Equal(t, "outdated.yml.link", linkedFiles[0].LinkFilePath) assert.Empty(t, linkedFiles[0].LinkChecksum) - assert.Equal(t, "internal/files/testdata/links/outdated.yml", linkedFiles[0].TargetFilePath) - assert.Equal(t, "internal/files/testdata/links/included.yml", linkedFiles[0].IncludedFilePath) + assert.Equal(t, "outdated.yml", linkedFiles[0].TargetFilePath("")) + assert.Equal(t, "./included.yml", linkedFiles[0].IncludedFilePath) assert.Equal(t, "d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e", linkedFiles[0].IncludedFileContentsChecksum) assert.False(t, linkedFiles[0].UpToDate) } func TestUpdateLinkedFilesChecksums(t *testing.T) { - root, err := linkedfiles.FindRepositoryRoot() + wd, err := os.Getwd() assert.NoError(t, err) - updated, err := UpdateLinkedFilesChecksums("internal/files/testdata/links") + basePath := filepath.Join(wd, filepath.FromSlash("testdata/links")) + updated, err := UpdateLinkedFilesChecksums(basePath) t.Cleanup(func() { - _ = linkedfiles.WriteFileToRoot(root, updated[0].LinkFilePath, []byte(updated[0].IncludedFilePath)) + _ = WriteFile(filepath.Join(updated[0].WorkDir, updated[0].LinkFilePath), []byte(updated[0].IncludedFilePath)) }) assert.NoError(t, err) assert.NotEmpty(t, updated) @@ -42,7 +110,10 @@ func TestUpdateLinkedFilesChecksums(t *testing.T) { } func TestLinkedFilesByPackageFrom(t *testing.T) { - m, err := LinkedFilesByPackageFrom("internal/files/testdata/links") + wd, err := os.Getwd() + assert.NoError(t, err) + basePath := filepath.Join(wd, filepath.FromSlash("testdata/links")) + m, err := LinkedFilesByPackageFrom(basePath) assert.NoError(t, err) assert.NotEmpty(t, m) assert.Len(t, m, 1) @@ -56,3 +127,42 @@ func TestLinkedFilesByPackageFrom(t *testing.T) { ) assert.True(t, match) } + +func TestIncludeLinkedFiles(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + fromDir := filepath.Join(wd, filepath.FromSlash("testdata/testpackage")) + toDir := t.TempDir() + linkedFiles, err := IncludeLinkedFiles(fromDir, toDir) + assert.NoError(t, err) + require.Equal(t, 1, len(linkedFiles)) + assert.FileExists(t, linkedFiles[0].TargetFilePath(toDir)) + equal, err := filesEqual( + filepath.Join(linkedFiles[0].WorkDir, filepath.FromSlash(linkedFiles[0].IncludedFilePath)), + linkedFiles[0].TargetFilePath(toDir), + ) + assert.NoError(t, err) + assert.True(t, equal, "files should be equal after copying") +} + +func createDummyFile(t *testing.T, filename, content string) { + file, err := os.Create(filename) + assert.NoError(t, err) + defer file.Close() + _, err = file.WriteString(content) + assert.NoError(t, err) +} + +func filesEqual(file1, file2 string) (bool, error) { + f1, err := os.ReadFile(file1) + if err != nil { + return false, err + } + + f2, err := os.ReadFile(file2) + if err != nil { + return false, err + } + + return bytes.Equal(f1, f2), nil +} diff --git a/internal/files/testdata/links/outdated.yml.link b/internal/files/testdata/links/outdated.yml.link index 8d5de3bbd4..76781e5392 100644 --- a/internal/files/testdata/links/outdated.yml.link +++ b/internal/files/testdata/links/outdated.yml.link @@ -1 +1 @@ -internal/files/testdata/links/included.yml \ No newline at end of file +./included.yml \ No newline at end of file diff --git a/internal/files/testdata/links/uptodate.yml.link b/internal/files/testdata/links/uptodate.yml.link index 3f0925ac7f..d0a9c517ea 100644 --- a/internal/files/testdata/links/uptodate.yml.link +++ b/internal/files/testdata/links/uptodate.yml.link @@ -1 +1 @@ -internal/files/testdata/links/included.yml d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e \ No newline at end of file +./included.yml d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e \ No newline at end of file diff --git a/internal/files/testdata/testpackage/included.yml.link b/internal/files/testdata/testpackage/included.yml.link index 3f0925ac7f..61dbe8caee 100644 --- a/internal/files/testdata/testpackage/included.yml.link +++ b/internal/files/testdata/testpackage/included.yml.link @@ -1 +1 @@ -internal/files/testdata/links/included.yml d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e \ No newline at end of file +../links/included.yml d709feed45b708c9548a18ca48f3ad4f41be8d3f691f83d7417ca902a20e6c1e \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link b/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link deleted file mode 100644 index 537ff60f76..0000000000 --- a/test/packages/other/with_includes/data_stream/first/agent/stream/stream.yml.hbs.link +++ /dev/null @@ -1 +0,0 @@ -test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs 069381d45bffbd532a4af8953766a053e75a2aceebdafdffc2264e800fcd1363 \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link b/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link deleted file mode 100644 index 99b4d232ab..0000000000 --- a/test/packages/other/with_includes/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link +++ /dev/null @@ -1 +0,0 @@ -test/packages/other/pipeline_tests/data_stream/test/elasticsearch/ingest_pipeline/default.yml f7c5f0c03aca8ef68c379a62447bdafbf0dcf32b1ff2de143fd6878ee01a91ad \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link b/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link deleted file mode 100644 index 537ff60f76..0000000000 --- a/test/packages/other/with_includes/data_stream/second/agent/stream/stream.yml.hbs.link +++ /dev/null @@ -1 +0,0 @@ -test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs 069381d45bffbd532a4af8953766a053e75a2aceebdafdffc2264e800fcd1363 \ No newline at end of file diff --git a/test/packages/other/with_includes/_dev/build/build.yml b/test/packages/other/with_links/_dev/build/build.yml similarity index 100% rename from test/packages/other/with_includes/_dev/build/build.yml rename to test/packages/other/with_links/_dev/build/build.yml diff --git a/test/packages/other/with_includes/_dev/build/docs/README.md b/test/packages/other/with_links/_dev/build/docs/README.md similarity index 100% rename from test/packages/other/with_includes/_dev/build/docs/README.md rename to test/packages/other/with_links/_dev/build/docs/README.md diff --git a/test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs b/test/packages/other/with_links/_dev/build/shared/stream.yml.hbs similarity index 100% rename from test/packages/other/with_includes/_dev/build/shared/stream.yml.hbs rename to test/packages/other/with_links/_dev/build/shared/stream.yml.hbs diff --git a/test/packages/other/with_includes/changelog.yml b/test/packages/other/with_links/changelog.yml similarity index 100% rename from test/packages/other/with_includes/changelog.yml rename to test/packages/other/with_links/changelog.yml diff --git a/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log b/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log new file mode 100644 index 0000000000..c8c9ffe960 --- /dev/null +++ b/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log @@ -0,0 +1 @@ +1.2.3.4 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-config.yml b/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-config.yml new file mode 100644 index 0000000000..958d74a23e --- /dev/null +++ b/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-config.yml @@ -0,0 +1,4 @@ +multiline: + first_line_pattern: "^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}" +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" diff --git a/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-expected.json b/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-expected.json new file mode 100644 index 0000000000..1c2f884a44 --- /dev/null +++ b/test/packages/other/with_links/data_stream/first/_dev/test/pipeline/test-access-raw.log-expected.json @@ -0,0 +1,5 @@ +{ + "expected": [ + null + ] +} \ No newline at end of file diff --git a/test/packages/other/with_links/data_stream/first/agent/stream/stream.yml.hbs.link b/test/packages/other/with_links/data_stream/first/agent/stream/stream.yml.hbs.link new file mode 100644 index 0000000000..ed8d9065d2 --- /dev/null +++ b/test/packages/other/with_links/data_stream/first/agent/stream/stream.yml.hbs.link @@ -0,0 +1 @@ +../../../../_dev/build/shared/stream.yml.hbs 069381d45bffbd532a4af8953766a053e75a2aceebdafdffc2264e800fcd1363 \ No newline at end of file diff --git a/test/packages/other/with_links/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link b/test/packages/other/with_links/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link new file mode 100644 index 0000000000..c8e0005e25 --- /dev/null +++ b/test/packages/other/with_links/data_stream/first/elasticsearch/ingest_pipeline/default.yml.link @@ -0,0 +1 @@ +../../../../../pipeline_tests/data_stream/test/elasticsearch/ingest_pipeline/default.yml f7c5f0c03aca8ef68c379a62447bdafbf0dcf32b1ff2de143fd6878ee01a91ad \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/first/fields/base-fields.yml b/test/packages/other/with_links/data_stream/first/fields/base-fields.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/first/fields/base-fields.yml rename to test/packages/other/with_links/data_stream/first/fields/base-fields.yml diff --git a/test/packages/other/with_includes/data_stream/first/fields/histogram-fields.yml b/test/packages/other/with_links/data_stream/first/fields/histogram-fields.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/first/fields/histogram-fields.yml rename to test/packages/other/with_links/data_stream/first/fields/histogram-fields.yml diff --git a/test/packages/other/with_includes/data_stream/first/manifest.yml b/test/packages/other/with_links/data_stream/first/manifest.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/first/manifest.yml rename to test/packages/other/with_links/data_stream/first/manifest.yml diff --git a/test/packages/other/with_includes/data_stream/first/sample_event.json b/test/packages/other/with_links/data_stream/first/sample_event.json similarity index 100% rename from test/packages/other/with_includes/data_stream/first/sample_event.json rename to test/packages/other/with_links/data_stream/first/sample_event.json diff --git a/test/packages/other/with_links/data_stream/second/agent/stream/stream.yml.hbs.link b/test/packages/other/with_links/data_stream/second/agent/stream/stream.yml.hbs.link new file mode 100644 index 0000000000..ed8d9065d2 --- /dev/null +++ b/test/packages/other/with_links/data_stream/second/agent/stream/stream.yml.hbs.link @@ -0,0 +1 @@ +../../../../_dev/build/shared/stream.yml.hbs 069381d45bffbd532a4af8953766a053e75a2aceebdafdffc2264e800fcd1363 \ No newline at end of file diff --git a/test/packages/other/with_includes/data_stream/second/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/with_links/data_stream/second/elasticsearch/ingest_pipeline/default.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/second/elasticsearch/ingest_pipeline/default.yml rename to test/packages/other/with_links/data_stream/second/elasticsearch/ingest_pipeline/default.yml diff --git a/test/packages/other/with_includes/data_stream/second/fields/base-fields.yml b/test/packages/other/with_links/data_stream/second/fields/base-fields.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/second/fields/base-fields.yml rename to test/packages/other/with_links/data_stream/second/fields/base-fields.yml diff --git a/test/packages/other/with_includes/data_stream/second/fields/geo-fields.yml b/test/packages/other/with_links/data_stream/second/fields/geo-fields.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/second/fields/geo-fields.yml rename to test/packages/other/with_links/data_stream/second/fields/geo-fields.yml diff --git a/test/packages/other/with_includes/data_stream/second/fields/histogram-fields.yml b/test/packages/other/with_links/data_stream/second/fields/histogram-fields.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/second/fields/histogram-fields.yml rename to test/packages/other/with_links/data_stream/second/fields/histogram-fields.yml diff --git a/test/packages/other/with_includes/data_stream/second/manifest.yml b/test/packages/other/with_links/data_stream/second/manifest.yml similarity index 100% rename from test/packages/other/with_includes/data_stream/second/manifest.yml rename to test/packages/other/with_links/data_stream/second/manifest.yml diff --git a/test/packages/other/with_includes/data_stream/second/sample_event.json b/test/packages/other/with_links/data_stream/second/sample_event.json similarity index 100% rename from test/packages/other/with_includes/data_stream/second/sample_event.json rename to test/packages/other/with_links/data_stream/second/sample_event.json diff --git a/test/packages/other/with_includes/docs/README.md b/test/packages/other/with_links/docs/README.md similarity index 100% rename from test/packages/other/with_includes/docs/README.md rename to test/packages/other/with_links/docs/README.md diff --git a/test/packages/other/with_includes/manifest.yml b/test/packages/other/with_links/manifest.yml similarity index 76% rename from test/packages/other/with_includes/manifest.yml rename to test/packages/other/with_links/manifest.yml index 838496d40b..3b0d50f567 100644 --- a/test/packages/other/with_includes/manifest.yml +++ b/test/packages/other/with_links/manifest.yml @@ -1,8 +1,8 @@ format_version: 2.3.0 -name: with_includes -title: "With Includes Tests" +name: with_links +title: "With Links Tests" version: 0.0.1 -description: "These are tests of field validation with includes." +description: "These are tests of field validation with links." type: integration categories: - custom From 6c943552bb2ba58e6e90ff2a4e147589f5071364 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 10 Apr 2025 12:11:02 +0200 Subject: [PATCH 19/20] replace package-spec --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index fd99f84359..1a7046cf90 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/elastic/elastic-package go 1.24.2 +replace github.com/elastic/package-spec/v3 => github.com/elastic/package-spec/v3 v3.0.0-20250409140721-851b65d4339d + require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Masterminds/semver/v3 v3.3.1 diff --git a/go.sum b/go.sum index 330fb2c426..26ecced323 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU= github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= -github.com/elastic/package-spec/v3 v3.3.5 h1:D0AXRiTNcF8Ue8gLIafF/BLOk7V2yqSFVUy/p0fwArM= -github.com/elastic/package-spec/v3 v3.3.5/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= +github.com/elastic/package-spec/v3 v3.0.0-20250409140721-851b65d4339d h1:jg8qN/0ZAxbo65coqJUFx01OC2PMkWc+6kaf9labTkc= +github.com/elastic/package-spec/v3 v3.0.0-20250409140721-851b65d4339d/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= From ed5a641154ce5ef8a379730ec90af477be99e22b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 10 Apr 2025 12:13:55 +0200 Subject: [PATCH 20/20] Update readme --- README.md | 2 +- cmd/links.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d6affc7976..915d904010 100644 --- a/README.md +++ b/README.md @@ -386,7 +386,7 @@ Use this command to check if linked files references inside the current director _Context: global_ -List packages linking files from this path. +Use this command to list all packages that have linked file references that include the current directory. ### `elastic-package links update` diff --git a/cmd/links.go b/cmd/links.go index e3f03d7373..843e366106 100644 --- a/cmd/links.go +++ b/cmd/links.go @@ -106,6 +106,7 @@ func getLinksListCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list", Short: "List packages linking files from this path", + Long: linksListLongDescription, Args: cobra.NoArgs, RunE: linksListCommandAction, }