diff --git a/internal/discovery/constructor.go b/internal/discovery/constructor.go index 9849e18c70..415e86eab3 100644 --- a/internal/discovery/constructor.go +++ b/internal/discovery/constructor.go @@ -1,6 +1,10 @@ package discovery import ( + "path/filepath" + "runtime" + + "github.com/gruntwork-io/terragrunt/config" "github.com/gruntwork-io/terragrunt/internal/component" "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/internal/filter" @@ -81,13 +85,17 @@ func NewForDiscoveryCommand(opts DiscoveryCommandOptions) (*Discovery, error) { }) } - if opts.Experiments.Evaluate(experiment.FilterFlag) && len(opts.FilterQueries) > 0 { - filters, err := filter.ParseFilterQueries(opts.FilterQueries, opts.WorkingDir) - if err != nil { - return nil, err - } + if opts.Experiments.Evaluate(experiment.FilterFlag) { + d = d.WithFilterFlagEnabled(true) + + if len(opts.FilterQueries) > 0 { + filters, err := filter.ParseFilterQueries(opts.FilterQueries, opts.WorkingDir) + if err != nil { + return nil, err + } - d = d.WithFilters(filters) + d = d.WithFilters(filters) + } } return d, nil @@ -97,14 +105,40 @@ func NewForDiscoveryCommand(opts DiscoveryCommandOptions) (*Discovery, error) { func NewForHCLCommand(opts HCLCommandOptions) (*Discovery, error) { d := NewDiscovery(opts.WorkingDir) - if opts.Experiments.Evaluate(experiment.FilterFlag) && len(opts.FilterQueries) > 0 { - filters, err := filter.ParseFilterQueries(opts.FilterQueries, opts.WorkingDir) - if err != nil { - return nil, err - } + if opts.Experiments.Evaluate(experiment.FilterFlag) { + d = d.WithFilterFlagEnabled(true) - d = d.WithFilters(filters) + if len(opts.FilterQueries) > 0 { + filters, err := filter.ParseFilterQueries(opts.FilterQueries, opts.WorkingDir) + if err != nil { + return nil, err + } + + d = d.WithFilters(filters) + } } return d, nil } + +// NewDiscovery creates a new Discovery. +func NewDiscovery(dir string, opts ...DiscoveryOption) *Discovery { + numWorkers := max(min(runtime.NumCPU(), maxDiscoveryWorkers), defaultDiscoveryWorkers) + + discovery := &Discovery{ + workingDir: dir, + hidden: false, + includeDirs: []string{ + config.StackDir, + filepath.Join(config.StackDir, "**"), + }, + numWorkers: numWorkers, + useDefaultExcludes: true, + } + + for _, opt := range opts { + opt(discovery) + } + + return discovery +} diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index c43f5f361d..d72401d40b 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -7,7 +7,6 @@ import ( "io/fs" "os" "path/filepath" - "runtime" "strings" "sync" @@ -154,6 +153,9 @@ type Discovery struct { // useDefaultExcludes determines whether to use default exclude patterns. useDefaultExcludes bool + + // filterFlagEnabled determines whether the filter flag experiment is active + filterFlagEnabled bool } // DiscoveryOption is a function that modifies a Discovery. @@ -168,28 +170,6 @@ type CompiledPattern struct { // DefaultConfigFilenames are the default Terragrunt config filenames used in discovery. var DefaultConfigFilenames = []string{config.DefaultTerragruntConfigPath, config.DefaultStackFile} -// NewDiscovery creates a new Discovery. -func NewDiscovery(dir string, opts ...DiscoveryOption) *Discovery { - numWorkers := max(min(runtime.NumCPU(), maxDiscoveryWorkers), defaultDiscoveryWorkers) - - discovery := &Discovery{ - workingDir: dir, - hidden: false, - includeDirs: []string{ - config.StackDir, - filepath.Join(config.StackDir, "**"), - }, - numWorkers: numWorkers, - useDefaultExcludes: true, - } - - for _, opt := range opts { - opt(discovery) - } - - return discovery -} - // WithHidden sets the Hidden flag to true. func (d *Discovery) WithHidden() *Discovery { d.hidden = true @@ -362,8 +342,27 @@ func (d *Discovery) WithoutDefaultExcludes() *Discovery { // WithFilters sets filter queries for component selection. // When filters are set, only components matching the filters will be included. +// +// WithFilters also determines whether certain aspects of the discovery configuration allows for optimizations or +// adjustments to discovery are required. e.g. exclude by default if there are any positive filters. func (d *Discovery) WithFilters(filters filter.Filters) *Discovery { d.filters = filters + + d.filterFlagEnabled = true + + // If there are any positive filters, we need to exclude by default, + // and only include components if they match filters. + if d.filters.HasPositiveFilter() { + d.excludeByDefault = true + } + + return d +} + +// WithFilterFlagEnabled sets whether the filter flag experiment is enabled. +// This changes how discovery processes components during file traversal. +func (d *Discovery) WithFilterFlagEnabled(enabled bool) *Discovery { + d.filterFlagEnabled = enabled return d } @@ -639,7 +638,6 @@ func (d *Discovery) walkDirectoryConcurrently( filePaths chan<- string, ) error { walkFn := filepath.WalkDir - if opts.Experiments.Evaluate(experiment.Symlinks) { walkFn = util.WalkDirWithSymlinks } @@ -680,6 +678,11 @@ func (d *Discovery) shouldSkipDirectory(path string, l log.Logger) error { return filepath.SkipDir } + // When filter flag is enabled, let filters control discovery instead of exclude patterns + if d.filterFlagEnabled { + return nil + } + canonicalDir, canErr := util.CanonicalPath(path, d.workingDir) if canErr == nil { for _, pattern := range d.compiledExcludePatterns { @@ -728,13 +731,62 @@ func (d *Discovery) processFile(path string, l log.Logger, filenames []string) c canonicalDir, canErr := util.CanonicalPath(dir, d.workingDir) if canErr == nil { - for _, pattern := range d.compiledExcludePatterns { - if pattern.Compiled.Match(canonicalDir) { - l.Debugf("Path %s excluded by glob %s", canonicalDir, pattern.Original) + // Eventually, this is going to be removed entirely, as filter evaluation + // will be all that's needed. + if !d.filterFlagEnabled { + for _, pattern := range d.compiledExcludePatterns { + if pattern.Compiled.Match(canonicalDir) { + l.Debugf("Path %s excluded by glob %s", canonicalDir, pattern.Original) + return nil + } + } + } + + if d.filterFlagEnabled { + cfg := d.createComponentFromPath(path, filenames) + if cfg == nil { return nil } + + // Check for hidden directories before returning + if !d.hidden && d.isInHiddenDirectory(path) { + allowHidden := false + + // Always allow .terragrunt-stack contents + cleanDir := util.CleanPath(canonicalDir) + for part := range strings.SplitSeq(cleanDir, "/") { + if part == config.StackDir { + allowHidden = true + break + } + } + + if !allowHidden { + return nil + } + } + + shouldEvaluateFiltersNow := !d.discoverDependencies + if shouldEvaluateFiltersNow { + if _, requiresParsing := d.filters.RequiresDiscovery(); !requiresParsing { + filtered, err := d.filters.Evaluate(component.Components{cfg}) + if err != nil { + l.Debugf("Error evaluating filters for %s: %v", cfg.Path(), err) + return nil + } + + if len(filtered) == 0 { + return nil + } + } + } + + return cfg } + // Everything after this point is only relevant when the filter flag is disabled. + // It should be removed once the filter flag is generally available. + // Enforce include patterns only when strictInclude or excludeByDefault are set if d.strictInclude || d.excludeByDefault { included := false @@ -760,8 +812,11 @@ func (d *Discovery) processFile(path string, l log.Logger, filenames []string) c if canErr == nil { // Always allow .terragrunt-stack contents cleanDir := util.CleanPath(canonicalDir) - if strings.Contains(cleanDir, "/"+config.StackDir+"/") || strings.HasSuffix(cleanDir, "/"+config.StackDir) { - allowHidden = true + for part := range strings.SplitSeq(cleanDir, "/") { + if part == config.StackDir { + allowHidden = true + break + } } if !allowHidden { @@ -780,6 +835,12 @@ func (d *Discovery) processFile(path string, l log.Logger, filenames []string) c } } + return d.createComponentFromPath(path, filenames) +} + +// createComponentFromPath creates a component from a file path if it matches one of the config filenames. +// Returns nil if the file doesn't match any of the provided filenames. +func (d *Discovery) createComponentFromPath(path string, filenames []string) component.Component { base := filepath.Base(path) for _, fname := range filenames { if base == fname { diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go index fc2543c0e6..462a9228e9 100644 --- a/internal/discovery/discovery_test.go +++ b/internal/discovery/discovery_test.go @@ -7,6 +7,7 @@ import ( "github.com/gruntwork-io/terragrunt/internal/component" "github.com/gruntwork-io/terragrunt/internal/discovery" + "github.com/gruntwork-io/terragrunt/internal/filter" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/test/helpers/logger" "github.com/stretchr/testify/assert" @@ -654,3 +655,69 @@ func TestDiscoveryPopulatesReadingField(t *testing.T) { assert.Contains(t, appComponent.Reading(), sharedHCL, "should contain shared.hcl") assert.Contains(t, appComponent.Reading(), sharedTFVars, "should contain shared.tfvars") } + +func TestDiscoveryExcludesByDefaultWhenFilterFlagIsEnabled(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + tmpDir, err := filepath.EvalSymlinks(tmpDir) + require.NoError(t, err) + + unit1Dir := filepath.Join(tmpDir, "unit1") + require.NoError(t, os.MkdirAll(unit1Dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(unit1Dir, "terragrunt.hcl"), []byte(""), 0644)) + + unit2Dir := filepath.Join(tmpDir, "unit2") + require.NoError(t, os.MkdirAll(unit2Dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(unit2Dir, "terragrunt.hcl"), []byte(""), 0644)) + + unit3Dir := filepath.Join(tmpDir, "unit3") + require.NoError(t, os.MkdirAll(unit3Dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(unit3Dir, "terragrunt.hcl"), []byte(""), 0644)) + + opts := options.NewTerragruntOptions() + opts.WorkingDir = tmpDir + + l := logger.CreateLogger() + + tt := []struct { + name string + filters []string + want []string + }{ + { + name: "include by default", + filters: []string{}, + want: []string{unit1Dir, unit2Dir, unit3Dir}, + }, + { + name: "exclude by default", + filters: []string{"unit1"}, + want: []string{unit1Dir}, + }, + { + name: "include by default when only negative filters are present", + filters: []string{"!unit2"}, + want: []string{unit1Dir, unit3Dir}, + }, + { + name: "exclude by default when positive and negative filters are present", + filters: []string{"unit1", "!unit2"}, + want: []string{unit1Dir}, + }, + } + + for _, tt := range tt { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + filters, err := filter.ParseFilterQueries(tt.filters, tmpDir) + require.NoError(t, err) + + d := discovery.NewDiscovery(tmpDir).WithFilters(filters) + components, err := d.Discover(t.Context(), l, opts) + require.NoError(t, err) + assert.ElementsMatch(t, tt.want, components.Filter(component.UnitKind).Paths()) + }) + } +} diff --git a/internal/discovery/filter_integration_test.go b/internal/discovery/filter_integration_test.go index af1348f414..65c11774e7 100644 --- a/internal/discovery/filter_integration_test.go +++ b/internal/discovery/filter_integration_test.go @@ -808,6 +808,8 @@ func TestDiscoveryWithReadingFiltersAndAbsolutePaths(t *testing.T) { t.Parallel() tmpDir := t.TempDir() + tmpDir, err := filepath.EvalSymlinks(tmpDir) + require.NoError(t, err) // Create a shared file with absolute path sharedFile := filepath.Join(tmpDir, "shared.hcl") diff --git a/internal/filter/ast.go b/internal/filter/ast.go index 2b49c2bdc5..fe45d4ae32 100644 --- a/internal/filter/ast.go +++ b/internal/filter/ast.go @@ -13,8 +13,8 @@ type Expression interface { expressionNode() // String returns a string representation of the expression for debugging. String() string - // RequiresHCLParsing returns true if the expression requires parsing Terragrunt HCL configurations. - RequiresHCLParsing() (Expression, bool) + // RequiresDiscovery returns true if the expression requires parsing Terragrunt HCL configurations. + RequiresDiscovery() (Expression, bool) } // PathFilter represents a path or glob filter (e.g., "./path/**/*" or "/absolute/path"). @@ -48,9 +48,9 @@ func (p *PathFilter) CompileGlob() (glob.Glob, error) { return p.compiledGlob, p.compileErr } -func (p *PathFilter) expressionNode() {} -func (p *PathFilter) String() string { return p.Value } -func (p *PathFilter) RequiresHCLParsing() (Expression, bool) { return p, false } +func (p *PathFilter) expressionNode() {} +func (p *PathFilter) String() string { return p.Value } +func (p *PathFilter) RequiresDiscovery() (Expression, bool) { return p, false } // AttributeFilter represents a key-value attribute filter (e.g., "name=my-app"). type AttributeFilter struct { @@ -93,9 +93,9 @@ func (a *AttributeFilter) supportsGlob() bool { return a.Key == AttributeReading || a.Key == AttributeName } -func (a *AttributeFilter) expressionNode() {} -func (a *AttributeFilter) String() string { return a.Key + "=" + a.Value } -func (a *AttributeFilter) RequiresHCLParsing() (Expression, bool) { return a, true } +func (a *AttributeFilter) expressionNode() {} +func (a *AttributeFilter) String() string { return a.Key + "=" + a.Value } +func (a *AttributeFilter) RequiresDiscovery() (Expression, bool) { return a, true } // PrefixExpression represents a prefix operator expression (e.g., "!name=foo"). type PrefixExpression struct { @@ -105,8 +105,8 @@ type PrefixExpression struct { func (p *PrefixExpression) expressionNode() {} func (p *PrefixExpression) String() string { return p.Operator + p.Right.String() } -func (p *PrefixExpression) RequiresHCLParsing() (Expression, bool) { - return p.Right.RequiresHCLParsing() +func (p *PrefixExpression) RequiresDiscovery() (Expression, bool) { + return p.Right.RequiresDiscovery() } // InfixExpression represents an infix operator expression (e.g., "./apps/* | name=bar"). @@ -120,12 +120,12 @@ func (i *InfixExpression) expressionNode() {} func (i *InfixExpression) String() string { return i.Left.String() + " " + i.Operator + " " + i.Right.String() } -func (i *InfixExpression) RequiresHCLParsing() (Expression, bool) { - if _, ok := i.Left.RequiresHCLParsing(); ok { +func (i *InfixExpression) RequiresDiscovery() (Expression, bool) { + if _, ok := i.Left.RequiresDiscovery(); ok { return i, true } - if _, ok := i.Right.RequiresHCLParsing(); ok { + if _, ok := i.Right.RequiresDiscovery(); ok { return i, true } diff --git a/internal/filter/errors.go b/internal/filter/errors.go index a37e77c7e5..9e830ebc48 100644 --- a/internal/filter/errors.go +++ b/internal/filter/errors.go @@ -45,14 +45,14 @@ func NewEvaluationErrorWithCause(message string, cause error) error { return errors.New(EvaluationError{Message: message, Cause: cause}) } -// FilterQueryRequiresHCLParsingError is an error that is returned when a filter query requires parsing Terragrunt configurations. -type FilterQueryRequiresHCLParsingError struct { +// FilterQueryRequiresDiscoveryError is an error that is returned when a filter query requires discovery of Terragrunt configurations. +type FilterQueryRequiresDiscoveryError struct { Query string } -func (e FilterQueryRequiresHCLParsingError) Error() string { +func (e FilterQueryRequiresDiscoveryError) Error() string { return fmt.Sprintf( - "Filter query '%s' requires parsing Terragrunt configurations, which is not supported when evaluating filters on files", + "Filter query '%s' requires discovery of Terragrunt configurations, which is not supported when evaluating filters on generic files", e.Query, ) } diff --git a/internal/filter/filters.go b/internal/filter/filters.go index ae64b215c2..dfdc564793 100644 --- a/internal/filter/filters.go +++ b/internal/filter/filters.go @@ -56,10 +56,10 @@ func (f Filters) HasPositiveFilter() bool { return false } -// RequiresHCLParsing returns the first expression that requires parsing Terragrunt HCL configurations if any do. -func (f Filters) RequiresHCLParsing() (Expression, bool) { +// RequiresDiscovery returns the first expression that requires discovery of Terragrunt components if any do. +func (f Filters) RequiresDiscovery() (Expression, bool) { for _, filter := range f { - if e, ok := filter.expr.RequiresHCLParsing(); ok { + if e, ok := filter.expr.RequiresDiscovery(); ok { return e, true } } @@ -114,8 +114,8 @@ func (f Filters) Evaluate(components component.Components) (component.Components // This is useful for the hcl format command, where we want to evaluate filters on files // rather than directories, like we do with components. func (f Filters) EvaluateOnFiles(files []string) (component.Components, error) { - if e, ok := f.RequiresHCLParsing(); ok { - return nil, FilterQueryRequiresHCLParsingError{Query: e.String()} + if e, ok := f.RequiresDiscovery(); ok { + return nil, FilterQueryRequiresDiscoveryError{Query: e.String()} } comps := make(component.Components, 0, len(files)) diff --git a/test/integration_hcl_filter_test.go b/test/integration_hcl_filter_test.go index abb3ba53c0..981fa9c5fc 100644 --- a/test/integration_hcl_filter_test.go +++ b/test/integration_hcl_filter_test.go @@ -120,31 +120,31 @@ func TestHCLFormatCheckWithFilter(t *testing.T) { name: "error: name=*.stack.hcl requires HCL parsing", filterArgs: []string{"name=*.stack.hcl"}, expectError: true, - errorAs: filter.FilterQueryRequiresHCLParsingError{Query: "name=*.stack.hcl"}, + errorAs: filter.FilterQueryRequiresDiscoveryError{Query: "name=*.stack.hcl"}, }, { name: "error: type=unit requires HCL parsing", filterArgs: []string{"type=unit"}, expectError: true, - errorAs: filter.FilterQueryRequiresHCLParsingError{Query: "type=unit"}, + errorAs: filter.FilterQueryRequiresDiscoveryError{Query: "type=unit"}, }, { name: "error: type=stack requires HCL parsing", filterArgs: []string{"type=stack"}, expectError: true, - errorAs: filter.FilterQueryRequiresHCLParsingError{Query: "type=stack"}, + errorAs: filter.FilterQueryRequiresDiscoveryError{Query: "type=stack"}, }, { name: "error: external=true requires HCL parsing", filterArgs: []string{"external=true"}, expectError: true, - errorAs: filter.FilterQueryRequiresHCLParsingError{Query: "external=true"}, + errorAs: filter.FilterQueryRequiresDiscoveryError{Query: "external=true"}, }, { name: "error: intersection with type filter", filterArgs: []string{"./needs-formatting/** | type=unit"}, expectError: true, - errorAs: filter.FilterQueryRequiresHCLParsingError{Query: "./needs-formatting/** | type=unit"}, + errorAs: filter.FilterQueryRequiresDiscoveryError{Query: "./needs-formatting/** | type=unit"}, }, }