diff --git a/cli/commands/run/context.go b/cli/commands/run/context.go index c41974d0e7..8b0f7f282a 100644 --- a/cli/commands/run/context.go +++ b/cli/commands/run/context.go @@ -14,7 +14,14 @@ const ( ) // WithRunVersionCache initializes the version cache in the context for the run package. +// If a cache already exists in the context, it reuses it. Otherwise, creates a new one. func WithRunVersionCache(ctx context.Context) context.Context { + // Check if cache already exists in context + if existingCache, ok := ctx.Value(versionCacheContextKey).(*cache.Cache[string]); ok && existingCache != nil { + return ctx // Reuse existing cache + } + + // Create new cache if none exists ctx = context.WithValue(ctx, versionCacheContextKey, cache.NewCache[string](versionCacheName)) return ctx } @@ -23,3 +30,8 @@ func WithRunVersionCache(ctx context.Context) context.Context { func GetRunVersionCache(ctx context.Context) *cache.Cache[string] { return cache.ContextCache[string](ctx, versionCacheContextKey) } + +// ClearVersionCache clears the version cache from the context. Useful during testing. +func ClearVersionCache(ctx context.Context) { + cache.ContextCache[string](ctx, versionCacheContextKey).Clear() +} diff --git a/config/config.go b/config/config.go index 40631dec3e..2725d84771 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,6 @@ import ( "github.com/gruntwork-io/terragrunt/pkg/log/writer" "github.com/gruntwork-io/terragrunt/tf" - "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/internal/ctyhelper" "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/internal/remotestate" @@ -51,8 +50,6 @@ const ( FoundInFile = "found_in_file" - iamRoleCacheName = "iamRoleCache" - logMsgSeparator = "\n" DefaultEngineType = "rpc" @@ -1170,7 +1167,7 @@ func ReadTerragruntConfig(ctx context.Context, l log.Logger, terragruntOptions * func ParseConfigFile(ctx *ParsingContext, l log.Logger, configPath string, includeFromChild *IncludeConfig) (*TerragruntConfig, error) { var config *TerragruntConfig - hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey) + hclCache := GetHclCache(ctx) err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "parse_config_file", map[string]any{ "config_path": configPath, @@ -1186,11 +1183,6 @@ func ParseConfigFile(ctx *ParsingContext, l log.Logger, configPath string, inclu decodeListKey = fmt.Sprintf("%v", ctx.PartialParseDecodeList) } - dir, err := os.Getwd() - if err != nil { - return err - } - fileInfo, err := os.Stat(configPath) if err != nil { if os.IsNotExist(err) { @@ -1200,11 +1192,16 @@ func ParseConfigFile(ctx *ParsingContext, l log.Logger, configPath string, inclu return errors.Errorf("failed to get file info: %w", err) } - var ( - file *hclparse.File - cacheKey = fmt.Sprintf("parse-config-%v-%v-%v-%v-%v-%v", configPath, childKey, decodeListKey, ctx.TerragruntOptions.WorkingDir, dir, fileInfo.ModTime().UnixMicro()) + cacheKey := fmt.Sprintf("%v-%v-%v-%v-%v", + configPath, + ctx.TerragruntOptions.WorkingDir, + childKey, + decodeListKey, + fileInfo.ModTime().UnixMicro(), ) + var file *hclparse.File + // TODO: Remove lint ignore if cacheConfig, found := hclCache.Get(ctx, cacheKey); found { //nolint:contextcheck file = cacheConfig @@ -1489,9 +1486,6 @@ func detectBareIncludeUsage(file *hclparse.File) bool { } } -// iamRoleCache - store for cached values of IAM roles -var iamRoleCache = cache.NewCache[options.IAMRoleOptions](iamRoleCacheName) - // setIAMRole - extract IAM role details from Terragrunt flags block func setIAMRole(ctx *ParsingContext, l log.Logger, file *hclparse.File, includeFromChild *IncludeConfig) error { // Prefer the IAM Role CLI args if they were passed otherwise lazily evaluate the IamRoleOptions using the config. @@ -1501,7 +1495,7 @@ func setIAMRole(ctx *ParsingContext, l log.Logger, file *hclparse.File, includeF // as key is considered HCL code and include configuration var ( key = fmt.Sprintf("%v-%v", file.Content(), includeFromChild) - config, found = iamRoleCache.Get(ctx, key) + config, found = GetIAMRoleCache(ctx).Get(ctx, key) ) if !found { @@ -1511,7 +1505,7 @@ func setIAMRole(ctx *ParsingContext, l log.Logger, file *hclparse.File, includeF } config = iamConfig.GetIAMRoleOptions() - iamRoleCache.Put(ctx, key, config) + GetIAMRoleCache(ctx).Put(ctx, key, config) } // We merge the OriginalIAMRoleOptions into the one from the config, because the CLI passed IAMRoleOptions has // precedence. diff --git a/config/config_helpers.go b/config/config_helpers.go index cb5ada28c3..53e5180065 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -25,7 +25,6 @@ import ( "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/internal/awshelper" - "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/internal/cli" "github.com/gruntwork-io/terragrunt/internal/ctyhelper" "github.com/gruntwork-io/terragrunt/internal/errors" @@ -79,8 +78,6 @@ const ( FuncNameTimeCmp = "timecmp" FuncNameMarkAsRead = "mark_as_read" FuncNameConstraintCheck = "constraint_check" - - sopsCacheName = "sopsCache" ) // TerraformCommandsNeedLocking is a list of terraform commands that accept -lock-timeout @@ -338,7 +335,7 @@ func parseGetEnvParameters(parameters []string) (EnvVar, error) { func RunCommand(ctx *ParsingContext, l log.Logger, args []string) (string, error) { // runCommandCache - cache of evaluated `run_cmd` invocations // see: https://github.com/gruntwork-io/terragrunt/issues/1427 - runCommandCache := cache.ContextCache[string](ctx, RunCmdCacheContextKey) + runCommandCache := GetRunCmdCache(ctx) if len(args) == 0 { return "", errors.New(EmptyStringNotAllowedError("parameter to the run_cmd function")) @@ -879,14 +876,6 @@ func getModulePathFromSourceURL(sourceURL string) (string, error) { return matches[1], nil } -// A cache of the results of a decrypt operation via sops. Each decryption -// operation can take several seconds, so this cache speeds up terragrunt executions -// where the same sops files are referenced multiple times. -// -// The cache keys are the canonical paths to the encrypted files, and the values are the -// plain-text result of the decrypt operation. -var sopsCache = cache.NewCache[string](sopsCacheName) - // decrypts and returns sops encrypted utf-8 yaml or json data as a string func sopsDecryptFile(ctx *ParsingContext, l log.Logger, params []string) (string, error) { numParams := len(params) @@ -939,7 +928,7 @@ func sopsDecryptFile(ctx *ParsingContext, l log.Logger, params []string) (string } } - if val, ok := sopsCache.Get(ctx, path); ok { + if val, ok := GetSopsCache(ctx).Get(ctx, path); ok { return val, nil } @@ -950,7 +939,7 @@ func sopsDecryptFile(ctx *ParsingContext, l log.Logger, params []string) (string if utf8.Valid(rawData) { value := string(rawData) - sopsCache.Put(ctx, path, value) + GetSopsCache(ctx).Put(ctx, path, value) return value, nil } diff --git a/config/config_partial.go b/config/config_partial.go index 9c8f454fe1..a78ad39efe 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -11,7 +11,6 @@ import ( "github.com/gruntwork-io/terragrunt/pkg/log" "github.com/huandu/go-clone" - "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/internal/strict/controls" "github.com/hashicorp/hcl/v2" @@ -277,7 +276,7 @@ func cliFlagsToCty(ctx *ParsingContext, flagByName map[string]*FeatureFlag) (map } func PartialParseConfigFile(ctx *ParsingContext, l log.Logger, configPath string, include *IncludeConfig) (*TerragruntConfig, error) { - hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey) + hclCache := GetHclCache(ctx) fileInfo, err := os.Stat(configPath) if err != nil { @@ -313,7 +312,7 @@ func PartialParseConfigFile(ctx *ParsingContext, l log.Logger, configPath string func TerragruntConfigFromPartialConfig(ctx *ParsingContext, l log.Logger, file *hclparse.File, includeFromChild *IncludeConfig) (*TerragruntConfig, error) { var cacheKey = fmt.Sprintf("%#v-%#v-%#v-%#v", file.ConfigPath, file.Content(), includeFromChild, ctx.PartialParseDecodeList) - terragruntConfigCache := cache.ContextCache[*TerragruntConfig](ctx, TerragruntConfigCacheContextKey) + terragruntConfigCache := GetTerragruntConfigCache(ctx) if ctx.TerragruntOptions.UsePartialParseConfigCache { if config, found := terragruntConfigCache.Get(ctx, cacheKey); found { l.Debugf("Cache hit for '%s' (partial parsing), decodeList: '%v'.", ctx.TerragruntOptions.TerragruntConfigPath, ctx.PartialParseDecodeList) diff --git a/config/context.go b/config/context.go index 2805d76a8a..67e1409527 100644 --- a/config/context.go +++ b/config/context.go @@ -2,31 +2,103 @@ package config import ( "context" + "sync" "github.com/gruntwork-io/terragrunt/config/hclparse" "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/gruntwork-io/terragrunt/options" ) type configKey byte const ( - HclCacheContextKey configKey = iota - TerragruntConfigCacheContextKey configKey = iota - RunCmdCacheContextKey configKey = iota - DependencyOutputCacheContextKey configKey = iota - - hclCacheName = "hclCache" - configCacheName = "configCache" - runCmdCacheName = "runCmdCache" - dependencyOutputCacheName = "dependencyOutputCache" + HclCacheContextKey configKey = iota + TerragruntConfigCacheContextKey configKey = iota + RunCmdCacheContextKey configKey = iota + DependencyOutputCacheContextKey configKey = iota + DependencyJSONOutputCacheContextKey configKey = iota + DependencyLocksContextKey configKey = iota + SopsCacheContextKey configKey = iota + IAMRoleCacheContextKey configKey = iota + + hclCacheName = "hclCache" + configCacheName = "configCache" + runCmdCacheName = "runCmdCache" + dependencyOutputCacheName = "dependencyOutputCache" + dependencyJSONOutputCacheName = "dependencyJSONOutputCache" + dependencyLocksCacheName = "dependencyLocksCache" + sopsCacheName = "sopsCache" + iamRoleCacheName = "iamRoleCache" ) +// GetSopsCache returns the SOPS cache instance from context +func GetSopsCache(ctx context.Context) *cache.Cache[string] { + return cache.ContextCache[string](ctx, SopsCacheContextKey) +} + +// GetIAMRoleCache returns the IAM role cache instance from context +func GetIAMRoleCache(ctx context.Context) *cache.Cache[options.IAMRoleOptions] { + return cache.ContextCache[options.IAMRoleOptions](ctx, IAMRoleCacheContextKey) +} + +// GetHclCache returns the HCL file cache instance from context +func GetHclCache(ctx context.Context) *cache.Cache[*hclparse.File] { + return cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey) +} + +// GetTerragruntConfigCache returns the Terragrunt config cache instance from context +func GetTerragruntConfigCache(ctx context.Context) *cache.Cache[*TerragruntConfig] { + return cache.ContextCache[*TerragruntConfig](ctx, TerragruntConfigCacheContextKey) +} + +// GetRunCmdCache returns the run command cache instance from context +func GetRunCmdCache(ctx context.Context) *cache.Cache[string] { + return cache.ContextCache[string](ctx, RunCmdCacheContextKey) +} + +// GetDependencyOutputCache returns the dependency output cache instance from context +func GetDependencyOutputCache(ctx context.Context) *cache.Cache[*dependencyOutputCache] { + return cache.ContextCache[*dependencyOutputCache](ctx, DependencyOutputCacheContextKey) +} + +// GetDependencyJSONOutputCache returns the dependency JSON output cache instance from context +func GetDependencyJSONOutputCache(ctx context.Context) *cache.Cache[[]byte] { + return cache.ContextCache[[]byte](ctx, DependencyJSONOutputCacheContextKey) +} + +// GetDependencyLocksCache returns the dependency locks cache instance from context +func GetDependencyLocksCache(ctx context.Context) *cache.Cache[*sync.Mutex] { + return cache.ContextCache[*sync.Mutex](ctx, DependencyLocksContextKey) +} + // WithConfigValues add to context default values for configuration. +// If caches already exist in the context, they are reused. Otherwise, new ones are created. func WithConfigValues(ctx context.Context) context.Context { - ctx = context.WithValue(ctx, HclCacheContextKey, cache.NewCache[*hclparse.File](hclCacheName)) - ctx = context.WithValue(ctx, TerragruntConfigCacheContextKey, cache.NewCache[*TerragruntConfig](configCacheName)) - ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName)) - ctx = context.WithValue(ctx, DependencyOutputCacheContextKey, cache.NewCache[*dependencyOutputCache](dependencyOutputCacheName)) + // Reuse existing caches if they exist, otherwise create new ones + if _, ok := ctx.Value(HclCacheContextKey).(*cache.Cache[*hclparse.File]); !ok { + ctx = context.WithValue(ctx, HclCacheContextKey, cache.NewCache[*hclparse.File](hclCacheName)) + } + if _, ok := ctx.Value(TerragruntConfigCacheContextKey).(*cache.Cache[*TerragruntConfig]); !ok { + ctx = context.WithValue(ctx, TerragruntConfigCacheContextKey, cache.NewCache[*TerragruntConfig](configCacheName)) + } + if _, ok := ctx.Value(RunCmdCacheContextKey).(*cache.Cache[string]); !ok { + ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName)) + } + if _, ok := ctx.Value(DependencyOutputCacheContextKey).(*cache.Cache[*dependencyOutputCache]); !ok { + ctx = context.WithValue(ctx, DependencyOutputCacheContextKey, cache.NewCache[*dependencyOutputCache](dependencyOutputCacheName)) + } + if _, ok := ctx.Value(DependencyJSONOutputCacheContextKey).(*cache.Cache[[]byte]); !ok { + ctx = context.WithValue(ctx, DependencyJSONOutputCacheContextKey, cache.NewCache[[]byte](dependencyJSONOutputCacheName)) + } + if _, ok := ctx.Value(DependencyLocksContextKey).(*cache.Cache[*sync.Mutex]); !ok { + ctx = context.WithValue(ctx, DependencyLocksContextKey, cache.NewCache[*sync.Mutex](dependencyLocksCacheName)) + } + if _, ok := ctx.Value(SopsCacheContextKey).(*cache.Cache[string]); !ok { + ctx = context.WithValue(ctx, SopsCacheContextKey, cache.NewCache[string](sopsCacheName)) + } + if _, ok := ctx.Value(IAMRoleCacheContextKey).(*cache.Cache[options.IAMRoleOptions]); !ok { + ctx = context.WithValue(ctx, IAMRoleCacheContextKey, cache.NewCache[options.IAMRoleOptions](iamRoleCacheName)) + } return ctx } diff --git a/config/dependency.go b/config/dependency.go index 5d1ac17ec5..7617804dae 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -3,6 +3,7 @@ package config import ( "bufio" "bytes" + "context" "encoding/json" "fmt" "io" @@ -14,7 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gruntwork-io/terragrunt/internal/awshelper" - "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/internal/experiment" "github.com/gruntwork-io/terragrunt/internal/remotestate" "github.com/gruntwork-io/terragrunt/internal/report" @@ -180,14 +180,6 @@ func (dep *Dependency) setRenderedOutputs(ctx *ParsingContext, l log.Logger) err return nil } -// jsonOutputCache is a map that maps config paths to the outputs so that they can be reused across calls for common -// modules. We use sync.Map to ensure atomic updates during concurrent access. -var jsonOutputCache = sync.Map{} - -// outputLocks is a map that maps config paths to mutex locks to ensure we only have a single instance of terragrunt -// output running for a given dependent config. We use sync.Map to ensure atomic updates during concurrent access. -var outputLocks = sync.Map{} - // Decode the dependency blocks from the file, and then retrieve all the outputs from the remote state. Then encode the // resulting map as a cty.Value object. // TODO: In the future, consider allowing importing dependency blocks from included config @@ -235,7 +227,7 @@ func decodeAndRetrieveOutputs(ctx *ParsingContext, l log.Logger, file *hclparse. // decodeDependencies decode dependencies and fetch inputs func decodeDependencies(ctx *ParsingContext, l log.Logger, decodedDependency TerragruntDependency) (*TerragruntDependency, error) { updatedDependencies := TerragruntDependency{} - depCache := cache.ContextCache[*dependencyOutputCache](ctx, DependencyOutputCacheContextKey) + depCache := GetDependencyOutputCache(ctx) for _, dep := range decodedDependency.Dependencies { depPath := getCleanedTargetConfigPath(dep.ConfigPath.AsString(), ctx.TerragruntOptions.TerragruntConfigPath) @@ -609,13 +601,22 @@ func isRenderCommand(ctx *ParsingContext) bool { // getOutputJSONWithCaching will run terragrunt output on the target config if it is not already cached. func getOutputJSONWithCaching(ctx *ParsingContext, l log.Logger, targetConfig string) ([]byte, error) { - // Acquire synchronization lock to ensure only one instance of output is called per config. - rawActualLock, _ := outputLocks.LoadOrStore(targetConfig, &sync.Mutex{}) + // Use context-based caches instead of global sync.Map + outputLocksCache := GetDependencyLocksCache(ctx) + jsonCache := GetDependencyJSONOutputCache(ctx) - actualLock := rawActualLock.(*sync.Mutex) - defer actualLock.Unlock() + // Acquire synchronization lock to ensure only one instance of output is called per config. + // Get or create a lock for this specific target config + var actualLock *sync.Mutex + if cachedLock, found := outputLocksCache.Get(ctx, targetConfig); found { + actualLock = cachedLock + } else { + actualLock = &sync.Mutex{} + outputLocksCache.Put(ctx, targetConfig, actualLock) + } actualLock.Lock() + defer actualLock.Unlock() // This debug log is useful for validating if the locking mechanism is working. If the locking mechanism is working, // we should only see one pair of logs at a time that begin with this statement, and then the relevant "terraform @@ -623,11 +624,10 @@ func getOutputJSONWithCaching(ctx *ParsingContext, l log.Logger, targetConfig st l.Debugf("Getting output of dependency %s for config %s", targetConfig, ctx.TerragruntOptions.TerragruntConfigPath) // Look up if we have already run terragrunt output for this target config - rawJSONBytes, hasRun := jsonOutputCache.Load(targetConfig) - if hasRun { + if cachedJSONBytes, hasRun := jsonCache.Get(ctx, targetConfig); hasRun { // Cache hit, so return cached output l.Debugf("%s was run before. Using cached output.", targetConfig) - return rawJSONBytes.([]byte), nil + return cachedJSONBytes, nil } // Cache miss, so look up the output and store in cache @@ -643,7 +643,7 @@ func getOutputJSONWithCaching(ctx *ParsingContext, l log.Logger, targetConfig st newJSONBytes = newJSONBytes[index:] } - jsonOutputCache.Store(targetConfig, newJSONBytes) + jsonCache.Put(ctx, targetConfig, newJSONBytes) return newJSONBytes, nil } @@ -1161,8 +1161,10 @@ func TerraformOutputJSONToCtyValueMap(targetConfigPath string, jsonBytes []byte) } // ClearOutputCache clears the output cache. Useful during testing. -func ClearOutputCache() { - jsonOutputCache = sync.Map{} +// Clears the persistent package-level caches directly. +func ClearOutputCache(ctx context.Context) { + persistentDependencyJSONOutputCache.Clear() + persistentDependencyLocksCache.Clear() } // runTerraformInitForDependencyOutput will run terraform init in a mode that doesn't pull down plugins or modules. Note diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 92ee0dceda..f2e13ed307 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -61,6 +61,22 @@ func (c *Cache[V]) Put(ctx context.Context, key string, value V) { c.Cache[cacheKey] = value } +// Size returns the current number of items in the cache +func (c *Cache[V]) Size() int { + c.Mutex.RLock() + defer c.Mutex.RUnlock() + + return len(c.Cache) +} + +// Clear removes all entries from the cache +func (c *Cache[V]) Clear() { + c.Mutex.Lock() + defer c.Mutex.Unlock() + + c.Cache = make(map[string]V) +} + // ExpiringItem - item with expiration time type ExpiringItem[V any] struct { Value V @@ -119,10 +135,11 @@ func (c *ExpiringCache[V]) Put(ctx context.Context, key string, value V, expirat // ContextCache returns cache from the context. If the cache is nil, it creates a new instance. func ContextCache[T any](ctx context.Context, key any) *Cache[T] { - cacheInstance, ok := ctx.Value(key).(*Cache[T]) - if !ok || cacheInstance == nil { - cacheInstance = NewCache[T](fmt.Sprintf("%v", key)) + // Try to fetch cache from context + if cache, ok := ctx.Value(key).(*Cache[T]); ok && cache != nil { + return cache } - return cacheInstance + // Create new cache if nothing found + return NewCache[T](fmt.Sprintf("%v", key)) } diff --git a/test/integration_aws_test.go b/test/integration_aws_test.go index 9d1e6fb65a..841f9fdd9e 100644 --- a/test/integration_aws_test.go +++ b/test/integration_aws_test.go @@ -953,7 +953,7 @@ func TestAwsDependencyOutputOptimization(t *testing.T) { // We need to bust the output cache that stores the dependency outputs so that the second run pulls the outputs. // This is only a problem during testing, where the process is shared across terragrunt runs. - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) // verify expected output stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt output -no-color -json --log-level trace --non-interactive --working-dir "+livePath) @@ -966,8 +966,8 @@ func TestAwsDependencyOutputOptimization(t *testing.T) { // If we want to force reinit, delete the relevant .terraform directories helpers.CleanupTerraformFolder(t, depPath) - // Now delete the deepdep state and verify still works (note we need to bust the cache again) - config.ClearOutputCache() + // Now delete the deepdep state, and verify still works (note we need to bust the cache again) + config.ClearOutputCache(t.Context()) require.NoError(t, os.Remove(filepath.Join(deepDepPath, "terraform.tfstate"))) fmt.Println("terragrunt output -no-color -json --log-level trace --non-interactive --working-dir " + livePath) @@ -1024,7 +1024,7 @@ func TestAwsDependencyOutputOptimizationDisableTest(t *testing.T) { // We need to bust the output cache that stores the dependency outputs so that the second run pulls the outputs. // This is only a problem during testing, where the process is shared across terragrunt runs. - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) // verify expected output stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt output -no-color -json --non-interactive --working-dir "+livePath) @@ -1035,7 +1035,7 @@ func TestAwsDependencyOutputOptimizationDisableTest(t *testing.T) { assert.Equal(t, expectedOutput, outputs["output"].Value) // Now delete the deepdep state and verify it no longer works, because it tries to fetch the deepdep dependency - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) require.NoError(t, os.Remove(filepath.Join(deepDepPath, "terraform.tfstate"))) require.NoError(t, os.RemoveAll(filepath.Join(deepDepPath, ".terraform"))) _, _, err = helpers.RunTerragruntCommandWithOutput(t, "terragrunt output -no-color -json --non-interactive --working-dir "+livePath) @@ -1272,7 +1272,7 @@ func TestAwsDependencyOutputSameOutputConcurrencyRegression(t *testing.T) { tt() // We need to bust the output cache that stores the dependency outputs so that the second run pulls the outputs. // This is only a problem during testing, where the process is shared across terragrunt runs. - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) } } @@ -1296,7 +1296,7 @@ func TestAwsRemoteStateCodegenGeneratesBackendBlockS3(t *testing.T) { } func TestAwsOutputFromRemoteState(t *testing.T) { //nolint: paralleltest - // NOTE: We can't run this test in parallel because there are other tests that also call `config.ClearOutputCache()`, but this function uses a global variable and sometimes it throws an unexpected error: + // NOTE: We can't run this test in parallel because there are other tests that also call `config.ClearOutputCache(context.Background())`, but this function uses a global variable and sometimes it throws an unexpected error: // "fixtures/output-from-remote-state/env1/app2/terragrunt.hcl:23,38-48: Unsupported attribute; This object does not have an attribute named "app3_text"." // t.Parallel() @@ -1313,7 +1313,7 @@ func TestAwsOutputFromRemoteState(t *testing.T) { //nolint: paralleltest helpers.RunTerragrunt(t, fmt.Sprintf("terragrunt apply --backend-bootstrap --dependency-fetch-output-from-state --auto-approve --non-interactive --working-dir %s/app1", environmentPath)) helpers.RunTerragrunt(t, fmt.Sprintf("terragrunt apply --backend-bootstrap --dependency-fetch-output-from-state --auto-approve --non-interactive --working-dir %s/app3", environmentPath)) // Now delete dependencies cached state - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) require.NoError(t, os.Remove(filepath.Join(environmentPath, "/app1/.terraform/terraform.tfstate"))) require.NoError(t, os.RemoveAll(filepath.Join(environmentPath, "/app1/.terraform"))) require.NoError(t, os.Remove(filepath.Join(environmentPath, "/app3/.terraform/terraform.tfstate"))) @@ -1337,7 +1337,7 @@ func TestAwsOutputFromRemoteState(t *testing.T) { //nolint: paralleltest } func TestAwsMockOutputsFromRemoteState(t *testing.T) { //nolint: paralleltest - // NOTE: We can't run this test in parallel because there are other tests that also call `config.ClearOutputCache()`, but this function uses a global variable and sometimes it throws an unexpected error: + // NOTE: We can't run this test in parallel because there are other tests that also call `config.ClearOutputCache(context.Background())`, but this function uses a global variable and sometimes it throws an unexpected error: // "fixtures/output-from-remote-state/env1/app2/terragrunt.hcl:23,38-48: Unsupported attribute; This object does not have an attribute named "app3_text"." // t.Parallel() @@ -1354,7 +1354,7 @@ func TestAwsMockOutputsFromRemoteState(t *testing.T) { //nolint: paralleltest // applying only the app1 dependency, the app3 dependency was purposely not applied and should be mocked when running the app2 module helpers.RunTerragrunt(t, fmt.Sprintf("terragrunt apply --dependency-fetch-output-from-state --auto-approve --backend-bootstrap --non-interactive --working-dir %s/app1", environmentPath)) // Now delete dependencies cached state - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) require.NoError(t, os.Remove(filepath.Join(environmentPath, "/app1/.terraform/terraform.tfstate"))) require.NoError(t, os.RemoveAll(filepath.Join(environmentPath, "/app1/.terraform"))) @@ -1661,10 +1661,10 @@ func dependencyOutputOptimizationTest(t *testing.T, moduleName string, forceInit // We need to bust the output cache that stores the dependency outputs so that the second run pulls the outputs. // This is only a problem during testing, where the process is shared across terragrunt runs. - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) // verify expected output - stdout, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt output -no-color -json --log-level trace --non-interactive --working-dir "+livePath) + stdout, _, err := helpers.RunTerragruntCommandWithOutputWithContext(t, t.Context(), "terragrunt output -no-color -json --log-level trace --non-interactive --working-dir "+livePath) require.NoError(t, err) outputs := map[string]helpers.TerraformOutput{} @@ -1676,13 +1676,13 @@ func dependencyOutputOptimizationTest(t *testing.T, moduleName string, forceInit helpers.CleanupTerraformFolder(t, depPath) } - // Now delete the deepdep state and verify still works (note we need to bust the cache again) - config.ClearOutputCache() + // Now delete the deepdep state, and verify still works (note we need to bust the cache again) + config.ClearOutputCache(t.Context()) require.NoError(t, os.Remove(filepath.Join(deepDepPath, "terraform.tfstate"))) fmt.Println("terragrunt output -no-color -json --log-level trace --non-interactive --working-dir " + livePath) - reout, reerr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt output -no-color -json --log-level trace --non-interactive --working-dir "+livePath) + reout, reerr, err := helpers.RunTerragruntCommandWithOutputWithContext(t, t.Context(), "terragrunt output -no-color -json --log-level trace --non-interactive --working-dir "+livePath) require.NoError(t, err) require.NoError(t, json.Unmarshal([]byte(reout), &outputs)) diff --git a/test/integration_destroy_test.go b/test/integration_destroy_test.go index f8bfe64cbf..d3f6152c90 100644 --- a/test/integration_destroy_test.go +++ b/test/integration_destroy_test.go @@ -91,7 +91,7 @@ func TestDestroyDependentModule(t *testing.T) { helpers.RunTerragrunt(t, "terragrunt apply -auto-approve --non-interactive --working-dir "+util.JoinPath(rootPath, "b")) helpers.RunTerragrunt(t, "terragrunt apply -auto-approve --non-interactive --working-dir "+util.JoinPath(rootPath, "c")) - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) // destroy module which have outputs from other modules stdout := bytes.Buffer{} diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 6b3e4793d8..f3eec9b97c 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -11,11 +11,13 @@ import ( "os" "path" "path/filepath" + "regexp" "runtime" "strings" "testing" "time" + "github.com/gruntwork-io/terragrunt/cli/commands/run" "github.com/gruntwork-io/terragrunt/test" "github.com/gruntwork-io/terragrunt/test/helpers" @@ -249,7 +251,7 @@ func TestTerragruntInputsFromDependency(t *testing.T) { appDir = filepath.Join(tc.rootPath, app) helpers.RunTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --non-interactive --working-dir %s --download-dir=%s", appDir, tc.downloadDir)) - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) } if tc.downloadDir != "" { @@ -824,3 +826,51 @@ func TestRunnerPoolTelemetry(t *testing.T) { assert.Contains(t, telemetryOutput, "\"Name\":\"runner_pool_controller\"") assert.Contains(t, telemetryOutput, "\"Name\":\"runner_pool_task\"") } + +func TestVersionIsInvokedInDifferentDirectory(t *testing.T) { + // Create a context with a fresh version cache for this test + ctx := run.WithRunVersionCache(t.Context()) + + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureVersionInvocation) + helpers.CleanupTerraformFolder(t, tmpEnvPath) + testPath := util.JoinPath(tmpEnvPath, testFixtureVersionInvocation) + + _, stderr, err := helpers.RunTerragruntCommandWithOutputWithContext(t, ctx, "terragrunt run --all --log-level trace --non-interactive --working-dir "+testPath+" -- apply") + require.NoError(t, err) + + versionCmdPattern := regexp.MustCompile(`Running command: ` + regexp.QuoteMeta(wrappedBinary()) + ` -version`) + matches := versionCmdPattern.FindAllStringIndex(stderr, -1) + + expected := 3 + + if expectExtraVersionCommandCall(t) { + expected++ + } + + assert.Len(t, matches, expected, "Expected exactly one occurrence of '-version' command, found %d", len(matches)) + assert.Contains(t, stderr, "prefix=dependency-with-custom-version msg=Running command: "+wrappedBinary()+" -version") +} + +func TestVersionIsInvokedOnlyOnce(t *testing.T) { + // Create a context with a fresh version cache for this test + ctx := run.WithRunVersionCache(t.Context()) + + tmpEnvPath := helpers.CopyEnvironment(t, testFixtureDependencyOutput) + helpers.CleanupTerraformFolder(t, tmpEnvPath) + testPath := util.JoinPath(tmpEnvPath, testFixtureDependencyOutput) + + _, stderr, err := helpers.RunTerragruntCommandWithOutputWithContext(t, ctx, "terragrunt run --all --log-level trace --non-interactive --working-dir "+testPath+" -- apply") + require.NoError(t, err) + + // check that version command was invoked only once -version + versionCmdPattern := regexp.MustCompile(`Running command: ` + regexp.QuoteMeta(wrappedBinary()) + ` -version`) + matches := versionCmdPattern.FindAllStringIndex(stderr, -1) + + expected := 2 + + if expectExtraVersionCommandCall(t) { + expected++ + } + + assert.Len(t, matches, expected, "Expected exactly one occurrence of '-version' command, found %d", len(matches)) +} diff --git a/test/integration_test.go b/test/integration_test.go index 6311525594..353122a6b6 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1538,9 +1538,9 @@ func TestDependencyMockOutput(t *testing.T) { // We need to bust the output cache that stores the dependency outputs so that the second run pulls the outputs. // This is only a problem during testing, where the process is shared across terragrunt runs. - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) - // Now run --all apply so that the dependency is applied, and verify it uses the dependency output + // Now run --all apply so that the dependency is applied and verify it uses the dependency output err = helpers.RunTerragruntCommand(t, "terragrunt run --all apply --non-interactive --working-dir "+rootPath, &showStdout, &showStderr) require.NoError(t, err) @@ -2188,13 +2188,13 @@ func TestDependencyOutputWithHooks(t *testing.T) { helpers.RunTerragrunt(t, "terragrunt run --all apply --non-interactive --working-dir "+rootPath) // We need to bust the output cache that stores the dependency outputs so that the second run pulls the outputs. // This is only a problem during testing, where the process is shared across terragrunt runs. - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) // The file should exist in the first run. assert.True(t, util.FileExists(depPathFileOut)) assert.False(t, util.FileExists(mainPathFileOut)) - // Now delete file and run plain main again. It should NOT create file.out. + // Now delete the file and run plain main again. It should NOT create file.out. require.NoError(t, os.Remove(depPathFileOut)) helpers.RunTerragrunt(t, "terragrunt plan --non-interactive --working-dir "+mainPath) assert.False(t, util.FileExists(depPathFileOut)) @@ -3110,7 +3110,7 @@ func TestDependenciesOptimisation(t *testing.T) { "Reading inputs from dependencies has been deprecated and will be removed in a future version of Terragrunt. If a value in a dependency is needed, use dependency outputs instead.", ) - config.ClearOutputCache() + config.ClearOutputCache(t.Context()) moduleC := util.JoinPath(tmpEnvPath, testFixtureDependenciesOptimisation, "module-c") @@ -4188,52 +4188,6 @@ func TestTfPathOverridesConfigWithTofuTerraform(t *testing.T) { } } -func TestVersionIsInvokedOnlyOnce(t *testing.T) { - t.Parallel() - - tmpEnvPath := helpers.CopyEnvironment(t, testFixtureDependencyOutput) - helpers.CleanupTerraformFolder(t, tmpEnvPath) - testPath := util.JoinPath(tmpEnvPath, testFixtureDependencyOutput) - - _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run --all --log-level trace --non-interactive --working-dir "+testPath+" -- apply") - require.NoError(t, err) - - // check that version command was invoked only once -version - versionCmdPattern := regexp.MustCompile(`Running command: ` + regexp.QuoteMeta(wrappedBinary()) + ` -version`) - matches := versionCmdPattern.FindAllStringIndex(stderr, -1) - - expected := 2 - - if expectExtraVersionCommandCall(t) { - expected++ - } - - assert.Len(t, matches, expected, "Expected exactly one occurrence of '-version' command, found %d", len(matches)) -} - -func TestVersionIsInvokedInDifferentDirectory(t *testing.T) { - t.Parallel() - - tmpEnvPath := helpers.CopyEnvironment(t, testFixtureVersionInvocation) - helpers.CleanupTerraformFolder(t, tmpEnvPath) - testPath := util.JoinPath(tmpEnvPath, testFixtureVersionInvocation) - - _, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run --all --log-level trace --non-interactive --working-dir "+testPath+" -- apply") - require.NoError(t, err) - - versionCmdPattern := regexp.MustCompile(`Running command: ` + regexp.QuoteMeta(wrappedBinary()) + ` -version`) - matches := versionCmdPattern.FindAllStringIndex(stderr, -1) - - expected := 3 - - if expectExtraVersionCommandCall(t) { - expected++ - } - - assert.Len(t, matches, expected, "Expected exactly one occurrence of '-version' command, found %d", len(matches)) - assert.Contains(t, stderr, "prefix=dependency-with-custom-version msg=Running command: "+wrappedBinary()+" -version") -} - // expectExtraVersionCommandCall returns true if we expect an extra version command to be invoked. // // We expect an extra version command to be invoked when the auto-provider-cache-dir experiment is enabled with OpenTofu,