diff --git a/README.md b/README.md index b061f6a1..2037c9ec 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,10 @@ The `boilerplate` binary supports the following options: * `--disable-shell`: If this flag is set, no `shell` helpers will execute. They will instead return the text "replace-me". * `--disable-dependency-prompt` (optional): Do not prompt for confirmation to include dependencies. Has the same effect as --non-interactive, without disabling variable prompts. Default: `false`. +* `--parallel-for-each` (optional): Process `for_each` dependencies in parallel instead of sequentially. This can +significantly improve performance when you have multiple dependencies that take time to process. **WARNING**: Ensure +that shell commands/hooks within your dependencies are thread-safe, as multiple dependencies may execute concurrently. +Operations like file I/O, network requests, or resource creation should be designed to handle concurrent execution. * `--help`: Show the help text and exit. * `--version`: Show the version and exit. @@ -214,6 +218,12 @@ Generate a project in ~/output from the templates in this repo's `include` examp boilerplate --template-url "git@github.com:gruntwork-io/boilerplate.git//examples/for-learning-and-testing/include?ref=main" --output-folder ~/output --var-file vars.yml ``` +Generate a project with parallel processing of dependencies which use `for_each` or `for_each_reference`: + +``` +boilerplate --template-url ~/templates --output-folder ~/output --var-file vars.yml --parallel-for-each +``` + #### The boilerplate.yml file @@ -373,6 +383,25 @@ executing the current one. Each dependency may contain the following keys: current value in the loop will be available as the variable `__each__`, available to both your Go templating and in other `dependencies` params: e.g., you could reference `{{ .__each__ }}` in `output-folder` to render to each iteration to a different folder. + + **Parallel Processing** + + By default, `for_each` dependencies are processed sequentially (one after another). You can enable parallel + processing using the `--parallel-for-each` flag, which will process all items in a `for_each` loop concurrently. + This can significantly improve performance when: + - You have many items in your `for_each` lists + - Each dependency takes significant time to process (e.g., complex templating, slow hooks) + - Dependencies are independent and don't rely on each other's output + + **Thread Safety Considerations** + + When using `--parallel-for-each`, be aware that multiple dependencies may execute simultaneously. Ensure your + templates and hooks are thread-safe: + - File operations should use unique file paths (leverage `{{ .__each__ }}` in `output-folder`) + - Shell commands should not conflict with each other (avoid shared resources) + - External API calls should handle concurrent requests appropriately + - Database operations should use proper locking or transactions + * `for_each_reference`: The name of another variable whose value should be used as the `for_each` value. See the [Dependencies](#dependencies) section for more info. diff --git a/cli/boilerplate_cli.go b/cli/boilerplate_cli.go index 69ca50b7..3f3d5deb 100644 --- a/cli/boilerplate_cli.go +++ b/cli/boilerplate_cli.go @@ -96,6 +96,10 @@ func CreateBoilerplateCli() *cli.App { Name: options.OptDisableDependencyPrompt, Usage: fmt.Sprintf("Do not prompt for confirmation to include dependencies. Has the same effect as --%s, without disabling variable prompts.", options.OptNonInteractive), }, + &cli.BoolFlag{ + Name: options.OptParallelForEach, + Usage: "If this flag is set, for_each'd dependencies will be processed in parallel instead of sequentially. This can significantly speed up template generation when dependencies take time to process. WARNING: Ensure that shell commands/hooks within dependencies are thread-safe, as multiple dependencies may execute concurrently.", + }, } // We pass JSON/YAML content to various CLI flags, such as --var, and this JSON/YAML content may contain commas or diff --git a/integration-tests/parallel_processing_test.go b/integration-tests/parallel_processing_test.go new file mode 100644 index 00000000..dc8e8ae3 --- /dev/null +++ b/integration-tests/parallel_processing_test.go @@ -0,0 +1,206 @@ +package integration_tests + +import ( + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/gruntwork-io/boilerplate/cli" +) + +// Test that parallel processing produces the same output as sequential processing +func TestParallelProcessingProducesSameOutput(t *testing.T) { + t.Parallel() + + templateFolder := "../examples/for-learning-and-testing/dependencies-for-each" + varFile := "../test-fixtures/examples-var-files/dependencies-for-each/vars.yml" + + sequentialOutput, err := os.MkdirTemp("", "boilerplate-test-sequential") + require.NoError(t, err) + defer os.RemoveAll(sequentialOutput) + + parallelOutput, err := os.MkdirTemp("", "boilerplate-test-parallel") + require.NoError(t, err) + defer os.RemoveAll(parallelOutput) + + // Run both sequential and parallel processing + runBoilerplateWithParallelFlag(t, templateFolder, sequentialOutput, varFile, false) + runBoilerplateWithParallelFlag(t, templateFolder, parallelOutput, varFile, true) + + // Compare outputs - they should be identical + assertDirectoriesEqual(t, sequentialOutput, parallelOutput) +} + + + +// Test that parallel processing handles errors correctly from multiple dependencies +func TestParallelProcessingErrorHandling(t *testing.T) { + t.Parallel() + + templateDir := createFailingTemplate(t) + defer os.RemoveAll(templateDir) + + outputDir, err := os.MkdirTemp("", "boilerplate-error-output") + require.NoError(t, err) + defer os.RemoveAll(outputDir) + + // Run with parallel processing - should fail but handle errors gracefully + app := cli.CreateBoilerplateCli() + args := []string{ + "boilerplate", + "--template-url", templateDir, + "--output-folder", outputDir, + "--non-interactive", + "--parallel-for-each", + } + + err = app.Run(args) + assert.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent-template") +} + + + +// Test that the --parallel-for-each flag is properly propagated to nested dependencies +func TestParallelFlagPropagation(t *testing.T) { + t.Parallel() + + templateDir := createNestedTemplate(t) + defer os.RemoveAll(templateDir) + + outputDir, err := os.MkdirTemp("", "boilerplate-propagation-output") + require.NoError(t, err) + defer os.RemoveAll(outputDir) + + // Run with parallel processing + runBoilerplateWithParallelFlag(t, templateDir, outputDir, "", true) + + // Verify that all nested files were created + expectedPaths := []string{ + "parent/parent1/child/child1/result.txt", + "parent/parent1/child/child2/result.txt", + "parent/parent1/child/child3/result.txt", + "parent/parent2/child/child1/result.txt", + "parent/parent2/child/child2/result.txt", + "parent/parent2/child/child3/result.txt", + } + + for _, expectedPath := range expectedPaths { + fullPath := path.Join(outputDir, expectedPath) + assert.FileExists(t, fullPath, "Nested file should exist at %s", expectedPath) + } +} + + + +// Helper function to run boilerplate with or without parallel flag +func runBoilerplateWithParallelFlag(t *testing.T, templateFolder, outputFolder, varFile string, parallel bool) { + app := cli.CreateBoilerplateCli() + + args := []string{ + "boilerplate", + "--template-url", templateFolder, + "--output-folder", outputFolder, + "--non-interactive", + } + + if varFile != "" { + args = append(args, "--var-file", varFile) + } + + if parallel { + args = append(args, "--parallel-for-each") + } + + err := app.Run(args) + require.NoError(t, err, "Boilerplate execution should succeed") +} + + + +// Helper function to create a template that will cause failures +func createFailingTemplate(t *testing.T) string { + templateDir, err := os.MkdirTemp("", "boilerplate-error-test") + require.NoError(t, err) + + boilerplateConfig := `dependencies: + - name: failing-dependency-1 + template-url: ./nonexistent-template-1 + for_each: + - item1 + - item2 + output-folder: "output/{{ .__each__ }}" + - name: failing-dependency-2 + template-url: ./nonexistent-template-2 + for_each: + - item3 + - item4 + output-folder: "output/{{ .__each__ }}" +` + + err = os.WriteFile(path.Join(templateDir, "boilerplate.yml"), []byte(boilerplateConfig), 0o644) + require.NoError(t, err) + + return templateDir +} + + + +// Helper function to create a template with nested dependencies +func createNestedTemplate(t *testing.T) string { + templateDir, err := os.MkdirTemp("", "boilerplate-propagation-test") + require.NoError(t, err) + + // Create main template with for_each dependency + err = os.WriteFile(path.Join(templateDir, "boilerplate.yml"), []byte(`dependencies: + - name: parent-dependency + template-url: ./nested-template + for_each: + - parent1 + - parent2 + output-folder: "parent/{{ .__each__ }}" +`), 0o644) + require.NoError(t, err) + + // Create nested template directory + nestedTemplateDir := path.Join(templateDir, "nested-template") + err = os.MkdirAll(nestedTemplateDir, 0o755) + require.NoError(t, err) + + // Create nested template with its own for_each dependencies + err = os.WriteFile(path.Join(nestedTemplateDir, "boilerplate.yml"), []byte(`dependencies: + - name: child-dependency + template-url: ./child-template + for_each: + - child1 + - child2 + - child3 + output-folder: "child/{{ .__each__ }}" +variables: + - name: ParentName + default: "{{ .__each__ }}" +`), 0o644) + require.NoError(t, err) + + // Create child template directory + childTemplateDir := path.Join(nestedTemplateDir, "child-template") + err = os.MkdirAll(childTemplateDir, 0o755) + require.NoError(t, err) + + // Create simple child template + err = os.WriteFile(path.Join(childTemplateDir, "boilerplate.yml"), []byte(`variables: + - name: ChildName + default: "{{ .__each__ }}" +`), 0o644) + require.NoError(t, err) + + err = os.WriteFile(path.Join(childTemplateDir, "result.txt"), []byte(`Parent: {{ .ParentName }} +Child: {{ .ChildName }} +`), 0o644) + require.NoError(t, err) + + return templateDir +} \ No newline at end of file diff --git a/options/options.go b/options/options.go index 12564d66..8708e4a8 100644 --- a/options/options.go +++ b/options/options.go @@ -20,6 +20,7 @@ const OptMissingConfigAction = "missing-config-action" const OptDisableHooks = "disable-hooks" const OptDisableShell = "disable-shell" const OptDisableDependencyPrompt = "disable-dependency-prompt" +const OptParallelForEach = "parallel-for-each" // The command-line options for the boilerplate app type BoilerplateOptions struct { @@ -36,6 +37,7 @@ type BoilerplateOptions struct { DisableHooks bool DisableShell bool DisableDependencyPrompt bool + ParallelForEach bool } // Validate that the options have reasonable values and return an error if they don't @@ -96,6 +98,7 @@ func ParseOptions(cliContext *cli.Context) (*BoilerplateOptions, error) { DisableHooks: cliContext.Bool(OptDisableHooks), DisableShell: cliContext.Bool(OptDisableShell), DisableDependencyPrompt: cliContext.Bool(OptDisableDependencyPrompt), + ParallelForEach: cliContext.Bool(OptParallelForEach), } if err := options.Validate(); err != nil { diff --git a/templates/template_processor.go b/templates/template_processor.go index 7f06cf85..177526e4 100644 --- a/templates/template_processor.go +++ b/templates/template_processor.go @@ -7,8 +7,10 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/gruntwork-io/go-commons/collections" + "github.com/hashicorp/go-multierror" "github.com/gruntwork-io/boilerplate/config" "github.com/gruntwork-io/boilerplate/errors" @@ -248,14 +250,45 @@ func processDependency( } if len(forEach) > 0 { - for _, item := range forEach { - updatedVars := collections.MergeMaps(originalVars, map[string]interface{}{eachVarName: item}) - if err := doProcess(updatedVars); err != nil { - return err + if opts.ParallelForEach { + // Process dependencies in parallel + util.Logger.Printf("Processing dependencies in parallel") + var wg sync.WaitGroup + errChan := make(chan error, len(forEach)) + + for _, item := range forEach { + wg.Add(1) + go func(forEachItem interface{}) { + defer wg.Done() + updatedVars := collections.MergeMaps(originalVars, map[string]interface{}{eachVarName: forEachItem}) + if err := doProcess(updatedVars); err != nil { + errChan <- err + } + }(item) } + + // Wait for all goroutines to complete + wg.Wait() + close(errChan) + + // Collect all errors + var errs *multierror.Error + for err := range errChan { + errs = multierror.Append(errs, err) + } + + return errs.ErrorOrNil() + } else { + util.Logger.Printf("Processing dependencies sequentially") + // Process dependencies sequentially (original behavior) + for _, item := range forEach { + updatedVars := collections.MergeMaps(originalVars, map[string]interface{}{eachVarName: item}) + if err := doProcess(updatedVars); err != nil { + return err + } + } + return nil } - - return nil } else { return doProcess(originalVars) } @@ -319,6 +352,7 @@ func cloneOptionsForDependency( DisableHooks: originalOpts.DisableHooks, DisableShell: originalOpts.DisableShell, DisableDependencyPrompt: originalOpts.DisableDependencyPrompt, + ParallelForEach: originalOpts.ParallelForEach, }, nil } diff --git a/templates/template_processor_test.go b/templates/template_processor_test.go index bc97ce50..6e905330 100644 --- a/templates/template_processor_test.go +++ b/templates/template_processor_test.go @@ -1,10 +1,14 @@ package templates import ( + "fmt" "os" "path/filepath" + "sync" + "sync/atomic" "testing" + "github.com/hashicorp/go-multierror" "github.com/gruntwork-io/boilerplate/options" "github.com/gruntwork-io/boilerplate/variables" "github.com/stretchr/testify/assert" @@ -204,3 +208,154 @@ func TestForEachReferenceRendersAsTemplate(t *testing.T) { assert.NoError(t, err) } } + +// Test both parallel and sequential processing modes, including __each__ variable handling +func TestProcessDependencyBothModes(t *testing.T) { + tempDir, templateDir := createTestTemplate(t, "boilerplate-both-modes-test") + defer os.RemoveAll(tempDir) + + dependency := variables.Dependency{ + Name: "test-dependency", + TemplateUrl: templateDir, + OutputFolder: "output/{{ .__each__ }}", + ForEach: []string{"alpha", "beta", "gamma"}, + } + + // Test both parallel and sequential modes + for _, parallel := range []bool{false, true} { + mode := "sequential" + if parallel { + mode = "parallel" + } + + t.Run(mode, func(t *testing.T) { + outputDir := filepath.Join(tempDir, mode) + err := os.MkdirAll(outputDir, 0o755) + require.NoError(t, err) + + opts := &options.BoilerplateOptions{ + TemplateFolder: templateDir, + OutputFolder: outputDir, + NonInteractive: true, + ParallelForEach: parallel, + OnMissingKey: options.ExitWithError, + } + + err = processDependency(dependency, opts, map[string]variables.Variable{}, map[string]interface{}{}) + require.NoError(t, err) + + // Verify all outputs were created correctly + for _, item := range dependency.ForEach { + outputPath := filepath.Join(outputDir, "output", item, "test.txt") + assert.FileExists(t, outputPath) + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + + // Verify __each__ variable was set correctly + assert.Contains(t, string(content), fmt.Sprintf("Value: %s", item)) + } + }) + } +} + +// Test error collection in parallel processing +func TestProcessDependencyParallelErrorCollection(t *testing.T) { + opts := &options.BoilerplateOptions{ + TemplateFolder: "/nonexistent", + OutputFolder: "/tmp", + NonInteractive: true, + ParallelForEach: true, + OnMissingKey: options.ExitWithError, + } + + dependency := variables.Dependency{ + Name: "failing-dependency", + TemplateUrl: "/nonexistent/template", + OutputFolder: "output/{{ .__each__ }}", + ForEach: []string{"item1", "item2", "item3"}, + } + + err := processDependency(dependency, opts, map[string]variables.Variable{}, map[string]interface{}{}) + + assert.Error(t, err) + + // Check if it's a multierror (parallel processing should collect multiple errors) + if multiErr, ok := err.(*multierror.Error); ok { + assert.GreaterOrEqual(t, len(multiErr.Errors), 1) + } +} + +// Test that race conditions don't occur during parallel processing +func TestProcessDependencyNoRaceConditions(t *testing.T) { + if testing.Short() { + t.Skip("Skipping race condition test in short mode") + } + + tempDir, templateDir := createTestTemplate(t, "boilerplate-race-test") + defer os.RemoveAll(tempDir) + + // Run multiple concurrent dependency processes + numConcurrentRuns := 5 + var wg sync.WaitGroup + var errorCount int32 + + for i := 0; i < numConcurrentRuns; i++ { + wg.Add(1) + go func(runIndex int) { + defer wg.Done() + + runOutputDir := filepath.Join(tempDir, fmt.Sprintf("run%d", runIndex)) + err := os.MkdirAll(runOutputDir, 0o755) + if err != nil { + atomic.AddInt32(&errorCount, 1) + return + } + + opts := &options.BoilerplateOptions{ + TemplateFolder: templateDir, + OutputFolder: runOutputDir, + NonInteractive: true, + ParallelForEach: true, + OnMissingKey: options.ExitWithError, + } + + dependency := variables.Dependency{ + Name: fmt.Sprintf("test-dependency-%d", runIndex), + TemplateUrl: templateDir, + OutputFolder: "output/{{ .__each__ }}", + ForEach: []string{"item1", "item2", "item3"}, + } + + if err := processDependency(dependency, opts, map[string]variables.Variable{}, map[string]interface{}{}); err != nil { + atomic.AddInt32(&errorCount, 1) + } + }(i) + } + + wg.Wait() + assert.Equal(t, int32(0), errorCount, "Race conditions should not cause errors") +} + +// Helper function to create a basic test template +func createTestTemplate(t *testing.T, prefix string) (tempDir, templateDir string) { + tempDir, err := os.MkdirTemp("", prefix) + require.NoError(t, err) + + templateDir = filepath.Join(tempDir, "template") + err = os.MkdirAll(templateDir, 0o755) + require.NoError(t, err) + + err = os.WriteFile(filepath.Join(templateDir, "boilerplate.yml"), []byte(`variables: + - name: TestVar + default: "{{ .__each__ }}" +`), 0o644) + require.NoError(t, err) + + err = os.WriteFile(filepath.Join(templateDir, "test.txt"), []byte(`Value: {{ .TestVar }}`), 0o644) + require.NoError(t, err) + + return tempDir, templateDir +} + +