allow parallel dependency processing.

README.md

@@ -185,6 +185,10 @@ The `boilerplate` binary supports the following options:
 * `--disable-shell`: If this flag is set, no `shell` helpers will execute. They will instead return the text "replace-me".
 * `--disable-dependency-prompt` (optional): Do not prompt for confirmation to include dependencies. Has the same effect as
   --non-interactive, without disabling variable prompts. Default: `false`.
+* `--parallel-for-each` (optional): Process `for_each` dependencies in parallel instead of sequentially. This can 
+significantly improve performance when you have multiple dependencies that take time to process. **WARNING**: Ensure 
+that shell commands/hooks within your dependencies are thread-safe, as multiple dependencies may execute concurrently.
+Operations like file I/O, network requests, or resource creation should be designed to handle concurrent execution.
 * `--help`: Show the help text and exit.
 * `--version`: Show the version and exit.
 
@@ -214,6 +218,12 @@ Generate a project in ~/output from the templates in this repo's `include` examp
 boilerplate --template-url "[email protected]:gruntwork-io/boilerplate.git//examples/for-learning-and-testing/include?ref=main" --output-folder ~/output --var-file vars.yml
 ```
 
+Generate a project with parallel processing of dependencies which use `for_each` or `for_each_reference`:
+
+```
+boilerplate --template-url ~/templates --output-folder ~/output --var-file vars.yml --parallel-for-each
+```
+
 
 #### The boilerplate.yml file
 
@@ -373,6 +383,25 @@ executing the current one. Each dependency may contain the following keys:
   current value in the loop will be available as the variable `__each__`, available to both your Go templating and
   in other `dependencies` params: e.g., you could reference `{{ .__each__ }}` in `output-folder` to render to each
   iteration to a different folder.
+
+  **Parallel Processing**
+
+  By default, `for_each` dependencies are processed sequentially (one after another). You can enable parallel 
+  processing using the `--parallel-for-each` flag, which will process all items in a `for_each` loop concurrently. 
+  This can significantly improve performance when:
+  - You have many items in your `for_each` lists
+  - Each dependency takes significant time to process (e.g., complex templating, slow hooks)
+  - Dependencies are independent and don't rely on each other's output
+
+  **Thread Safety Considerations**
+
+  When using `--parallel-for-each`, be aware that multiple dependencies may execute simultaneously. Ensure your 
+  templates and hooks are thread-safe:
+  - File operations should use unique file paths (leverage `{{ .__each__ }}` in `output-folder`)
+  - Shell commands should not conflict with each other (avoid shared resources)
+  - External API calls should handle concurrent requests appropriately
+  - Database operations should use proper locking or transactions
+
 * `for_each_reference`: The name of another variable whose value should be used as the `for_each` value.
 
 See the [Dependencies](#dependencies) section for more info.

cli/boilerplate_cli.go

@@ -96,6 +96,10 @@ func CreateBoilerplateCli() *cli.App {
 			Name:  options.OptDisableDependencyPrompt,
 			Usage: fmt.Sprintf("Do not prompt for confirmation to include dependencies. Has the same effect as --%s, without disabling variable prompts.", options.OptNonInteractive),
 		},
+		&cli.BoolFlag{
+			Name:  options.OptParallelForEach,
+			Usage: "If this flag is set, for_each'd dependencies will be processed in parallel instead of sequentially. This can significantly speed up template generation when dependencies take time to process. WARNING: Ensure that shell commands/hooks within dependencies are thread-safe, as multiple dependencies may execute concurrently.",
+		},
 	}
 
 	// We pass JSON/YAML content to various CLI flags, such as --var, and this JSON/YAML content may contain commas or

integration-tests/parallel_processing_test.go

@@ -0,0 +1,206 @@
+package integration_tests
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/gruntwork-io/boilerplate/cli"
+)
+
+// Test that parallel processing produces the same output as sequential processing
+func TestParallelProcessingProducesSameOutput(t *testing.T) {
+	t.Parallel()
+
+	templateFolder := "../examples/for-learning-and-testing/dependencies-for-each"
+	varFile := "../test-fixtures/examples-var-files/dependencies-for-each/vars.yml"
+
+	sequentialOutput, err := os.MkdirTemp("", "boilerplate-test-sequential")
+	require.NoError(t, err)
+	defer os.RemoveAll(sequentialOutput)
+
+	parallelOutput, err := os.MkdirTemp("", "boilerplate-test-parallel")
+	require.NoError(t, err)
+	defer os.RemoveAll(parallelOutput)
+
+	// Run both sequential and parallel processing
+	runBoilerplateWithParallelFlag(t, templateFolder, sequentialOutput, varFile, false)
+	runBoilerplateWithParallelFlag(t, templateFolder, parallelOutput, varFile, true)
+
+	// Compare outputs - they should be identical
+	assertDirectoriesEqual(t, sequentialOutput, parallelOutput)
+}
+
+
+
+// Test that parallel processing handles errors correctly from multiple dependencies
+func TestParallelProcessingErrorHandling(t *testing.T) {
+	t.Parallel()
+
+	templateDir := createFailingTemplate(t)
+	defer os.RemoveAll(templateDir)
+
+	outputDir, err := os.MkdirTemp("", "boilerplate-error-output")
+	require.NoError(t, err)
+	defer os.RemoveAll(outputDir)
+
+	// Run with parallel processing - should fail but handle errors gracefully
+	app := cli.CreateBoilerplateCli()
+	args := []string{
+		"boilerplate",
+		"--template-url", templateDir,
+		"--output-folder", outputDir,
+		"--non-interactive",
+		"--parallel-for-each",
+	}
+
+	err = app.Run(args)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "nonexistent-template")
+}
+
+
+
+// Test that the --parallel-for-each flag is properly propagated to nested dependencies
+func TestParallelFlagPropagation(t *testing.T) {
+	t.Parallel()
+
+	templateDir := createNestedTemplate(t)
+	defer os.RemoveAll(templateDir)
+
+	outputDir, err := os.MkdirTemp("", "boilerplate-propagation-output")
+	require.NoError(t, err)
+	defer os.RemoveAll(outputDir)
+
+	// Run with parallel processing
+	runBoilerplateWithParallelFlag(t, templateDir, outputDir, "", true)
+
+	// Verify that all nested files were created
+	expectedPaths := []string{
+		"parent/parent1/child/child1/result.txt",
+		"parent/parent1/child/child2/result.txt", 
+		"parent/parent1/child/child3/result.txt",
+		"parent/parent2/child/child1/result.txt",
+		"parent/parent2/child/child2/result.txt",
+		"parent/parent2/child/child3/result.txt",
+	}
+
+	for _, expectedPath := range expectedPaths {
+		fullPath := path.Join(outputDir, expectedPath)
+		assert.FileExists(t, fullPath, "Nested file should exist at %s", expectedPath)
+	}
+}
+
+
+
+// Helper function to run boilerplate with or without parallel flag
+func runBoilerplateWithParallelFlag(t *testing.T, templateFolder, outputFolder, varFile string, parallel bool) {
+	app := cli.CreateBoilerplateCli()
+
+	args := []string{
+		"boilerplate",
+		"--template-url", templateFolder,
+		"--output-folder", outputFolder,
+		"--non-interactive",
+	}
+
+	if varFile != "" {
+		args = append(args, "--var-file", varFile)
+	}
+
+	if parallel {
+		args = append(args, "--parallel-for-each")
+	}
+
+	err := app.Run(args)
+	require.NoError(t, err, "Boilerplate execution should succeed")
+}
+
+
+
+// Helper function to create a template that will cause failures
+func createFailingTemplate(t *testing.T) string {
+	templateDir, err := os.MkdirTemp("", "boilerplate-error-test")
+	require.NoError(t, err)
+
+	boilerplateConfig := `dependencies:
+  - name: failing-dependency-1
+    template-url: ./nonexistent-template-1
+    for_each:
+      - item1
+      - item2
+    output-folder: "output/{{ .__each__ }}"
+  - name: failing-dependency-2
+    template-url: ./nonexistent-template-2
+    for_each:
+      - item3
+      - item4
+    output-folder: "output/{{ .__each__ }}"
+`
+
+	err = os.WriteFile(path.Join(templateDir, "boilerplate.yml"), []byte(boilerplateConfig), 0o644)
+	require.NoError(t, err)
+
+	return templateDir
+}
+
+
+
+// Helper function to create a template with nested dependencies
+func createNestedTemplate(t *testing.T) string {
+	templateDir, err := os.MkdirTemp("", "boilerplate-propagation-test")
+	require.NoError(t, err)
+
+	// Create main template with for_each dependency
+	err = os.WriteFile(path.Join(templateDir, "boilerplate.yml"), []byte(`dependencies:
+  - name: parent-dependency
+    template-url: ./nested-template
+    for_each:
+      - parent1
+      - parent2
+    output-folder: "parent/{{ .__each__ }}"
+`), 0o644)
+	require.NoError(t, err)
+
+	// Create nested template directory
+	nestedTemplateDir := path.Join(templateDir, "nested-template")
+	err = os.MkdirAll(nestedTemplateDir, 0o755)
+	require.NoError(t, err)
+
+	// Create nested template with its own for_each dependencies
+	err = os.WriteFile(path.Join(nestedTemplateDir, "boilerplate.yml"), []byte(`dependencies:
+  - name: child-dependency
+    template-url: ./child-template
+    for_each:
+      - child1
+      - child2
+      - child3
+    output-folder: "child/{{ .__each__ }}"
+variables:
+  - name: ParentName
+    default: "{{ .__each__ }}"
+`), 0o644)
+	require.NoError(t, err)
+
+	// Create child template directory
+	childTemplateDir := path.Join(nestedTemplateDir, "child-template")
+	err = os.MkdirAll(childTemplateDir, 0o755)
+	require.NoError(t, err)
+
+	// Create simple child template
+	err = os.WriteFile(path.Join(childTemplateDir, "boilerplate.yml"), []byte(`variables:
+  - name: ChildName
+    default: "{{ .__each__ }}"
+`), 0o644)
+	require.NoError(t, err)
+
+	err = os.WriteFile(path.Join(childTemplateDir, "result.txt"), []byte(`Parent: {{ .ParentName }}
+Child: {{ .ChildName }}
+`), 0o644)
+	require.NoError(t, err)
+
+	return templateDir
+} 

options/options.go

@@ -20,6 +20,7 @@ const OptMissingConfigAction = "missing-config-action"
 const OptDisableHooks = "disable-hooks"
 const OptDisableShell = "disable-shell"
 const OptDisableDependencyPrompt = "disable-dependency-prompt"
+const OptParallelForEach = "parallel-for-each"
 
 // The command-line options for the boilerplate app
 type BoilerplateOptions struct {
@@ -36,6 +37,7 @@ type BoilerplateOptions struct {
 	DisableHooks            bool
 	DisableShell            bool
 	DisableDependencyPrompt bool
+	ParallelForEach         bool
 }
 
 // Validate that the options have reasonable values and return an error if they don't
@@ -96,6 +98,7 @@ func ParseOptions(cliContext *cli.Context) (*BoilerplateOptions, error) {
 		DisableHooks:            cliContext.Bool(OptDisableHooks),
 		DisableShell:            cliContext.Bool(OptDisableShell),
 		DisableDependencyPrompt: cliContext.Bool(OptDisableDependencyPrompt),
+		ParallelForEach:         cliContext.Bool(OptParallelForEach),
 	}
 
 	if err := options.Validate(); err != nil {

templates/template_processor.go

@@ -7,8 +7,10 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/gruntwork-io/go-commons/collections"
+	"github.com/hashicorp/go-multierror"
 
 	"github.com/gruntwork-io/boilerplate/config"
 	"github.com/gruntwork-io/boilerplate/errors"
@@ -248,14 +250,45 @@ func processDependency(
 		}
 
 		if len(forEach) > 0 {
-			for _, item := range forEach {
-				updatedVars := collections.MergeMaps(originalVars, map[string]interface{}{eachVarName: item})
-				if err := doProcess(updatedVars); err != nil {
-					return err
+			if opts.ParallelForEach {
+				// Process dependencies in parallel
+				util.Logger.Printf("Processing dependencies in parallel")
+				var wg sync.WaitGroup
+				errChan := make(chan error, len(forEach))
+
+				for _, item := range forEach {
+					wg.Add(1)
+					go func(forEachItem interface{}) {
+						defer wg.Done()
+						updatedVars := collections.MergeMaps(originalVars, map[string]interface{}{eachVarName: forEachItem})
+						if err := doProcess(updatedVars); err != nil {
+							errChan <- err
+						}
+					}(item)
 				}
+
+				// Wait for all goroutines to complete
+				wg.Wait()
+				close(errChan)
+
+				// Collect all errors
+				var errs *multierror.Error
+				for err := range errChan {
+					errs = multierror.Append(errs, err)
+				}
+
+				return errs.ErrorOrNil()
+			} else {
+				util.Logger.Printf("Processing dependencies sequentially")
+				// Process dependencies sequentially (original behavior)
+				for _, item := range forEach {
+					updatedVars := collections.MergeMaps(originalVars, map[string]interface{}{eachVarName: item})
+					if err := doProcess(updatedVars); err != nil {
+						return err
+					}
+				}
+				return nil
 			}
-
-			return nil
 		} else {
 			return doProcess(originalVars)
 		}
@@ -319,6 +352,7 @@ func cloneOptionsForDependency(
 		DisableHooks:            originalOpts.DisableHooks,
 		DisableShell:            originalOpts.DisableShell,
 		DisableDependencyPrompt: originalOpts.DisableDependencyPrompt,
+		ParallelForEach:         originalOpts.ParallelForEach,
 	}, nil
 }