Add new functions and template example for ILM and tranforms

internal/docs/ilm.go

@@ -0,0 +1,122 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package docs
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// flattenNestedMap flattens a nested JSON-like structure (maps and slices) into
+// a flat map with dot-separated keys.
+func flattenNestedMap(prefix string, nested map[string]interface{}, flatMap map[string]string) {
+	for k, v := range nested {
+		key := k
+		if prefix != "" {
+			key = fmt.Sprintf("%s.%s", prefix, k)
+		}
+
+		switch child := v.(type) {
+		case map[string]interface{}:
+			flattenNestedMap(key, child, flatMap)
+		case []interface{}:
+			for i, val := range child {
+				// handle slices with index
+				newKey := fmt.Sprintf("%s.%d", key, i)
+				if nextMap, ok := val.(map[string]interface{}); ok {
+					flattenNestedMap(newKey, nextMap, flatMap)
+				} else {
+					flatMap[newKey] = fmt.Sprintf("%v", val)
+				}
+			}
+		default:
+			flatMap[key] = fmt.Sprintf("%v", v)
+		}
+	}
+}
+
+func getILMPolicyMap(path string) (map[string]string, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("reading ILM policy file failed: %w", err)
+	}
+	var policy map[string]interface{}
+	err = json.Unmarshal(content, &policy)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshalling ILM policy failed: %w", err)
+	}
+
+	flatMap := make(map[string]string)
+	flattenNestedMap("", policy, flatMap)
+	return flatMap, nil
+}
+
+func renderILMPolicyMap(output *strings.Builder, policyMap map[string]string) {
+	keys := make([]string, 0, len(policyMap))
+	for key := range policyMap {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		output.WriteString(fmt.Sprintf("| %s | %s |\n", key, policyMap[key]))
+	}
+}
+
+func renderILMPaths(packageRoot string) (string, error) {
+	// gather the list of data streams that have ILM policies defined
+	// if the list is empty, return ""
+	// if the list is not empty, format the list as a markdown list
+	ilmPaths, err := findILMPaths(packageRoot)
+	if err != nil {
+		return "", fmt.Errorf("finding ILM paths failed: %w", err)
+	}
+	if len(ilmPaths) == 0 {
+		return "", nil
+	}
+
+	sort.Strings(ilmPaths)
+	var renderedDocs strings.Builder
+	renderedDocs.WriteString("\n### Data streams using ILM policies:\n")
+	for _, ilmPath := range ilmPaths {
+		ilmPolicyPath := filepath.Join(packageRoot, "data_stream", ilmPath, "elasticsearch", "ilm", "default_policy.json")
+		// get the policy map
+		policyMap, err := getILMPolicyMap(ilmPolicyPath)
+		if err != nil {
+			return "", fmt.Errorf("getting ILM policy map for path %s failed: %w", ilmPolicyPath, err)
+		}
+		renderedDocs.WriteString(fmt.Sprintf("#### %s\n", ilmPath))
+
+		// render the policy map as a markdown table
+		renderedDocs.WriteString("| Key | Value |\n")
+		renderedDocs.WriteString("|---|---|\n")
+		renderILMPolicyMap(&renderedDocs, policyMap)
+	}
+	return renderedDocs.String(), nil
+}
+
+// findILMPaths scans a given package path for data streams that have ILM policies
+// and returns a list of all data stream names that have ILM policies defined.
+func findILMPaths(packageRoot string) ([]string, error) {
+	// look for ilm/ from the packageRoot/data_stream/<data_stream_name>/elasticsearch/ilm/
+	// add the data_stream_name to the list
+	ilmPaths, err := filepath.Glob(filepath.Join(packageRoot, "data_stream", "*", "elasticsearch", "ilm"))
+	if err != nil {
+		return nil, fmt.Errorf("finding ILM paths failed: %w", err)
+	}
+
+	result := make([]string, 0, len(ilmPaths))
+
+	// return the list of globbed paths
+	for _, ilmPath := range ilmPaths {
+		// get the data_stream_name from the ilmPath
+		dataStreamName := filepath.Base(filepath.Dir(filepath.Dir(ilmPath)))
+		result = append(result, dataStreamName)
+	}
+	return result, nil
+}

internal/docs/readme.go

@@ -222,6 +222,14 @@ func renderReadme(repositoryRoot *os.Root, fileName, packageRoot, templatePath s
 		"inputDocs": func() (string, error) {
 			return renderInputDocs(packageRoot)
 		},
+		"ilm": func() (string, error) {
+			logger.Debug("renderILMPaths")
+			return renderILMPaths(packageRoot)
+		},
+		"transform": func() (string, error) {
+			logger.Debug("renderTransformPaths")
+			return renderTransformPaths(packageRoot)
+		},
 		"generatedHeader": func() string {
 			return doNotModifyStr
 		},

internal/docs/readme_test.go

@@ -191,6 +191,7 @@ var renderCases = []struct {
 	title                  string
 	templatePath           string
 	dataStreamName         string
+	transformName          string
 	readmeTemplateContents string
 	fieldsContents         string
 	expected               string
@@ -214,6 +215,7 @@ Introduction to the package
 | data_stream.type | Data stream type package. | constant_keyword |
 `,
 		dataStreamName: "",
+		transformName:  "",
 		fieldsContents: `
 - name: data_stream.type
   type: constant_keyword
@@ -234,6 +236,7 @@ Introduction to the package
 | data_stream.type | Data stream type. | constant_keyword |
 `,
 		dataStreamName: "example",
+		transformName:  "",
 		fieldsContents: `
 - name: data_stream.type
   type: constant_keyword
@@ -257,6 +260,7 @@ Introduction to the package
 | dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's ` + "`name`" + ` should be the one that corresponds with the answer's ` + "`data`" + `. It should not simply be the original ` + "`question.name`" + ` repeated. | keyword |
 `,
 		dataStreamName: "example",
+		transformName:  "",
 		fieldsContents: `
 - name: emptygroup
   type: group
@@ -284,8 +288,51 @@ Introduction to the package
 (no fields available)
 `,
 		dataStreamName: "example",
+		transformName:  "",
 		fieldsContents: "",
 	},
+	{
+		title:        "README with ilm",
+		templatePath: "_dev/build/docs/README.md",
+		readmeTemplateContents: `# README
+Introduction to the package
+{{ fields "example" }}
+
+{{ ilm }}
+{{ transform }}
+`,
+		expected: `# README
+Introduction to the package
+**Exported fields**
+
+| Field | Description | Type |
+|---|---|---|
+| data_stream.type | Data stream type. | constant_keyword |
+
+
+
+### Data streams using ILM policies:
+#### example
+| Key | Value |
+|---|---|
+| policy.phases.delete.min_age | 30d |
+| policy.phases.hot.actions.rollover.max_age | 30d |
+| policy.phases.hot.actions.rollover.max_primary_shard_size | 50gb |
+
+
+### Transforms used:
+| Name | Description | Source | Dest |
+|---|---|---|---|
+| latest | Latest Findings from Cloud Synergy | logs-cloud_synergy.findings-\* | cloud_solution-cloud_synergy.latest-v1 |
+
+`,
+		dataStreamName: "example",
+		transformName:  "latest",
+		fieldsContents: `
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.`,
+	},
 }
 
 func TestRenderReadmeWithFields(t *testing.T) {
@@ -300,6 +347,8 @@ func TestRenderReadmeWithFields(t *testing.T) {
 			createBuildFile(t, packageRoot)
 			createReadmeTemplateFile(t, packageRoot, c.readmeTemplateContents)
 			createFieldsFile(t, packageRoot, c.dataStreamName, c.fieldsContents)
+			createILMFile(t, packageRoot, c.dataStreamName)
+			createTransformFile(t, packageRoot, c.transformName)
 
 			root, err := os.OpenRoot(packageRoot)
 			require.NoError(t, err)
@@ -324,6 +373,8 @@ func TestUpdateReadmeWithFields(t *testing.T) {
 			createBuildFile(t, packageRoot)
 			createReadmeTemplateFile(t, packageRoot, c.readmeTemplateContents)
 			createFieldsFile(t, packageRoot, c.dataStreamName, c.fieldsContents)
+			createILMFile(t, packageRoot, c.dataStreamName)
+			createTransformFile(t, packageRoot, c.transformName)
 
 			buildPackageRoot := t.TempDir()
 			createManifestFile(t, buildPackageRoot)
@@ -438,3 +489,107 @@ func createBuildFile(t *testing.T, packageRoot string) {
 	err = os.WriteFile(buildFile, []byte(content), 0644)
 	require.NoError(t, err)
 }
+
+// create ilm file
+func createILMFile(t *testing.T, packageRoot, dataStreamName string) {
+	t.Helper()
+	if dataStreamName == "" {
+		return
+	}
+
+	ilmFolder := createILMFolder(t, packageRoot, dataStreamName)
+	ilmFile := filepath.Join(ilmFolder, "default_policy.json")
+	err := os.WriteFile(ilmFile, []byte(`{
+	"policy": {
+		"phases": {
+			"hot": {
+				"actions": {
+					"rollover": {
+						"max_age": "30d",
+						"max_primary_shard_size": "50gb"
+					}
+				}
+			},
+			"delete": {
+				"min_age": "30d",
+				"actions": {
+					"delete": {}
+				}
+			}
+		}
+	}
+}
+`), 0644)
+	require.NoError(t, err)
+}
+
+func createILMFolder(t *testing.T, packageRoot, dataStreamName string) string {
+	t.Helper()
+
+	if dataStreamName == "" {
+		return ""
+	}
+
+	ilmFolder := filepath.Join(packageRoot, "data_stream", dataStreamName, "elasticsearch", "ilm")
+	err := os.MkdirAll(ilmFolder, 0755)
+	require.NoError(t, err)
+	return ilmFolder
+}
+
+func createTransformFile(t *testing.T, packageRoot, transformName string) {
+	t.Helper()
+	if transformName == "" {
+		return
+	}
+	contents := `source:
+  index:
+    - "logs-cloud_synergy.findings-*"
+  query:
+    bool:
+      must_not:
+        exists:
+          field: error.message
+dest:
+  index: "cloud_solution-cloud_synergy.latest-v1"
+  aliases:
+    - alias: "cloud_solution-cloud_synergy.latest"
+      move_on_creation: true
+latest:
+  unique_key:
+    - rule.id
+    - resource.id
+    - data_stream.namespace
+  sort: "@timestamp"
+description: Latest Findings from Cloud Synergy
+frequency: 5m
+sync:
+  time:
+    field: event.ingested
+retention_policy:
+  time:
+    field: "@timestamp"
+    max_age: 26h
+settings:
+  unattended: true
+_meta:
+  managed: true
+  # Bump this version to delete, reinstall, and restart the transform during package.
+  # Version bump is needed if there is any code change in transform.
+  fleet_transform_version: 0.2.1
+`
+	transformFolder := createTransformFolder(t, packageRoot, transformName)
+	transformFile := filepath.Join(transformFolder, "transform.yml")
+	err := os.WriteFile(transformFile, []byte(contents), 0644)
+	require.NoError(t, err)
+}
+
+func createTransformFolder(t *testing.T, packageRoot, transformName string) string {
+	t.Helper()
+	if transformName == "" {
+		return ""
+	}
+	transformFolder := filepath.Join(packageRoot, "elasticsearch", "transform", transformName)
+	err := os.MkdirAll(transformFolder, 0755)
+	require.NoError(t, err)
+	return transformFolder
+}