Merge pull request #1370 from devstream-io/main

bugfix for v0.10.3

Author: daniel-hutao

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -28,8 +28,8 @@ body:
     label: 'DevStream Version'
     description: "To find out the version run: `dtm version`"
     options:
-      - < v0.10.0
-      - v0.10.0
+      - < v0.10.3
+      - v0.10.3
       - latest
   validations:
     required: true

.github/sync-staging-repo.yml

@@ -1,5 +1,5 @@
-devstream-io/dtm-gitlab-share-library:
-  - source: staging/dtm-gitlab-share-library/
+devstream-io/dtm-jenkins-share-library:
+  - source: staging/dtm-jenkins-share-library/
     dest: .
     deleteOrphaned: true
 devstream-io/dtm-pipeline-templates:

.github/workflows/automated-release.yml

@@ -41,6 +41,8 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ap-southeast-1
+      - name: check md5 & commit
+        run: git rev-parse HEAD && md5sum dtm*
       - name: upload core
         run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
       - name: upload plugin
@@ -83,4 +85,3 @@ jobs:
         run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
       - name: upload plugins to s3
         run: aws s3 cp ~/.devstream/plugins/ s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read
-

cmd/devstream/init.go

@@ -91,18 +91,18 @@ func GetPluginsFromFlags() (tools configmanager.Tools, err error) {
 		// check if plugins to download are supported by dtm
 		for _, plugin := range pluginsName {
 			if _, ok := list.PluginNamesMap()[plugin]; !ok {
-				return nil, fmt.Errorf("Plugin %s is not supported by dtm", plugin)
+				return nil, fmt.Errorf("plugin %s is not supported by dtm", plugin)
 			}
 		}
 	}
 
 	if len(pluginsName) == 0 {
-		return nil, errors.New("Please use --plugins to specify plugins to download or use --all to download all plugins.")
+		return nil, errors.New("please use --plugins to specify plugins to download or use --all to download all plugins")
 	}
 	log.Debugf("plugins to download: %v", pluginsName)
 
 	if initOS == "" || initArch == "" {
-		return nil, fmt.Errorf("Once you use the --all flag, you must specify the --os and --arch flags")
+		return nil, fmt.Errorf("once you use the --all flag, you must specify the --os and --arch flags")
 	}
 
 	log.Infof("Plugins to download: %v", pluginsName)

hack/release/auto-release-darwin-arm64.sh

@@ -73,7 +73,7 @@ pip3 install awscli
 aws s3 cp $plugin_dir $STORAGE_URL_WITH_TAG --recursive --acl public-read
 
 # check if the number of plugins on s3 is correct
-local_plugin_nums=$(../../dtm list plugins |wc -l)
+local_plugin_nums=$(./dtm list plugins |wc -l)
 ((local_plugin_file_nums=local_plugin_nums*6))
 s3_plugin_file_total_nums=$(aws s3 ls download.devstream.io/"$tag"/|awk '{print $NF}'|uniq|wc -l)
 ((s3_plugin_file_nums=s3_plugin_file_total_nums-3))

internal/pkg/configmanager/pipelineDefault.go

@@ -83,13 +83,15 @@ func pipelineArgocdAppGenerator(options RawOptions, globalVars *pipelineGlobalOp
 		sourceMap := source.(RawOptions)
 		if _, repoURLExist := sourceMap["repoURL"]; !repoURLExist {
 			sourceMap["repoURL"] = globalVars.Repo.GetCloneURL()
+			sourceMap["repoBranch"] = globalVars.Repo.Branch
 		}
 		options["source"] = sourceMap
 	} else {
 		options["source"] = RawOptions{
-			"valuefile": "values.yaml",
-			"path":      fmt.Sprintf("helm/%s", globalVars.AppName),
-			"repoURL":   string(globalVars.Repo.GetCloneURL()),
+			"valuefile":  "values.yaml",
+			"path":       fmt.Sprintf("helm/%s", globalVars.AppName),
+			"repoURL":    string(globalVars.Repo.GetCloneURL()),
+			"repoBranch": globalVars.Repo.Branch,
 		}
 	}
 	// config imageRepo default options

internal/pkg/configmanager/pipelineDefault_test.go

@@ -69,6 +69,7 @@ var _ = Describe("pipelineArgocdAppGenerator func", func() {
 			},
 			Repo: &git.RepoInfo{
 				CloneURL: "scm.test.com",
+				Branch:   "testBranch",
 			},
 			AppName: "test_app",
 		}
@@ -85,9 +86,10 @@ var _ = Describe("pipelineArgocdAppGenerator func", func() {
 				"namespace": "default",
 			},
 			"source": RawOptions{
-				"valuefile": "values.yaml",
-				"path":      "helm/test_app",
-				"repoURL":   "https://scm.test.com",
+				"valuefile":  "values.yaml",
+				"path":       "helm/test_app",
+				"repoURL":    "https://scm.test.com",
+				"repoBranch": "testBranch",
 			},
 			"imageRepo": RawOptions{
 				"owner": "test_user",

internal/pkg/configmanager/variables.go

@@ -8,12 +8,10 @@ import (
 func renderConfigWithVariables(fileContent string, variables map[string]interface{}) ([]byte, error) {
 	logFunc(fileContent, variables)
 
-	str, err := template.New().
-		FromContent(fileContent).
-		AddDotForVariablesInConfigProcessor().
-		AddQuoteForVariablesInConfigProcessor().
-		SetDefaultRender(fileContent, variables).
-		Render()
+	str, err := template.NewRenderClient(&template.TemplateOption{
+		Name: "configmanager",
+	}, template.ContentGetter, template.AddDotForVariablesInConfigProcessor, template.AddQuoteForVariablesInConfigProcessor,
+	).Render(fileContent, variables)
 
 	if err != nil {
 		return nil, err

internal/pkg/develop/plugin/plugin.go

@@ -79,7 +79,12 @@ func (p *Plugin) renderTplString(tplStr string) (string, error) {
 		"dirFormat": pluginTpl.FormatPackageDirName,
 	}
 
-	res, err := templateUtil.New().FromContent(tplStr).SetDefaultRender("default", *p, funcMap).Render()
+	res, err := templateUtil.NewRenderClient(
+		&templateUtil.TemplateOption{
+			Name:    "plugins",
+			FuncMap: funcMap,
+		}, templateUtil.ContentGetter,
+	).Render(tplStr, *p)
 	if err != nil {
 		log.Debugf("Template execute failed: %s.", err)
 		log.Debugf("Template content: %s.", tplStr)

internal/pkg/plugin/argocdapp/argocdapp.go

@@ -22,7 +22,10 @@ func pushArgocdConfigFiles(rawOptions configmanager.RawOptions) error {
 	}
 
 	// 1. init scm client for check argocdapp config exists and push argocdapp files
-	repoInfo := &git.RepoInfo{CloneURL: git.ScmURL(opts.Source.RepoURL)}
+	repoInfo := &git.RepoInfo{
+		CloneURL: git.ScmURL(opts.Source.RepoURL),
+		Branch:   opts.Source.RepoBranch,
+	}
 	scmClient, err := scm.NewClientWithAuth(repoInfo)
 	if err != nil {
 		return err

internal/pkg/plugin/argocdapp/options.go

@@ -43,9 +43,10 @@ type destination struct {
 
 // source is the struct for the source of an ArgoCD app.
 type source struct {
-	Valuefile string `mapstructure:"valuefile"`
-	Path      string `mapstructure:"path" validate:"required"`
-	RepoURL   string `mapstructure:"repoURL" validate:"required"`
+	Valuefile  string `mapstructure:"valuefile"`
+	Path       string `mapstructure:"path" validate:"required"`
+	RepoURL    string `mapstructure:"repoURL" validate:"required"`
+	RepoBranch string `mapstructure:"repoBranch"`
 }
 
 // / newOptions create options by raw options
@@ -82,7 +83,9 @@ func (o *options) getArgocdDefaultConfigFiles(configLocation downloader.Resource
 	}
 	// 3. get file content
 	fContentFunc := func(filePath string) ([]byte, error) {
-		renderContent, err := template.New().FromLocalFile(filePath).SetDefaultRender("argocd config", o).Render()
+		renderContent, err := template.NewRenderClient(&template.TemplateOption{
+			Name: "argocd",
+		}, template.LocalFileGetter).Render(filePath, o)
 		if err != nil {
 			return nil, err
 		}

internal/pkg/plugin/gitlabcedocker/options.go

@@ -58,7 +58,9 @@ func (opts *Options) setGitLabURL() {
 
 func showHelpMsg(options configmanager.RawOptions) error {
 	log.Infof("GitLab access URL: %s", gitlabURL)
-	log.Infof("GitLab initial root password: execute the command -> docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password")
+	log.Info("Execute these two command to get/set GitLab root password: ")
+	log.Info("1. docker exec -it gitlab bash")
+	log.Info(`2. gitlab-rake "gitlab:password:reset"`)
 
 	return nil
 }

internal/pkg/plugin/gitlabci/gitlabci.go

@@ -64,12 +64,12 @@ func createGitlabRunnerByHelm(repoInfo *git.RepoInfo) error {
 		return err
 	}
 	// 2. else create runner for this project
-	valuesYaml, err := template.New().FromContent(helmValueTpl).SetDefaultRender(
-		"gitlab-runner tpl", map[string]any{
+	valuesYaml, err := template.NewRenderClient(&template.TemplateOption{
+		Name: "gitlab-runner"}, template.ContentGetter).Render(
+		helmValueTpl, map[string]any{
 			"GitlabURL":     repoInfo.BaseURL,
 			"RegisterToken": runnerToken,
-		},
-	).Render()
+		})
 	if err != nil {
 		return err
 	}

internal/pkg/plugin/harbordocker/harbordocker.go

@@ -57,7 +57,9 @@ func renderConfig(options configmanager.RawOptions) (configmanager.RawOptions, e
 		return nil, err
 	}
 
-	content, err := template.New().FromContent(HarborConfigTemplate).SetDefaultRender("compose", opts).Render()
+	content, err := template.NewRenderClient(&template.TemplateOption{
+		Name: "docker-compose",
+	}, template.ContentGetter).Render(HarborConfigTemplate, opts)
 	if err != nil {
 		return nil, err
 	}

internal/pkg/plugin/installer/ci/cifile/cifile.go

@@ -68,7 +68,9 @@ func (c *CIFileConfig) newCIServerClient() (ciClient server.CIServerOptions) {
 func (c *CIFileConfig) renderContent(ciFileContent string) (string, error) {
 	needRenderContent := len(c.Vars) > 0
 	if needRenderContent {
-		return template.New().FromContent(ciFileContent).SetDefaultRender(ciTemplateName, c.Vars).Render()
+		return template.NewRenderClient(&template.TemplateOption{
+			Name: fmt.Sprintf("ci-%s", ciTemplateName),
+		}, template.ContentGetter).Render(ciFileContent, c.Vars)
 	}
 	return ciFileContent, nil
 

internal/pkg/plugin/installer/ci/cifile/utils.go

@@ -1,6 +1,7 @@
 package cifile
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/devstream-io/devstream/pkg/util/file"
@@ -16,7 +17,9 @@ func processCIFilesFunc(vars map[string]interface{}) file.DirFileContentFunc {
 		if len(vars) == 0 {
 			return os.ReadFile(filePath)
 		}
-		renderContent, err := template.New().FromLocalFile(filePath).SetDefaultRender(ciTemplateName, vars).Render()
+		renderContent, err := template.NewRenderClient(&template.TemplateOption{
+			Name: fmt.Sprintf("ci-%s", ciTemplateName),
+		}, template.LocalFileGetter).Render(filePath, vars)
 		if err != nil {
 			return nil, err
 		}

internal/pkg/plugin/installer/goclient/installer.go

@@ -1,7 +1,7 @@
 package goclient
 
 import (
-	"errors"
+	"fmt"
 	"strings"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -27,8 +27,7 @@ func DealWithNsWhenInstall(options configmanager.RawOptions) error {
 
 	// Namespace should not be empty
 	if opts.Namespace == "" {
-		log.Debugf("No namespace is given.")
-		return errors.New("No namespace is given.")
+		return fmt.Errorf("config options should set namespace config")
 	}
 	log.Debugf("Prepare to create the namespace: %s.", opts.Namespace)
 

internal/pkg/plugin/installer/kubectl/installer.go

@@ -25,7 +25,9 @@ func ProcessByContent(action, content string) installer.BaseOperation {
 }
 
 func renderKubectlContent(content string, options configmanager.RawOptions) (io.Reader, error) {
-	content, err := template.New().FromContent(content).SetDefaultRender("kubectl", options).Render()
+	content, err := template.NewRenderClient(&template.TemplateOption{
+		Name: "kubectl",
+	}, template.ContentGetter).Render(content, options)
 	if err != nil {
 		return nil, err
 	}
@@ -38,7 +40,9 @@ func renderKubectlContent(content string, options configmanager.RawOptions) (io.
 
 func ProcessByURL(action, url string) installer.BaseOperation {
 	return func(options configmanager.RawOptions) error {
-		content, err := template.New().FromURL(url).SetDefaultRender("kubectl", options).Render()
+		content, err := template.NewRenderClient(&template.TemplateOption{
+			Name: "kubectl",
+		}, template.URLGetter).Render(url, options)
 		if err != nil {
 			return err
 		}

internal/pkg/plugin/installer/reposcaffolding/utils.go

@@ -49,7 +49,9 @@ func processRepoFileFunc(appName string, vars map[string]interface{}) file.DirFi
 		log.Debugf("Walk: found file: %s.", filePath)
 		// if file endswith tpl, render this file, else copy this file directly
 		if strings.Contains(filePath, "tpl") {
-			fileContentStr, err := template.New().FromLocalFile(filePath).SetDefaultRender(appName, vars).Render()
+			fileContentStr, err := template.NewRenderClient(&template.TemplateOption{
+				Name: "repo-scaffolding",
+			}, template.LocalFileGetter).Render(filePath, vars)
 			if err != nil {
 				log.Warnf("repo render file failed: %s", err)
 				return fileContent, err

internal/pkg/plugin/jiragithub/render.go

@@ -14,5 +14,7 @@ func renderTemplate(workflow *github.Workflow, options *Options) (string, error)
 		return "", err
 	}
 
-	return template.New().FromContent(workflow.WorkflowContent).SetDefaultRender("jiragithub", opts).Render()
+	return template.NewRenderClient(&template.TemplateOption{
+		Name: "jiragithub",
+	}, template.ContentGetter).Render(workflow.WorkflowContent, opts)
 }

internal/pkg/show/config/gen_embed_var.go

@@ -77,10 +77,10 @@ func copyTemplates(src, dst string) error {
 
 // generate generates the code for Option `o` into a file named by `o.Path`.
 func generate(o *Option) {
-	content, err := templateUtil.New().
-		FromContent(templateCode).
-		SetDefaultRender("gen", o, o.Funcs).
-		Render()
+	content, err := templateUtil.NewRenderClient(&templateUtil.TemplateOption{
+		Name:    "gen_embed",
+		FuncMap: o.Funcs,
+	}, templateUtil.ContentGetter).Render(templateCode, o)
 
 	if err != nil {
 		log.Fatal("template Execute error:", err)

internal/pkg/show/config/plugins/argocdapp.yaml

@@ -27,6 +27,8 @@ tools:
       path: charts/go-hello-http
       # Helm chart repo URL, this is only an example, do not use this
       repoURL: YOUR_CHART_REPO_URL
+      # Helm chart repo branch
+      repoBranch: YOUR_CHART_REPO_BRANCH
     # if repo doesn't contain path, use imageRepo to create a helm config
     imageRepo:
       # imageRepo address