Merge branch 'main' into release-0.6

Tiexin Guo · Tiexin Guo · commit 54bacbd27f82 · 2022-05-21T13:53:51.000+08:00
diff --git a/.github/workflows/automated-release.yml b/.github/workflows/automated-release.yml
@@ -45,7 +45,7 @@ jobs:
       - name: upload core
         run: bash -e ./hack/release/upload_assets.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }}
       - name: upload plugin
-        run: aws s3 cp .devstream/  s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read
+        run: aws s3 cp .devstream/ s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read
 
   # release for darwin-amd64
   build-and-release-on-darwin-amd64:
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -63,14 +63,19 @@ jobs:
           aws eks update-kubeconfig --region ap-southeast-1 --name dtm-test
       - name: copy config files
         run: cp ./test/e2e/yaml/e2e-*.yaml ./
-      - name: apply
+      - name: apply git-ops
         run: ./dtm apply -f e2e-config.yaml -y
-      - name: apply twice
+      - name: apply twice git-ops
         run: ./dtm apply -f e2e-config.yaml -y
+      - name: apply trello-github
+        run: ./dtm apply -f e2e-trello-github-config.yaml -y
+      - name: apply twice trello-github
+        run: ./dtm apply -f e2e-trello-github-config.yaml -y
       - name: check if pod is ready
         run: while [[ $(kubectl get pods -l app=dtm-e2e-go -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "pod not ready yet..."; sleep 3; done
         timeout-minutes: 10
       - name: verify
         run: ./dtm verify -f e2e-config.yaml
       - name: clean
         run: ./dtm delete -f e2e-config.yaml -y
+
diff --git a/Makefile b/Makefile
@@ -24,7 +24,9 @@ ROOT_DIR := $(abspath $(shell cd $(SELF_DIR) && pwd -P))
 endif
 
 ifeq ($(origin VERSION), undefined)
-VERSION := $(shell git describe --tags --always --match='v*')
+# the VERSION is a number, like 0.6.0
+# it doesn't contain the prefix v, not v0.6.0, but 0.6.0
+VERSION := $(shell git describe --tags --always --match='v*' | cut -c 2-)
 endif
 
 ifeq ($(origin PLUGINS_DIR),undefined)
diff --git a/cmd/devstream/list/list.go b/cmd/devstream/list/list.go
@@ -14,7 +14,7 @@ import (
 
 var PluginsName string
 
-// List all of plugins name
+// List all plugins name
 func List(pluginFilter string) {
 	listPluginsName := strings.Fields(PluginsName)
 	r, _ := regexp.Compile(pluginFilter)
diff --git a/docs/development/creating-a-plugin.md b/docs/development/creating-a-plugin.md
@@ -53,4 +53,4 @@ DevStream uses [go plugin](https://pkg.go.dev/plugin) to implement custom DevOps
 
 When you execute a command which calls any of the interfaces(`Create`, `Read`, `Update`, `Delete`), devstream's pluginengine will call the [`plugin.Lookup("DevStreamPlugin")` function](https://github.com/devstream-io/devstream/blob/38307894bbc08f691b2c5015366d9e45cc87970c/internal/pkg/pluginengine/plugin_helper.go#L28) to load the plugin, get the variable `DevStreamPlugin` that implements the ` DevStreamPlugin` interface, and then you can call the corresponding plugin logic functions.  This is why it is not recommended to modify the `/cmd/plugin/YOUR-PLUGIN-NAME/main.go` file directly.
 
-Note: The `main()` in `/cmd/plugin/YOUR-PLUGIN-NAME/main.go` file will not executed, it is only used to avoid the goclangci-lint error.
+Note: The `main()` in `/cmd/plugin/YOUR-PLUGIN-NAME/main.go` file will not be executed, it is only used to avoid the goclangci-lint error.
diff --git a/docs/quickstart_en.md b/docs/quickstart_en.md
@@ -15,7 +15,7 @@ For Linux/Macos users:
 
 ## 2 Prepare a Config File
 
-Before you start: for an example of DevStream config, see [examples/tools-quickstart.yaml](./examples/tools-quickstart.yaml). Remember to open this configuration file, modify all FULL_UPPER_CASE_STRINGS (like YOUR_GITHUB_USERNAME, for example) in it to your own. Pay attention to the meaning of each item to ensure that it is what you want. For other plugins, checkout the "Plugins" section in our [doc](https://docs.devstream.io) for detailed usage.
+Before you start: for an example of DevStream config, see [examples/tools-quickstart.yaml](https://github.com/devstream-io/devstream/blob/main/examples/tools-quickstart.yaml). Remember to open this configuration file, modify all FULL_UPPER_CASE_STRINGS (like YOUR_GITHUB_USERNAME, for example) in it to your own. Pay attention to the meaning of each item to ensure that it is what you want. For other plugins, checkout the "Plugins" section in our [doc](https://docs.devstream.io) for detailed usage.
 
 Download the [examples/quickstart.yaml](https://raw.githubusercontent.com/devstream-io/devstream/main/examples/quickstart.yaml) and [examples/tools-quickstart.yaml](https://raw.githubusercontent.com/devstream-io/devstream/main/examples/tools-quickstart.yaml) to your working directory and rename `quickstart.yaml` to `config.yaml`:
 
diff --git a/docs/quickstart_zh.md b/docs/quickstart_zh.md
@@ -12,7 +12,7 @@
 
 ## 2 准备一个配置文件
 
-开始之前：这是一个DevStream配置的例子：[examples/tools-quickstart.yaml](../examples/tools-quickstart.yaml)。记得打开这个配置文件，把里面所有的 `FULL_UPPER_CASE_STRINGS`（比如说 `YOUR_GITHUB_USERNAME` ）修改成你自己的。注意每一项的含义，并确保它是你要的。对于其他插件，请查看我们的 [文档](https://docs.devstream.io) 中的"插件"部分，以了解详细用法。
+开始之前：这是一个DevStream配置的例子：[examples/tools-quickstart.yaml](https://github.com/devstream-io/devstream/blob/main/examples/tools-quickstart.yaml) 。记得打开这个配置文件，把里面所有的 `FULL_UPPER_CASE_STRINGS`（比如说 `YOUR_GITHUB_USERNAME` ）修改成你自己的。注意每一项的含义，并确保它是你要的。对于其他插件，请查看我们的 [文档](https://docs.devstream.io) 中的"插件"部分，以了解详细用法。
 
 将 [examples/quickstart.yaml](https://raw.githubusercontent.com/devstream-io/devstream/main/examples/quickstart.yaml) 和 [examples/tools-quickstart.yaml](https://raw.githubusercontent.com/devstream-io/devstream/main/examples/tools-quickstart.yaml) 文件下载到你到工作目录下，然后重命名`quickstart.yaml` 成 `config.yaml`：
 
diff --git a/hack/release/auto-release-darwin-arm64.sh b/hack/release/auto-release-darwin-arm64.sh
@@ -18,7 +18,7 @@ while getopts "t:" opt; do
 done
 
 if [ "${tag}" == "invalid" ]; then
-  echo "Maybe you forgot to use -t flag. E.g. sh auto-release-darwin-arm64.sh -t v0.6.0"
+  echo "Maybe you forgot to use -t flag. E.g. sh auto-release-darwin-arm64.sh -t 0.6.0"
   exit 1
 fi
 echo "tag: ${tag}"
diff --git a/hack/terraform/bucket-policy.tf b/hack/terraform/bucket-policy.tf
@@ -0,0 +1,46 @@
+resource "aws_s3_bucket_policy" "policy_iterable" {
+  bucket = "download.devstream.io"
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Sid" : "PublicReadGetObjectOnlyForCloudFlareIps111",
+        "Effect" : "Deny",
+        "Principal" : "*",
+        "Action" : "s3:GetObject",
+        "Resource" : [
+          "arn:aws:s3:::download.devstream.io",
+          "arn:aws:s3:::download.devstream.io/*"
+        ],
+        "Condition" : {
+          "NotIpAddress" : {
+            "aws:SourceIp" : [
+              "2400:cb00::/32",
+              "2606:4700::/32",
+              "2803:f800::/32",
+              "2405:b500::/32",
+              "2405:8100::/32",
+              "2a06:98c0::/29",
+              "2c0f:f248::/32",
+              "173.245.48.0/20",
+              "103.21.244.0/22",
+              "103.22.200.0/22",
+              "103.31.4.0/22",
+              "141.101.64.0/18",
+              "108.162.192.0/18",
+              "190.93.240.0/20",
+              "188.114.96.0/20",
+              "197.234.240.0/22",
+              "198.41.128.0/17",
+              "162.158.0.0/15",
+              "104.16.0.0/13",
+              "104.24.0.0/14",
+              "172.64.0.0/13",
+              "131.0.72.0/22"
+            ]
+          }
+        }
+      }
+    ]
+  })
+}
diff --git a/hack/terraform/devstream-iam.tf b/hack/terraform/devstream-iam.tf
@@ -141,7 +141,9 @@ resource "aws_iam_group_policy" "DevStream-Download-Bucket-RW-Policy" {
         "Effect": "Allow",
         "Action": [
           "s3:GetObject",
+          "s3:GetObjectAcl",
           "s3:PutObject",
+          "s3:PutObjectAcl",
           "s3:DeleteObject",
           "s3:RestoreObject",
           "s3:ListBucket",
diff --git a/hack/terraform/github-actions-iam.tf b/hack/terraform/github-actions-iam.tf
@@ -19,6 +19,39 @@ resource "aws_iam_user_policy" "githubactions" {
   policy = data.aws_iam_policy_document.githubactions.json
 }
 
+resource "aws_iam_user_policy" "GitHubActions-Download-Bucket-RW-Policy" {
+  name  = "GitHubActions-Download-Bucket-RW-Policy"
+  user = aws_iam_user.githubactions.name
+
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Sid": "PermissionForDownloadUploadForTheBucket",
+        "Effect": "Allow",
+        "Action": [
+          "s3:GetObject",
+          "s3:GetObjectAcl",
+          "s3:PutObject",
+          "s3:PutObjectAcl",
+          "s3:DeleteObject",
+          "s3:RestoreObject",
+          "s3:ListBucket",
+          "s3:GetBucketPolicy",
+          "s3:ReplicateObject",
+          "s3:GetBucketWebsite",
+          "s3:PutBucketWebsite",
+          "s3:GetBucketCORS",
+        ],
+        "Resource": [
+          "arn:aws:s3:::download.devstream.io",
+          "arn:aws:s3:::download.devstream.io/*"
+        ]
+      }
+    ]
+  })
+}
+
 resource "aws_iam_access_key" "githubactions" {
   user = aws_iam_user.githubactions.name
 }
diff --git a/internal/pkg/backend/backend.go b/internal/pkg/backend/backend.go
@@ -18,9 +18,9 @@ const (
 
 // Backend is used to persist data, it can be local file/etcd/s3/...
 type Backend interface {
-	// Read is used to reads data from persistent storage.
+	// Read is used to read data from persistent storage.
 	Read() ([]byte, error)
-	// Write is used to writes data to persistent storage.
+	// Write is used to write data to persistent storage.
 	Write(data []byte) error
 }
 
diff --git a/internal/pkg/backend/local/local.go b/internal/pkg/backend/local/local.go
@@ -52,7 +52,7 @@ func (l *Local) Read() ([]byte, error) {
 	return data, nil
 }
 
-// Write is used to writes the data to local file.
+// Write is used to write the data to local file.
 func (l *Local) Write(data []byte) error {
 	l.mu.Lock()
 	defer l.mu.Unlock()
diff --git a/internal/pkg/configloader/validation.go b/internal/pkg/configloader/validation.go
@@ -82,10 +82,6 @@ func validateDependency(tools []Tool) []error {
 func validateConfigFile(c *ConfigFile) []error {
 	errors := make([]error, 0)
 
-	if c.VarFile == "" {
-		errors = append(errors, fmt.Errorf("variables file is empty"))
-	}
-
 	if c.ToolFile == "" {
 		errors = append(errors, fmt.Errorf("tool file is empty"))
 	}
diff --git a/internal/pkg/configloader/variables.go b/internal/pkg/configloader/variables.go
@@ -12,6 +12,9 @@ import (
 )
 
 func renderVariables(varFileName string, configFileBytes []byte) ([]byte, error) {
+	if varFileName == "" {
+		return configFileBytes, nil
+	}
 	// load variables file
 	variables, err := loadVariablesFilesIntoMap(varFileName)
 	if err != nil {
diff --git a/internal/pkg/plugin/githubactions/golang/jobs.go b/internal/pkg/plugin/githubactions/golang/jobs.go
@@ -9,20 +9,20 @@ const (
 	RegistryHarbor    RegistryType = "harbor"
 )
 
-// Build is the struct for githubacions job.
+// Build is the struct for githubactions job.
 type Build struct {
 	Enable  bool
 	Command string
 }
 
-// Test is the struct for githubacions job.
+// Test is the struct for githubactions job.
 type Test struct {
 	Enable   bool
 	Command  string
 	Coverage Coverage
 }
 
-// Docker is the struct for githubacions job.
+// Docker is the struct for githubactions job.
 type Docker struct {
 	Enable   bool
 	Registry Registry
@@ -35,18 +35,18 @@ type Registry struct {
 	Repository string
 }
 
-// Coverage is the struct for githubacions job.
+// Coverage is the struct for githubactions job.
 type Coverage struct {
 	Enable  bool
 	Profile string
 	Output  string
 }
 
-// Tag is the struct for githubacions job.
+// Tag is the struct for githubactions job.
 type Tag struct {
 }
 
-// Image is the struct for githubacions job.
+// Image is the struct for githubactions job.
 type Image struct {
 }
 
diff --git a/internal/pkg/pluginengine/change.go b/internal/pkg/pluginengine/change.go
@@ -38,13 +38,13 @@ const (
 )
 
 // GetChangesForApply takes "State Manager" & "Config" then do some calculate and return a Plan.
-// All actions should be execute is included in this Plan.changes.
+// All actions should be executed is included in this Plan.changes.
 func GetChangesForApply(smgr statemanager.Manager, cfg *configloader.Config) ([]*Change, error) {
 	return getChanges(smgr, cfg, CommandApply, false)
 }
 
 // GetChangesForDelete takes "State Manager" & "Config" then do some calculation and return a Plan to delete all plugins in the Config.
-// All actions should be execute is included in this Plan.changes.
+// All actions should be executed is included in this Plan.changes.
 func GetChangesForDelete(smgr statemanager.Manager, cfg *configloader.Config, isForceDelete bool) ([]*Change, error) {
 	return getChanges(smgr, cfg, CommandDelete, isForceDelete)
 }
diff --git a/internal/pkg/pluginengine/outputs.go b/internal/pkg/pluginengine/outputs.go
@@ -43,7 +43,7 @@ func HandleOutputsReferences(smgr statemanager.Manager, options map[string]inter
 }
 
 // getToolNamePluginKindAndOutputReferenceKey returns (false, "", "", "") if regex doesn't match
-// if match, returns (true, name, instanceID, key)
+// if matched, returns (true, name, instanceID, key)
 func getToolNamePluginOutputKey(s string) (bool, string, string, string) {
 	regex := `.*\${{\s*([^.]*)\.([^.]*)\.outputs\.([^.\s]*)\s*}}.*`
 	r := regexp.MustCompile(regex)
diff --git a/test/e2e/yaml/e2e-trello-github-config.yaml b/test/e2e/yaml/e2e-trello-github-config.yaml
@@ -0,0 +1,6 @@
+varFile: e2e-variables.yaml
+toolFile: e2e-trello-github-tools.yaml
+state:
+  backend: local
+  options:
+    stateFile: devstream.state
diff --git a/test/e2e/yaml/e2e-trello-github-tools.yaml b/test/e2e/yaml/e2e-trello-github-tools.yaml
@@ -0,0 +1,19 @@
+tools:
+  - name: trello
+    instanceID: default
+    dependsOn: [ ]
+    options:
+      org: [[ githubOrganization ]]
+      repo: [[ RepoName ]]
+      kanbanBoardName: [[ kanbanBoardName ]]
+  - name: trello-github-integ
+    instanceID: default
+    dependsOn: [ "trello.default" ]
+    options:
+      org: [[ githubOrganization ]]
+      repo: [[ RepoName ]]
+      boardId: ${{ trello.default.outputs.boardId }}
+      todoListId: ${{ trello.default.outputs.todoListId }}
+      doingListId: ${{ trello.default.outputs.doingListId }}
+      doneListId: ${{ trello.default.outputs.doneListId }}
+      branch: main
diff --git a/test/e2e/yaml/e2e-variables.yaml b/test/e2e/yaml/e2e-variables.yaml
@@ -4,3 +4,4 @@ repoName: dtm-e2e-go
 dockerRegistryUserName: dtme2etest
 argocdNameSpace: argocd
 argocdDeployTimeout: 10m
+kanbanBoardName: dtmKanbanTest

Original file line number	Diff line number	Diff line change
`@@ -53,4 +53,4 @@ DevStream uses [go plugin](https://pkg.go.dev/plugin) to implement custom DevOps`
`53`	`53`
`54`	`54`	When you execute a command which calls any of the interfaces(`Create`, `Read`, `Update`, `Delete`), devstream's pluginengine will call the [`plugin.Lookup("DevStreamPlugin")` function](https://github.com/devstream-io/devstream/blob/38307894bbc08f691b2c5015366d9e45cc87970c/internal/pkg/pluginengine/plugin_helper.go#L28) to load the plugin, get the variable `DevStreamPlugin` that implements the ` DevStreamPlugin` interface, and then you can call the corresponding plugin logic functions. This is why it is not recommended to modify the `/cmd/plugin/YOUR-PLUGIN-NAME/main.go` file directly.
`55`	`55`
`56`		-Note: The `main()` in `/cmd/plugin/YOUR-PLUGIN-NAME/main.go` file will not executed, it is only used to avoid the goclangci-lint error.
	`56`	+Note: The `main()` in `/cmd/plugin/YOUR-PLUGIN-NAME/main.go` file will not be executed, it is only used to avoid the goclangci-lint error.
Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ func (l *Local) Read() ([]byte, error) {`
`52`	`52`	`return data, nil`
`53`	`53`	`}`
`54`	`54`
`55`		`-// Write is used to writes the data to local file.`
	`55`	`+// Write is used to write the data to local file.`
`56`	`56`	`func (l *Local) Write(data []byte) error {`
`57`	`57`	`l.mu.Lock()`
`58`	`58`	`defer l.mu.Unlock()`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,9 @@ import (`
`12`	`12`	`)`
`13`	`13`
`14`	`14`	`func renderVariables(varFileName string, configFileBytes []byte) ([]byte, error) {`
	`15`	`+ if varFileName == "" {`
	`16`	`+ return configFileBytes, nil`
	`17`	`+ }`
`15`	`18`	`// load variables file`
`16`	`19`	`variables, err := loadVariablesFilesIntoMap(varFileName)`
`17`	`20`	`if err != nil {`
Original file line number	Diff line number	Diff line change
`@@ -9,20 +9,20 @@ const (`
`9`	`9`	`RegistryHarbor RegistryType = "harbor"`
`10`	`10`	`)`
`11`	`11`
`12`		`-// Build is the struct for githubacions job.`
	`12`	`+// Build is the struct for githubactions job.`
`13`	`13`	`type Build struct {`
`14`	`14`	`Enable bool`
`15`	`15`	`Command string`
`16`	`16`	`}`
`17`	`17`
`18`		`-// Test is the struct for githubacions job.`
	`18`	`+// Test is the struct for githubactions job.`
`19`	`19`	`type Test struct {`
`20`	`20`	`Enable bool`
`21`	`21`	`Command string`
`22`	`22`	`Coverage Coverage`
`23`	`23`	`}`
`24`	`24`
`25`		`-// Docker is the struct for githubacions job.`
	`25`	`+// Docker is the struct for githubactions job.`
`26`	`26`	`type Docker struct {`
`27`	`27`	`Enable bool`
`28`	`28`	`Registry Registry`
`@@ -35,18 +35,18 @@ type Registry struct {`
`35`	`35`	`Repository string`
`36`	`36`	`}`
`37`	`37`
`38`		`-// Coverage is the struct for githubacions job.`
	`38`	`+// Coverage is the struct for githubactions job.`
`39`	`39`	`type Coverage struct {`
`40`	`40`	`Enable bool`
`41`	`41`	`Profile string`
`42`	`42`	`Output string`
`43`	`43`	`}`
`44`	`44`
`45`		`-// Tag is the struct for githubacions job.`
	`45`	`+// Tag is the struct for githubactions job.`
`46`	`46`	`type Tag struct {`
`47`	`47`	`}`
`48`	`48`
`49`		`-// Image is the struct for githubacions job.`
	`49`	`+// Image is the struct for githubactions job.`
`50`	`50`	`type Image struct {`
`51`	`51`	`}`
`52`	`52`
Original file line number	Diff line number	Diff line change
`@@ -38,13 +38,13 @@ const (`
`38`	`38`	`)`
`39`	`39`
`40`	`40`	`// GetChangesForApply takes "State Manager" & "Config" then do some calculate and return a Plan.`
`41`		`-// All actions should be execute is included in this Plan.changes.`
	`41`	`+// All actions should be executed is included in this Plan.changes.`
`42`	`42`	`func GetChangesForApply(smgr statemanager.Manager, cfg configloader.Config) ([]Change, error) {`
`43`	`43`	`return getChanges(smgr, cfg, CommandApply, false)`
`44`	`44`	`}`
`45`	`45`
`46`	`46`	`// GetChangesForDelete takes "State Manager" & "Config" then do some calculation and return a Plan to delete all plugins in the Config.`
`47`		`-// All actions should be execute is included in this Plan.changes.`
	`47`	`+// All actions should be executed is included in this Plan.changes.`
`48`	`48`	`func GetChangesForDelete(smgr statemanager.Manager, cfg configloader.Config, isForceDelete bool) ([]Change, error) {`
`49`	`49`	`return getChanges(smgr, cfg, CommandDelete, isForceDelete)`
`50`	`50`	`}`
Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,7 @@ func HandleOutputsReferences(smgr statemanager.Manager, options map[string]inter`
`43`	`43`	`}`
`44`	`44`
`45`	`45`	`// getToolNamePluginKindAndOutputReferenceKey returns (false, "", "", "") if regex doesn't match`
`46`		`-// if match, returns (true, name, instanceID, key)`
	`46`	`+// if matched, returns (true, name, instanceID, key)`
`47`	`47`	`func getToolNamePluginOutputKey(s string) (bool, string, string, string) {`
`48`	`48`	regex := `.\${{\s([^.])\.([^.])\.outputs\.([^.\s])\s}}.*`
`49`	`49`	`r := regexp.MustCompile(regex)`