Skip to content

Commit 4906a0c

Browse files
daniel-hutaodaniel-hutao
authored
Merge branch 'main' into fix-options-nil-pointer
2 parents 6263405 + 19e1a31 commit 4906a0c

17 files changed

+526
-319
lines changed

Diff for: ‎.github/workflows/automated-release.yml

+7
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,13 @@ jobs:
2828
run: make build -j8
2929
- name: Install Github-release
3030
run: go install github.com/github-release/github-release@latest
31+
- name: Install AWS CLI v2
32+
run: |
33+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
34+
unzip -q /tmp/awscliv2.zip -d /tmp
35+
rm /tmp/awscliv2.zip
36+
sudo /tmp/aws/install --update
37+
rm -rf /tmp/aws/
3138
- name: Configure AWS credentials
3239
uses: aws-actions/configure-aws-credentials@v1
3340
with:

Diff for: ‎internal/pkg/configmanager/app.go

+18-20
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ package configmanager
33
import (
44
"fmt"
55

6-
"gopkg.in/yaml.v3"
7-
8-
"github.com/devstream-io/devstream/pkg/util/file"
6+
"github.com/devstream-io/devstream/pkg/util/log"
97
"github.com/devstream-io/devstream/pkg/util/scm"
108
)
119

@@ -27,33 +25,31 @@ type app struct {
2725
CDRawConfigs []pipelineRaw `yaml:"cd" mapstructure:"cd"`
2826
}
2927

30-
func getAppsFromConfigFileWithVarsRendered(fileBytes []byte, vars map[string]any) ([]*app, error) {
31-
yamlPath := "$.apps[*]"
32-
yamlStrArray, err := file.GetYamlNodeArrayByPath(fileBytes, yamlPath)
28+
func (a *app) getTools(vars map[string]any, templateMap map[string]string) (Tools, error) {
29+
// generate app repo and template repo from scmInfo
30+
a.setDefault()
31+
repoScaffoldingTool, err := a.getRepoTemplateTool()
3332
if err != nil {
34-
return nil, err
35-
}
36-
37-
if yamlStrArray == nil {
38-
return make([]*app, 0), nil
33+
return nil, fmt.Errorf("app[%s] can't get valid repo config: %w", a.Name, err)
3934
}
4035

41-
yamlWithVars, err := renderConfigWithVariables(yamlStrArray.StrOrigin, vars)
36+
// get ci/cd pipelineTemplates
37+
appVars := a.Spec.merge(vars)
38+
tools, err := a.generateCICDTools(templateMap, appVars)
4239
if err != nil {
43-
return nil, err
40+
return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
4441
}
45-
46-
var retTApps = make([]*app, 0)
47-
err = yaml.Unmarshal(yamlWithVars, &retTApps)
48-
if err != nil {
49-
return nil, err
42+
if repoScaffoldingTool != nil {
43+
tools = append(tools, repoScaffoldingTool)
5044
}
5145

52-
return retTApps, nil
46+
log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)
47+
48+
return tools, nil
5349
}
5450

5551
// getAppPipelineTool generate ci/cd tools from app config
56-
func (a *app) generateCICDToolsFromAppConfig(templateMap map[string]string, appVars map[string]any) (Tools, error) {
52+
func (a *app) generateCICDTools(templateMap map[string]string, appVars map[string]any) (Tools, error) {
5753
allPipelineRaw := append(a.CIRawConfigs, a.CDRawConfigs...)
5854
var tools Tools
5955
for _, p := range allPipelineRaw {
@@ -81,7 +77,9 @@ func (a *app) getRepoTemplateTool() (*Tool, error) {
8177
return nil, fmt.Errorf("configmanager[app] parse repo failed: %w", err)
8278
}
8379
if a.RepoTemplate != nil {
80+
// templateRepo doesn't need auth info
8481
templateRepo, err := a.RepoTemplate.BuildRepoInfo()
82+
templateRepo.NeedAuth = false
8583
if err != nil {
8684
return nil, fmt.Errorf("configmanager[app] parse repoTemplate failed: %w", err)
8785
}

Diff for: ‎internal/pkg/configmanager/app_test.go

+73-17
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,83 @@
11
package configmanager
22

33
import (
4+
"fmt"
5+
46
. "github.com/onsi/ginkgo/v2"
57
. "github.com/onsi/gomega"
8+
9+
"github.com/devstream-io/devstream/pkg/util/scm"
610
)
711

8-
var _ = Describe("getToolsFromConfigFileWithVarsRendered", func() {
9-
const appsConfig = `---
10-
apps:
11-
- name: app-1
12-
cd:
13-
- type: template
14-
vars:
15-
app: [[ appName ]]
16-
`
17-
When("get apps from config file", func() {
18-
It("should return config with vars", func() {
19-
apps, err := getAppsFromConfigFileWithVarsRendered([]byte(appsConfig), map[string]any{"appName": interface{}("app-1")})
20-
Expect(err).NotTo(HaveOccurred())
21-
Expect(apps).NotTo(BeNil())
22-
Expect(len(apps)).To(Equal(1))
23-
Expect(len(apps[0].CDRawConfigs)).To(Equal(1))
24-
Expect(apps[0].CDRawConfigs[0].Vars["app"]).To(Equal(interface{}("app-1")))
12+
var _ = Describe("app struct", func() {
13+
var (
14+
a *app
15+
appName string
16+
vars map[string]any
17+
templateMap map[string]string
18+
)
19+
BeforeEach(func() {
20+
appName = "test_app"
21+
vars = map[string]any{}
22+
templateMap = map[string]string{}
23+
})
24+
Context("getTools method", func() {
25+
When("repo is not valid", func() {
26+
BeforeEach(func() {
27+
a = &app{Name: appName}
28+
})
29+
It("should return error", func() {
30+
_, err := a.getTools(vars, templateMap)
31+
Expect(err).Should(HaveOccurred())
32+
Expect(err.Error()).Should(ContainSubstring(fmt.Sprintf("app[%s] can't get valid repo config", appName)))
33+
})
34+
})
35+
When("ci/cd template is not valid", func() {
36+
BeforeEach(func() {
37+
a = &app{
38+
Repo: &scm.SCMInfo{
39+
CloneURL: "http://test.com/test/test_app",
40+
},
41+
CIRawConfigs: []pipelineRaw{
42+
{
43+
Type: "template",
44+
TemplateName: "not_exist",
45+
},
46+
},
47+
}
48+
})
49+
It("should return error", func() {
50+
_, err := a.getTools(vars, templateMap)
51+
Expect(err).Should(HaveOccurred())
52+
Expect(err.Error()).Should(ContainSubstring("not found in pipelineTemplates"))
53+
})
54+
})
55+
})
56+
57+
Context("generateCICDTools method", func() {
58+
When("template type not exist", func() {
59+
BeforeEach(func() {
60+
a = &app{
61+
Repo: &scm.SCMInfo{
62+
CloneURL: "http://test.com/test/test_app",
63+
},
64+
CIRawConfigs: []pipelineRaw{
65+
{
66+
Type: "template",
67+
TemplateName: "not_valid",
68+
},
69+
},
70+
}
71+
})
72+
It("should return error", func() {
73+
templateMap = map[string]string{
74+
"not_valid": `
75+
name: not_valid,
76+
type: not_valid`}
77+
_, err := a.generateCICDTools(templateMap, vars)
78+
Expect(err).Should(HaveOccurred())
79+
Expect(err.Error()).Should(ContainSubstring("pipeline type [not_valid] not supported for now"))
80+
})
2581
})
2682
})
2783
})

Diff for: ‎internal/pkg/configmanager/config.go

-52
Original file line numberDiff line numberDiff line change
@@ -4,65 +4,13 @@ import (
44
"fmt"
55

66
"gopkg.in/yaml.v3"
7-
8-
"github.com/devstream-io/devstream/pkg/util/log"
97
)
108

119
// Config is a general config in DevStream.
1210
type Config struct {
1311
Config CoreConfig `yaml:"config"`
1412
Vars map[string]any `yaml:"vars"`
1513
Tools Tools `yaml:"tools"`
16-
Apps []*app `yaml:"apps"`
17-
// We'll read the pipeline templates from config file and render it to pipelineTemplateMap in no time.
18-
//PipelineTemplates []*pipelineTemplate `yaml:"-"`
19-
pipelineTemplateMap map[string]string `yaml:"-"`
20-
}
21-
22-
func (c *Config) getToolsFromApps() (Tools, error) {
23-
var tools Tools
24-
for _, a := range c.Apps {
25-
appTools, err := c.getToolsFromApp(a)
26-
if err != nil {
27-
return nil, err
28-
}
29-
tools = append(tools, appTools...)
30-
}
31-
return tools, nil
32-
}
33-
34-
func (c *Config) getToolsFromApp(a *app) (Tools, error) {
35-
// generate app repo and template repo from scmInfo
36-
a.setDefault()
37-
repoScaffoldingTool, err := a.getRepoTemplateTool()
38-
if err != nil {
39-
return nil, fmt.Errorf("app[%s] get repo failed: %w", a.Name, err)
40-
}
41-
42-
// get ci/cd pipelineTemplates
43-
appVars := a.Spec.merge(c.Vars)
44-
tools, err := a.generateCICDToolsFromAppConfig(c.pipelineTemplateMap, appVars)
45-
if err != nil {
46-
return nil, fmt.Errorf("app[%s] get pipeline tools failed: %w", a.Name, err)
47-
}
48-
if repoScaffoldingTool != nil {
49-
tools = append(tools, repoScaffoldingTool)
50-
}
51-
52-
// all tools from apps should depend on the original tools,
53-
// because dtm will execute all original tools first, then execute all tools from apps
54-
for _, toolFromApps := range tools {
55-
for _, t := range c.Tools {
56-
toolFromApps.DependsOn = append(toolFromApps.DependsOn, t.KeyWithNameAndInstanceID())
57-
}
58-
}
59-
60-
log.Debugf("Have got %d tools from app %s.", len(tools), a.Name)
61-
for i, t := range tools {
62-
log.Debugf("Tool %d: %v", i+1, t)
63-
}
64-
65-
return tools, nil
6614
}
6715

6816
func (c *Config) renderInstanceIDtoOptions() {

Diff for: ‎internal/pkg/configmanager/configmanager.go

+26-34
Original file line numberDiff line numberDiff line change
@@ -20,25 +20,17 @@ func NewManager(configFilePath string) *Manager {
2020
// LoadConfig is the only method that the caller of Manager needs to be concerned with, and this method returns a *Config finally.
2121
// The main workflow of this method is:
2222
// 1. Get the original config from the config file specified by ConfigFilePath;
23-
// 2. Parsing tools from the apps, and merge that tools to Config.Tools;
24-
// 3. Validation.
23+
// 2. Validation.
2524
func (m *Manager) LoadConfig() (*Config, error) {
26-
// step 1
25+
// step 1: get config
2726
c, err := m.getConfigFromFileWithGlobalVars()
2827
if err != nil {
2928
return nil, err
3029
}
31-
32-
// step 2
33-
appTools, err := c.getToolsFromApps()
34-
if err != nil {
35-
return nil, err
36-
}
37-
38-
c.Tools = append(c.Tools, appTools...)
30+
// set instanceID in options
3931
c.renderInstanceIDtoOptions()
4032

41-
// step 3
33+
// step 2: check config is valid
4234
if err = c.validate(); err != nil {
4335
return nil, err
4436
}
@@ -50,50 +42,50 @@ func (m *Manager) LoadConfig() (*Config, error) {
5042
// 1. render the global variables to Config.Tools and Config.Apps
5143
// 2. transfer the PipelineTemplates to Config.pipelineTemplateMap, it's map[string]string type.
5244
// We can't render the original config file to Config.PipelineTemplates directly for the:
53-
// 1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;
54-
// 2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;
55-
// 3. pipeline templates are used in apps, so it would be more appropriate to refer to pipeline templates when dealing with apps
45+
// 1. variables rendered must be before the yaml.Unmarshal() called for the [[ foo ]] will be treated as a two-dimensional array by the yaml parser;
46+
// 2. the variables used([[ foo ]]) in the Config.PipelineTemplates can be defined in the Config.Apps or Config.Vars;
5647
func (m *Manager) getConfigFromFileWithGlobalVars() (*Config, error) {
5748
configBytes, err := os.ReadFile(m.ConfigFilePath)
5849
if err != nil {
5950
return nil, err
6051
}
6152

62-
// global variables
63-
vars, err := getVarsFromConfigFile(configBytes)
53+
// extract top raw config struct from config text
54+
r, err := newRawConfigFromConfigBytes(configBytes)
55+
if err != nil {
56+
return nil, err
57+
}
58+
// 1. get global variables
59+
vars, err := r.getVars()
6460
if err != nil {
6561
return nil, fmt.Errorf("failed to get variables from config file. Error: %w", err)
6662
}
6763

68-
// tools with global variables rendered
69-
tools, err := getToolsFromConfigFileWithVarsRendered(configBytes, vars)
64+
// 2. tools with global variables rendered
65+
tools, err := r.getToolsWithVars(vars)
7066
if err != nil {
7167
return nil, fmt.Errorf("failed to get tools from config file. Error: %w", err)
7268
}
7369

74-
// apps with global variables rendered
75-
apps, err := getAppsFromConfigFileWithVarsRendered(configBytes, vars)
70+
// 3. apps tools with global variables rendered
71+
appTools, err := r.getAppToolsWithVars(vars)
7672
if err != nil {
7773
return nil, fmt.Errorf("failed to get apps from config file. Error: %w", err)
7874
}
75+
// all tools from apps should depend on the original tools,
76+
// because dtm will execute all original tools first, then execute all tools from apps
77+
appTools.updateToolDepends(tools)
78+
tools = append(tools, appTools...)
7979

80-
// pipelineTemplateMap transfer from PipelineTemplates
81-
pipelineTemplateMap, err := getPipelineTemplatesMapFromConfigFile(configBytes)
82-
if err != nil {
83-
return nil, fmt.Errorf("failed to get pipelineTemplatesMap from config file. Error: %w", err)
84-
}
85-
86-
// coreConfig without any changes
87-
coreConfig, err := getCoreConfigFromConfigFile(configBytes)
80+
// 4. coreConfig without any changes
81+
coreConfig, err := r.getConfig()
8882
if err != nil {
8983
return nil, fmt.Errorf("failed to get coreConfig from config file. Error: %w", err)
9084
}
9185

9286
return &Config{
93-
Config: *coreConfig,
94-
Vars: vars,
95-
Tools: tools,
96-
Apps: apps,
97-
pipelineTemplateMap: pipelineTemplateMap,
87+
Config: *coreConfig,
88+
Vars: vars,
89+
Tools: tools,
9890
}, nil
9991
}

Diff for: ‎internal/pkg/configmanager/configmanager_test.go

+2-5
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,6 @@ var _ = Describe("LoadConfig", func() {
202202
"sourceRepo": RawOptions{
203203
"repoType": "github",
204204
"url": "github.com/devstream-io/dtm-scaffolding-golang",
205-
"needAuth": true,
206205
"org": "devstream-io",
207206
"repo": "dtm-scaffolding-golang",
208207
"branch": "main",
@@ -218,7 +217,7 @@ var _ = Describe("LoadConfig", func() {
218217
})
219218

220219
When("load a config file", func() {
221-
It("should return 5 tools", func() {
220+
It("should return tools", func() {
222221
mgr := NewManager(filepath.Join(tmpWorkDir, "config.yaml"))
223222
cfg, err := mgr.LoadConfig()
224223
Expect(err).NotTo(HaveOccurred())
@@ -274,9 +273,7 @@ var _ = Describe("getConfigFromFileWithGlobalVars", func() {
274273
Expect(err).NotTo(HaveOccurred())
275274
Expect(cfg.Config.State.Backend).To(Equal("local"))
276275
Expect(cfg.Vars["foo1"]).To(Equal("bar1"))
277-
Expect(len(cfg.Apps)).To(Equal(1))
278-
Expect(cfg.Apps[0].Name).To(Equal("service-a"))
279-
Expect(len(cfg.Tools)).To(Equal(2))
276+
Expect(len(cfg.Tools)).To(Equal(5))
280277
Expect(cfg.Tools[1].Name).To(Equal("plugin2"))
281278
})
282279
})

0 commit comments

Comments
 (0)