Skip to content

Commit 6b9c2cd

Browse files
liu-hm19liu-hm19
authored
feat: support runtime configurations in workspace (#1211)
1 parent a89353c commit 6b9c2cd

File tree

18 files changed

+237
-38
lines changed

18 files changed

+237
-38
lines changed

‎pkg/apis/api.kusion.io/v1/types.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ type Workspace struct {
9292
// SecretStore represents a secure external location for storing secrets.
9393
SecretStore *SecretStore `yaml:"secretStore,omitempty" json:"secretStore,omitempty"`
9494

95-
// Context contains workspace-level configurations, such as topologies, server endpoints, metadata, etc.
95+
// Context contains workspace-level configurations, such as runtimes, topologies, and metadata, etc.
9696
Context GenericConfig `yaml:"context,omitempty" json:"context,omitempty"`
9797
}
9898

@@ -475,6 +475,9 @@ const (
475475
EnvAwsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
476476
EnvAwsDefaultRegion = "AWS_DEFAULT_REGION"
477477
EnvAwsRegion = "AWS_REGION"
478+
EnvAlicloudAccessKey = "ALICLOUD_ACCESS_KEY"
479+
EnvAlicloudSecretKey = "ALICLOUD_SECRET_KEY"
480+
EnvAlicloudRegion = "ALICLOUD_REGION"
478481
)
479482

480483
// BackendConfigs contains the configuration of multiple backends and the current backend.
@@ -861,6 +864,8 @@ type Spec struct {
861864
Resources Resources `yaml:"resources" json:"resources"`
862865
// SecretSore represents a external secret store location for storing secrets.
863866
SecretStore *SecretStore `yaml:"secretStore" json:"secretStore"`
867+
// Context contains workspace-level configurations, such as runtimes, topologies, and metadata, etc.
868+
Context GenericConfig `yaml:"context" json:"context"`
864869
}
865870

866871
// State is a record of an operation's result. It is a mapping between resources in KCL and the actual

‎pkg/cmd/apply/apply.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -713,7 +713,7 @@ func Watch(
713713
watchErrCh <- *err
714714
}()
715715
// Init the runtimes according to the resource types.
716-
runtimes, s := runtimeinit.Runtimes(toBeWatched)
716+
runtimes, s := runtimeinit.Runtimes(*rel.Spec)
717717
if v1.IsErr(s) {
718718
panic(fmt.Errorf("failed to init runtimes: %s", s.String()))
719719
}

‎pkg/engine/operation/apply.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,7 @@ func (ao *ApplyOperation) Apply(req *ApplyRequest) (rsp *ApplyResponse, s v1.Sta
7171
stateResourceIndex[k] = v
7272
}
7373

74-
resources := req.Release.Spec.Resources
75-
resources = append(resources, priorState.Resources...)
76-
runtimesMap, s := runtimeinit.Runtimes(resources)
74+
runtimesMap, s := runtimeinit.Runtimes(*req.Release.Spec)
7775
if v1.IsErr(s) {
7876
return nil, s
7977
}

‎pkg/engine/operation/apply_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func TestApplyOperation_Apply(t *testing.T) {
148148
return nil
149149
}).Build()
150150
mockey.Mock(runtimeinit.Runtimes).To(func(
151-
resources apiv1.Resources,
151+
spec apiv1.Spec,
152152
) (map[apiv1.Type]runtime.Runtime, v1.Status) {
153153
return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: &kubernetes.KubernetesRuntime{}}, nil
154154
}).Build()

‎pkg/engine/operation/destory.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func (do *DestroyOperation) Destroy(req *DestroyRequest) (rsp *DestroyResponse,
4848

4949
// only destroy resources we have recorded
5050
resources := priorState.Resources
51-
runtimesMap, s := runtimeinit.Runtimes(resources)
51+
runtimesMap, s := runtimeinit.Runtimes(*req.Release.Spec)
5252
if v1.IsErr(s) {
5353
return nil, s
5454
}

‎pkg/engine/operation/graph/resource_node.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (rn *ResourceNode) replaceK8sSecretRefs(o *models.Operation) v1.Status {
106106
continue
107107
}
108108

109-
externalSecretRef, err := parseExternalSecretDataRef(ref)
109+
externalSecretRef, err := ParseExternalSecretDataRef(ref)
110110
if err != nil {
111111
return v1.NewErrorStatus(err)
112112
}
@@ -513,8 +513,8 @@ func ReplaceRef(
513513
return result, v, nil
514514
}
515515

516-
// parseExternalSecretDataRef knows how to parse the remote data ref string, returns the corresponding ExternalSecretRef object.
517-
func parseExternalSecretDataRef(dataRefStr string) (*apiv1.ExternalSecretRef, error) {
516+
// ParseExternalSecretDataRef knows how to parse the remote data ref string, returns the corresponding ExternalSecretRef object.
517+
func ParseExternalSecretDataRef(dataRefStr string) (*apiv1.ExternalSecretRef, error) {
518518
uri, err := url.Parse(dataRefStr)
519519
if err != nil {
520520
return nil, err

‎pkg/engine/operation/graph/resource_node_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ func TestParseExternalSecretDataRef(t *testing.T) {
303303
}
304304
for _, tt := range tests {
305305
t.Run(tt.name, func(t *testing.T) {
306-
got, err := parseExternalSecretDataRef(tt.dataRefStr)
306+
got, err := ParseExternalSecretDataRef(tt.dataRefStr)
307307
if (err != nil) != tt.wantErr {
308308
t.Errorf("parseExternalSecretDataRef() error = %v, wantErr %v", err, tt.wantErr)
309309
return

‎pkg/engine/operation/preview.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,7 @@ func (po *PreviewOperation) Preview(req *PreviewRequest) (rsp *PreviewResponse,
5858
priorState := req.State
5959

6060
// Kusion is a multi-runtime system. We initialize runtimes dynamically by resource types
61-
resources := req.Spec.Resources
62-
resources = append(resources, priorState.Resources...)
63-
runtimesMap, s := runtimeinit.Runtimes(resources)
61+
runtimesMap, s := runtimeinit.Runtimes(*req.Spec)
6462
if v1.IsErr(s) {
6563
return nil, s
6664
}
@@ -75,7 +73,7 @@ func (po *PreviewOperation) Preview(req *PreviewRequest) (rsp *PreviewResponse,
7573
priorStateResourceIndex = priorState.Resources.Index()
7674
ag, s = newApplyGraph(req.Spec, priorState)
7775
case models.DestroyPreview:
78-
resources = req.Spec.Resources
76+
resources := req.Spec.Resources
7977
priorStateResourceIndex = resources.Index()
8078
ag, s = newDestroyGraph(resources)
8179
}

‎pkg/engine/operation/preview_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ func TestPreviewOperation_Preview(t *testing.T) {
253253
}
254254

255255
mockey.Mock(runtimeinit.Runtimes).To(func(
256-
resources apiv1.Resources,
256+
spec apiv1.Spec,
257257
) (map[apiv1.Type]runtime.Runtime, v1.Status) {
258258
return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: &fakePreviewRuntime{}}, nil
259259
}).Build()

‎pkg/engine/operation/watch.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func (wo *WatchOperation) Watch(req *WatchRequest) error {
4343

4444
// init runtimes
4545
resources := req.Spec.Resources
46-
runtimes, s := runtimeinit.Runtimes(resources)
46+
runtimes, s := runtimeinit.Runtimes(*req.Spec)
4747
if v1.IsErr(s) {
4848
return errors.New(s.Message())
4949
}

‎pkg/engine/operation/watch_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ func TestWatchOperation_Watch(t *testing.T) {
9494
},
9595
}
9696
mockey.Mock(runtimeinit.Runtimes).To(func(
97-
resources apiv1.Resources,
97+
spec apiv1.Spec,
9898
) (map[apiv1.Type]runtime.Runtime, v1.Status) {
9999
return map[apiv1.Type]runtime.Runtime{runtime.Kubernetes: fooRuntime}, nil
100100
}).Build()

‎pkg/engine/release/util.go

+13-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,19 @@ func CreateDestroyRelease(storage Storage, project, stack, workspace string) (*v
113113

114114
resources := make([]v1.Resource, len(lastRelease.State.Resources))
115115
copy(resources, lastRelease.State.Resources)
116-
spec := &v1.Spec{Resources: resources}
116+
117+
secretStore := &v1.SecretStore{}
118+
if lastRelease.Spec != nil && lastRelease.Spec.SecretStore != nil {
119+
secretStore = lastRelease.Spec.SecretStore
120+
}
121+
122+
specContext := v1.GenericConfig{}
123+
if lastRelease.Spec != nil && lastRelease.Spec.Context != nil {
124+
specContext = lastRelease.Spec.Context
125+
}
126+
127+
spec := &v1.Spec{Resources: resources, SecretStore: secretStore, Context: specContext}
128+
117129
// if no resource managed, set phase to Succeeded directly.
118130
phase := v1.ReleasePhasePreviewing
119131
if len(resources) == 0 {

‎pkg/engine/runtime/init/init.go

+70-3
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,45 @@
11
package init
22

33
import (
4+
"context"
5+
"errors"
46
"fmt"
57
"reflect"
8+
"strings"
69

710
apiv1 "kusionstack.io/kusion/pkg/apis/api.kusion.io/v1"
811
v1 "kusionstack.io/kusion/pkg/apis/status/v1"
12+
"kusionstack.io/kusion/pkg/engine/operation/graph"
913
"kusionstack.io/kusion/pkg/engine/runtime"
1014
"kusionstack.io/kusion/pkg/engine/runtime/kubernetes"
1115
"kusionstack.io/kusion/pkg/engine/runtime/kubernetes/kubeops"
1216
"kusionstack.io/kusion/pkg/engine/runtime/terraform"
17+
"kusionstack.io/kusion/pkg/secrets"
18+
"kusionstack.io/kusion/pkg/workspace"
1319
)
1420

1521
var SupportRuntimes = map[apiv1.Type]InitFn{
1622
runtime.Kubernetes: kubernetes.NewKubernetesRuntime,
1723
runtime.Terraform: terraform.NewTerraformRuntime,
1824
}
1925

26+
var contextKeys = []string{
27+
kubeops.KubeConfigContentKey,
28+
apiv1.EnvAwsAccessKeyID,
29+
apiv1.EnvAwsSecretAccessKey,
30+
apiv1.EnvAlicloudAccessKey,
31+
apiv1.EnvAlicloudSecretKey,
32+
}
33+
2034
// InitFn runtime init func
21-
type InitFn func(resource *apiv1.Resource) (runtime.Runtime, error)
35+
type InitFn func(spec apiv1.Spec) (runtime.Runtime, error)
2236

23-
func Runtimes(resources apiv1.Resources) (map[apiv1.Type]runtime.Runtime, v1.Status) {
37+
func Runtimes(spec apiv1.Spec) (map[apiv1.Type]runtime.Runtime, v1.Status) {
38+
// Parse the secret ref in the Context of Spec.
39+
if err := parseContextSecretRef(&spec); err != nil {
40+
return nil, v1.NewErrorStatus(err)
41+
}
42+
resources := spec.Resources
2443
runtimesMap := map[apiv1.Type]runtime.Runtime{}
2544
if resources == nil {
2645
return runtimesMap, nil
@@ -32,7 +51,7 @@ func Runtimes(resources apiv1.Resources) (map[apiv1.Type]runtime.Runtime, v1.Sta
3251
for _, resource := range resources {
3352
rt := resource.Type
3453
if runtimesMap[rt] == nil {
35-
r, err := SupportRuntimes[rt](&resource)
54+
r, err := SupportRuntimes[rt](spec)
3655
if err != nil {
3756
return nil, v1.NewErrorStatus(fmt.Errorf("init %s runtime failed. %w", rt, err))
3857
}
@@ -65,3 +84,51 @@ func validResources(resources apiv1.Resources) v1.Status {
6584
}
6685
return nil
6786
}
87+
88+
// parseContextSecretRef parses the external secret ref of the credentials
89+
// in the Context of Spec.
90+
func parseContextSecretRef(spec *apiv1.Spec) error {
91+
// Copy the Context of Spec.
92+
parsedContext := apiv1.GenericConfig{}
93+
for k, v := range spec.Context {
94+
parsedContext[k] = v
95+
}
96+
97+
// Retrieve the context with the specified keys from spec and parse the external secret ref.
98+
for _, key := range contextKeys {
99+
contextStr, err := workspace.GetStringFromGenericConfig(spec.Context, key)
100+
if err != nil {
101+
return err
102+
}
103+
104+
if contextStr != "" {
105+
// Replace the secret store ref.
106+
if strings.HasPrefix(contextStr, graph.SecretRefPrefix) {
107+
externalSecretRef, err := graph.ParseExternalSecretDataRef(contextStr)
108+
if err != nil {
109+
return err
110+
}
111+
112+
provider, exist := secrets.GetProvider(spec.SecretStore.Provider)
113+
if !exist {
114+
return errors.New("no matched secret store found, please check workspace yaml")
115+
}
116+
secretStore, err := provider.NewSecretStore(spec.SecretStore)
117+
if err != nil {
118+
return err
119+
}
120+
secretData, err := secretStore.GetSecret(context.Background(), *externalSecretRef)
121+
if err != nil {
122+
return err
123+
}
124+
125+
parsedContext[key] = string(secretData)
126+
}
127+
}
128+
}
129+
130+
// Reset the Context with the parsed values.
131+
spec.Context = parsedContext
132+
133+
return nil
134+
}

‎pkg/engine/runtime/kubernetes/kubeops/config.go

+5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ const (
1515
RecommendedKubeConfigFileName = "config"
1616
)
1717

18+
const (
19+
KubeConfigPathKey = "kubeconfig_path"
20+
KubeConfigContentKey = "kubeconfig_content"
21+
)
22+
1823
var (
1924
RecommendedConfigDir = filepath.Join(homedir.HomeDir(), RecommendedHomeDir)
2025
RecommendedKubeConfigFile = filepath.Join(RecommendedConfigDir, RecommendedKubeConfigFileName)

‎pkg/engine/runtime/kubernetes/kubernetes_runtime.go

+46-6
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"kusionstack.io/kusion/pkg/engine/runtime/kubernetes/kubeops"
3636
"kusionstack.io/kusion/pkg/log"
3737
jsonutil "kusionstack.io/kusion/pkg/util/json"
38+
"kusionstack.io/kusion/pkg/workspace"
3839
)
3940

4041
var _ runtime.Runtime = (*KubernetesRuntime)(nil)
@@ -45,8 +46,8 @@ type KubernetesRuntime struct {
4546
}
4647

4748
// NewKubernetesRuntime create a new KubernetesRuntime
48-
func NewKubernetesRuntime(resource *apiv1.Resource) (runtime.Runtime, error) {
49-
client, mapper, err := getKubernetesClient(resource)
49+
func NewKubernetesRuntime(spec apiv1.Spec) (runtime.Runtime, error) {
50+
client, mapper, err := getKubernetesClient(spec)
5051
if err != nil {
5152
return nil, err
5253
}
@@ -394,11 +395,50 @@ func (k *KubernetesRuntime) Watch(ctx context.Context, request *runtime.WatchReq
394395
}
395396

396397
// getKubernetesClient get kubernetes client
397-
func getKubernetesClient(resource *apiv1.Resource) (dynamic.Interface, meta.RESTMapper, error) {
398+
func getKubernetesClient(spec apiv1.Spec) (dynamic.Interface, meta.RESTMapper, error) {
398399
// build config
399-
cfg, err := clientcmd.BuildConfigFromFlags("", kubeops.GetKubeConfig(resource))
400-
if err != nil {
401-
return nil, nil, err
400+
var err error
401+
var cfg *rest.Config
402+
403+
if len(spec.Context) != 0 {
404+
kubeConfigPath, err := workspace.GetStringFromGenericConfig(spec.Context, kubeops.KubeConfigPathKey)
405+
if err != nil {
406+
return nil, nil, err
407+
}
408+
kubeConfigContent, err := workspace.GetStringFromGenericConfig(spec.Context, kubeops.KubeConfigContentKey)
409+
if err != nil {
410+
return nil, nil, err
411+
}
412+
if kubeConfigContent != "" {
413+
clientCfg, err := clientcmd.NewClientConfigFromBytes([]byte(kubeConfigContent))
414+
if err != nil {
415+
return nil, nil, err
416+
}
417+
418+
cfg, err = clientCfg.ClientConfig()
419+
if err != nil {
420+
return nil, nil, err
421+
}
422+
} else if kubeConfigPath != "" {
423+
cfg, err = clientcmd.BuildConfigFromFlags("", kubeConfigPath)
424+
if err != nil {
425+
return nil, nil, err
426+
}
427+
}
428+
} else {
429+
var kubeConfigFromRes string
430+
for _, res := range spec.Resources {
431+
if res.Type == apiv1.Kubernetes {
432+
kubeConfigFromRes = kubeops.GetKubeConfig(&res)
433+
}
434+
if kubeConfigFromRes != "" {
435+
break
436+
}
437+
}
438+
cfg, err = clientcmd.BuildConfigFromFlags("", kubeConfigFromRes)
439+
if err != nil {
440+
return nil, nil, err
441+
}
402442
}
403443

404444
// DynamicRESTMapper can discover resource types at runtime dynamically

0 commit comments

Comments
 (0)