-
Notifications
You must be signed in to change notification settings - Fork 5k
New flag to allow option for passing a config file for addon configure command. #20255
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
22 commits
Select commit
Hold shift + click to select a range
d105076
Addon configuration now takes an optional config file to load from
panyam 5cbc860
Using a rarer value for default instead of changeme so it has less ch…
panyam 0f0c407
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 1485c75
Fixing expected values of test cases
panyam f5776e7
making addonConfigFile private
panyam 966e8b9
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 5a169eb
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 98e61df
Merge branch 'master' into addon-cfg-from-file
panyam b81fd3f
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam b39c649
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 5217ca3
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 177432b
Keeping registry_creds configs together.
panyam e5d3df2
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 47add6e
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 49b651d
Making registry creds structs private and other PR feedback cleanups
panyam 3adcb9b
Changing default value back to 'changeme'
panyam 27a9403
Merge branch 'kubernetes:master' into addon-cfg-from-file
panyam 7150b83
Simplifying duplicate consts
panyam 423be4d
Adding missed comma
panyam 0a7f10c
lint fixes
panyam 3e5176e
Generating docs
panyam f083400
Copyright for new file
panyam File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,273 @@ | ||
/* | ||
Copyright 2025 The Kubernetes Authors All rights reserved. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package config | ||
|
||
import ( | ||
"os" | ||
|
||
"k8s.io/minikube/pkg/minikube/exit" | ||
"k8s.io/minikube/pkg/minikube/out" | ||
"k8s.io/minikube/pkg/minikube/reason" | ||
"k8s.io/minikube/pkg/minikube/service" | ||
) | ||
|
||
const configDefaultValue = "changeme" | ||
|
||
// Top level configs for RegistryCreds addons | ||
panyam marked this conversation as resolved.
Show resolved
Hide resolved
|
||
type registryCredsAddonConfig struct { | ||
EnableAWSEcr string `json:"enableAWSEcr"` | ||
EcrConfigs registryCredsAddonConfigAWSEcr `json:"awsEcrConfigs"` | ||
|
||
EnableGCR string `json:"enableGCR"` | ||
GcrConfigs registryCredsAddonConfigGCR `json:"gcrConfigs"` | ||
|
||
EnableDockerRegistry string `json:"enableDockerRegistry"` | ||
DockerConfigs registryCredsAddonConfigDocker `json:"dockerConfigs"` | ||
|
||
EnableACR string `json:"enableACR"` | ||
AcrConfigs registryCredsAddonConfigACR `json:"acrConfigs"` | ||
} | ||
|
||
// Registry Creds addon config for AWS ECR | ||
type registryCredsAddonConfigAWSEcr struct { | ||
AccessID string `json:"awsAccessID"` | ||
AccessKey string `json:"awsAccessKey"` | ||
SessionToken string `json:"awsSessionToken"` | ||
Region string `json:"awsRegion"` | ||
Account string `json:"awsAccount"` | ||
Role string `json:"awsRole"` | ||
} | ||
|
||
// Registry Creds addon config for GCR | ||
type registryCredsAddonConfigGCR struct { | ||
GcrPath string `json:"gcrPath"` | ||
GcrURL string `json:"gcrURL"` | ||
} | ||
|
||
// Registry Creds addon config for Docker Registry | ||
type registryCredsAddonConfigDocker struct { | ||
DockerServer string `json:"dockerServer"` | ||
DockerUser string `json:"dockerUser"` | ||
DockerPass string `json:"dockerPass"` | ||
} | ||
|
||
// Registry Creds addon config for Docker Azure container registry | ||
type registryCredsAddonConfigACR struct { | ||
AcrURL string `json:"acrURL"` | ||
AcrClientID string `json:"acrClientID"` | ||
AcrPassword string `json:"acrPassword"` | ||
} | ||
|
||
// Processes registry-creds addon config from configFile if it exists otherwise resorts to default behavior | ||
func processRegistryCredsConfig(profile string, ac *addonConfig) { | ||
// Default values | ||
awsAccessID := configDefaultValue | ||
awsAccessKey := configDefaultValue | ||
awsSessionToken := "" | ||
awsRegion := configDefaultValue | ||
awsAccount := configDefaultValue | ||
awsRole := configDefaultValue | ||
gcrApplicationDefaultCredentials := configDefaultValue | ||
dockerServer := configDefaultValue | ||
dockerUser := configDefaultValue | ||
dockerPass := configDefaultValue | ||
gcrURL := "https://gcr.io" | ||
acrURL := configDefaultValue | ||
acrClientID := configDefaultValue | ||
acrPassword := configDefaultValue | ||
|
||
regCredsConf := &ac.RegistryCreds | ||
awsEcrAction := regCredsConf.EnableAWSEcr // regCredsConf. "enableAWSEcr") | ||
if awsEcrAction == "prompt" || awsEcrAction == "" { | ||
enableAWSECR := AskForYesNoConfirmation("\nDo you want to enable AWS Elastic Container Registry?", posResponses, negResponses) | ||
if enableAWSECR { | ||
awsAccessID = AskForStaticValue("-- Enter AWS Access Key ID: ") | ||
awsAccessKey = AskForStaticValue("-- Enter AWS Secret Access Key: ") | ||
awsSessionToken = AskForStaticValueOptional("-- (Optional) Enter AWS Session Token: ") | ||
awsRegion = AskForStaticValue("-- Enter AWS Region: ") | ||
awsAccount = AskForStaticValue("-- Enter 12 digit AWS Account ID (Comma separated list): ") | ||
awsRole = AskForStaticValueOptional("-- (Optional) Enter ARN of AWS role to assume: ") | ||
} | ||
} else if awsEcrAction == "enable" { | ||
out.Ln("Loading AWS ECR configs from: %s", addonConfigFile) | ||
// Then read the configs | ||
awsAccessID = regCredsConf.EcrConfigs.AccessID | ||
awsAccessKey = regCredsConf.EcrConfigs.AccessKey | ||
awsSessionToken = regCredsConf.EcrConfigs.SessionToken | ||
awsRegion = regCredsConf.EcrConfigs.Region | ||
awsAccount = regCredsConf.EcrConfigs.Account | ||
awsRole = regCredsConf.EcrConfigs.Role | ||
} else if awsEcrAction == "disable" { | ||
out.Ln("Ignoring AWS ECR configs") | ||
} else { | ||
out.Ln("Disabling AWS ECR. Invalid value for enableAWSEcr (%s). Must be one of 'disable', 'enable' or 'prompt'", awsEcrAction) | ||
} | ||
|
||
gcrPath := "" | ||
gcrAction := regCredsConf.EnableGCR | ||
if gcrAction == "prompt" || gcrAction == "" { | ||
enableGCR := AskForYesNoConfirmation("\nDo you want to enable Google Container Registry?", posResponses, negResponses) | ||
if enableGCR { | ||
gcrPath = AskForStaticValue("-- Enter path to credentials (e.g. /home/user/.config/gcloud/application_default_credentials.json):") | ||
gcrchangeURL := AskForYesNoConfirmation("-- Do you want to change the GCR URL (Default https://gcr.io)?", posResponses, negResponses) | ||
|
||
if gcrchangeURL { | ||
gcrURL = AskForStaticValue("-- Enter GCR URL (e.g. https://asia.gcr.io):") | ||
} | ||
} | ||
} else if gcrAction == "enable" { | ||
out.Ln("Loading GCR configs from: %s", addonConfigFile) | ||
// Then read the configs | ||
gcrPath = regCredsConf.GcrConfigs.GcrPath | ||
gcrURL = regCredsConf.GcrConfigs.GcrURL | ||
} else if gcrAction == "disable" { | ||
out.Ln("Ignoring GCR configs") | ||
} else { | ||
out.Ln("Disabling GCR. Invalid value for enableGCR (%s). Must be one of 'disable', 'enable' or 'prompt'", gcrAction) | ||
} | ||
|
||
if gcrPath != "" { | ||
// Read file from disk | ||
dat, err := os.ReadFile(gcrPath) | ||
|
||
if err != nil { | ||
exit.Message(reason.Usage, "Error reading {{.path}}: {{.error}}", out.V{"path": gcrPath, "error": err}) | ||
} else { | ||
gcrApplicationDefaultCredentials = string(dat) | ||
} | ||
} | ||
|
||
dockerRegistryAction := regCredsConf.EnableDockerRegistry | ||
if dockerRegistryAction == "prompt" || dockerRegistryAction == "" { | ||
enableDR := AskForYesNoConfirmation("\nDo you want to enable Docker Registry?", posResponses, negResponses) | ||
if enableDR { | ||
dockerServer = AskForStaticValue("-- Enter docker registry server url: ") | ||
dockerUser = AskForStaticValue("-- Enter docker registry username: ") | ||
dockerPass = AskForPasswordValue("-- Enter docker registry password: ") | ||
} | ||
} else if dockerRegistryAction == "enable" { | ||
out.Ln("Loading Docker Registry configs from: %s", addonConfigFile) | ||
dockerServer = regCredsConf.DockerConfigs.DockerServer | ||
dockerUser = regCredsConf.DockerConfigs.DockerUser | ||
dockerPass = regCredsConf.DockerConfigs.DockerPass | ||
} else if dockerRegistryAction == "disable" { | ||
out.Ln("Ignoring Docker Registry configs") | ||
} else { | ||
out.Ln("Disabling Docker Registry. Invalid value for enableDockerRegistry (%s). Must be one of 'disable', 'enable' or 'prompt'", dockerRegistryAction) | ||
} | ||
|
||
acrAction := regCredsConf.EnableACR | ||
if acrAction == "prompt" || acrAction == "" { | ||
enableACR := AskForYesNoConfirmation("\nDo you want to enable Azure Container Registry?", posResponses, negResponses) | ||
if enableACR { | ||
acrURL = AskForStaticValue("-- Enter Azure Container Registry (ACR) URL: ") | ||
acrClientID = AskForStaticValue("-- Enter client ID (service principal ID) to access ACR: ") | ||
acrPassword = AskForPasswordValue("-- Enter service principal password to access Azure Container Registry: ") | ||
} | ||
} else if acrAction == "enable" { | ||
out.Ln("Loading ACR configs from: ", addonConfigFile) | ||
acrURL = regCredsConf.AcrConfigs.AcrURL | ||
acrClientID = regCredsConf.AcrConfigs.AcrClientID | ||
acrPassword = regCredsConf.AcrConfigs.AcrPassword | ||
} else if acrAction == "disable" { | ||
out.Ln("Ignoring ACR configs") | ||
} else { | ||
out.Stringf("Disabling ACR. Invalid value for enableACR (%s). Must be one of 'disable', 'enable' or 'prompt'", acrAction) | ||
} | ||
|
||
namespace := "kube-system" | ||
|
||
// Create ECR Secret | ||
err := service.CreateSecret( | ||
profile, | ||
namespace, | ||
"registry-creds-ecr", | ||
map[string]string{ | ||
"AWS_ACCESS_KEY_ID": awsAccessID, | ||
"AWS_SECRET_ACCESS_KEY": awsAccessKey, | ||
"AWS_SESSION_TOKEN": awsSessionToken, | ||
"aws-account": awsAccount, | ||
"aws-region": awsRegion, | ||
"aws-assume-role": awsRole, | ||
}, | ||
map[string]string{ | ||
"app": "registry-creds", | ||
"cloud": "ecr", | ||
"kubernetes.io/minikube-addons": "registry-creds", | ||
}) | ||
if err != nil { | ||
exit.Message(reason.InternalCommandRunner, "ERROR creating `registry-creds-ecr` secret: {{.error}}", out.V{"error": err}) | ||
} | ||
|
||
// Create GCR Secret | ||
err = service.CreateSecret( | ||
profile, | ||
namespace, | ||
"registry-creds-gcr", | ||
map[string]string{ | ||
"application_default_credentials.json": gcrApplicationDefaultCredentials, | ||
"gcrurl": gcrURL, | ||
}, | ||
map[string]string{ | ||
"app": "registry-creds", | ||
"cloud": "gcr", | ||
"kubernetes.io/minikube-addons": "registry-creds", | ||
}) | ||
|
||
if err != nil { | ||
exit.Message(reason.InternalCommandRunner, "ERROR creating `registry-creds-gcr` secret: {{.error}}", out.V{"error": err}) | ||
} | ||
|
||
// Create Docker Secret | ||
err = service.CreateSecret( | ||
profile, | ||
namespace, | ||
"registry-creds-dpr", | ||
map[string]string{ | ||
"DOCKER_PRIVATE_REGISTRY_SERVER": dockerServer, | ||
"DOCKER_PRIVATE_REGISTRY_USER": dockerUser, | ||
"DOCKER_PRIVATE_REGISTRY_PASSWORD": dockerPass, | ||
}, | ||
map[string]string{ | ||
"app": "registry-creds", | ||
"cloud": "dpr", | ||
"kubernetes.io/minikube-addons": "registry-creds", | ||
}) | ||
|
||
if err != nil { | ||
out.WarningT("ERROR creating `registry-creds-dpr` secret") | ||
} | ||
|
||
// Create Azure Container Registry Secret | ||
err = service.CreateSecret( | ||
profile, | ||
namespace, | ||
"registry-creds-acr", | ||
map[string]string{ | ||
"ACR_URL": acrURL, | ||
"ACR_CLIENT_ID": acrClientID, | ||
"ACR_PASSWORD": acrPassword, | ||
}, | ||
map[string]string{ | ||
"app": "registry-creds", | ||
"cloud": "acr", | ||
"kubernetes.io/minikube-addons": "registry-creds", | ||
}) | ||
if err != nil { | ||
out.WarningT("ERROR creating `registry-creds-acr` secret") | ||
} | ||
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,6 +21,7 @@ package integration | |
import ( | ||
"bytes" | ||
"context" | ||
"encoding/base64" | ||
"encoding/json" | ||
"errors" | ||
"fmt" | ||
|
@@ -100,7 +101,7 @@ func TestAddons(t *testing.T) { | |
// so we override that here to let minikube auto-detect appropriate cgroup driver | ||
os.Setenv(constants.MinikubeForceSystemdEnv, "") | ||
|
||
args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget", "--addons=nvidia-device-plugin", "--addons=yakd", "--addons=volcano", "--addons=amd-gpu-device-plugin"}, StartArgs()...) | ||
args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=registry-creds", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget", "--addons=nvidia-device-plugin", "--addons=yakd", "--addons=volcano", "--addons=amd-gpu-device-plugin"}, StartArgs()...) | ||
if !NoneDriver() { | ||
args = append(args, "--addons=ingress", "--addons=ingress-dns", "--addons=storage-provisioner-rancher") | ||
} | ||
|
@@ -140,6 +141,7 @@ func TestAddons(t *testing.T) { | |
t.Run("parallel", func(t *testing.T) { | ||
tests := []TestCase{ | ||
{"Registry", validateRegistryAddon}, | ||
{"RegistryCreds", validateRegistryCredsAddon}, | ||
{"Ingress", validateIngressAddon}, | ||
{"InspektorGadget", validateInspektorGadgetAddon}, | ||
{"MetricsServer", validateMetricsServerAddon}, | ||
|
@@ -304,6 +306,65 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { | |
} | ||
} | ||
|
||
// validateRegistryCredsAddon tests the registry-creds addon by trying to load its configs | ||
func validateRegistryCredsAddon(ctx context.Context, t *testing.T, profile string) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. lets use a struct for the regsitry cred addons so it can be typed checked |
||
defer disableAddon(t, "registry-creds", profile) | ||
defer PostMortemLogs(t, profile) | ||
|
||
client, err := kapi.Client(profile) | ||
if err != nil { | ||
t.Fatalf("failed to get Kubernetes client for %s : %v", profile, err) | ||
} | ||
|
||
start := time.Now() | ||
if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "registry-creds", Minutes(6)); err != nil { | ||
t.Errorf("failed waiting for registry-creds deployment to stabilize: %v", err) | ||
} | ||
t.Logf("registry-creds stabilized in %s", time.Since(start)) | ||
|
||
rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "configure", "registry-creds", "-f", "./testdata/addons_testconfig.json", "-p", profile)) | ||
if err != nil { | ||
t.Errorf("failed to configure addon. args %q : %v", rr.Command(), err) | ||
} | ||
|
||
// Check a few secrets exists that match our test data | ||
// In our test aws and gcp are set, docker and acr are disabled - so they will be set to "changeme" | ||
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "-n", "kube-system", "get", "secret", "-o", "yaml")) | ||
if err != nil { | ||
t.Errorf("failed to get secrets. args %q : %v", rr.Command(), err) | ||
} | ||
|
||
base64OfDefaultValue := base64.StdEncoding.EncodeToString([]byte("changeme")) | ||
expected := []string{ | ||
fmt.Sprintf("DOCKER_PRIVATE_REGISTRY_PASSWORD: %s", base64OfDefaultValue), | ||
fmt.Sprintf("DOCKER_PRIVATE_REGISTRY_SERVER: %s", base64OfDefaultValue), | ||
fmt.Sprintf("DOCKER_PRIVATE_REGISTRY_USER: %s", base64OfDefaultValue), | ||
|
||
fmt.Sprintf("ACR_CLIENT_ID: %s", base64OfDefaultValue), | ||
fmt.Sprintf("ACR_PASSWORD: %s", base64OfDefaultValue), | ||
fmt.Sprintf("ACR_URL: %s", base64OfDefaultValue), | ||
|
||
"AWS_ACCESS_KEY_ID: dGVzdF9hd3NfYWNjZXNzaWQ=", | ||
"AWS_SECRET_ACCESS_KEY: dGVzdF9hd3NfYWNjZXNza2V5", | ||
"AWS_SESSION_TOKEN: dGVzdF9hd3Nfc2Vzc2lvbl90b2tlbg==", | ||
"aws-account: dGVzdF9hd3NfYWNjb3VudA==", | ||
"aws-assume-role: dGVzdF9hd3Nfcm9sZQ==", | ||
"aws-region: dGVzdF9hd3NfcmVnaW9u", | ||
|
||
"application_default_credentials.json: ewogICJjbGllbnRfaWQiOiAiaGFoYSIsCiAgImNsaWVudF9zZWNyZXQiOiAibmljZV90cnkiLAogICJxdW90YV9wcm9qZWN0X2lkIjogInRoaXNfaXNfZmFrZSIsCiAgInJlZnJlc2hfdG9rZW4iOiAibWF5YmVfbmV4dF90aW1lIiwKICAidHlwZSI6ICJhdXRob3JpemVkX3VzZXIiCn0K", | ||
"gcrurl: aHR0cHM6Ly9nY3IuaW8=", | ||
} | ||
|
||
rrout := strings.TrimSpace(rr.Stdout.String()) | ||
for _, exp := range expected { | ||
re := regexp.MustCompile(fmt.Sprintf(".*%s.*", exp)) | ||
secret := re.FindString(rrout) | ||
if secret == "" { | ||
t.Errorf("Did not find expected secret: '%s'", secret) | ||
} | ||
} | ||
} | ||
|
||
// validateRegistryAddon tests the registry addon | ||
func validateRegistryAddon(ctx context.Context, t *testing.T, profile string) { | ||
defer disableAddon(t, "registry", profile) | ||
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this new files needs copyright boiler plate to be added to the header