forked from Azure/AgentBaker
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Revert "Revert "Support dynamic kubelet (Azure#139)""
This reverts commit 7250ec2.
- Loading branch information
Tony Xu
committed
Jul 15, 2020
1 parent
7250ec2
commit 397223c
Showing
94 changed files
with
14,103 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,7 @@ _logs/ | |
| cmd/_test_output | ||
| *.test | ||
| *.coverprofile | ||
| **/junit.xml | ||
|
|
||
| .idea | ||
| .vs | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT license. | ||
|
|
||
| package agent | ||
|
|
||
| import ( | ||
| . "github.com/onsi/ginkgo" | ||
| "github.com/onsi/ginkgo/reporters" | ||
| . "github.com/onsi/gomega" | ||
|
|
||
| "testing" | ||
| ) | ||
|
|
||
| var _ = BeforeSuite(func() { | ||
| }) | ||
|
|
||
| func TestUtils(t *testing.T) { | ||
| RegisterFailHandler(Fail) | ||
| junitReporter := reporters.NewJUnitReporter("junit.xml") | ||
| RunSpecsWithDefaultAndCustomReporters(t, "Agent Suite", []Reporter{junitReporter}) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| package agent | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/ioutil" | ||
| "os" | ||
| "os/exec" | ||
|
|
||
| "github.com/Azure/aks-engine/pkg/api" | ||
| "github.com/Azure/aks-engine/pkg/api/common" | ||
| . "github.com/onsi/ginkgo" | ||
| . "github.com/onsi/ginkgo/extensions/table" | ||
| . "github.com/onsi/gomega" | ||
| ) | ||
|
|
||
| var _ = Describe("Assert generated customData and cseCmd", func() { | ||
| DescribeTable("Generated customData and CSE", func(folder, k8sVersion string, containerServiceUpdator func(*api.ContainerService)) { | ||
| cs := &api.ContainerService{ | ||
| Location: "southcentralus", | ||
| Type: "Microsoft.ContainerService/ManagedClusters", | ||
| Properties: &api.Properties{ | ||
| OrchestratorProfile: &api.OrchestratorProfile{ | ||
| OrchestratorType: api.Kubernetes, | ||
| OrchestratorVersion: k8sVersion, | ||
| KubernetesConfig: &api.KubernetesConfig{ | ||
| KubeletConfig: map[string]string{ | ||
| "--feature-gates": "RotateKubeletServerCertificate=true,a=b, PodPriority=true, x=y", | ||
| }, | ||
| }, | ||
| }, | ||
| HostedMasterProfile: &api.HostedMasterProfile{ | ||
| DNSPrefix: "uttestdom", | ||
| }, | ||
| AgentPoolProfiles: []*api.AgentPoolProfile{ | ||
| { | ||
| Name: "agent2", | ||
| Count: 3, | ||
| VMSize: "Standard_DS1_v2", | ||
| StorageProfile: "ManagedDisks", | ||
| OSType: api.Linux, | ||
| VnetSubnetID: "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/aks-vnet-07752737/subnet/subnet1", | ||
| AvailabilityProfile: api.VirtualMachineScaleSets, | ||
| KubernetesConfig: &api.KubernetesConfig{ | ||
| KubeletConfig: map[string]string{ | ||
| "--address": "0.0.0.0", | ||
| "--pod-manifest-path": "/etc/kubernetes/manifests", | ||
| "--cluster-domain": "cluster.local", | ||
| "--cluster-dns": "10.0.0.10", | ||
| "--cgroups-per-qos": "true", | ||
| "--tls-cert-file": "/etc/kubernetes/certs/kubeletserver.crt", | ||
| "--tls-private-key-file": "/etc/kubernetes/certs/kubeletserver.key", | ||
| "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", | ||
| "--max-pods": "110", | ||
| "--node-status-update-frequency": "10s", | ||
| "--image-gc-high-threshold": "85", | ||
| "--image-gc-low-threshold": "80", | ||
| "--event-qps": "0", | ||
| "--pod-max-pids": "-1", | ||
| "--enforce-node-allocatable": "pods", | ||
| "--streaming-connection-idle-timeout": "4h0m0s", | ||
| "--rotate-certificates": "true", | ||
| "--read-only-port": "10255", | ||
| "--protect-kernel-defaults": "true", | ||
| "--resolv-conf": "/etc/resolv.conf", | ||
| "--anonymous-auth": "false", | ||
| "--client-ca-file": "/etc/kubernetes/certs/ca.crt", | ||
| "--authentication-token-webhook": "true", | ||
| "--authorization-mode": "Webhook", | ||
| "--eviction-hard": "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%", | ||
| "--feature-gates": "RotateKubeletServerCertificate=true,a=b,PodPriority=true,x=y", | ||
| "--system-reserved": "cpu=2,memory=1Gi", | ||
| "--kube-reserved": "cpu=100m,memory=1638Mi", | ||
| }, | ||
| }, | ||
| Distro: api.AKSUbuntu1604, | ||
| }, | ||
| }, | ||
| LinuxProfile: &api.LinuxProfile{ | ||
| AdminUsername: "azureuser", | ||
| }, | ||
| ServicePrincipalProfile: &api.ServicePrincipalProfile{ | ||
| ClientID: "ClientID", | ||
| Secret: "Secret", | ||
| }, | ||
| }, | ||
| } | ||
| cs.Properties.LinuxProfile.SSH.PublicKeys = []api.PublicKey{{ | ||
| KeyData: string("testsshkey"), | ||
| }} | ||
|
|
||
| // AKS always pass in te customHyperKubeImage to aks-e, so we don't really rely on | ||
| // the default component version for "hyperkube", which is not set since 1.17 | ||
| if IsKubernetesVersionGe(k8sVersion, "1.17.0") { | ||
| cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage = fmt.Sprintf("k8s.gcr.io/hyperkube-amd64:v%v", k8sVersion) | ||
| } | ||
|
|
||
| if containerServiceUpdator != nil { | ||
| containerServiceUpdator(cs) | ||
| } | ||
|
|
||
| agentPool := cs.Properties.AgentPoolProfiles[0] | ||
| baker := InitializeTemplateGenerator() | ||
|
|
||
| // customData | ||
| customData := baker.GetNodeBootstrappingPayload(cs, agentPool) | ||
| // Uncomment below line to generate test data in local if agentbaker is changed in generating customData | ||
| // backfillCustomData(folder, customData) | ||
| expectedCustomData, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s/CustomData", folder)) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| Expect(customData).To(Equal(string(expectedCustomData))) | ||
|
|
||
| // CSE | ||
| cseCommand := baker.GetNodeBootstrappingCmd(cs, agentPool, | ||
| "tenantID", "subID", "resourceGroupName", "userAssignedID", true, true) | ||
| // Uncomment below line to generate test data in local if agentbaker is changed in generating customData | ||
| // ioutil.WriteFile(fmt.Sprintf("./testdata/%s/CSECommand", folder), []byte(cseCommand), 0644) | ||
| expectedCSECommand, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s/CSECommand", folder)) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| Expect(cseCommand).To(Equal(string(expectedCSECommand))) | ||
|
|
||
| }, Entry("AKSUbuntu1604 with k8s version less than 1.18", "AKSUbuntu1604+K8S115", "1.15.7", nil), | ||
| Entry("AKSUbuntu1604 with k8s version 1.18", "AKSUbuntu1604+K8S118", "1.18.2", nil), | ||
| Entry("AKSUbuntu1604 with k8s version 1.17", "AKSUbuntu1604+K8S117", "1.17.7", nil), | ||
| Entry("AKSUbuntu1604 with Temp Disk", "AKSUbuntu1604+TempDisk", "1.15.7", func(cs *api.ContainerService) { | ||
| cs.Properties.OrchestratorProfile.KubernetesConfig = &api.KubernetesConfig{ | ||
| ContainerRuntimeConfig: map[string]string{ | ||
| common.ContainerDataDirKey: "/mnt/containers", | ||
| }, | ||
| } | ||
| }), | ||
| Entry("AKSUbuntu1604 with Temp Disk and containerd", "AKSUbuntu1604+TempDisk+Containerd", "1.15.7", func(cs *api.ContainerService) { | ||
| cs.Properties.OrchestratorProfile.KubernetesConfig = &api.KubernetesConfig{ | ||
| ContainerRuntimeConfig: map[string]string{ | ||
| common.ContainerDataDirKey: "/mnt/containers", | ||
| }, | ||
| } | ||
| cs.Properties.AgentPoolProfiles[0].KubernetesConfig = &api.KubernetesConfig{ | ||
| KubeletConfig: map[string]string{}, | ||
| ContainerRuntime: api.Containerd, | ||
| } | ||
| }), | ||
| Entry("AKSUbuntu1604 with RawUbuntu", "RawUbuntu", "1.15.7", func(cs *api.ContainerService) { | ||
| // cs.Properties.OrchestratorProfile.KubernetesConfig = nil | ||
| cs.Properties.AgentPoolProfiles[0].Distro = api.Ubuntu | ||
| }), | ||
| Entry("AKSUbuntu1604 with AKSCustomCloud", "AKSUbuntu1604+AKSCustomCloud", "1.15.7", func(cs *api.ContainerService) { | ||
| // cs.Properties.OrchestratorProfile.KubernetesConfig = nil | ||
| cs.Location = "usnat" | ||
| })) | ||
| }) | ||
|
|
||
| func backfillCustomData(folder, customData string) { | ||
| if _, err := os.Stat(fmt.Sprintf("./testdata/%s", folder)); os.IsNotExist(err) { | ||
| e := os.MkdirAll(fmt.Sprintf("./testdata/%s", folder), 0755) | ||
| Expect(e).To(BeNil()) | ||
| } | ||
| ioutil.WriteFile(fmt.Sprintf("./testdata/%s/CustomData", folder), []byte(customData), 0644) | ||
| err := exec.Command("/bin/sh", "-c", fmt.Sprintf("./testdata/convert.sh testdata/%s", folder)).Run() | ||
| Expect(err).To(BeNil()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| echo $(date),$(hostname); retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz mcr.microsoft.com 443 2>&1 || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ADMINUSER=azureuser CONTAINERD_VERSION= MOBY_VERSION= TENANT_ID=tenantID KUBERNETES_VERSION=1.15.7 HYPERKUBE_URL=hyperkube-amd64:v1.15.7 APISERVER_PUBLIC_KEY= SUBSCRIPTION_ID=subID RESOURCE_GROUP=resourceGroupName LOCATION=usnat VM_TYPE=vmss SUBNET=subnet1 NETWORK_SECURITY_GROUP=aks-agentpool-36873793-nsg VIRTUAL_NETWORK=aks-vnet-07752737 VIRTUAL_NETWORK_RESOURCE_GROUP=MC_rg ROUTE_TABLE=aks-agentpool-36873793-routetable PRIMARY_AVAILABILITY_SET= PRIMARY_SCALE_SET=aks-agent2-36873793-vmss SERVICE_PRINCIPAL_CLIENT_ID=ClientID SERVICE_PRINCIPAL_CLIENT_SECRET='Secret' KUBELET_PRIVATE_KEY= NETWORK_PLUGIN= NETWORK_POLICY= VNET_CNI_PLUGINS_URL=https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-linux-amd64-v1.1.3.tgz CNI_PLUGINS_URL=https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-v0.7.6.tgz CLOUDPROVIDER_BACKOFF=<nil> CLOUDPROVIDER_BACKOFF_MODE= CLOUDPROVIDER_BACKOFF_RETRIES=0 CLOUDPROVIDER_BACKOFF_EXPONENT=0 CLOUDPROVIDER_BACKOFF_DURATION=0 CLOUDPROVIDER_BACKOFF_JITTER=0 CLOUDPROVIDER_RATELIMIT=<nil> CLOUDPROVIDER_RATELIMIT_QPS=0 CLOUDPROVIDER_RATELIMIT_QPS_WRITE=0 CLOUDPROVIDER_RATELIMIT_BUCKET=0 CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE=0 LOAD_BALANCER_DISABLE_OUTBOUND_SNAT=<nil> USE_MANAGED_IDENTITY_EXTENSION=false USE_INSTANCE_METADATA=false LOAD_BALANCER_SKU= EXCLUDE_MASTER_FROM_STANDARD_LB=true MAXIMUM_LOADBALANCER_RULE_COUNT=0 CONTAINER_RUNTIME= CONTAINERD_DOWNLOAD_URL_BASE=https://storage.googleapis.com/cri-containerd-release/ NETWORK_MODE= KUBE_BINARY_URL= USER_ASSIGNED_IDENTITY_ID=userAssignedID API_SERVER_NAME= IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false CONFIG_GPU_DRIVER_IF_NEEDED=true ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED=true /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1; systemctl --no-pager -l status kubelet 2>&1 | head -n 100" |
Oops, something went wrong.