Skip to content

Commit 4ba3218

Browse files
liztioliztio
committed
Basic E2E tests for kubeadm
Run with `go run test/e2e_kubeadm/runner/local/run_local.go -build -test-flags="--kubeconfig=$KUBECONFIG"`
1 parent a80b545 commit 4ba3218

File tree

7 files changed

+332
-1
lines changed

7 files changed

+332
-1
lines changed

‎build/BUILD

-1
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ filegroup(
148148
"//cmd/kubemark", # TODO: server platforms only
149149
"//cmd/linkcheck",
150150
"//test/e2e:e2e.test",
151-
"//test/e2e_kubeadm:e2e_kubeadm.test", # TODO: server platforms only
152151
"//test/e2e_node:e2e_node.test", # TODO: server platforms only
153152
"//vendor/github.com/onsi/ginkgo/ginkgo",
154153
],

‎hack/.golint_failures

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
cluster/images/etcd-version-monitor
23
cmd/hyperkube
34
cmd/kube-apiserver/app
@@ -699,6 +700,7 @@ test/e2e/ui
699700
test/e2e/upgrades
700701
test/e2e/upgrades/apps
701702
test/e2e/upgrades/storage
703+
test/e2e_kubeadm
702704
test/e2e_node
703705
test/e2e_node/builder
704706
test/e2e_node/environment

‎test/e2e_kubeadm/BUILD

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
package(default_visibility = ["//visibility:public"])
2+
13
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
24

35
go_test(
@@ -7,6 +9,7 @@ go_test(
79
"kubeadm_test.go",
810
],
911
embed = [":go_default_library"],
12+
tags = ["e2e"],
1013
deps = [
1114
"//test/e2e/framework:go_default_library",
1215
"//vendor/github.com/onsi/ginkgo:go_default_library",

‎test/e2e_kubeadm/e2e_kubeadm_suite_test.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
"flag"
21+
"os"
22+
"testing"
23+
24+
"github.com/onsi/ginkgo"
25+
"github.com/onsi/gomega"
26+
"github.com/spf13/pflag"
27+
28+
"k8s.io/kubernetes/test/e2e/framework"
29+
)
30+
31+
func init() {
32+
framework.RegisterCommonFlags()
33+
framework.RegisterClusterFlags()
34+
35+
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
36+
}
37+
38+
func TestMain(m *testing.M) {
39+
pflag.Parse()
40+
framework.AfterReadingAllFlags(&framework.TestContext)
41+
os.Exit(m.Run())
42+
}
43+
44+
func TestE2E(t *testing.T) {
45+
reporters := []ginkgo.Reporter{}
46+
gomega.RegisterFailHandler(ginkgo.Fail)
47+
ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "E2EKubadm suite", reporters)
48+
}

‎test/e2e_kubeadm/kubeadm_test.go

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
corev1 "k8s.io/api/core/v1"
21+
rbacv1 "k8s.io/api/rbac/v1"
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
"k8s.io/apimachinery/pkg/labels"
24+
"k8s.io/client-go/kubernetes"
25+
"k8s.io/client-go/rest"
26+
bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
27+
"k8s.io/kubernetes/test/e2e/framework"
28+
29+
. "github.com/onsi/ginkgo"
30+
. "github.com/onsi/gomega"
31+
)
32+
33+
const (
34+
masterTaint = "node-role.kubernetes.io/master"
35+
kubeadmConfigNamespace = "kube-system"
36+
kubeadmConfigName = "kubeadm-config"
37+
clusterInfoNamespace = "kube-public"
38+
clusterInfoName = "cluster-info"
39+
bootstrapSignerRoleNamespace = "kube-system"
40+
bootstrapSignerRoleName = "system:controller:bootstrap-signer"
41+
)
42+
43+
var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() {
44+
f := framework.NewDefaultFramework("kubeadm")
45+
46+
Describe("kubeadm master", func() {
47+
It("should be labelled and tainted", func() {
48+
selector := labels.Set{masterTaint: ""}.AsSelector()
49+
master, err := f.ClientSet.CoreV1().Nodes().
50+
List(metav1.ListOptions{LabelSelector: selector.String()})
51+
framework.ExpectNoError(err, "couldn't find a master node")
52+
Expect(master.Items).NotTo(BeEmpty())
53+
for _, master := range master.Items {
54+
Expect(master.Spec.Taints).To(
55+
ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)),
56+
)
57+
}
58+
})
59+
})
60+
61+
Describe("kubeadm-config config map", func() {
62+
It("should exist", func() {
63+
_, err := f.ClientSet.CoreV1().
64+
ConfigMaps(kubeadmConfigNamespace).
65+
Get(kubeadmConfigName, metav1.GetOptions{})
66+
framework.ExpectNoError(err)
67+
})
68+
})
69+
70+
Describe("cluster-info", func() {
71+
It("should have expected keys", func() {
72+
clientInfo, err := f.ClientSet.CoreV1().
73+
ConfigMaps(clusterInfoNamespace).
74+
Get(clusterInfoName, metav1.GetOptions{})
75+
framework.ExpectNoError(err, "couldn't find config map")
76+
77+
Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix)))
78+
Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey))
79+
})
80+
81+
It("should be public", func() {
82+
cfg, err := framework.LoadConfig()
83+
framework.ExpectNoError(err, "couldn't get config")
84+
cfg = rest.AnonymousClientConfig(cfg)
85+
client, err := kubernetes.NewForConfig(cfg)
86+
framework.ExpectNoError(err, "couldn't create client")
87+
88+
_, err = client.CoreV1().ConfigMaps(clusterInfoNamespace).
89+
Get(clusterInfoName, metav1.GetOptions{})
90+
framework.ExpectNoError(err, "couldn't anonymously access config")
91+
})
92+
})
93+
94+
Describe("bootstrap signer RBAC role", func() {
95+
It("should exist", func() {
96+
_, err := f.ClientSet.RbacV1().
97+
Roles(bootstrapSignerRoleNamespace).
98+
Get(bootstrapSignerRoleName, metav1.GetOptions{})
99+
framework.ExpectNoError(err, "doesn't exist")
100+
})
101+
})
102+
103+
Describe("kubeadm:kubelet-bootstrap cluster role binding", func() {
104+
It("should exist", func() {
105+
binding, err := f.ClientSet.RbacV1().
106+
ClusterRoleBindings().
107+
Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{})
108+
framework.ExpectNoError(err, "couldn't get clusterrolebinding")
109+
Expect(binding.Subjects).To(
110+
ContainElement(subject(
111+
"system:bootstrappers:kubeadm:default-node-token",
112+
rbacv1.GroupKind,
113+
)),
114+
)
115+
Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper"))
116+
})
117+
})
118+
119+
Describe("autoapproval for new bootstrap token", func() {
120+
It("should create a clusterrolebinding", func() {
121+
binding, err := f.ClientSet.RbacV1().
122+
ClusterRoleBindings().
123+
Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{})
124+
framework.ExpectNoError(err, "couldn't get clusterrolebinding")
125+
Expect(binding.Subjects).To(
126+
ContainElement(subject(
127+
"system:bootstrappers:kubeadm:default-node-token",
128+
rbacv1.GroupKind,
129+
)),
130+
)
131+
Expect(binding.RoleRef.Name).To(
132+
Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"),
133+
)
134+
})
135+
})
136+
})

‎test/e2e_kubeadm/matchers.go

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e_kubeadm
18+
19+
import (
20+
"github.com/onsi/gomega"
21+
"github.com/onsi/gomega/gstruct"
22+
corev1 "k8s.io/api/core/v1"
23+
)
24+
25+
func subject(name, kind string) gomega.OmegaMatcher {
26+
return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
27+
"Name": gomega.Equal(name),
28+
"Kind": gomega.Equal(kind),
29+
})
30+
}
31+
32+
func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher {
33+
return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
34+
"Key": gomega.Equal(key),
35+
"Effect": gomega.Equal(effect),
36+
})
37+
}

‎test/e2e_kubeadm/runner/local/run_local.go

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
Copyright 2016 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"flag"
21+
"fmt"
22+
"os"
23+
"os/exec"
24+
"path/filepath"
25+
"runtime"
26+
"strings"
27+
28+
"github.com/golang/glog"
29+
"k8s.io/kubernetes/test/utils"
30+
)
31+
32+
func bazelBuild() error {
33+
targets := []string{
34+
"//vendor/github.com/onsi/ginkgo/ginkgo",
35+
"//test/e2e_kubeadm:e2e_kubeadm.test",
36+
}
37+
38+
args := append([]string{"build"}, targets...)
39+
40+
return execCommand("bazel", args...)
41+
}
42+
43+
var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
44+
var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
45+
var build = flag.Bool("build", false, "use Bazel to build binaries before testing")
46+
47+
func main() {
48+
flag.Parse()
49+
50+
if *build {
51+
if err := bazelBuild(); err != nil {
52+
glog.Exitf("couldn't build with bazel: %v", err)
53+
}
54+
}
55+
56+
ginkgo, err := getBazelGinkgo()
57+
if err != nil {
58+
glog.Fatalf("Failed to get ginkgo binary: %v", err)
59+
}
60+
61+
test, err := getBazelTestBin()
62+
if err != nil {
63+
glog.Fatalf("Failed to get test file: %v", err)
64+
}
65+
66+
args := append(strings.Split(*ginkgoFlags, " "), test, "--")
67+
args = append(args, strings.Split(*testFlags, " ")...)
68+
69+
if execCommand(ginkgo, args...); err != nil {
70+
glog.Exitf("Test failed: %v", err)
71+
}
72+
73+
}
74+
75+
func getBazelTestBin() (string, error) {
76+
k8sRoot, err := utils.GetK8sRootDir()
77+
if err != nil {
78+
return "", err
79+
}
80+
buildFile := filepath.Join(k8sRoot, "bazel-bin/test/e2e_kubeadm/e2e_kubeadm.test")
81+
if _, err := os.Stat(buildFile); err != nil {
82+
return "", err
83+
}
84+
return buildFile, nil
85+
86+
}
87+
88+
func getBazelGinkgo() (string, error) {
89+
k8sRoot, err := utils.GetK8sRootDir()
90+
if err != nil {
91+
return "", err
92+
}
93+
buildOutputDir := filepath.Join(k8sRoot, "bazel-bin", "vendor/github.com/onsi/ginkgo/ginkgo", fmt.Sprintf("%s_%s_stripped", runtime.GOOS, runtime.GOARCH), "ginkgo")
94+
if _, err := os.Stat(buildOutputDir); err != nil {
95+
return "", err
96+
}
97+
return buildOutputDir, nil
98+
}
99+
100+
func execCommand(binary string, args ...string) error {
101+
fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " "))
102+
cmd := exec.Command(binary, args...)
103+
cmd.Stdout = os.Stdout
104+
cmd.Stderr = os.Stderr
105+
return cmd.Run()
106+
}

0 commit comments

Comments
 (0)