From c52d103ac2c649f6a24aa0c0a09062b1632d6b50 Mon Sep 17 00:00:00 2001 From: Maxim Babushkin Date: Sun, 27 Oct 2024 21:29:40 +0200 Subject: [PATCH] Refactor logs gather flow to support multiple clusters - Refactor logs gathering flow to support logs gather from multiple clusters. This is used in multi cluster scenario - Add "getSecret" function to fetch certs that are used in multicluster test. - Add name cluster to be specified during log gathering. Signed-off-by: Maxim Babushkin --- .../controlplane/control_plane_suite_test.go | 2 +- tests/e2e/controlplane/control_plane_test.go | 4 +- tests/e2e/dualstack/dualstack_suite_test.go | 2 +- tests/e2e/dualstack/dualstack_test.go | 4 +- .../multicluster_multiprimary_test.go | 6 +- .../multicluster_primaryremote_test.go | 6 +- .../multicluster/multicluster_suite_test.go | 4 +- tests/e2e/operator/operator_install_test.go | 4 +- tests/e2e/operator/operator_suite_test.go | 2 +- tests/e2e/util/common/e2e_utils.go | 59 ++++++++++--------- tests/e2e/util/kubectl/kubectl.go | 21 ++++++- 11 files changed, 70 insertions(+), 44 deletions(-) diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index a1b573984..aca69e1a7 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -63,5 +63,5 @@ func setup() { cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) - k = kubectl.New() + k = kubectl.New("clControlPlane") } diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go index 20c52e872..e55967f80 100644 --- a/tests/e2e/controlplane/control_plane_test.go +++ b/tests/e2e/controlplane/control_plane_test.go @@ -295,7 +295,7 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { - common.LogDebugInfo() + common.LogDebugInfo(k) debugInfoLogged = true } @@ -314,7 +314,7 @@ spec: AfterAll(func() { if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo() + common.LogDebugInfo(k) debugInfoLogged = true } diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go index 3dd477ce2..000345978 100644 --- a/tests/e2e/dualstack/dualstack_suite_test.go +++ b/tests/e2e/dualstack/dualstack_suite_test.go @@ -63,5 +63,5 @@ func setup() { cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) - k = kubectl.New() + k = kubectl.New("clDualStack") } diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index 9d4fb4425..9a7768438 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -280,7 +280,7 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { - common.LogDebugInfo() + common.LogDebugInfo(k) debugInfoLogged = true } @@ -295,7 +295,7 @@ spec: AfterAll(func() { if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo() + common.LogDebugInfo(k) debugInfoLogged = true } diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index e6274d0c0..8d647504a 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -253,7 +253,8 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { - common.LogDebugInfo() + common.LogDebugInfo(k1) + common.LogDebugInfo(k2) debugInfoLogged = true } @@ -277,7 +278,8 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo() + common.LogDebugInfo(k1) + common.LogDebugInfo(k2) debugInfoLogged = true } diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index 1ebfc799e..7ff6db158 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -295,7 +295,8 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { - common.LogDebugInfo() + common.LogDebugInfo(k1) + common.LogDebugInfo(k2) debugInfoLogged = true } @@ -319,7 +320,8 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo() + common.LogDebugInfo(k1) + common.LogDebugInfo(k2) debugInfoLogged = true } diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index bf08d58d5..d00b9c64e 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -99,6 +99,6 @@ func setup(t *testing.T) { exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir) // Initialize kubectl utilities, one for each cluster - k1 = kubectl.New().WithKubeconfig(kubeconfig) - k2 = kubectl.New().WithKubeconfig(kubeconfig2) + k1 = kubectl.New("clPrimary").WithKubeconfig(kubeconfig) + k2 = kubectl.New("clRemote").WithKubeconfig(kubeconfig2) } diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index 0bb5b887a..efd880949 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -100,14 +100,14 @@ var _ = Describe("Operator", Ordered, func() { AfterAll(func() { if CurrentSpecReport().Failed() { - common.LogDebugInfo() + common.LogDebugInfo(k) } }) }) AfterAll(func() { if CurrentSpecReport().Failed() { - common.LogDebugInfo() + common.LogDebugInfo(k) } if skipDeploy { diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go index 5995af641..144cd885c 100644 --- a/tests/e2e/operator/operator_suite_test.go +++ b/tests/e2e/operator/operator_suite_test.go @@ -62,5 +62,5 @@ func setup() { GinkgoWriter.Println("Running on Kubernetes") } - k = kubectl.New() + k = kubectl.New("clOperator") } diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index 3f5e9a4c4..b513452ee 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -57,8 +57,6 @@ var ( // - 1.24-alpha.feabc1234 // matching only the version before first '_' which is used in the downstream builds, e.g. "1.23.2_ossm_tp.2" istiodVersionRegex = regexp.MustCompile(`Version:"([^"_]*)[^"]*"`) - - k = kubectl.New() ) // GetObject returns the object with the given key @@ -131,73 +129,80 @@ func CheckNamespaceEmpty(ctx SpecContext, cl client.Client, ns string) { }).Should(BeEmpty(), "No Services should be present in the namespace") } -func LogDebugInfo() { +func LogDebugInfo(k kubectl.Kubectl) { // General debugging information to help diagnose the failure // TODO: Add the creation of file with this information to be attached to the test report GinkgoWriter.Println() - GinkgoWriter.Println("The test run has failures and the debug information is as follows:") + GinkgoWriter.Println("The test run has failures and the debug information is as follows from cluster: %q:", k.GetClusterName()) + GinkgoWriter.Println("=========================================================") + logOperatorDebugInfo(k) GinkgoWriter.Println("=========================================================") - logOperatorDebugInfo() + logIstioDebugInfo(k) GinkgoWriter.Println("=========================================================") - logIstioDebugInfo() + logCNIDebugInfo(k) GinkgoWriter.Println("=========================================================") - logCNIDebugInfo() + logCertsDebugInfo(k) GinkgoWriter.Println("=========================================================") } -func logOperatorDebugInfo() { - k := k.WithNamespace(OperatorNamespace) +func logOperatorDebugInfo(k kubectl.Kubectl) { + k = k.WithNamespace(OperatorNamespace) operator, err := k.GetYAML("deployment", deploymentName) - logDebugElement("Operator Deployment YAML", operator, err) + logDebugElement("=====Operator Deployment YAML=====", operator, err) logs, err := k.Logs("deploy/"+deploymentName, ptr.Of(120*time.Second)) - logDebugElement("Operator logs", logs, err) + logDebugElement("=====Operator logs=====", logs, err) events, err := k.GetEvents() - logDebugElement("Events in "+OperatorNamespace, events, err) + logDebugElement("=====Events in "+OperatorNamespace+"=====", events, err) // Temporary information to gather more details about failure pods, err := k.GetPods("", "-o wide") - logDebugElement("Pods in "+OperatorNamespace, pods, err) + logDebugElement("=====Pods in "+OperatorNamespace+"=====", pods, err) describe, err := k.Describe("deployment", deploymentName) - logDebugElement("Operator Deployment describe", describe, err) + logDebugElement("=====Operator Deployment describe=====", describe, err) } -func logIstioDebugInfo() { +func logIstioDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istio", istioName) - logDebugElement("Istio YAML", resource, err) + logDebugElement("=====Istio YAML=====", resource, err) output, err := k.WithNamespace(controlPlaneNamespace).GetPods("", "-o wide") - logDebugElement("Pods in "+controlPlaneNamespace, output, err) + logDebugElement("=====Pods in "+controlPlaneNamespace+"=====", output, err) logs, err := k.WithNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) - logDebugElement("Istiod logs", logs, err) + logDebugElement("=====Istiod logs=====", logs, err) events, err := k.WithNamespace(controlPlaneNamespace).GetEvents() - logDebugElement("Events in "+controlPlaneNamespace, events, err) + logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err) } -func logCNIDebugInfo() { +func logCNIDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istiocni", istioCniName) - logDebugElement("IstioCNI YAML", resource, err) + logDebugElement("=====IstioCNI YAML=====", resource, err) ds, err := k.WithNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node") - logDebugElement("Istio CNI DaemonSet YAML", ds, err) + logDebugElement("=====Istio CNI DaemonSet YAML=====", ds, err) events, err := k.WithNamespace(istioCniNamespace).GetEvents() - logDebugElement("Events in "+istioCniNamespace, events, err) + logDebugElement("=====Events in "+istioCniNamespace+"=====", events, err) // Temporary information to gather more details about failure pods, err := k.WithNamespace(istioCniNamespace).GetPods("", "-o wide") - logDebugElement("Pods in "+istioCniNamespace, pods, err) + logDebugElement("=====Pods in "+istioCniNamespace+"=====", pods, err) describe, err := k.WithNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node") - logDebugElement("Istio CNI DaemonSet describe", describe, err) + logDebugElement("=====Istio CNI DaemonSet describe=====", describe, err) logs, err := k.WithNamespace(istioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) - logDebugElement("Istio CNI logs", logs, err) + logDebugElement("=====Istio CNI logs=====", logs, err) +} + +func logCertsDebugInfo(k kubectl.Kubectl) { + certs, err := k.WithNamespace(controlPlaneNamespace).GetSecret("cacerts") + logDebugElement("=====CA certs=====", certs, err) } func logDebugElement(caption string, info string, err error) { @@ -211,7 +216,7 @@ func logDebugElement(caption string, info string, err error) { } func GetVersionFromIstiod() (*semver.Version, error) { - k := kubectl.New() + k := kubectl.New("testCluster") output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") if err != nil { return nil, fmt.Errorf("error getting version from istiod: %w", err) diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index 5c92fbee0..d7addba21 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -25,14 +25,15 @@ import ( ) type Kubectl struct { + name string binary string namespace string kubeconfig string } // New creates a new kubectl.Kubectl -func New() Kubectl { - return Kubectl{}.WithBinary(os.Getenv("COMMAND")) +func New(name string) Kubectl { + return Kubectl{name: name}.WithBinary(os.Getenv("COMMAND")) } func (k Kubectl) build(cmd string) string { @@ -217,6 +218,11 @@ func (k Kubectl) ForceDelete(kind, name string) error { return k.Delete(kind, name) } +// Gets cluster name defined during initialization +func (k Kubectl) GetClusterName() string { + return k.name +} + // GetYAML returns the yaml of a resource func (k Kubectl) GetYAML(kind, name string) (string, error) { cmd := k.build(fmt.Sprintf(" get %s %s -o yaml", kind, name)) @@ -250,6 +256,17 @@ func (k Kubectl) GetInternalIP(label string) (string, error) { return output, nil } +// GetSecret returns the secret of a namespace +func (k Kubectl) GetSecret(secret string) (string, error) { + cmd := k.build(fmt.Sprintf(" get secret %s -o yaml", secret)) + output, err := k.executeCommand(cmd) + if err != nil { + return "", fmt.Errorf("error getting secret: %w, output %s", err, output) + } + + return output, nil +} + // Exec executes a command in the pod or specific container func (k Kubectl) Exec(pod, container, command string) (string, error) { cmd := k.build(fmt.Sprintf(" exec %s %s -- %s", pod, containerFlag(container), command))