Add multi control plane e2e tests

Signed-off-by: Marko Lukša <[email protected]>

tests/e2e/multicontrolplane/multi_control_plane_suite_test.go

@@ -0,0 +1,71 @@
+//go:build e2e
+
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+	"testing"
+
+	"github.com/istio-ecosystem/sail-operator/pkg/test/util/supportedversion"
+	k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/env"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	cl                     client.Client
+	err                    error
+	version                = supportedversion.New
+	namespace              = common.OperatorNamespace
+	deploymentName         = env.Get("DEPLOYMENT_NAME", "sail-operator")
+	controlPlaneNamespace1 = env.Get("CONTROL_PLANE_NS1", "istio-system1")
+	controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2")
+	istioName1             = env.Get("ISTIO_NAME1", "mesh1")
+	istioName2             = env.Get("ISTIO_NAME2", "mesh2")
+	istioCniNamespace      = env.Get("ISTIOCNI_NAMESPACE", "istio-cni")
+	istioCniName           = env.Get("ISTIOCNI_NAME", "default")
+	skipDeploy             = env.GetBool("SKIP_DEPLOY", false)
+	appNamespace1          = env.Get("APP_NAMESPACE1", "app1")
+	appNamespace2a         = env.Get("APP_NAMESPACE2A", "app2a")
+	appNamespace2b         = env.Get("APP_NAMESPACE2B", "app2b")
+	multicluster           = env.GetBool("MULTICLUSTER", false)
+	ipFamily               = env.Get("IP_FAMILY", "ipv4")
+
+	k kubectl.Kubectl
+)
+
+func TestInstall(t *testing.T) {
+	if ipFamily == "dual" || multicluster {
+		t.Skip("Skipping the multi control plane tests")
+	}
+	RegisterFailHandler(Fail)
+	setup()
+	RunSpecs(t, "Multiple Control Planes Test Suite")
+}
+
+func setup() {
+	GinkgoWriter.Println("************ Running Setup ************")
+
+	GinkgoWriter.Println("Initializing k8s client")
+	cl, err = k8sclient.InitK8sClient("")
+	Expect(err).NotTo(HaveOccurred())
+
+	k = kubectl.New("clControlPlane")
+}

tests/e2e/multicontrolplane/multi_control_plane_test.go

@@ -0,0 +1,226 @@
+//go:build e2e
+
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR Condition OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	v1 "github.com/istio-ecosystem/sail-operator/api/v1"
+	"github.com/istio-ecosystem/sail-operator/pkg/kube"
+	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
+	"github.com/istio-ecosystem/sail-operator/pkg/test/util/supportedversion"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Multiple Control Planes", Ordered, func() {
+	SetDefaultEventuallyTimeout(180 * time.Second)
+	SetDefaultEventuallyPollingInterval(time.Second)
+	debugInfoLogged := false
+
+	BeforeAll(func(ctx SpecContext) {
+		Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created")
+
+		if skipDeploy {
+			Success("Skipping operator installation because it was deployed externally")
+		} else {
+			Expect(common.InstallOperatorViaHelm()).
+				To(Succeed(), "Operator failed to be deployed")
+		}
+
+		Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
+			Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD")
+		Success("Operator is deployed in the namespace and Running")
+	})
+
+	Context("Two control planes", func() {
+		BeforeAll(func() {
+			Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created")
+			Expect(k.CreateNamespace(controlPlaneNamespace1)).To(Succeed(), "Istio namespace failed to be created")
+			Expect(k.CreateNamespace(controlPlaneNamespace2)).To(Succeed(), "Istio namespace failed to be created")
+
+			Expect(k.Label("namespace", controlPlaneNamespace1, "mesh", istioName1)).To(Succeed(), "Failed to label namespace")
+			Expect(k.Label("namespace", controlPlaneNamespace2, "mesh", istioName2)).To(Succeed(), "Failed to label namespace")
+		})
+
+		It("Installs IstioCNI", func(ctx SpecContext) {
+			yaml := `
+apiVersion: sailoperator.io/v1
+kind: IstioCNI
+metadata:
+  name: default
+spec:
+  version: %s
+  namespace: %s`
+			yaml = fmt.Sprintf(yaml, version, istioCniNamespace)
+			Expect(k.CreateFromString(yaml)).To(Succeed(), "failed to create IstioCNI")
+			Success("IstioCNI created")
+
+			Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}).
+				Should(HaveCondition(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition")
+			Success("IstioCNI is Ready")
+		})
+
+		DescribeTable("Installs Istios",
+			Entry("Mesh 1", istioName1, controlPlaneNamespace1),
+			Entry("Mesh 2", istioName2, controlPlaneNamespace2),
+			func(ctx SpecContext, name, ns string) {
+				Expect(k.CreateFromString(`
+apiVersion: sailoperator.io/v1
+kind: Istio
+metadata:
+  name: `+name+`
+spec:
+  version: `+version+`
+  namespace: `+ns+`
+  values:
+    meshConfig:
+      discoverySelectors:
+      - matchLabels:
+          mesh: `+name)).To(Succeed(), "failed to create Istio CR")
+
+				Expect(k.CreateFromString(`
+apiVersion: security.istio.io/v1
+kind: PeerAuthentication
+metadata:
+  name: default
+  namespace: `+ns+`
+spec:
+  mtls:
+    mode: STRICT`)).To(Succeed(), "failed to create PeerAuthentication")
+			})
+
+		DescribeTable("Waits for Istios",
+			Entry("Mesh 1", istioName1),
+			Entry("Mesh 2", istioName2),
+			func(ctx SpecContext, name string) {
+				Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(name), &v1.Istio{}).
+					Should(
+						And(
+							HaveCondition(v1.IstioConditionReconciled, metav1.ConditionTrue),
+							HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue),
+						), "Istio is not Reconciled and Ready; unexpected Condition")
+				Success(fmt.Sprintf("Istio %s ready", name))
+			})
+
+		DescribeTable("Deploys applications",
+			Entry("App 1", appNamespace1, istioName1),
+			Entry("App 2a", appNamespace2a, istioName2),
+			Entry("App 2b", appNamespace2b, istioName2),
+			func(ns, mesh string) {
+				Expect(k.CreateNamespace(ns)).To(Succeed(), "Failed to create namespace")
+				Expect(k.Label("namespace", ns, "mesh", mesh)).To(Succeed(), "Failed to label namespace")
+				Expect(k.Label("namespace", ns, "istio.io/rev", mesh)).To(Succeed(), "Failed to label namespace")
+				for _, appName := range []string{"sleep", "httpbin"} {
+					Expect(k.WithNamespace(ns).
+						Apply(common.GetSampleYAML(supportedversion.Map[version], appName))).
+						To(Succeed(), "Failed to deploy application")
+				}
+				Success(fmt.Sprintf("Applications in namespace %s deployed", ns))
+			})
+
+		DescribeTable("Waits for apps to be ready",
+			Entry("App 1", appNamespace1),
+			Entry("App 2a", appNamespace2a),
+			Entry("App 2b", appNamespace2b),
+			func(ctx SpecContext, ns string) {
+				for _, deployment := range []string{"sleep", "httpbin"} {
+					Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deployment, ns), &appsv1.Deployment{}).
+						Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error waiting for deployment to be available")
+				}
+				Success(fmt.Sprintf("Applications in namespace %s ready", ns))
+			})
+
+		It("Verifies app2a cannot connect to app1", func(ctx SpecContext) {
+			// time.Sleep(10 * time.Minute)
+			output, err := k.WithNamespace(appNamespace2a).
+				Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace1))
+			Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod")
+			Expect(output).To(ContainSubstring("503 Service Unavailable"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace1))
+		})
+
+		It("Verifies app2a can connect to app2b", func(ctx SpecContext) {
+			output, err := k.WithNamespace(appNamespace2a).
+				Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace2b))
+			Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod")
+			Expect(output).To(ContainSubstring("200 OK"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace2b))
+		})
+
+		It("Cleans up the resources", func() {
+			By("Cleaning up the application namespaces")
+			Expect(k.DeleteNamespace(appNamespace1, appNamespace2a, appNamespace2b)).To(Succeed())
+
+			By("Cleaning up the Istio namespace")
+			Expect(k.DeleteNamespace(controlPlaneNamespace1, controlPlaneNamespace2)).To(Succeed(), "Istio Namespaces failed to be deleted")
+
+			By("Cleaning up the IstioCNI namespace")
+			Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted")
+
+			By("Deleting any left-over Istio and IstioRevision resources")
+			Expect(forceDeleteIstioResources()).To(Succeed())
+			Success("Resources deleted")
+			Success("Cleanup done")
+		})
+	})
+
+	AfterAll(func() {
+		if CurrentSpecReport().Failed() && !debugInfoLogged {
+			common.LogDebugInfo(k)
+			debugInfoLogged = true
+		}
+
+		if skipDeploy {
+			Success("Skipping operator undeploy because it was deployed externally")
+			return
+		}
+
+		By("Deleting operator deployment")
+		Expect(common.UninstallOperator()).
+			To(Succeed(), "Operator failed to be deleted")
+		GinkgoWriter.Println("Operator uninstalled")
+
+		Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted")
+		Success("Namespace deleted")
+	})
+})
+
+func forceDeleteIstioResources() error {
+	// This is a workaround to delete the Istio CRs that are left in the cluster
+	// This will be improved by splitting the tests into different Nodes with their independent setups and cleanups
+	err := k.ForceDelete("istio", istioName1)
+	if err != nil && !strings.Contains(err.Error(), "not found") {
+		return fmt.Errorf("failed to delete %s CR: %w", "istio", err)
+	}
+
+	err = k.ForceDelete("istiorevision", "default")
+	if err != nil && !strings.Contains(err.Error(), "not found") {
+		return fmt.Errorf("failed to delete %s CR: %w", "istiorevision", err)
+	}
+
+	err = k.Delete("istiocni", istioCniName)
+	if err != nil && !strings.Contains(err.Error(), "not found") {
+		return fmt.Errorf("failed to delete %s CR: %w", "istiocni", err)
+	}
+
+	return nil
+}

tests/e2e/util/kubectl/kubectl.go

@@ -309,6 +309,12 @@ func (k Kubectl) Logs(pod string, since *time.Duration) (string, error) {
 	return output, nil
 }
 
+// Label adds a label to the specified resource
+func (k Kubectl) Label(kind, name, labelKey, labelValue string) error {
+	_, err := k.executeCommand(k.build(fmt.Sprintf(" label %s %s %s=%s", kind, name, labelKey, labelValue)))
+	return err
+}
+
 // executeCommand handles running the command and then resets the namespace automatically
 func (k Kubectl) executeCommand(cmd string) (string, error) {
 	return shell.ExecuteCommand(cmd)