Add multi control plane e2e tests

tests/e2e/ambient/ambient_suite_test.go

@@ -62,5 +62,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New("clAmbient")
+	k = kubectl.New()
 }

tests/e2e/controlplane/control_plane_suite_test.go

@@ -62,5 +62,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New("clControlPlane")
+	k = kubectl.New()
 }

tests/e2e/dualstack/dualstack_suite_test.go

@@ -62,5 +62,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New("clDualStack")
+	k = kubectl.New()
 }

tests/e2e/multicluster/multicluster_multiprimary_test.go

@@ -274,8 +274,7 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
-						common.LogDebugInfo(k1)
-						common.LogDebugInfo(k2)
+						common.LogDebugInfo(k1, k2)
 						debugInfoLogged = true
 					}
 
@@ -299,8 +298,7 @@ spec:
 
 	AfterAll(func(ctx SpecContext) {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo(k1)
-			common.LogDebugInfo(k2)
+			common.LogDebugInfo(k1, k2)
 			debugInfoLogged = true
 		}
 

tests/e2e/multicluster/multicluster_primaryremote_test.go

@@ -318,8 +318,7 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
-						common.LogDebugInfo(k1)
-						common.LogDebugInfo(k2)
+						common.LogDebugInfo(k1, k2)
 						debugInfoLogged = true
 					}
 
@@ -349,8 +348,7 @@ spec:
 
 	AfterAll(func(ctx SpecContext) {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo(k1)
-			common.LogDebugInfo(k2)
+			common.LogDebugInfo(k1, k2)
 			debugInfoLogged = true
 		}
 

tests/e2e/multicluster/multicluster_suite_test.go

@@ -99,6 +99,6 @@ func setup(t *testing.T) {
 	exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir)
 
 	// Initialize kubectl utilities, one for each cluster
-	k1 = kubectl.New("clPrimary").WithKubeconfig(kubeconfig)
-	k2 = kubectl.New("clRemote").WithKubeconfig(kubeconfig2)
+	k1 = kubectl.New().WithClusterName("primary").WithKubeconfig(kubeconfig)
+	k2 = kubectl.New().WithClusterName("remote").WithKubeconfig(kubeconfig2)
 }

tests/e2e/multicontrolplane/multi_control_plane_suite_test.go

@@ -0,0 +1,71 @@
+//go:build e2e
+
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+	"testing"
+
+	"github.com/istio-ecosystem/sail-operator/pkg/test/util/supportedversion"
+	k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/env"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	cl                     client.Client
+	err                    error
+	version                = supportedversion.New
+	namespace              = common.OperatorNamespace
+	deploymentName         = env.Get("DEPLOYMENT_NAME", "sail-operator")
+	controlPlaneNamespace1 = env.Get("CONTROL_PLANE_NS1", "istio-system1")
+	controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2")
+	istioName1             = env.Get("ISTIO_NAME1", "mesh1")
+	istioName2             = env.Get("ISTIO_NAME2", "mesh2")
+	istioCniNamespace      = env.Get("ISTIOCNI_NAMESPACE", "istio-cni")
+	istioCniName           = env.Get("ISTIOCNI_NAME", "default")
+	skipDeploy             = env.GetBool("SKIP_DEPLOY", false)
+	appNamespace1          = env.Get("APP_NAMESPACE1", "app1")
+	appNamespace2a         = env.Get("APP_NAMESPACE2A", "app2a")
+	appNamespace2b         = env.Get("APP_NAMESPACE2B", "app2b")
+	multicluster           = env.GetBool("MULTICLUSTER", false)
+	ipFamily               = env.Get("IP_FAMILY", "ipv4")
+
+	k kubectl.Kubectl
+)
+
+func TestInstall(t *testing.T) {
+	if ipFamily == "dual" || multicluster {
+		t.Skip("Skipping the multi control plane tests")
+	}
+	RegisterFailHandler(Fail)
+	setup()
+	RunSpecs(t, "Multiple Control Planes Test Suite")
+}
+
+func setup() {
+	GinkgoWriter.Println("************ Running Setup ************")
+
+	GinkgoWriter.Println("Initializing k8s client")
+	cl, err = k8sclient.InitK8sClient("")
+	Expect(err).NotTo(HaveOccurred())
+
+	k = kubectl.New()
+}

tests/e2e/multicontrolplane/multi_control_plane_test.go

@@ -0,0 +1,209 @@
+//go:build e2e
+
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR Condition OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+	"fmt"
+	"time"
+
+	v1 "github.com/istio-ecosystem/sail-operator/api/v1"
+	"github.com/istio-ecosystem/sail-operator/pkg/kube"
+	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
+	"github.com/istio-ecosystem/sail-operator/pkg/test/util/supportedversion"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Multi control plane deployment model", Ordered, func() {
+	SetDefaultEventuallyTimeout(180 * time.Second)
+	SetDefaultEventuallyPollingInterval(time.Second)
+	debugInfoLogged := false
+
+	BeforeAll(func(ctx SpecContext) {
+		Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created")
+
+		if skipDeploy {
+			Success("Skipping operator installation because it was deployed externally")
+		} else {
+			Expect(common.InstallOperatorViaHelm()).
+				To(Succeed(), "Operator failed to be deployed")
+		}
+
+		Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deploymentName, namespace), &appsv1.Deployment{}).
+			Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD")
+		Success("Operator is deployed in the namespace and Running")
+	})
+
+	Describe("Installation", func() {
+		It("Sets up namespaces", func(ctx SpecContext) {
+			Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created")
+			Expect(k.CreateNamespace(controlPlaneNamespace1)).To(Succeed(), "Istio namespace failed to be created")
+			Expect(k.CreateNamespace(controlPlaneNamespace2)).To(Succeed(), "Istio namespace failed to be created")
+
+			Expect(k.Label("namespace", controlPlaneNamespace1, "mesh", istioName1)).To(Succeed(), "Failed to label namespace")
+			Expect(k.Label("namespace", controlPlaneNamespace2, "mesh", istioName2)).To(Succeed(), "Failed to label namespace")
+		})
+
+		It("Installs IstioCNI", func(ctx SpecContext) {
+			yaml := `
+apiVersion: sailoperator.io/v1
+kind: IstioCNI
+metadata:
+  name: default
+spec:
+  version: %s
+  namespace: %s`
+			yaml = fmt.Sprintf(yaml, version, istioCniNamespace)
+			Expect(k.CreateFromString(yaml)).To(Succeed(), "failed to create IstioCNI")
+			Success("IstioCNI created")
+
+			Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}).
+				Should(HaveCondition(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition")
+			Success("IstioCNI is Ready")
+		})
+
+		DescribeTable("Installs Istios",
+			Entry("Mesh 1", istioName1, controlPlaneNamespace1),
+			Entry("Mesh 2", istioName2, controlPlaneNamespace2),
+			func(ctx SpecContext, name, ns string) {
+				Expect(k.CreateFromString(`
+apiVersion: sailoperator.io/v1
+kind: Istio
+metadata:
+  name: `+name+`
+spec:
+  version: `+version+`
+  namespace: `+ns+`
+  values:
+    meshConfig:
+      discoverySelectors:
+      - matchLabels:
+          mesh: `+name)).To(Succeed(), "failed to create Istio CR")
+
+				Expect(k.CreateFromString(`
+apiVersion: security.istio.io/v1
+kind: PeerAuthentication
+metadata:
+  name: default
+  namespace: `+ns+`
+spec:
+  mtls:
+    mode: STRICT`)).To(Succeed(), "failed to create PeerAuthentication")
+			})
+
+		DescribeTable("Waits for Istios",
+			Entry("Mesh 1", istioName1),
+			Entry("Mesh 2", istioName2),
+			func(ctx SpecContext, name string) {
+				Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(name), &v1.Istio{}).
+					Should(
+						And(
+							HaveCondition(v1.IstioConditionReconciled, metav1.ConditionTrue),
+							HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue),
+						), "Istio is not Reconciled and Ready; unexpected Condition")
+				Success(fmt.Sprintf("Istio %s ready", name))
+			})
+
+		DescribeTable("Deploys applications",
+			Entry("App 1", appNamespace1, istioName1),
+			Entry("App 2a", appNamespace2a, istioName2),
+			Entry("App 2b", appNamespace2b, istioName2),
+			func(ns, mesh string) {
+				Expect(k.CreateNamespace(ns)).To(Succeed(), "Failed to create namespace")
+				Expect(k.Label("namespace", ns, "mesh", mesh)).To(Succeed(), "Failed to label namespace")
+				Expect(k.Label("namespace", ns, "istio.io/rev", mesh)).To(Succeed(), "Failed to label namespace")
+				for _, appName := range []string{"sleep", "httpbin"} {
+					Expect(k.WithNamespace(ns).
+						Apply(common.GetSampleYAML(supportedversion.Map[version], appName))).
+						To(Succeed(), "Failed to deploy application")
+				}
+				Success(fmt.Sprintf("Applications in namespace %s deployed", ns))
+			})
+
+		DescribeTable("Waits for apps to be ready",
+			Entry("App 1", appNamespace1),
+			Entry("App 2a", appNamespace2a),
+			Entry("App 2b", appNamespace2b),
+			func(ctx SpecContext, ns string) {
+				for _, deployment := range []string{"sleep", "httpbin"} {
+					Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deployment, ns), &appsv1.Deployment{}).
+						Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error waiting for deployment to be available")
+				}
+				Success(fmt.Sprintf("Applications in namespace %s ready", ns))
+			})
+	})
+
+	Describe("Verification", func() {
+		It("Verifies app2a cannot connect to app1", func(ctx SpecContext) {
+			output, err := k.WithNamespace(appNamespace2a).
+				Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace1))
+			Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod")
+			Expect(output).To(ContainSubstring("503 Service Unavailable"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace1))
+			Success("As expected, app2a in mesh2 is not allowed to communicate with app1 in mesh1")
+		})
+
+		It("Verifies app2a can connect to app2b", func(ctx SpecContext) {
+			output, err := k.WithNamespace(appNamespace2a).
+				Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace2b))
+			Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod")
+			Expect(output).To(ContainSubstring("200 OK"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace2b))
+			Success("As expected, app2a in mesh2 can communicate with app2b in the same mesh")
+		})
+	})
+
+	AfterAll(func() {
+		By("Cleaning up the application namespaces")
+		Expect(k.DeleteNamespace(appNamespace1, appNamespace2a, appNamespace2b)).To(Succeed())
+
+		By("Cleaning up the Istio namespace")
+		Expect(k.DeleteNamespace(controlPlaneNamespace1, controlPlaneNamespace2)).To(Succeed(), "Istio Namespaces failed to be deleted")
+
+		By("Cleaning up the IstioCNI namespace")
+		Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted")
+
+		By("Deleting any left-over Istio and IstioRevision resources")
+		Expect(k.Delete("istio", istioName1)).To(Succeed(), "Failed to delete Istio")
+		Expect(k.Delete("istio", istioName2)).To(Succeed(), "Failed to delete Istio")
+		Expect(k.Delete("istiocni", istioCniName)).To(Succeed(), "Failed to delete IstioCNI")
+		Success("Istio Resources deleted")
+		Success("Cleanup done")
+	})
+
+	AfterAll(func() {
+		if CurrentSpecReport().Failed() && !debugInfoLogged {
+			common.LogDebugInfo(k)
+			debugInfoLogged = true
+		}
+
+		if skipDeploy {
+			Success("Skipping operator undeploy because it was deployed externally")
+			return
+		}
+
+		By("Deleting operator deployment")
+		Expect(common.UninstallOperator()).
+			To(Succeed(), "Operator failed to be deleted")
+		GinkgoWriter.Println("Operator uninstalled")
+
+		Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted")
+		Success("Namespace deleted")
+	})
+})

tests/e2e/operator/operator_suite_test.go

@@ -57,5 +57,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New("clOperator")
+	k = kubectl.New()
 }