diff --git a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go new file mode 100644 index 000000000..26f063b61 --- /dev/null +++ b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go @@ -0,0 +1,71 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "testing" + + "github.com/istio-ecosystem/sail-operator/pkg/test/util/supportedversion" + k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/env" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + cl client.Client + err error + version = supportedversion.New + namespace = common.OperatorNamespace + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + controlPlaneNamespace1 = env.Get("CONTROL_PLANE_NS1", "istio-system1") + controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2") + istioName1 = env.Get("ISTIO_NAME1", "mesh1") + istioName2 = env.Get("ISTIO_NAME2", "mesh2") + istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniName = env.Get("ISTIOCNI_NAME", "default") + skipDeploy = env.GetBool("SKIP_DEPLOY", false) + appNamespace1 = env.Get("APP_NAMESPACE1", "app1") + appNamespace2a = env.Get("APP_NAMESPACE2A", "app2a") + appNamespace2b = env.Get("APP_NAMESPACE2B", "app2b") + multicluster = env.GetBool("MULTICLUSTER", false) + ipFamily = env.Get("IP_FAMILY", "ipv4") + + k kubectl.Kubectl +) + +func TestInstall(t *testing.T) { + if ipFamily == "dual" || multicluster { + t.Skip("Skipping the multi control plane tests") + } + RegisterFailHandler(Fail) + setup() + RunSpecs(t, "Multiple Control Planes Test Suite") +} + +func setup() { + GinkgoWriter.Println("************ Running Setup ************") + + GinkgoWriter.Println("Initializing k8s client") + cl, err = k8sclient.InitK8sClient("") + Expect(err).NotTo(HaveOccurred()) + + k = kubectl.New("clControlPlane") +} diff --git a/tests/e2e/multicontrolplane/multi_control_plane_test.go b/tests/e2e/multicontrolplane/multi_control_plane_test.go new file mode 100644 index 000000000..cd75de9de --- /dev/null +++ b/tests/e2e/multicontrolplane/multi_control_plane_test.go @@ -0,0 +1,226 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR Condition OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "fmt" + "strings" + "time" + + v1 "github.com/istio-ecosystem/sail-operator/api/v1" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/pkg/test/util/supportedversion" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" + . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("Multiple Control Planes", Ordered, func() { + SetDefaultEventuallyTimeout(180 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + debugInfoLogged := false + + BeforeAll(func(ctx SpecContext) { + Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") + + if skipDeploy { + Success("Skipping operator installation because it was deployed externally") + } else { + Expect(common.InstallOperatorViaHelm()). + To(Succeed(), "Operator failed to be deployed") + } + + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Success("Operator is deployed in the namespace and Running") + }) + + Context("Two control planes", func() { + BeforeAll(func() { + Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") + Expect(k.CreateNamespace(controlPlaneNamespace1)).To(Succeed(), "Istio namespace failed to be created") + Expect(k.CreateNamespace(controlPlaneNamespace2)).To(Succeed(), "Istio namespace failed to be created") + + Expect(k.Label("namespace", controlPlaneNamespace1, "mesh", istioName1)).To(Succeed(), "Failed to label namespace") + Expect(k.Label("namespace", controlPlaneNamespace2, "mesh", istioName2)).To(Succeed(), "Failed to label namespace") + }) + + It("Installs IstioCNI", func(ctx SpecContext) { + yaml := ` +apiVersion: sailoperator.io/v1 +kind: IstioCNI +metadata: + name: default +spec: + version: %s + namespace: %s` + yaml = fmt.Sprintf(yaml, version, istioCniNamespace) + Expect(k.CreateFromString(yaml)).To(Succeed(), "failed to create IstioCNI") + Success("IstioCNI created") + + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). + Should(HaveCondition(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") + Success("IstioCNI is Ready") + }) + + DescribeTable("Installs Istios", + Entry("Mesh 1", istioName1, controlPlaneNamespace1), + Entry("Mesh 2", istioName2, controlPlaneNamespace2), + func(ctx SpecContext, name, ns string) { + Expect(k.CreateFromString(` +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: `+name+` +spec: + version: `+version+` + namespace: `+ns+` + values: + meshConfig: + discoverySelectors: + - matchLabels: + mesh: `+name)).To(Succeed(), "failed to create Istio CR") + + Expect(k.CreateFromString(` +apiVersion: security.istio.io/v1 +kind: PeerAuthentication +metadata: + name: default + namespace: `+ns+` +spec: + mtls: + mode: STRICT`)).To(Succeed(), "failed to create PeerAuthentication") + }) + + DescribeTable("Waits for Istios", + Entry("Mesh 1", istioName1), + Entry("Mesh 2", istioName2), + func(ctx SpecContext, name string) { + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(name), &v1.Istio{}). + Should( + And( + HaveCondition(v1.IstioConditionReconciled, metav1.ConditionTrue), + HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue), + ), "Istio is not Reconciled and Ready; unexpected Condition") + Success(fmt.Sprintf("Istio %s ready", name)) + }) + + DescribeTable("Deploys applications", + Entry("App 1", appNamespace1, istioName1), + Entry("App 2a", appNamespace2a, istioName2), + Entry("App 2b", appNamespace2b, istioName2), + func(ns, mesh string) { + Expect(k.CreateNamespace(ns)).To(Succeed(), "Failed to create namespace") + Expect(k.Label("namespace", ns, "mesh", mesh)).To(Succeed(), "Failed to label namespace") + Expect(k.Label("namespace", ns, "istio.io/rev", mesh)).To(Succeed(), "Failed to label namespace") + for _, appName := range []string{"sleep", "httpbin"} { + Expect(k.WithNamespace(ns). + Apply(common.GetSampleYAML(supportedversion.Map[version], appName))). + To(Succeed(), "Failed to deploy application") + } + Success(fmt.Sprintf("Applications in namespace %s deployed", ns)) + }) + + DescribeTable("Waits for apps to be ready", + Entry("App 1", appNamespace1), + Entry("App 2a", appNamespace2a), + Entry("App 2b", appNamespace2b), + func(ctx SpecContext, ns string) { + for _, deployment := range []string{"sleep", "httpbin"} { + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deployment, ns), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error waiting for deployment to be available") + } + Success(fmt.Sprintf("Applications in namespace %s ready", ns)) + }) + + It("Verifies app2a cannot connect to app1", func(ctx SpecContext) { + // time.Sleep(10 * time.Minute) + output, err := k.WithNamespace(appNamespace2a). + Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace1)) + Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod") + Expect(output).To(ContainSubstring("503 Service Unavailable"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace1)) + }) + + It("Verifies app2a can connect to app2b", func(ctx SpecContext) { + output, err := k.WithNamespace(appNamespace2a). + Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace2b)) + Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod") + Expect(output).To(ContainSubstring("200 OK"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace2b)) + }) + + It("Cleans up the resources", func() { + By("Cleaning up the application namespaces") + Expect(k.DeleteNamespace(appNamespace1, appNamespace2a, appNamespace2b)).To(Succeed()) + + By("Cleaning up the Istio namespace") + Expect(k.DeleteNamespace(controlPlaneNamespace1, controlPlaneNamespace2)).To(Succeed(), "Istio Namespaces failed to be deleted") + + By("Cleaning up the IstioCNI namespace") + Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") + + By("Deleting any left-over Istio and IstioRevision resources") + Expect(forceDeleteIstioResources()).To(Succeed()) + Success("Resources deleted") + Success("Cleanup done") + }) + }) + + AfterAll(func() { + if CurrentSpecReport().Failed() && !debugInfoLogged { + common.LogDebugInfo(k) + debugInfoLogged = true + } + + if skipDeploy { + Success("Skipping operator undeploy because it was deployed externally") + return + } + + By("Deleting operator deployment") + Expect(common.UninstallOperator()). + To(Succeed(), "Operator failed to be deleted") + GinkgoWriter.Println("Operator uninstalled") + + Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") + Success("Namespace deleted") + }) +}) + +func forceDeleteIstioResources() error { + // This is a workaround to delete the Istio CRs that are left in the cluster + // This will be improved by splitting the tests into different Nodes with their independent setups and cleanups + err := k.ForceDelete("istio", istioName1) + if err != nil && !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("failed to delete %s CR: %w", "istio", err) + } + + err = k.ForceDelete("istiorevision", "default") + if err != nil && !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("failed to delete %s CR: %w", "istiorevision", err) + } + + err = k.Delete("istiocni", istioCniName) + if err != nil && !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("failed to delete %s CR: %w", "istiocni", err) + } + + return nil +} diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index d7addba21..d01e5e9f5 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -309,6 +309,12 @@ func (k Kubectl) Logs(pod string, since *time.Duration) (string, error) { return output, nil } +// Label adds a label to the specified resource +func (k Kubectl) Label(kind, name, labelKey, labelValue string) error { + _, err := k.executeCommand(k.build(fmt.Sprintf(" label %s %s %s=%s", kind, name, labelKey, labelValue))) + return err +} + // executeCommand handles running the command and then resets the namespace automatically func (k Kubectl) executeCommand(cmd string) (string, error) { return shell.ExecuteCommand(cmd)