diff --git a/e2e/aws_helpers.go b/e2e/aws_helpers.go new file mode 100644 index 000000000..1a8fcac57 --- /dev/null +++ b/e2e/aws_helpers.go @@ -0,0 +1,242 @@ +package e2e + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/e2e/framework" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + yaml "sigs.k8s.io/yaml" +) + +const ( + awsMachineTemplateName = "aws-machine-template" + machineSetOpenshiftLabelKey = "machine.openshift.io/cluster-api-machineset" +) + +// getDefaultAWSMAPIProviderSpec retrieves the first MAPI MachineSet and extracts its AWS providerSpec for use as a template. +func getDefaultAWSMAPIProviderSpec() (*mapiv1beta1.MachineSet, *mapiv1beta1.AWSMachineProviderConfig) { + machineSetList := &mapiv1beta1.MachineSetList{} + Eventually(komega.List(machineSetList, client.InNamespace(framework.MAPINamespace))).Should(Succeed(), "Failed to list MachineSets in MAPI namespace") + + Expect(machineSetList.Items).ToNot(BeNil()) + Expect(machineSetList.Items).ToNot(HaveLen(0), "No MachineSets found in namespace %s", framework.MAPINamespace) + machineSet := &machineSetList.Items[0] + Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil()) + + providerSpec := &mapiv1beta1.AWSMachineProviderConfig{} + Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed()) + + return machineSet, providerSpec +} + +// newAWSMachineTemplate creates an AWSMachineTemplate from a MAPI AWS providerSpec by converting relevant fields. +func newAWSMachineTemplate(mapiProviderSpec *mapiv1beta1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate { + By("Creating AWS machine template") + + Expect(mapiProviderSpec).ToNot(BeNil()) + Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil()) + Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil()) + Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty()) + Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty()) + Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil()) + Expect(mapiProviderSpec.Subnet.Filters).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.Subnet.Filters[0].Values).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0)) + + awsMachineSpec := awsv1.AWSMachineSpec{ + IAMInstanceProfile: *mapiProviderSpec.IAMInstanceProfile.ID, + InstanceType: mapiProviderSpec.InstanceType, + AMI: awsv1.AMIReference{ + ID: mapiProviderSpec.AMI.ID, + }, + Ignition: &awsv1.Ignition{ + StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData, + }, + Subnet: &awsv1.AWSResourceReference{ + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.Subnet.Filters[0].Values, + }, + }, + }, + AdditionalSecurityGroups: []awsv1.AWSResourceReference{ + { + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values, + }, + }, + }, + }, + } + + awsMachineTemplate := &awsv1.AWSMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: awsMachineTemplateName, + Namespace: framework.CAPINamespace, + }, + Spec: awsv1.AWSMachineTemplateSpec{ + Template: awsv1.AWSMachineTemplateResource{ + Spec: awsMachineSpec, + }, + }, + } + + return awsMachineTemplate +} + +// createAWSClient creates an AWS EC2 client using credentials from the CAPI bootstrap credentials secret. +func createAWSClient(region string) *ec2.EC2 { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "capa-manager-bootstrap-credentials", + Namespace: framework.CAPINamespace, + }, + } + Eventually(komega.Get(secret)).Should(Succeed(), "Failed to get AWS credentials secret") + + accessKey := secret.Data["aws_access_key_id"] + Expect(accessKey).ToNot(BeNil()) + secretAccessKey := secret.Data["aws_secret_access_key"] + Expect(secretAccessKey).ToNot(BeNil()) + + awsConfig := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials( + string(accessKey), + string(secretAccessKey), + "", + ), + } + + sess, err := session.NewSession(awsConfig) + Expect(err).ToNot(HaveOccurred()) + + return ec2.New(sess) +} + +// getMAPICreatedInstance retrieves the EC2 instance created by a MAPI MachineSet using the AWS API. +func getMAPICreatedInstance(awsClient *ec2.EC2, msName string) ec2.Instance { + Expect(awsClient).ToNot(BeNil()) + Expect(msName).ToNot(BeEmpty()) + mapiMachineList := &mapiv1beta1.MachineList{} + Eventually(komega.ObjectList(mapiMachineList, client.InNamespace(framework.MAPINamespace), client.MatchingLabels{ + machineSetOpenshiftLabelKey: msName, + })).Should(HaveField("Items", Not(BeEmpty())), "Failed to find MAPI machines for MachineSet %s", msName) + + mapiMachine := mapiMachineList.Items[0] + Expect(mapiMachine.Status.ProviderStatus).ToNot(BeNil()) + + mapiProviderStatus := &mapiv1beta1.AWSMachineProviderStatus{} + Expect(yaml.Unmarshal(mapiMachine.Status.ProviderStatus.Raw, mapiProviderStatus)).To(Succeed()) + + Expect(mapiProviderStatus.InstanceID).ToNot(BeNil()) + Expect(*mapiProviderStatus.InstanceID).ToNot(BeEmpty()) + + request := &ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{*mapiProviderStatus.InstanceID}), + } + + result, err := awsClient.DescribeInstances(request) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.Reservations).To(HaveLen(1)) + Expect(result.Reservations[0].Instances).To(HaveLen(1)) + + return *result.Reservations[0].Instances[0] +} + +// getCAPICreatedInstance retrieves the EC2 instance created by a CAPI MachineSet using the AWS API. +func getCAPICreatedInstance(awsClient *ec2.EC2, msName string) ec2.Instance { + Expect(awsClient).ToNot(BeNil()) + Expect(msName).ToNot(BeEmpty()) + capiMachineList := &awsv1.AWSMachineList{} + + Eventually(komega.ObjectList(capiMachineList, client.InNamespace(framework.CAPINamespace), client.MatchingLabels{ + machineSetOpenshiftLabelKey: msName, + })).Should(HaveField("Items", HaveLen(1)), "Failed to find exactly one CAPI AWSMachine for MachineSet %s", msName) + + capiMachine := capiMachineList.Items[0] + Expect(capiMachine.Spec.InstanceID).ToNot(BeNil(), "AWSMachine InstanceID not set in Spec") + Expect(*capiMachine.Spec.InstanceID).ToNot(BeEmpty(), "AWSMachine InstanceID is empty") + + request := &ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{*capiMachine.Spec.InstanceID}), + } + + result, err := awsClient.DescribeInstances(request) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.Reservations).To(HaveLen(1)) + Expect(result.Reservations[0].Instances).To(HaveLen(1)) + + return *result.Reservations[0].Instances[0] +} + +// compareInstances compares EC2 instances created by MAPI and CAPI MachineSets, logging any differences while ignoring instance-specific fields. +func compareInstances(awsClient *ec2.EC2, mapiMsName, capiMsName string) { + By("Comparing instances created by MAPI and CAPI") + mapiEC2Instance := getMAPICreatedInstance(awsClient, mapiMsName) + capiEC2Instance := getCAPICreatedInstance(awsClient, capiMsName) + + // Ignore fields that are unique for each instance + ignoreInstanceFields := cmpopts.IgnoreFields(ec2.Instance{}, + "InstanceId", + "ClientToken", + "LaunchTime", + "PrivateDnsName", + "PrivateIpAddress", + "UsageOperationUpdateTime", + "Tags", // tags won't match we should write a set of tests for comparing them manually + ) + + ignoreNicFields := cmpopts.IgnoreFields(ec2.InstanceNetworkInterface{}, + "MacAddress", + "NetworkInterfaceId", + "PrivateDnsName", + "PrivateIpAddress", + ) + ignorePrivateIpFields := cmpopts.IgnoreFields(ec2.InstancePrivateIpAddress{}, + "PrivateDnsName", + "PrivateIpAddress", + ) + // Ignore variable fields on nested types explicitly + ignoreEbsFields := cmpopts.IgnoreFields(ec2.EbsInstanceBlockDevice{}, + "AttachTime", + "VolumeId", + ) + ignoreNicAttachmentFields := cmpopts.IgnoreFields(ec2.NetworkInterfaceAttachment{}, + "AttachTime", + "AttachmentId", + ) + // Tags won't match we should write a set of tests for comparing them manually + ignoreTags := cmpopts.IgnoreTypes(ec2.Tag{}) + cmpOpts := []cmp.Option{ + ignoreInstanceFields, + ignoreEbsFields, + ignoreNicFields, + ignoreNicAttachmentFields, + ignorePrivateIpFields, + ignoreTags, + cmpopts.EquateEmpty(), + } + + if !cmp.Equal(mapiEC2Instance, capiEC2Instance, cmpOpts...) { + GinkgoWriter.Print("Instances created by MAPI and CAPI are not equal\n" + cmp.Diff(mapiEC2Instance, capiEC2Instance, cmpOpts...)) + } +} diff --git a/e2e/aws_test.go b/e2e/aws_test.go index 356325a29..70b92e36a 100644 --- a/e2e/aws_test.go +++ b/e2e/aws_test.go @@ -1,14 +1,7 @@ package e2e import ( - "context" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" configv1 "github.com/openshift/api/config/v1" @@ -16,16 +9,8 @@ import ( "github.com/openshift/cluster-capi-operator/e2e/framework" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - yaml "sigs.k8s.io/yaml" -) - -const ( - awsMachineTemplateName = "aws-machine-template" - machineSetOpenshiftLabelKey = "machine.openshift.io/cluster-api-machineset" ) var _ = Describe("Cluster API AWS MachineSet", Ordered, func() { @@ -41,7 +26,7 @@ var _ = Describe("Cluster API AWS MachineSet", Ordered, func() { if platform != configv1.AWSPlatformType { Skip("Skipping AWS E2E tests") } - mapiDefaultMS, mapiDefaultProviderSpec = getDefaultAWSMAPIProviderSpec(cl) + mapiDefaultMS, mapiDefaultProviderSpec = getDefaultAWSMAPIProviderSpec() awsClient = createAWSClient(mapiDefaultProviderSpec.Placement.Region) }) @@ -80,211 +65,3 @@ var _ = Describe("Cluster API AWS MachineSet", Ordered, func() { compareInstances(awsClient, mapiDefaultMS.Name, "aws-machineset") }) }) - -func getDefaultAWSMAPIProviderSpec(cl client.Client) (*mapiv1.MachineSet, *mapiv1.AWSMachineProviderConfig) { - machineSetList := &mapiv1.MachineSetList{} - Expect(cl.List(ctx, machineSetList, client.InNamespace(framework.MAPINamespace))).To(Succeed()) - - Expect(machineSetList.Items).ToNot(HaveLen(0)) - machineSet := &machineSetList.Items[0] - Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil()) - - providerSpec := &mapiv1.AWSMachineProviderConfig{} - Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed()) - - return machineSet, providerSpec -} - -func newAWSMachineTemplate(mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate { - By("Creating AWS machine template") - - Expect(mapiProviderSpec).ToNot(BeNil()) - Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil()) - Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil()) - Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty()) - Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty()) - Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil()) - Expect(mapiProviderSpec.Subnet.Filters).ToNot(HaveLen(0)) - Expect(mapiProviderSpec.Subnet.Filters[0].Values).ToNot(HaveLen(0)) - Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0)) - Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0)) - Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0)) - - awsMachineSpec := awsv1.AWSMachineSpec{ - IAMInstanceProfile: *mapiProviderSpec.IAMInstanceProfile.ID, - InstanceType: mapiProviderSpec.InstanceType, - AMI: awsv1.AMIReference{ - ID: mapiProviderSpec.AMI.ID, - }, - Ignition: &awsv1.Ignition{ - StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData, - }, - Subnet: &awsv1.AWSResourceReference{ - Filters: []awsv1.Filter{ - { - Name: "tag:Name", - Values: mapiProviderSpec.Subnet.Filters[0].Values, - }, - }, - }, - AdditionalSecurityGroups: []awsv1.AWSResourceReference{ - { - Filters: []awsv1.Filter{ - { - Name: "tag:Name", - Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values, - }, - }, - }, - }, - } - - awsMachineTemplate := &awsv1.AWSMachineTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Name: awsMachineTemplateName, - Namespace: framework.CAPINamespace, - }, - Spec: awsv1.AWSMachineTemplateSpec{ - Template: awsv1.AWSMachineTemplateResource{ - Spec: awsMachineSpec, - }, - }, - } - - return awsMachineTemplate -} - -func createAWSClient(region string) *ec2.EC2 { - var secret corev1.Secret - Expect(cl.Get(context.Background(), client.ObjectKey{ - Namespace: framework.CAPINamespace, - Name: "capa-manager-bootstrap-credentials", - }, &secret)).To(Succeed()) - - accessKey := secret.Data["aws_access_key_id"] - Expect(accessKey).ToNot(BeNil()) - secretAccessKey := secret.Data["aws_secret_access_key"] - Expect(secretAccessKey).ToNot(BeNil()) - - awsConfig := &aws.Config{ - Region: aws.String(region), - Credentials: credentials.NewStaticCredentials( - string(accessKey), - string(secretAccessKey), - "", - ), - } - - sess, err := session.NewSession(awsConfig) - Expect(err).ToNot(HaveOccurred()) - - return ec2.New(sess) -} - -func getMAPICreatedInstance(awsClient *ec2.EC2, msName string) ec2.Instance { - Expect(awsClient).ToNot(BeNil()) - Expect(msName).ToNot(BeEmpty()) - mapiMachineList := &mapiv1.MachineList{} - Expect(cl.List(ctx, mapiMachineList, client.InNamespace(framework.MAPINamespace), client.MatchingLabels{ - machineSetOpenshiftLabelKey: msName, - })).To(Succeed()) - Expect(len(mapiMachineList.Items)).To(BeNumerically(">", 0)) - - mapiMachine := mapiMachineList.Items[0] - Expect(mapiMachine.Status.ProviderStatus).ToNot(BeNil()) - - mapiProviderStatus := &mapiv1.AWSMachineProviderStatus{} - Expect(yaml.Unmarshal(mapiMachine.Status.ProviderStatus.Raw, mapiProviderStatus)).To(Succeed()) - - Expect(mapiProviderStatus.InstanceID).ToNot(BeNil()) - Expect(*mapiProviderStatus.InstanceID).ToNot(BeEmpty()) - - request := &ec2.DescribeInstancesInput{ - InstanceIds: aws.StringSlice([]string{*mapiProviderStatus.InstanceID}), - } - - result, err := awsClient.DescribeInstances(request) - Expect(err).ToNot(HaveOccurred()) - - Expect(result.Reservations).To(HaveLen(1)) - Expect(result.Reservations[0].Instances).To(HaveLen(1)) - - return *result.Reservations[0].Instances[0] -} - -func getCAPICreatedInstance(awsClient *ec2.EC2, msName string) ec2.Instance { - Expect(awsClient).ToNot(BeNil()) - Expect(msName).ToNot(BeEmpty()) - capiMachineList := &awsv1.AWSMachineList{} - - Expect(cl.List(ctx, capiMachineList, client.InNamespace(framework.CAPINamespace), client.MatchingLabels{ - machineSetOpenshiftLabelKey: msName, - })).To(Succeed()) - Expect(capiMachineList.Items).To(HaveLen(1)) - - capiMachine := capiMachineList.Items[0] - Expect(capiMachine.Status).ToNot(BeNil()) - - request := &ec2.DescribeInstancesInput{ - InstanceIds: aws.StringSlice([]string{*capiMachine.Spec.InstanceID}), - } - - result, err := awsClient.DescribeInstances(request) - Expect(err).ToNot(HaveOccurred()) - - Expect(result.Reservations).To(HaveLen(1)) - Expect(result.Reservations[0].Instances).To(HaveLen(1)) - - return *result.Reservations[0].Instances[0] -} - -func compareInstances(awsClient *ec2.EC2, mapiMsName, capiMsName string) { - By("Comparing instances created by MAPI and CAPI") - mapiEC2Instance := getMAPICreatedInstance(awsClient, mapiMsName) - capiEC2Instance := getCAPICreatedInstance(awsClient, capiMsName) - - // Ignore fields that are unique for each instance - ignoreInstanceFields := cmpopts.IgnoreFields(ec2.Instance{}, - "InstanceId", - "ClientToken", - "LaunchTime", - "PrivateDnsName", - "PrivateIpAddress", - "UsageOperationUpdateTime", - "Tags", // tags won't match we should write a set of tests for comparing them manually - ) - - ignoreBlockDeviceFields := cmpopts.IgnoreFields(ec2.InstanceBlockDeviceMapping{}, - "Ebs.AttachTime", - "Ebs.VolumeId", - ) - - ignoreNicFields := cmpopts.IgnoreFields(ec2.InstanceNetworkInterface{}, - "Attachment.AttachTime", - "Attachment.AttachmentId", - "MacAddress", - "NetworkInterfaceId", - "PrivateDnsName", - "PrivateIpAddress", - ) - - ignorePrivateIpFields := cmpopts.IgnoreFields(ec2.InstancePrivateIpAddress{}, - "PrivateDnsName", - "PrivateIpAddress", - ) - - // Tags won't match we should write a set of tests for comparing them manually - ignoreTags := cmpopts.IgnoreTypes(ec2.Tag{}) - - cmpOpts := []cmp.Option{ - ignoreInstanceFields, - ignoreBlockDeviceFields, - ignoreNicFields, - ignorePrivateIpFields, - ignoreTags, - } - - if !cmp.Equal(mapiEC2Instance, capiEC2Instance, cmpOpts...) { - GinkgoWriter.Print("Instances created by MAPI and CAPI are not equal\n" + cmp.Diff(mapiEC2Instance, capiEC2Instance, cmpOpts...)) - } -} diff --git a/e2e/e2e_common.go b/e2e/e2e_common.go new file mode 100644 index 000000000..0f66bc7ed --- /dev/null +++ b/e2e/e2e_common.go @@ -0,0 +1,72 @@ +package e2e + +import ( + "context" + + . "github.com/onsi/gomega" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + metal3v1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1" + + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + gcpv1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + ibmpowervsv1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta2" + vspherev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + configv1 "github.com/openshift/api/config/v1" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +const ( + infrastructureName = "cluster" + infraAPIVersion = "infrastructure.cluster.x-k8s.io/v1beta2" + managedByAnnotationValueClusterCAPIOperatorInfraClusterController = "cluster-capi-operator-infracluster-controller" +) + +var ( + cl client.Client + ctx = context.Background() + platform configv1.PlatformType + clusterName string +) + +func init() { + utilruntime.Must(configv1.Install(scheme.Scheme)) + utilruntime.Must(awsv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(gcpv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(azurev1.AddToScheme(scheme.Scheme)) + utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(mapiv1beta1.AddToScheme(scheme.Scheme)) + utilruntime.Must(ibmpowervsv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(vspherev1.AddToScheme(scheme.Scheme)) + utilruntime.Must(metal3v1.AddToScheme(scheme.Scheme)) + utilruntime.Must(bmov1alpha1.AddToScheme(scheme.Scheme)) +} + +// InitCommonVariables initializes global variables used across test cases. +func InitCommonVariables() { + cfg, err := config.GetConfig() + Expect(err).ToNot(HaveOccurred()) + + cl, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + + infra := &configv1.Infrastructure{} + infraName := client.ObjectKey{ + Name: infrastructureName, + } + Expect(cl.Get(ctx, infraName, infra)).To(Succeed()) + Expect(infra.Status.PlatformStatus).ToNot(BeNil()) + clusterName = infra.Status.InfrastructureName + platform = infra.Status.PlatformStatus.Type + + komega.SetClient(cl) + komega.SetContext(ctx) +} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 84ded9b6a..2fe912e8a 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -1,79 +1,17 @@ package e2e import ( - "context" "testing" - bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" - metal3v1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes/scheme" - awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" - azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" - gcpv1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" - ibmpowervsv1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta2" - vspherev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - - configv1 "github.com/openshift/api/config/v1" - mapiv1 "github.com/openshift/api/machine/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/envtest/komega" -) - -const ( - infrastructureName = "cluster" - infraAPIVersion = "infrastructure.cluster.x-k8s.io/v1beta1" - managedByAnnotationValueClusterCAPIOperatorInfraClusterController = "cluster-capi-operator-infracluster-controller" -) - -var ( - cl runtimeclient.Client - ctx = context.Background() - platform configv1.PlatformType - clusterName string - mapiInfrastructure *configv1.Infrastructure ) -func init() { - utilruntime.Must(configv1.Install(scheme.Scheme)) - utilruntime.Must(awsv1.AddToScheme(scheme.Scheme)) - utilruntime.Must(gcpv1.AddToScheme(scheme.Scheme)) - utilruntime.Must(azurev1.AddToScheme(scheme.Scheme)) - utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) - utilruntime.Must(mapiv1.AddToScheme(scheme.Scheme)) - utilruntime.Must(ibmpowervsv1.AddToScheme(scheme.Scheme)) - utilruntime.Must(vspherev1.AddToScheme(scheme.Scheme)) - utilruntime.Must(metal3v1.AddToScheme(scheme.Scheme)) - utilruntime.Must(bmov1alpha1.AddToScheme(scheme.Scheme)) -} - func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Cluster API Suite") } var _ = BeforeSuite(func() { - cfg, err := config.GetConfig() - Expect(err).ToNot(HaveOccurred()) - - cl, err = runtimeclient.New(cfg, runtimeclient.Options{}) - Expect(err).ToNot(HaveOccurred()) - komega.SetClient(cl) - - infra := &configv1.Infrastructure{} - infraName := runtimeclient.ObjectKey{ - Name: infrastructureName, - } - Expect(cl.Get(ctx, infraName, infra)).To(Succeed()) - Expect(infra.Status.PlatformStatus).ToNot(BeNil()) - mapiInfrastructure = infra - clusterName = infra.Status.InfrastructureName - platform = infra.Status.PlatformStatus.Type - - komega.SetClient(cl) - komega.SetContext(ctx) + InitCommonVariables() }) diff --git a/e2e/machineset_migration_capi_authoritative_test.go b/e2e/machineset_migration_capi_authoritative_test.go new file mode 100644 index 000000000..94bc567f4 --- /dev/null +++ b/e2e/machineset_migration_capi_authoritative_test.go @@ -0,0 +1,279 @@ +package e2e + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + mapiframework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + capiframework "github.com/openshift/cluster-capi-operator/e2e/framework" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] MachineSet Migration CAPI Authoritative Tests", Ordered, func() { + BeforeAll(func() { + if platform != configv1.AWSPlatformType { + Skip(fmt.Sprintf("Skipping tests on %s, this is only supported on AWS", platform)) + } + + if !capiframework.IsMachineAPIMigrationEnabled(ctx, cl) { + Skip("Skipping, this feature is only supported on MachineAPIMigration enabled clusters") + } + }) + + var _ = Describe("Create MAPI MachineSets", Ordered, func() { + var mapiMSAuthCAPIName = "ms-authoritativeapi-capi" + var existingCAPIMSAuthorityCAPIName = "capi-machineset-authoritativeapi-capi" + + var awsMachineTemplate *awsv1.AWSMachineTemplate + var capiMachineSet *clusterv1.MachineSet + var mapiMachineSet *mapiv1beta1.MachineSet + var instanceType = "m5.large" + + Context("with spec.authoritativeAPI: ClusterAPI and existing CAPI MachineSet with same name", func() { + BeforeAll(func() { + capiMachineSet = createCAPIMachineSet(ctx, cl, 0, existingCAPIMSAuthorityCAPIName, instanceType) + + By("Creating a same name MAPI MachineSet") + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, existingCAPIMSAuthorityCAPIName, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + awsMachineTemplate = waitForAWSMachineTemplate(cl, existingCAPIMSAuthorityCAPIName) + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: ClusterAPI and existing CAPI MachineSet with same name' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("should verify that MAPI MachineSet has Paused condition True", func() { + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + // bug https://issues.redhat.com/browse/OCPBUGS-55337 + PIt("should verify that the non-authoritative MAPI MachineSet providerSpec has been updated to reflect the authoritative CAPI MachineSet mirror values", func() { + verifyMAPIMachineSetProviderSpec(ctx, cl, mapiMachineSet, HaveField("InstanceType", Equal(instanceType))) + }) + }) + + Context("with spec.authoritativeAPI: ClusterAPI and no existing CAPI MachineSet with same name", func() { + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, mapiMSAuthCAPIName, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + capiMachineSet = waitForCAPIMachineSetMirror(cl, mapiMSAuthCAPIName) + awsMachineTemplate = waitForAWSMachineTemplate(cl, mapiMSAuthCAPIName) + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: ClusterAPI and no existing CAPI MachineSet with same name' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("should find MAPI MachineSet .status.authoritativeAPI to equal CAPI", func() { + verifyMachineSetAuthoritative(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should verify that MAPI MachineSet Paused condition is True", func() { + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should verify that MAPI MachineSet Synchronized condition is True", func() { + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should verify that the non-authoritative MAPI MachineSet has an authoritative CAPI MachineSet mirror", func() { + waitForCAPIMachineSetMirror(cl, mapiMSAuthCAPIName) + }) + + It("should verify that CAPI MachineSet has Paused condition False", func() { + verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + }) + }) + + var _ = Describe("Scale MAPI MachineSets", Ordered, func() { + var mapiMSAuthCAPIName = "ms-authoritativeapi-capi" + + var awsMachineTemplate *awsv1.AWSMachineTemplate + var capiMachineSet *clusterv1.MachineSet + var mapiMachineSet *mapiv1beta1.MachineSet + var firstMAPIMachine *mapiv1beta1.Machine + var secondMAPIMachine *mapiv1beta1.Machine + + Context("with spec.authoritativeAPI: ClusterAPI", Ordered, func() { + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 1, mapiMSAuthCAPIName, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + capiMachineSet, awsMachineTemplate = waitForMAPIMachineSetMirrors(cl, mapiMSAuthCAPIName) + + mapiMachines, err := mapiframework.GetMachinesFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + Expect(mapiMachines).ToNot(BeEmpty(), "no MAPI Machines found") + + capiMachines := capiframework.GetMachinesFromMachineSet(cl, capiMachineSet) + Expect(capiMachines).ToNot(BeEmpty(), "no CAPI Machines found") + Expect(capiMachines[0].Name).To(Equal(mapiMachines[0].Name)) + firstMAPIMachine = mapiMachines[0] + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: ClusterAPI' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("should succeed scaling CAPI MachineSet to 2 replicas", func() { + By("Scaling up CAPI MachineSet to 2 replicas") + capiframework.ScaleCAPIMachineSet(mapiMSAuthCAPIName, 2, capiframework.CAPINamespace) + + By("Verifying a new CAPI Machine is created and Paused condition is False") + capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthCAPIName, capiframework.CAPINamespace) + capiMachine := capiframework.GetNewestMachineFromMachineSet(cl, capiMachineSet) + verifyMachineRunning(cl, capiMachine) + verifyMachinePausedCondition(capiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Verifying MAPI MachineSet status.replicas is set to 2") + verifyMachinesetReplicas(mapiMachineSet, 2) + + By("Verifying there is a non-authoritative, paused MAPI Machine mirror for the new CAPI Machine") + var err error + secondMAPIMachine, err = mapiframework.GetLatestMachineFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + verifyMachineAuthoritative(secondMAPIMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachinePausedCondition(secondMAPIMachine, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should succeed switching MachineSet's AuthoritativeAPI to MachineAPI", func() { + switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) + + It("should succeed scaling up MAPI MachineSet to 3, after switching AuthoritativeAPI to MachineAPI", func() { + By("Scaling up MAPI MachineSet to 3 replicas") + Expect(mapiframework.ScaleMachineSet(mapiMSAuthCAPIName, 3)).To(Succeed(), "should be able to scale up MAPI MachineSet") + + By("Verifying the newly requested MAPI Machine has been created and its status.authoritativeAPI is MachineAPI and its Paused condition is False") + mapiMachine, err := mapiframework.GetLatestMachineFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + verifyMachineRunning(cl, mapiMachine) + verifyMachineAuthoritative(mapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachinePausedCondition(mapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Verifying MachineSet status.replicas is set to 3") + verifyMachinesetReplicas(mapiMachineSet, 3) + verifyMachinesetReplicas(capiMachineSet, 3) + + By("Verifying there is a non-authoritative, paused CAPI Machine mirror for the new MAPI Machine") + capiMachine := capiframework.GetNewestMachineFromMachineSet(cl, capiMachineSet) + verifyMachinePausedCondition(capiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Verifying old Machines still exist and authority on them is still ClusterAPI") + verifyMachineAuthoritative(firstMAPIMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineAuthoritative(secondMAPIMachine, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should succeed scaling down MAPI MachineSet to 1, after the switch of AuthoritativeAPI to MachineAPI", func() { + By("Scaling down MAPI MachineSet to 1 replicas") + Expect(mapiframework.ScaleMachineSet(mapiMSAuthCAPIName, 1)).To(Succeed(), "should be able to scale down MAPI MachineSet") + }) + + It("should succeed switching back MachineSet's AuthoritativeAPI to ClusterAPI, after the initial switch to AuthoritativeAPI: MachineAPI", func() { + switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should delete both MAPI and CAPI MachineSets/Machines and InfraMachineTemplate when deleting CAPI MachineSet", func() { + capiframework.DeleteMachineSets(ctx, cl, capiMachineSet) + mapiframework.WaitForMachineSetsDeleted(ctx, cl, mapiMachineSet) + capiframework.WaitForMachineSetsDeleted(cl, capiMachineSet) + verifyResourceRemoved(awsMachineTemplate) + }) + }) + }) + + var _ = Describe("Delete MachineSets", Ordered, func() { + var mapiMSAuthMAPIName = "ms-authoritativeapi-mapi" + var mapiMachineSet *mapiv1beta1.MachineSet + var capiMachineSet *clusterv1.MachineSet + var awsMachineTemplate *awsv1.AWSMachineTemplate + + Context("when removing non-authoritative MAPI MachineSet", Ordered, func() { + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 1, mapiMSAuthMAPIName, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + capiMachineSet, awsMachineTemplate = waitForMAPIMachineSetMirrors(cl, mapiMSAuthMAPIName) + + mapiMachines, err := mapiframework.GetMachinesFromMachineSet(ctx, cl, mapiMachineSet) + Expect(mapiMachines).ToNot(BeEmpty(), "no MAPI Machines found") + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + + capiMachines := capiframework.GetMachinesFromMachineSet(cl, capiMachineSet) + Expect(capiMachines).ToNot(BeEmpty(), "no CAPI Machines found") + Expect(capiMachines[0].Name).To(Equal(mapiMachines[0].Name)) + + DeferCleanup(func() { + By("Cleaning up Context 'when removing non-authoritative MAPI MachineSet' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("shouldn't delete its authoritative CAPI MachineSet", func() { + By("Switching AuthoritativeAPI to ClusterAPI") + switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Scaling up CAPI MachineSet to 2 replicas") + capiframework.ScaleCAPIMachineSet(mapiMachineSet.GetName(), 2, capiframework.CAPINamespace) + + By("Verifying new CAPI Machine is running") + capiMachine := capiframework.GetNewestMachineFromMachineSet(cl, capiMachineSet) + verifyMachineRunning(cl, capiMachine) + verifyMachinePausedCondition(capiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Verifying MAPI MachineSet status.replicas is set to 2") + verifyMachinesetReplicas(mapiMachineSet, 2) + + By("Verifying there is a non-authoritative, paused MAPI Machine mirror for the new CAPI Machine") + mapiMachine, err := mapiframework.GetLatestMachineFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + verifyMachineAuthoritative(mapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachinePausedCondition(mapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Deleting non-authoritative MAPI MachineSet") + mapiMachineSet, err = mapiframework.GetMachineSet(ctx, cl, mapiMSAuthMAPIName) + Expect(err).ToNot(HaveOccurred(), "failed to get mapiMachineSet") + mapiframework.DeleteMachineSets(cl, mapiMachineSet) + + By("Verifying CAPI MachineSet not removed, both MAPI Machines and Mirrors remain") + // TODO: Add full verification once OCPBUGS-56897 is fixed + capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) + Expect(capiMachineSet).ToNot(BeNil(), "CAPI MachineSet should still exist after deleting non-authoritative MAPI MachineSet") + }) + }) + }) +}) diff --git a/e2e/machineset_migration_helpers.go b/e2e/machineset_migration_helpers.go new file mode 100644 index 000000000..bf6c2ebaa --- /dev/null +++ b/e2e/machineset_migration_helpers.go @@ -0,0 +1,367 @@ +package e2e + +import ( + "context" + "encoding/json" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + mapiframework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + capiframework "github.com/openshift/cluster-capi-operator/e2e/framework" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + yaml "sigs.k8s.io/yaml" +) + +// createCAPIMachineSet creates a CAPI MachineSet with an AWSMachineTemplate and waits for it to be ready. +func createCAPIMachineSet(ctx context.Context, cl client.Client, replicas int32, machineSetName string, instanceType string) *clusterv1.MachineSet { + By(fmt.Sprintf("Creating CAPI MachineSet %s with %d replicas", machineSetName, replicas)) + _, mapiDefaultProviderSpec := getDefaultAWSMAPIProviderSpec() + createAWSClient(mapiDefaultProviderSpec.Placement.Region) + awsMachineTemplate := newAWSMachineTemplate(mapiDefaultProviderSpec) + awsMachineTemplate.Name = machineSetName + if instanceType != "" { + awsMachineTemplate.Spec.Template.Spec.InstanceType = instanceType + } + + Eventually(cl.Create(ctx, awsMachineTemplate), capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "Failed to create a new awsMachineTemplate %s", awsMachineTemplate.Name) + + machineSet := capiframework.CreateMachineSet(ctx, cl, capiframework.NewMachineSetParams( + machineSetName, + clusterName, + "", + replicas, + corev1.ObjectReference{ + Kind: "AWSMachineTemplate", + APIVersion: infraAPIVersion, + Name: machineSetName, + }, + "worker-user-data", + )) + + capiframework.WaitForMachineSet(cl, machineSet.Name, machineSet.Namespace) + return machineSet +} + +// createMAPIMachineSetWithAuthoritativeAPI creates a MAPI MachineSet with specified authoritativeAPI and waits for the CAPI mirror to be created. +func createMAPIMachineSetWithAuthoritativeAPI(ctx context.Context, cl client.Client, replicas int, machineSetName string, machineSetAuthority mapiv1beta1.MachineAuthority, machineAuthority mapiv1beta1.MachineAuthority) *mapiv1beta1.MachineSet { + By(fmt.Sprintf("Creating MAPI MachineSet with spec.authoritativeAPI: %s, spec.template.spec.authoritativeAPI: %s, replicas=%d", machineSetAuthority, machineAuthority, replicas)) + machineSetParams := mapiframework.BuildMachineSetParams(ctx, cl, replicas) + machineSetParams.Name = machineSetName + machineSetParams.Labels[mapiframework.MachineSetKey] = machineSetName + machineSetParams.MachinesetAuthoritativeAPI = machineSetAuthority + machineSetParams.MachineAuthoritativeAPI = machineAuthority + // Remove taints as CAPI MachineSets don't support them yet. This is a known limitation tracked in https://issues.redhat.com/browse/OCPCLOUD-2861 + machineSetParams.Taints = []corev1.Taint{} + mapiMachineSet, err := mapiframework.CreateMachineSet(cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "MAPI MachineSet %s creation should succeed", machineSetName) + + capiMachineSet := &clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: machineSetName, + Namespace: capiframework.CAPINamespace, + }, + } + Eventually(komega.Get(capiMachineSet), capiframework.WaitShort, capiframework.RetryShort).Should( + Succeed(), "Should have mirror CAPI MachineSet created within 1 minute") + + switch machineAuthority { + case mapiv1beta1.MachineAuthorityMachineAPI: + mapiframework.WaitForMachineSet(ctx, cl, machineSetName) + case mapiv1beta1.MachineAuthorityClusterAPI: + capiframework.WaitForMachineSet(cl, machineSetName, capiframework.CAPINamespace) + } + return mapiMachineSet +} + +// switchMachineSetAuthoritativeAPI updates the authoritativeAPI fields of a MAPI MachineSet and its template. +func switchMachineSetAuthoritativeAPI(mapiMachineSet *mapiv1beta1.MachineSet, machineSetAuthority mapiv1beta1.MachineAuthority, machineAuthority mapiv1beta1.MachineAuthority) { + By(fmt.Sprintf("Switching MachineSet %s AuthoritativeAPI to spec.authoritativeAPI: %s, spec.template.spec.authoritativeAPI: %s", mapiMachineSet.Name, machineSetAuthority, machineAuthority)) + Eventually(komega.Update(mapiMachineSet, func() { + mapiMachineSet.Spec.AuthoritativeAPI = machineSetAuthority + mapiMachineSet.Spec.Template.Spec.AuthoritativeAPI = machineAuthority + }), capiframework.WaitShort, capiframework.RetryShort).Should(Succeed(), "Failed to update MachineSet %s AuthoritativeAPI", mapiMachineSet.Name) +} + +// verifyMachineSetAuthoritative verifies that a MAPI MachineSet's status.authoritativeAPI matches the expected authority. +func verifyMachineSetAuthoritative(mapiMachineSet *mapiv1beta1.MachineSet, authority mapiv1beta1.MachineAuthority) { + By(fmt.Sprintf("Verifying the MachineSet authoritative is %s", authority)) + Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( + HaveField("Status.AuthoritativeAPI", Equal(authority)), + fmt.Sprintf("Expected MachineSet with correct status.AuthoritativeAPI %s", authority), + ) +} + +// verifyMachineSetPausedCondition verifies the Paused condition of a MachineSet (MAPI or CAPI) based on its authoritative API. +func verifyMachineSetPausedCondition(machineSet client.Object, authority mapiv1beta1.MachineAuthority) { + Expect(machineSet).NotTo(BeNil(), "MachineSet parameter cannot be nil") + Expect(machineSet.GetName()).NotTo(BeEmpty(), "MachineSet name cannot be empty") + var conditionMatcher types.GomegaMatcher + + switch ms := machineSet.(type) { + case *mapiv1beta1.MachineSet: + // This is a MAPI MachineSet + switch authority { + case mapiv1beta1.MachineAuthorityMachineAPI: + By("Verifying MAPI MachineSet is unpaused") + conditionMatcher = SatisfyAll( + HaveField("Type", Equal(MAPIPausedCondition)), + HaveField("Status", Equal(corev1.ConditionFalse)), + HaveField("Reason", Equal("AuthoritativeAPIMachineAPI")), + HaveField("Message", ContainSubstring("MachineAPI")), + ) + case mapiv1beta1.MachineAuthorityClusterAPI: + By("Verifying MAPI MachineSet is paused") + conditionMatcher = SatisfyAll( + HaveField("Type", Equal(MAPIPausedCondition)), + HaveField("Status", Equal(corev1.ConditionTrue)), + HaveField("Reason", Equal("AuthoritativeAPINotMachineAPI")), + HaveField("Message", ContainSubstring("ClusterAPI")), + ) + default: + Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) + } + + Eventually(komega.Object(ms), capiframework.WaitMedium, capiframework.RetryMedium).Should( + HaveField("Status.Conditions", ContainElement(conditionMatcher)), + fmt.Sprintf("Should have the expected Paused condition for MAPI MachineSet %s with authority: %s", ms.Name, authority), + ) + + case *clusterv1.MachineSet: + // This is a CAPI MachineSet + switch authority { + case mapiv1beta1.MachineAuthorityClusterAPI: + By("Verifying CAPI MachineSet is unpaused") + conditionMatcher = SatisfyAll( + HaveField("Type", Equal(CAPIPausedCondition)), + HaveField("Status", Equal(metav1.ConditionFalse)), + HaveField("Reason", Equal("NotPaused")), + ) + case mapiv1beta1.MachineAuthorityMachineAPI: + By("Verifying CAPI MachineSet is paused") + conditionMatcher = SatisfyAll( + HaveField("Type", Equal(CAPIPausedCondition)), + HaveField("Status", Equal(metav1.ConditionTrue)), + HaveField("Reason", Equal("Paused")), + ) + default: + Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) + } + + Eventually(komega.Object(ms), capiframework.WaitMedium, capiframework.RetryMedium).Should( + HaveField("Status.V1Beta2.Conditions", ContainElement(conditionMatcher)), + fmt.Sprintf("Should have the expected Paused condition for CAPI MachineSet %s with authority: %s", ms.Name, authority), + ) + + default: + Fail(fmt.Sprintf("unsupported MachineSet type: %T", machineSet)) + } +} + +// verifyMachinesetReplicas verifies that a MachineSet (MAPI or CAPI) has the expected number of replicas in its status. +func verifyMachinesetReplicas(machineSet client.Object, replicas int) { + Expect(machineSet).NotTo(BeNil(), "Machine parameter cannot be nil") + Expect(machineSet.GetName()).NotTo(BeEmpty(), "Machine name cannot be empty") + switch ms := machineSet.(type) { + case *mapiv1beta1.MachineSet: + By(fmt.Sprintf("Verifying MAPI MachineSet status.Replicas is %d", replicas)) + Eventually(komega.Object(ms), capiframework.WaitLong, capiframework.RetryLong).Should( + HaveField("Status.Replicas", Equal(int32(replicas))), + "Should have MAPI MachineSet %q replicas status eventually be %d", ms.Name, replicas) + case *clusterv1.MachineSet: + By(fmt.Sprintf("Verifying CAPI MachineSet status.Replicas is %d", replicas)) + Eventually(komega.Object(ms), capiframework.WaitLong, capiframework.RetryLong).Should( + HaveField("Status.Replicas", Equal(int32(replicas))), + "Should have CAPI MachineSet %q replicas status eventually be %d", ms.Name, replicas) + default: + Fail(fmt.Sprintf("unsupported MachineSet type: %T", machineSet)) + } +} + +// verifyMAPIMachineSetSynchronizedCondition verifies that a MAPI MachineSet has the Synchronized condition set to True with the correct message. +func verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet *mapiv1beta1.MachineSet, authority mapiv1beta1.MachineAuthority) { + By("Verifying the MAPI MachineSet Synchronized condition is True") + var expectedMessage string + + switch authority { + case mapiv1beta1.MachineAuthorityMachineAPI: + expectedMessage = "Successfully synchronized MAPI MachineSet to CAPI" + case mapiv1beta1.MachineAuthorityClusterAPI: + expectedMessage = "Successfully synchronized CAPI MachineSet to MAPI" + default: + Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) + } + + Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( + WithTransform( + func(ms *mapiv1beta1.MachineSet) []mapiv1beta1.Condition { + return ms.Status.Conditions + }, + ContainElement( + SatisfyAll( + HaveField("Type", Equal(SynchronizedCondition)), + HaveField("Status", Equal(corev1.ConditionTrue)), + HaveField("Reason", Equal("ResourceSynchronized")), + HaveField("Message", Equal(expectedMessage)), + ), + ), + ), + fmt.Sprintf("Should have Synchronized condition for %s", authority), + ) +} + +// verifyMAPIMachineSetProviderSpec verifies that a MAPI MachineSet's providerSpec matches the given Gomega matcher. +func verifyMAPIMachineSetProviderSpec(ctx context.Context, cl client.Client, mapiMachineSet *mapiv1beta1.MachineSet, matcher types.GomegaMatcher) { + By(fmt.Sprintf("Verifying MAPI MachineSet %s ProviderSpec", mapiMachineSet.Name)) + Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryShort).Should( + WithTransform(getAWSProviderSpecFromMachineSet, matcher), + ) +} + +// getAWSProviderSpecFromMachineSet extracts and unmarshals the AWSMachineProviderConfig from a MAPI MachineSet. +func getAWSProviderSpecFromMachineSet(mapiMachineSet *mapiv1beta1.MachineSet) *mapiv1beta1.AWSMachineProviderConfig { + Expect(mapiMachineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil()) + + providerSpec := &mapiv1beta1.AWSMachineProviderConfig{} + Expect(yaml.Unmarshal(mapiMachineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed()) + + return providerSpec +} + +// updateAWSMachineSetProviderSpec updates a MAPI MachineSet's AWS providerSpec using the provided update function. +func updateAWSMachineSetProviderSpec(ctx context.Context, cl client.Client, mapiMachineSet *mapiv1beta1.MachineSet, updateFunc func(*mapiv1beta1.AWSMachineProviderConfig)) { + By(fmt.Sprintf("Updating MachineSet %s providerSpec", mapiMachineSet.Name)) + providerSpec := getAWSProviderSpecFromMachineSet(mapiMachineSet) + + updateFunc(providerSpec) + + rawProviderSpec, err := json.Marshal(providerSpec) + Expect(err).ToNot(HaveOccurred(), "failed to marshal updated provider spec") + + original := mapiMachineSet.DeepCopy() + mapiMachineSet.Spec.Template.Spec.ProviderSpec.Value.Raw = rawProviderSpec + + patch := client.MergeFrom(original) + Expect(cl.Patch(ctx, mapiMachineSet, patch)).To(Succeed(), "failed to patch MachineSet provider spec") +} + +// waitForMAPIMachineSetMirrors waits for the corresponding CAPI MachineSet and AWSMachineTemplate mirrors to be created for a MAPI MachineSet. +func waitForMAPIMachineSetMirrors(cl client.Client, machineSetNameMAPI string) (*clusterv1.MachineSet, *awsv1.AWSMachineTemplate) { + By(fmt.Sprintf("Verifying there is a CAPI MachineSet mirror and AWSMachineTemplate for MAPI MachineSet %s", machineSetNameMAPI)) + var err error + var capiMachineSet *clusterv1.MachineSet + var awsMachineTemplate *awsv1.AWSMachineTemplate + + Eventually(func() error { + capiMachineSet = capiframework.GetMachineSet(cl, machineSetNameMAPI, capiframework.CAPINamespace) + if capiMachineSet == nil { + return fmt.Errorf("CAPI MachineSet %s/%s not found", capiframework.CAPINamespace, machineSetNameMAPI) + } + return nil + }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "Should have CAPI MachineSet %s/%s exist", capiframework.CAPINamespace, machineSetNameMAPI) + + Eventually(func() error { + awsMachineTemplate, err = capiframework.GetAWSMachineTemplateByPrefix(cl, machineSetNameMAPI, capiframework.CAPINamespace) + return err + }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "Should have AWSMachineTemplate with prefix %s exist", machineSetNameMAPI) + + return capiMachineSet, awsMachineTemplate +} + +// waitForCAPIMachineSetMirror waits for a CAPI MachineSet mirror to be created for a MAPI MachineSet. +func waitForCAPIMachineSetMirror(cl client.Client, machineName string) *clusterv1.MachineSet { + By(fmt.Sprintf("Verifying there is a CAPI MachineSet mirror for MAPI MachineSet %s", machineName)) + var capiMachineSet *clusterv1.MachineSet + Eventually(func() error { + capiMachineSet = capiframework.GetMachineSet(cl, machineName, capiframework.CAPINamespace) + if capiMachineSet == nil { + return fmt.Errorf("CAPI MachineSet %s/%s not found", capiframework.CAPINamespace, machineName) + } + return nil + }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "Should have CAPI MachineSet %s/%s exist", capiframework.CAPINamespace, machineName) + return capiMachineSet +} + +// waitForAWSMachineTemplate waits for an AWSMachineTemplate with the specified name prefix to be created. +func waitForAWSMachineTemplate(cl client.Client, prefix string) *awsv1.AWSMachineTemplate { + By(fmt.Sprintf("Verifying there is an AWSMachineTemplate with prefix %s", prefix)) + var awsMachineTemplate *awsv1.AWSMachineTemplate + Eventually(func() error { + var err error + awsMachineTemplate, err = capiframework.GetAWSMachineTemplateByPrefix(cl, prefix, capiframework.CAPINamespace) + if err != nil { + return fmt.Errorf("failed to get AWSMachineTemplate with prefix %s in %s: %w", prefix, capiframework.CAPINamespace, err) + } + return nil + }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), + "Should have AWSMachineTemplate with prefix %s exist", prefix) + return awsMachineTemplate +} + +// createAWSMachineTemplate creates a new AWSMachineTemplate with an optional update function to modify the spec. +func createAWSMachineTemplate(ctx context.Context, cl client.Client, originalName string, updateFunc func(*awsv1.AWSMachineSpec)) *awsv1.AWSMachineTemplate { + By("Creating a new awsMachineTemplate") + _, mapiDefaultProviderSpec := getDefaultAWSMAPIProviderSpec() + createAWSClient(mapiDefaultProviderSpec.Placement.Region) + + newTemplate := newAWSMachineTemplate(mapiDefaultProviderSpec) + newTemplate.Name = "new-" + originalName + + if updateFunc != nil { + updateFunc(&newTemplate.Spec.Template.Spec) + } + + Eventually(cl.Create(ctx, newTemplate), capiframework.WaitMedium, capiframework.RetryMedium).Should( + Succeed(), "Failed to create a new awsMachineTemplate %s", newTemplate.Name) + + return newTemplate +} + +// updateCAPIMachineSetInfraTemplate updates a CAPI MachineSet's infrastructureRef to point to a new template. +func updateCAPIMachineSetInfraTemplate(capiMachineSet *clusterv1.MachineSet, newInfraTemplateName string) { + By(fmt.Sprintf("Updating CAPI MachineSet %s to point to new InfraTemplate %s", capiMachineSet.Name, newInfraTemplateName)) + Eventually(komega.Update(capiMachineSet, func() { + capiMachineSet.Spec.Template.Spec.InfrastructureRef.Name = newInfraTemplateName + }), capiframework.WaitShort, capiframework.RetryShort).Should( + Succeed(), + "Failed to update CAPI MachineSet %s to point to new InfraTemplate %s", + capiMachineSet.Name, newInfraTemplateName, + ) +} + +// cleanupMachineSetTestResources deletes CAPI MachineSets, MAPI MachineSets, and AWSMachineTemplates created during tests. +func cleanupMachineSetTestResources(ctx context.Context, cl client.Client, capiMachineSets []*clusterv1.MachineSet, awsMachineTemplates []*awsv1.AWSMachineTemplate, mapiMachineSets []*mapiv1beta1.MachineSet) { + for _, ms := range capiMachineSets { + if ms == nil { + continue + } + By(fmt.Sprintf("Deleting CAPI MachineSet %s", ms.Name)) + capiframework.DeleteMachineSets(ctx, cl, ms) + capiframework.WaitForMachineSetsDeleted(cl, ms) + } + + for _, ms := range mapiMachineSets { + if ms == nil { + continue + } + By(fmt.Sprintf("Deleting MAPI MachineSet %s", ms.Name)) + mapiframework.DeleteMachineSets(cl, ms) + mapiframework.WaitForMachineSetsDeleted(ctx, cl, ms) + } + + for _, template := range awsMachineTemplates { + if template == nil { + continue + } + By(fmt.Sprintf("Deleting awsMachineTemplate %s", template.Name)) + capiframework.DeleteAWSMachineTemplates(ctx, cl, template) + } +} diff --git a/e2e/machineset_migration_mapi_authoritative_test.go b/e2e/machineset_migration_mapi_authoritative_test.go new file mode 100644 index 000000000..eb3ad71a1 --- /dev/null +++ b/e2e/machineset_migration_mapi_authoritative_test.go @@ -0,0 +1,372 @@ +package e2e + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + mapiframework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + capiframework "github.com/openshift/cluster-capi-operator/e2e/framework" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] MachineSet Migration MAPI Authoritative Tests", Ordered, func() { + var k komega.Komega + + BeforeAll(func() { + if platform != configv1.AWSPlatformType { + Skip(fmt.Sprintf("Skipping tests on %s, this is only supported on AWS", platform)) + } + + if !capiframework.IsMachineAPIMigrationEnabled(ctx, cl) { + Skip("Skipping, this feature is only supported on MachineAPIMigration enabled clusters") + } + + k = komega.New(cl) + }) + + var _ = Describe("Create MAPI MachineSets", Ordered, func() { + var mapiMSAuthMAPIName = "ms-authoritativeapi-mapi" + var existingCAPIMSAuthorityMAPIName = "capi-machineset-authoritativeapi-mapi" + + var awsMachineTemplate *awsv1.AWSMachineTemplate + var capiMachineSet *clusterv1.MachineSet + var mapiMachineSet *mapiv1beta1.MachineSet + + Context("with spec.authoritativeAPI: MachineAPI and existing CAPI MachineSet with same name", func() { + BeforeAll(func() { + capiMachineSet = createCAPIMachineSet(ctx, cl, 0, existingCAPIMSAuthorityMAPIName, "") + awsMachineTemplate = waitForAWSMachineTemplate(cl, existingCAPIMSAuthorityMAPIName) + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: MachineAPI and existing CAPI MachineSet with same name' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{}, + ) + }) + }) + + // https://issues.redhat.com/browse/OCPCLOUD-3188 + PIt("should reject creation of MAPI MachineSet with same name as existing CAPI MachineSet", func() { + By("Creating a same name MAPI MachineSet") + createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, existingCAPIMSAuthorityMAPIName, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + }) + }) + + Context("with spec.authoritativeAPI: MachineAPI and when no existing CAPI MachineSet with same name", func() { + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, mapiMSAuthMAPIName, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + capiMachineSet = waitForCAPIMachineSetMirror(cl, mapiMSAuthMAPIName) + awsMachineTemplate = waitForAWSMachineTemplate(cl, mapiMSAuthMAPIName) + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: MachineAPI and when no existing CAPI MachineSet with same name' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("should find MAPI MachineSet .status.authoritativeAPI to equal MAPI", func() { + verifyMachineSetAuthoritative(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) + + It("should verify that MAPI MachineSet Paused condition is False", func() { + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) + + It("should verify that MAPI MachineSet Synchronized condition is True", func() { + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) + + It("should find that MAPI MachineSet has a CAPI MachineSet mirror", func() { + waitForCAPIMachineSetMirror(cl, mapiMSAuthMAPIName) + }) + + It("should verify that the mirror CAPI MachineSet has Paused condition True", func() { + verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) + }) + }) + + var _ = Describe("Scale MAPI MachineSets", Ordered, func() { + var mapiMSAuthMAPIName = "ms-authoritativeapi-mapi" + var mapiMSAuthMAPICAPI = "ms-mapi-machine-capi" + + var awsMachineTemplate *awsv1.AWSMachineTemplate + var capiMachineSet *clusterv1.MachineSet + var mapiMachineSet *mapiv1beta1.MachineSet + var firstMAPIMachine *mapiv1beta1.Machine + var secondMAPIMachine *mapiv1beta1.Machine + + Context("with spec.authoritativeAPI: MachineAPI", Ordered, func() { + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 1, mapiMSAuthMAPIName, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + capiMachineSet, awsMachineTemplate = waitForMAPIMachineSetMirrors(cl, mapiMSAuthMAPIName) + + mapiMachines, err := mapiframework.GetMachinesFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + Expect(mapiMachines).ToNot(BeEmpty(), "no MAPI Machines found") + + capiMachines := capiframework.GetMachinesFromMachineSet(cl, capiMachineSet) + Expect(capiMachines).ToNot(BeEmpty(), "no CAPI Machines found") + Expect(capiMachines[0].Name).To(Equal(mapiMachines[0].Name)) + firstMAPIMachine = mapiMachines[0] + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: MachineAPI' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("should be able scale MAPI MachineSet to 2 replicas successfully", func() { + By("Scaling up MAPI MachineSet to 2 replicas") + Expect(mapiframework.ScaleMachineSet(mapiMachineSet.GetName(), 2)).To(Succeed(), "should be able to scale up MAPI MachineSet") + mapiframework.WaitForMachineSet(ctx, cl, mapiMSAuthMAPIName) + verifyMachinesetReplicas(mapiMachineSet, 2) + verifyMachinesetReplicas(capiMachineSet, 2) + + By("Verifying a new MAPI Machine is created and Paused condition is False") + var err error + secondMAPIMachine, err = mapiframework.GetLatestMachineFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + verifyMachineRunning(cl, secondMAPIMachine) + verifyMachineAuthoritative(secondMAPIMachine, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachinePausedCondition(secondMAPIMachine, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Verifying there is a non-authoritative CAPI Machine mirror for the MAPI Machine and its Paused condition is True") + capiMachine := capiframework.GetNewestMachineFromMachineSet(cl, capiMachineSet) + verifyMachinePausedCondition(capiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Verifying CAPI MachineSet status.replicas is set to 2") + verifyMachinesetReplicas(capiMachineSet, 2) + }) + + It("should succeed switching MAPI MachineSet AuthoritativeAPI to ClusterAPI", func() { + switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should succeed scaling up CAPI MachineSet to 3, after the switch of AuthoritativeAPI to ClusterAPI", func() { + By("Scaling up CAPI MachineSet to 3") + capiframework.ScaleCAPIMachineSet(mapiMSAuthMAPIName, 3, capiframework.CAPINamespace) + + By("Verifying a new CAPI Machine is running and Paused condition is False") + capiMachine := capiframework.GetNewestMachineFromMachineSet(cl, capiMachineSet) + verifyMachineRunning(cl, capiMachine) + verifyMachinePausedCondition(capiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Verifying there is a non-authoritative, paused MAPI Machine mirror for the new CAPI Machine") + mapiMachine, err := mapiframework.GetLatestMachineFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + verifyMachineAuthoritative(mapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachinePausedCondition(mapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Verifying old Machines still exist and authority on them is still MachineAPI") + verifyMachineAuthoritative(firstMAPIMachine, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineAuthoritative(secondMAPIMachine, mapiv1beta1.MachineAuthorityMachineAPI) + }) + + It("should succeed scaling down CAPI MachineSet to 1, after the switch of AuthoritativeAPI to ClusterAPI", func() { + By("Scaling down CAPI MachineSet to 1") + capiframework.ScaleCAPIMachineSet(mapiMSAuthMAPIName, 1, capiframework.CAPINamespace) + capiframework.WaitForMachineSet(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) + + By("Verifying both CAPI MachineSet and its MAPI MachineSet mirror are scaled down to 1") + verifyMachinesetReplicas(capiMachineSet, 1) + verifyMachinesetReplicas(mapiMachineSet, 1) + }) + + It("should succeed in switching back the AuthoritativeAPI to MachineAPI after the initial switch to ClusterAPI", func() { + switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) + + It("should delete both MAPI and CAPI MachineSets/Machines and InfraMachineTemplate when deleting MAPI MachineSet", func() { + Expect(mapiframework.DeleteMachineSets(cl, mapiMachineSet)).To(Succeed(), "Should be able to delete test MachineSet") + capiframework.WaitForMachineSetsDeleted(cl, capiMachineSet) + mapiframework.WaitForMachineSetsDeleted(ctx, cl, mapiMachineSet) + verifyResourceRemoved(awsMachineTemplate) + }) + }) + + Context("with spec.authoritativeAPI: MachineAPI, spec.template.spec.authoritativeAPI: ClusterAPI", Ordered, func() { + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, mapiMSAuthMAPICAPI, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityClusterAPI) + capiMachineSet, awsMachineTemplate = waitForMAPIMachineSetMirrors(cl, mapiMSAuthMAPICAPI) + + DeferCleanup(func() { + By("Cleaning up Context 'with spec.authoritativeAPI: MachineAPI, spec.template.spec.authoritativeAPI: ClusterAPI' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + It("should create an authoritative CAPI Machine when scaling MAPI MachineSet to 1 replicas", func() { + By("Scaling up MAPI MachineSet to 1 replicas") + Expect(mapiframework.ScaleMachineSet(mapiMachineSet.GetName(), 1)).To(Succeed(), "should be able to scale up MAPI MachineSet") + capiframework.WaitForMachineSet(cl, mapiMSAuthMAPICAPI, capiframework.CAPINamespace) + verifyMachinesetReplicas(mapiMachineSet, 1) + verifyMachinesetReplicas(capiMachineSet, 1) + + By("Verifying MAPI Machine is created and .status.authoritativeAPI to equal CAPI") + mapiMachine, err := mapiframework.GetLatestMachineFromMachineSet(ctx, cl, mapiMachineSet) + Expect(err).ToNot(HaveOccurred(), "failed to get MAPI Machines from MachineSet") + verifyMachineAuthoritative(mapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachinePausedCondition(mapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Verifying CAPI Machine is created and Paused condition is False and provisions a running Machine") + capiMachine := capiframework.GetNewestMachineFromMachineSet(cl, capiMachineSet) + verifyMachineRunning(cl, capiMachine) + verifyMachinePausedCondition(capiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should delete both MAPI and CAPI MachineSets/Machines and InfraMachineTemplate when deleting MAPI MachineSet", func() { + Expect(mapiframework.DeleteMachineSets(cl, mapiMachineSet)).To(Succeed(), "Should be able to delete test MachineSet") + capiframework.WaitForMachineSetsDeleted(cl, capiMachineSet) + mapiframework.WaitForMachineSetsDeleted(ctx, cl, mapiMachineSet) + verifyResourceRemoved(awsMachineTemplate) + }) + }) + }) + + var _ = Describe("Update MachineSets", Ordered, func() { + var mapiMSAuthMAPIName = "ms-authoritativeapi-mapi" + var mapiMachineSet *mapiv1beta1.MachineSet + var capiMachineSet *clusterv1.MachineSet + var awsMachineTemplate *awsv1.AWSMachineTemplate + var newAWSMachineTemplate *awsv1.AWSMachineTemplate + + BeforeAll(func() { + mapiMachineSet = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, mapiMSAuthMAPIName, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI) + capiMachineSet, awsMachineTemplate = waitForMAPIMachineSetMirrors(cl, mapiMSAuthMAPIName) + + DeferCleanup(func() { + By("Cleaning up 'Update MachineSet' resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{capiMachineSet}, + []*awsv1.AWSMachineTemplate{awsMachineTemplate, newAWSMachineTemplate}, + []*mapiv1beta1.MachineSet{mapiMachineSet}, + ) + }) + }) + + Context("when MAPI MachineSet with spec.authoritativeAPI: MachineAPI and replicas 0", Ordered, func() { + It("should reject update when attempting scaling of the CAPI MachineSet mirror", func() { + By("Scaling up CAPI MachineSet to 1 should be rejected") + capiframework.ScaleCAPIMachineSet(mapiMSAuthMAPIName, 1, capiframework.CAPINamespace) + capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) + verifyMachinesetReplicas(capiMachineSet, 0) + }) + + It("should reject update when attempting to change the spec of the CAPI MachineSet mirror", func() { + By("Updating CAPI mirror spec (such as DeletePolicy)") + Eventually(k.Update(capiMachineSet, func() { + capiMachineSet.Spec.DeletePolicy = "Oldest" + }), capiframework.WaitMedium, capiframework.RetryShort).Should(Succeed(), "Failed to update CAPI MachineSet DeletePolicy") + + By("Verifying both MAPI and CAPI MachineSet spec value are restored to original value") + Eventually(k.Object(mapiMachineSet), capiframework.WaitShort, capiframework.RetryShort).Should(HaveField("Spec.DeletePolicy", SatisfyAny(BeEmpty(), Equal("Random"))), "Should have DeletePolicy be either empty or 'Random'") + Eventually(k.Object(capiMachineSet), capiframework.WaitShort, capiframework.RetryShort).Should(HaveField("Spec.DeletePolicy", Equal("Random")), "Should have DeletePolicy be 'Random'") + }) + + It("should create a new InfraTemplate when update MAPI MachineSet providerSpec", func() { + By("Updating MAPI MachineSet providerSpec InstanceType to m5.large") + newInstanceType := "m5.large" + updateAWSMachineSetProviderSpec(ctx, cl, mapiMachineSet, func(providerSpec *mapiv1beta1.AWSMachineProviderConfig) { + providerSpec.InstanceType = newInstanceType + }) + + By("Waiting for new InfraTemplate to be created") + originalAWSMachineTemplateName := capiMachineSet.Spec.Template.Spec.InfrastructureRef.Name + capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) + Eventually(k.Object(capiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should(HaveField("Spec.Template.Spec.InfrastructureRef.Name", Not(Equal(originalAWSMachineTemplateName))), "Should have InfraTemplate name changed") + + By("Verifying new InfraTemplate has the updated InstanceType") + newAWSMachineTemplate, err := capiframework.GetAWSMachineTemplateByPrefix(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) + Expect(err).ToNot(HaveOccurred(), "Failed to get new awsMachineTemplate %s", newAWSMachineTemplate) + Expect(newAWSMachineTemplate.Spec.Template.Spec.InstanceType).To(Equal(newInstanceType)) + + By("Verifying the old InfraTemplate is deleted") + verifyResourceRemoved(awsMachineTemplate) + }) + }) + + Context("when switching MAPI MachineSet spec.authoritativeAPI to ClusterAPI", Ordered, func() { + BeforeAll(func() { + switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + }) + + It("should be rejected when scaling MAPI MachineSet", func() { + By("Scaling up MAPI MachineSet to 1") + mapiframework.ScaleMachineSet(mapiMSAuthMAPIName, 1) + + By("Verifying MAPI MachineSet replicas is restored to original value 0") + verifyMachinesetReplicas(mapiMachineSet, 0) + }) + + It("should be rejected when when updating providerSpec of MAPI MachineSet", func() { + By("Getting the current MAPI MachineSet providerSpec InstanceType") + originalSpec := getAWSProviderSpecFromMachineSet(mapiMachineSet) + + By("Updating the MAPI MachineSet providerSpec InstanceType") + updateAWSMachineSetProviderSpec(ctx, cl, mapiMachineSet, func(providerSpec *mapiv1beta1.AWSMachineProviderConfig) { + providerSpec.InstanceType = "m5.xlarge" + }) + + By("Verifying MAPI MachineSet instanceType is restored to original value") + verifyMAPIMachineSetProviderSpec(ctx, cl, mapiMachineSet, HaveField("InstanceType", Equal(originalSpec.InstanceType))) + }) + + It("should update MAPI MachineSet and remove old InfraTemplate when CAPI MachineSet points to new InfraTemplate", func() { + By("Creating a new awsMachineTemplate with different spec") + newInstanceType := "m6.xlarge" + originalAWSMachineTemplateName := capiMachineSet.Spec.Template.Spec.InfrastructureRef.Name + newAWSMachineTemplate = createAWSMachineTemplate(ctx, cl, originalAWSMachineTemplateName, func(spec *awsv1.AWSMachineSpec) { + spec.InstanceType = newInstanceType + }) + + By("Updating CAPI MachineSet to point to the new InfraTemplate") + updateCAPIMachineSetInfraTemplate(capiMachineSet, newAWSMachineTemplate.Name) + + By("Verifying the MAPI MachineSet is updated to reflect the new template") + mapiMachineSet, _ = mapiframework.GetMachineSet(ctx, cl, mapiMSAuthMAPIName) + Eventually(k.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( + HaveField("Spec.Template.Spec.ProviderSpec.Value.Raw", ContainSubstring(newInstanceType)), + "Should have MAPI MachineSet providerSpec updated to reflect the new InfraTemplate with InstanceType %s", newInstanceType, + ) + }) + }) + }) +}) diff --git a/e2e/machineset_migration_test.go b/e2e/machineset_migration_test.go deleted file mode 100644 index 8efb6ec4a..000000000 --- a/e2e/machineset_migration_test.go +++ /dev/null @@ -1,382 +0,0 @@ -package e2e - -import ( - "context" - "encoding/json" - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/types" - - configv1 "github.com/openshift/api/config/v1" - machinev1beta1 "github.com/openshift/api/machine/v1beta1" - mapiframework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" - capiframework "github.com/openshift/cluster-capi-operator/e2e/framework" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - capav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" - capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest/komega" -) - -var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] MachineSet Migration Tests", Ordered, func() { - BeforeAll(func() { - if platform != configv1.AWSPlatformType { - Skip(fmt.Sprintf("Skipping tests on %s, this is only supported on AWS", platform)) - } - - if !capiframework.IsMachineAPIMigrationEnabled(ctx, cl) { - Skip("Skipping, this feature is only supported on MachineAPIMigration enabled clusters") - } - }) - - var _ = Describe("Create MAPI MachineSets", Ordered, func() { - var mapiMSAuthMAPIName = "ms-authoritativeapi-mapi" - var mapiMSAuthCAPIName = "ms-authoritativeapi-capi" - var existingCAPIMSAuthorityMAPIName = "capi-machineset-authoritativeapi-mapi" - var existingCAPIMSAuthorityCAPIName = "capi-machineset-authoritativeapi-capi" - - var awsMachineTemplate *capav1.AWSMachineTemplate - var capiMachineSet *capiv1beta1.MachineSet - var mapiMachineSet *machinev1beta1.MachineSet - var err error - - Context("with spec.authoritativeAPI: MAPI and existing CAPI MachineSet with same name", func() { - BeforeAll(func() { - capiMachineSet, err = createCAPIMachineSet(ctx, cl, 0, existingCAPIMSAuthorityMAPIName, "") - Expect(err).ToNot(HaveOccurred(), "CAPI MachineSet %s creation should succeed", capiMachineSet.GetName()) - - Eventually(func() error { - awsMachineTemplate, err = capiframework.GetAWSMachineTemplateByPrefix(cl, existingCAPIMSAuthorityMAPIName, capiframework.CAPINamespace) - return err - }, capiframework.WaitMedium, capiframework.RetryShort).Should(Succeed(), "awsMachineTemplate should exist") - - DeferCleanup(func() { - By("Cleaning up Context 'with spec.authoritativeAPI: MAPI and existing CAPI MachineSet with same name' resources") - cleanupTestResources( - ctx, - cl, - []*capiv1beta1.MachineSet{capiMachineSet}, - []*capav1.AWSMachineTemplate{awsMachineTemplate}, - []*machinev1beta1.MachineSet{}, - ) - }) - }) - - // https://issues.redhat.com/browse/OCPCLOUD-2641 - PIt("should reject creation of MAPI MachineSet with same name as existing CAPI MachineSet", func() { - By("Creating a same name MAPI MachineSet") - mapiMachineSet, err = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, existingCAPIMSAuthorityMAPIName, machinev1beta1.MachineAuthorityMachineAPI, machinev1beta1.MachineAuthorityMachineAPI) - Expect(err).To(HaveOccurred(), "denied request to create MAPI MachineSet %s", mapiMachineSet.GetName()) - }) - }) - - Context("with spec.authoritativeAPI: MAPI and when no existing CAPI MachineSet with same name", func() { - BeforeAll(func() { - mapiMachineSet, err = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, mapiMSAuthMAPIName, machinev1beta1.MachineAuthorityMachineAPI, machinev1beta1.MachineAuthorityMachineAPI) - Expect(err).ToNot(HaveOccurred(), "MAPI MachineSet %s creation should succeed", mapiMachineSet.GetName()) - - Eventually(func() error { - awsMachineTemplate, err = capiframework.GetAWSMachineTemplateByPrefix(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) - return err - }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "awsMachineTemplate should exist") - - DeferCleanup(func() { - By("Cleaning up Context 'with spec.authoritativeAPI: MAPI and when no existing CAPI MachineSet with same name' resources") - cleanupTestResources( - ctx, - cl, - []*capiv1beta1.MachineSet{}, - []*capav1.AWSMachineTemplate{awsMachineTemplate}, - []*machinev1beta1.MachineSet{mapiMachineSet}, - ) - }) - }) - - It("should find MAPI MachineSet .status.authoritativeAPI to equal MAPI", func() { - Eventually(komega.Object(mapiMachineSet)).Should(HaveField("Status.AuthoritativeAPI", Equal(machinev1beta1.MachineAuthorityMachineAPI))) - }) - - It("should verify that MAPI MachineSet paused condition is False", func() { - verifyMAPIPausedCondition(mapiMachineSet, machinev1beta1.MachineAuthorityMachineAPI) - }) - - It("should verify that MAPI MachineSet Synchronized condition is True", func() { - verifySynchronizedCondition(mapiMachineSet, machinev1beta1.MachineAuthorityMachineAPI) - }) - - It("should find that MAPI MachineSet has a CAPI MachineSet mirror", func() { - capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthMAPIName, capiframework.CAPINamespace) - }) - - It("should verify that the mirror CAPI MachineSet has Paused condition True", func() { - verifyCAPIPausedCondition(capiMachineSet, machinev1beta1.MachineAuthorityMachineAPI) - }) - }) - - Context("with spec.authoritativeAPI: CAPI and existing CAPI MachineSet with same name", func() { - BeforeAll(func() { - capiMachineSet, err = createCAPIMachineSet(ctx, cl, 0, existingCAPIMSAuthorityCAPIName, "m5.large") - Expect(err).ToNot(HaveOccurred(), "CAPI MachineSet %s creation should succeed", capiMachineSet.GetName()) - - By("Creating a same name MAPI MachineSet") - mapiMachineSet, err = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, existingCAPIMSAuthorityCAPIName, machinev1beta1.MachineAuthorityClusterAPI, machinev1beta1.MachineAuthorityClusterAPI) - Expect(err).ToNot(HaveOccurred(), "failed to create MAPI MachineSet %s", existingCAPIMSAuthorityCAPIName) - - Eventually(func() error { - awsMachineTemplate, err = capiframework.GetAWSMachineTemplateByPrefix(cl, existingCAPIMSAuthorityCAPIName, capiframework.CAPINamespace) - return err - }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "awsMachineTemplate should exist") - - DeferCleanup(func() { - By("Cleaning up Context 'with spec.authoritativeAPI: CAPI and existing CAPI MachineSet with same name' resources") - cleanupTestResources( - ctx, - cl, - []*capiv1beta1.MachineSet{capiMachineSet}, - []*capav1.AWSMachineTemplate{awsMachineTemplate}, - []*machinev1beta1.MachineSet{mapiMachineSet}, - ) - }) - }) - - It("should verify that MAPI MachineSet has Paused condition True", func() { - verifyMAPIPausedCondition(mapiMachineSet, machinev1beta1.MachineAuthorityClusterAPI) - }) - - // bug https://issues.redhat.com/browse/OCPBUGS-55337 - PIt("should verify that the non-authoritative MAPI MachineSet providerSpec has been updated to reflect the authoritative CAPI MachineSet mirror values", func() { - Eventually(func() string { - providerSpec := mapiMachineSet.Spec.Template.Spec.ProviderSpec - var awsConfig machinev1beta1.AWSMachineProviderConfig - _ = json.Unmarshal(providerSpec.Value.Raw, &awsConfig) - return awsConfig.InstanceType - }, capiframework.WaitMedium, capiframework.RetryShort).Should(Equal("m5.large"), "MAPI MSet Sepc should be updated to reflect existing CAPI mirror") - }) - }) - - Context("with spec.authoritativeAPI: CAPI and no existing CAPI MachineSet with same name", func() { - BeforeAll(func() { - mapiMachineSet, err = createMAPIMachineSetWithAuthoritativeAPI(ctx, cl, 0, mapiMSAuthCAPIName, machinev1beta1.MachineAuthorityClusterAPI, machinev1beta1.MachineAuthorityClusterAPI) - Expect(err).ToNot(HaveOccurred(), "MAPI MachineSet %s creation should succeed", mapiMachineSet.GetName()) - capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthCAPIName, capiframework.CAPINamespace) - - Eventually(func() error { - awsMachineTemplate, err = capiframework.GetAWSMachineTemplateByPrefix(cl, mapiMSAuthCAPIName, capiframework.CAPINamespace) - return err - }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed(), "awsMachineTemplate should exist") - - DeferCleanup(func() { - By("Cleaning up Context 'with spec.authoritativeAPI: CAPI and no existing CAPI MachineSet with same name' resources") - cleanupTestResources( - ctx, - cl, - []*capiv1beta1.MachineSet{capiMachineSet}, - []*capav1.AWSMachineTemplate{awsMachineTemplate}, - []*machinev1beta1.MachineSet{mapiMachineSet}, - ) - }) - }) - - It("should find MAPI MachineSet .status.authoritativeAPI to equal CAPI", func() { - Eventually(komega.Object(mapiMachineSet)).Should(HaveField("Status.AuthoritativeAPI", Equal(machinev1beta1.MachineAuthorityClusterAPI))) - }) - - It("should verify that MAPI MachineSet paused condition is True", func() { - verifyMAPIPausedCondition(mapiMachineSet, machinev1beta1.MachineAuthorityClusterAPI) - }) - - It("should verify that MAPI MachineSet Synchronized condition is True", func() { - verifySynchronizedCondition(mapiMachineSet, machinev1beta1.MachineAuthorityClusterAPI) - }) - - It("should verify that the non-authoritative MAPI MachineSet has an authoritative CAPI MachineSet mirror", func() { - capiMachineSet = capiframework.GetMachineSet(cl, mapiMSAuthCAPIName, capiframework.CAPINamespace) - }) - - It("should verify that CAPI MachineSet has Paused condition False", func() { - verifyCAPIPausedCondition(capiMachineSet, machinev1beta1.MachineAuthorityClusterAPI) - }) - }) - }) -}) - -func createMAPIMachineSetWithAuthoritativeAPI(ctx context.Context, cl client.Client, replicas int, machineSetName string, machineSetAuthority machinev1beta1.MachineAuthority, machineAuthority machinev1beta1.MachineAuthority) (*machinev1beta1.MachineSet, error) { - By(fmt.Sprintf("Creating MAPI MachineSet with spec.authoritativeAPI: %s, spec.template.spec.authoritativeAPI: %s, replicas=%d", machineSetAuthority, machineAuthority, replicas)) - machineSetParams := mapiframework.BuildMachineSetParams(ctx, cl, replicas) - machineSetParams.Name = machineSetName - machineSetParams.MachinesetAuthoritativeAPI = machineSetAuthority - machineSetParams.MachineAuthoritativeAPI = machineAuthority - // Now CAPI machineSet doesn't support taint, remove it. card https://issues.redhat.com/browse/OCPCLOUD-2861 - machineSetParams.Taints = []corev1.Taint{} - mapiMachineSet, err := mapiframework.CreateMachineSet(cl, machineSetParams) - Expect(err).ToNot(HaveOccurred(), "MAPI MachineSet %s creation should succeed", machineSetName) - - capiMachineSet := &capiv1beta1.MachineSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: machineSetName, - Namespace: capiframework.CAPINamespace, - }, - } - Eventually(komega.Get(capiMachineSet), capiframework.WaitShort, capiframework.RetryShort).Should( - Succeed(), "Mirror CAPI MachineSet should be created within 1 minute") - - switch machineAuthority { - case machinev1beta1.MachineAuthorityMachineAPI: - mapiframework.WaitForMachineSet(ctx, cl, machineSetName) - case machinev1beta1.MachineAuthorityClusterAPI: - capiframework.WaitForMachineSet(cl, machineSetName, capiframework.CAPINamespace) - } - return mapiMachineSet, nil -} - -func createCAPIMachineSet(ctx context.Context, cl client.Client, replicas int32, machineSetName string, instanceType string) (*capiv1beta1.MachineSet, error) { - By(fmt.Sprintf("Creating CAPI MachineSet %s with %d replicas", machineSetName, replicas)) - - _, mapiDefaultProviderSpec := getDefaultAWSMAPIProviderSpec(cl) - createAWSClient(mapiDefaultProviderSpec.Placement.Region) - awsMachineTemplate := newAWSMachineTemplate(mapiDefaultProviderSpec) - awsMachineTemplate.Name = machineSetName - if instanceType != "" { - awsMachineTemplate.Spec.Template.Spec.InstanceType = instanceType - } - - if err := cl.Create(ctx, awsMachineTemplate); err != nil { - Expect(err).ToNot(HaveOccurred()) - } - - machineSet := capiframework.CreateMachineSet(ctx, cl, capiframework.NewMachineSetParams( - machineSetName, - clusterName, - "", - replicas, - corev1.ObjectReference{ - Kind: "AWSMachineTemplate", - APIVersion: infraAPIVersion, - Name: machineSetName, - }, - "worker-user-data", - )) - - capiframework.WaitForMachineSet(cl, machineSet.Name, machineSet.Namespace) - return machineSet, nil -} - -func verifySynchronizedCondition(mapiMachineSet *machinev1beta1.MachineSet, authority machinev1beta1.MachineAuthority) { - By("Verify the MAPI MachineSet Synchronized condition is True") - var expectedMessage string - - switch authority { - case machinev1beta1.MachineAuthorityMachineAPI: - expectedMessage = "Successfully synchronized MAPI MachineSet to CAPI" - case machinev1beta1.MachineAuthorityClusterAPI: - expectedMessage = "Successfully synchronized CAPI MachineSet to MAPI" - default: - Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) - } - - Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( - WithTransform( - func(ms *machinev1beta1.MachineSet) []machinev1beta1.Condition { - return ms.Status.Conditions - }, - ContainElement( - SatisfyAll( - HaveField("Type", Equal(SynchronizedCondition)), - HaveField("Status", Equal(corev1.ConditionTrue)), - HaveField("Reason", Equal("ResourceSynchronized")), - HaveField("Message", Equal(expectedMessage)), - ), - ), - ), - fmt.Sprintf("Expected Synchronized condition for %s not found or incorrect", authority), - ) -} - -func verifyMAPIPausedCondition(mapiMachineSet *machinev1beta1.MachineSet, authority machinev1beta1.MachineAuthority) { - var conditionMatcher types.GomegaMatcher - - switch authority { - case machinev1beta1.MachineAuthorityMachineAPI: - By("Verifying MAPI MachineSet is unpaused") - conditionMatcher = SatisfyAll( - HaveField("Type", Equal(MAPIPausedCondition)), - HaveField("Status", Equal(corev1.ConditionFalse)), - HaveField("Reason", Equal("AuthoritativeAPIMachineAPI")), - HaveField("Message", ContainSubstring("MachineAPI")), - ) - case machinev1beta1.MachineAuthorityClusterAPI: - By("Verifying MAPI MachineSet is paused") - conditionMatcher = SatisfyAll( - HaveField("Type", Equal(MAPIPausedCondition)), - HaveField("Status", Equal(corev1.ConditionTrue)), - HaveField("Reason", Equal("AuthoritativeAPINotMachineAPI")), - HaveField("Message", ContainSubstring("ClusterAPI")), - ) - default: - Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) - } - - Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( - HaveField("Status.Conditions", ContainElement(conditionMatcher)), - fmt.Sprintf("Expected MAPI MachineSet with correct paused condition for %s", authority), - ) -} - -func verifyCAPIPausedCondition(capiMachineSet *capiv1beta1.MachineSet, authority machinev1beta1.MachineAuthority) { - var conditionMatcher types.GomegaMatcher - - switch authority { - case machinev1beta1.MachineAuthorityClusterAPI: - By("Verifying CAPI MachineSet is unpaused (ClusterAPI is authoritative)") - conditionMatcher = SatisfyAll( - HaveField("Type", Equal(CAPIPausedCondition)), - HaveField("Status", Equal(metav1.ConditionFalse)), - HaveField("Reason", Equal("NotPaused")), - ) - case machinev1beta1.MachineAuthorityMachineAPI: - By("Verifying CAPI MachineSet is paused (MachineAPI is authoritative)") - conditionMatcher = SatisfyAll( - HaveField("Type", Equal(CAPIPausedCondition)), - HaveField("Status", Equal(metav1.ConditionTrue)), - HaveField("Reason", Equal("Paused")), - ) - default: - Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) - } - - Eventually(komega.Object(capiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( - HaveField("Status.V1Beta2.Conditions", ContainElement(conditionMatcher)), - fmt.Sprintf("Expected CAPI MachineSet with correct paused condition for %s", authority), - ) -} - -func cleanupTestResources(ctx context.Context, cl client.Client, capiMachineSets []*capiv1beta1.MachineSet, awsMachineTemplates []*capav1.AWSMachineTemplate, mapiMachineSets []*machinev1beta1.MachineSet) { - for _, ms := range capiMachineSets { - if ms == nil { - continue - } - By(fmt.Sprintf("Deleting CAPI MachineSet %s", ms.Name)) - capiframework.DeleteMachineSets(ctx, cl, ms) - capiframework.WaitForMachineSetsDeleted(cl, ms) - } - - for _, ms := range mapiMachineSets { - if ms == nil { - continue - } - By(fmt.Sprintf("Deleting MAPI MachineSet %s", ms.Name)) - mapiframework.DeleteMachineSets(cl, ms) - mapiframework.WaitForMachineSetsDeleted(ctx, cl, ms) - } - - for _, template := range awsMachineTemplates { - if template == nil { - continue - } - By(fmt.Sprintf("Deleting awsMachineTemplate %s", template.Name)) - capiframework.DeleteAWSMachineTemplates(ctx, cl, template) - } -} diff --git a/e2e/migration_common.go b/e2e/migration_common.go index 5b52b3b34..42fcba872 100644 --- a/e2e/migration_common.go +++ b/e2e/migration_common.go @@ -1,15 +1,15 @@ package e2e import ( - machinev1beta1 "github.com/openshift/api/machine/v1beta1" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) const ( // SynchronizedCondition indicates that a machine resource has been successfully synchronized between MAPI and CAPI during migration. - SynchronizedCondition machinev1beta1.ConditionType = "Synchronized" + SynchronizedCondition mapiv1beta1.ConditionType = "Synchronized" // MAPIPausedCondition represents the paused state for MAPI machines. - MAPIPausedCondition machinev1beta1.ConditionType = "Paused" + MAPIPausedCondition mapiv1beta1.ConditionType = "Paused" // CAPIPausedCondition represents the paused state for CAPI machines. CAPIPausedCondition = clusterv1.PausedV1Beta2Condition )