diff --git a/test/extended/networking/kubevirt/client.go b/test/extended/networking/kubevirt/client.go index 77b813b68284..d6a05b7a83cf 100644 --- a/test/extended/networking/kubevirt/client.go +++ b/test/extended/networking/kubevirt/client.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "sigs.k8s.io/yaml" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" @@ -93,6 +95,48 @@ func (c *Client) GetJSONPath(resource, name, jsonPath string) (string, error) { } return strings.TrimSuffix(strings.TrimPrefix(output, `"`), `"`), nil } + +func (c *Client) GetPodsByLabel(labelKey, labelValue string) ([]string, error) { + output, err := c.oc.AsAdmin().Run("get").Args("pods", "-n", c.oc.Namespace(), "-l", fmt.Sprintf("%s=%s", labelKey, labelValue), "-o", "jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return nil, err + } + if output == "" { + return []string{}, nil + } + return strings.Fields(output), nil +} + +func (c *Client) GetEventsForPod(podName string) ([]string, error) { + output, err := c.oc.AsAdmin().Run("get").Args("events", "-n", c.oc.Namespace(), "--field-selector", fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod", podName), "-o", "jsonpath={.items[*].message}").Output() + if err != nil { + return nil, err + } + if output == "" { + return []string{}, nil + } + return strings.Fields(output), nil +} + +func (c *Client) CreateVMIFromSpec(vmNamespace, vmName string, vmiSpec map[string]interface{}) error { + newVMI := map[string]interface{}{ + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachineInstance", + "metadata": map[string]interface{}{ + "name": vmName, + "namespace": vmNamespace, + }, + "spec": vmiSpec, + } + + newVMIYAML, err := yaml.Marshal(newVMI) + if err != nil { + return err + } + + return c.Apply(string(newVMIYAML)) +} + func ensureVirtctl(oc *exutil.CLI, dir string) (string, error) { filepath := filepath.Join(dir, "virtctl") _, err := os.Stat(filepath) diff --git a/test/extended/networking/kubevirt/template.go b/test/extended/networking/kubevirt/template.go index 55901a04d08a..3c668345c011 100644 --- a/test/extended/networking/kubevirt/template.go +++ b/test/extended/networking/kubevirt/template.go @@ -2,7 +2,7 @@ package kubevirt import ( "bytes" - "html/template" + "text/template" ) const ( @@ -196,6 +196,63 @@ spec: password: fedora chpasswd: { expire: False } name: cloudinitdisk +` + FedoraVMWithPreconfiguredPrimaryUDNAttachment = ` +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: {{ .VMName }} + namespace: {{ .VMNamespace }} +spec: + runStrategy: Always + template: + {{- if .PreconfiguredIP }} + metadata: + annotations: + network.kubevirt.io/addresses: {{ printf "%q" .PreconfiguredIP }} + {{- end }} + spec: + domain: + devices: + disks: + - name: containerdisk + disk: + bus: virtio + - name: cloudinitdisk + disk: + bus: virtio + interfaces: + - name: overlay + binding: + name: {{ .NetBindingName }} + {{- if .PreconfiguredMAC }} + macAddress: "{{ .PreconfiguredMAC }}" + {{- end }} + machine: + type: "" + resources: + requests: + memory: 2048M + networks: + - name: overlay + pod: {} + terminationGracePeriodSeconds: 0 + volumes: + - name: containerdisk + containerDisk: + image: {{ .FedoraContainterDiskImage }} + - name: cloudinitdisk + cloudInitNoCloud: + networkData: | + version: 2 + ethernets: + eth0: + dhcp4: true + dhcp6: true + userData: |- + #cloud-config + password: fedora + chpasswd: { expire: False } ` vmimTemplate = ` apiVersion: kubevirt.io/v1 @@ -214,6 +271,8 @@ type CreationTemplateParams struct { FedoraContainterDiskImage string NetBindingName string NetworkName string + PreconfiguredIP string + PreconfiguredMAC string } func renderVMTemplate(vmTemplateString string, params CreationTemplateParams) (string, error) { diff --git a/test/extended/networking/livemigration.go b/test/extended/networking/livemigration.go index 9bddeee2ce84..7a7cd48184e1 100644 --- a/test/extended/networking/livemigration.go +++ b/test/extended/networking/livemigration.go @@ -70,8 +70,12 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F DescribeTableSubtree("created using", func(createNetworkFn func(netConfig networkAttachmentConfigParams) networkAttachmentConfig) { - DescribeTable("[Suite:openshift/network/virtualization] should keep ip", func(netConfig networkAttachmentConfigParams, vmResource string, opCmd func(cli *kubevirt.Client, vmNamespace, vmName string)) { + DescribeTable("[Suite:openshift/network/virtualization] should keep ip", func(netConfig networkAttachmentConfigParams, vmResource string, opCmd func(cli *kubevirt.Client, vmNamespace, vmName string), wlConfig ...workloadNetworkConfig) { var err error + var workloadConfig workloadNetworkConfig + if len(wlConfig) > 0 { + workloadConfig = wlConfig[0] + } l := map[string]string{ "e2e-framework": f.BaseName, } @@ -128,6 +132,14 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F vmCreationParams.NetworkName = nadName } + if len(workloadConfig.preconfiguredIPs) > 0 { + var err error + vmCreationParams.PreconfiguredIP, err = formatAddressesAnnotation(workloadConfig.preconfiguredIPs) + Expect(err).NotTo(HaveOccurred()) + } + if workloadConfig.preconfiguredMAC != "" { + vmCreationParams.PreconfiguredMAC = workloadConfig.preconfiguredMAC + } Expect(virtClient.CreateVM(vmResource, vmCreationParams)).To(Succeed()) waitForVMReadiness(virtClient, vmCreationParams.VMNamespace, vmCreationParams.VMName) @@ -151,6 +163,17 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F } Expect(initialAddresses).To(HaveLen(expectedNumberOfAddresses)) + if len(workloadConfig.preconfiguredIPs) > 0 { + By("Verifying VM received the preconfigured IP address(es)") + for _, expectedIP := range workloadConfig.preconfiguredIPs { + expectedIP = strings.TrimSpace(expectedIP) + Expect(initialAddresses).To(ContainElement(expectedIP), fmt.Sprintf("Expected IP %s not found in VM addresses %v", expectedIP, initialAddresses)) + } + } + if workloadConfig.preconfiguredMAC != "" { + By("Verifying VM received the preconfigured MAC address") + verifyVMMAC(virtClient, vmName, workloadConfig.preconfiguredMAC) + } httpServerPodsIPs := httpServerTestPodsMultusNetworkIPs(netConfig, httpServerPods) By(fmt.Sprintf("Check east/west traffic before test operation using IPs: %v", httpServerPodsIPs)) @@ -173,6 +196,10 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F ShouldNot(BeEmpty()) Expect(obtainedAddresses).To(ConsistOf(initialAddresses)) + if workloadConfig.preconfiguredMAC != "" { + By("Verifying VM MAC address persisted after test operation") + verifyVMMAC(virtClient, vmName, workloadConfig.preconfiguredMAC) + } By("Check east/west after test operation") checkEastWestTraffic(virtClient, vmName, httpServerPodsIPs) }, @@ -241,7 +268,79 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F }, kubevirt.FedoraVMWithSecondaryNetworkAttachment, restartVM, - )) + ), + Entry( + "[OCPFeatureGate:PreconfiguredUDNAddresses] when the VM with preconfigured IPs attached to a primary UDN is restarted", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + role: "primary", + allowPersistentIPs: true, + }, + kubevirt.FedoraVMWithPreconfiguredPrimaryUDNAttachment, + restartVM, + workloadNetworkConfig{ + preconfiguredIPs: []string{"203.203.0.50", "2014:100:200::50"}, + }, + ), + Entry( + "[OCPFeatureGate:PreconfiguredUDNAddresses] when the VM with preconfigured MAC attached to a primary UDN is restarted", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + role: "primary", + allowPersistentIPs: true, + }, + kubevirt.FedoraVMWithPreconfiguredPrimaryUDNAttachment, + restartVM, + workloadNetworkConfig{ + preconfiguredMAC: "02:0A:0B:0C:0D:50", + }, + ), + Entry( + "[OCPFeatureGate:PreconfiguredUDNAddresses] when the VM with preconfigured IP and MAC attached to a primary UDN is migrated between nodes", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + role: "primary", + allowPersistentIPs: true, + }, + kubevirt.FedoraVMWithPreconfiguredPrimaryUDNAttachment, + migrateVM, + workloadNetworkConfig{ + preconfiguredIPs: []string{"203.203.0.51", "2014:100:200::51"}, + preconfiguredMAC: "02:0A:0B:0C:0D:51", + }, + ), + Entry( + "[OCPFeatureGate:PreconfiguredUDNAddresses] when the VM with preconfigured IP address is created when the address is already taken", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + role: "primary", + allowPersistentIPs: true, + }, + kubevirt.FedoraVMWithPreconfiguredPrimaryUDNAttachment, + duplicateVM, + workloadNetworkConfig{ + preconfiguredIPs: []string{"203.203.0.100", "2014:100:200::100"}, + }, + ), + Entry( + "[OCPFeatureGate:PreconfiguredUDNAddresses] when the VM with preconfigured MAC address is created when the address is already taken", + networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + role: "primary", + allowPersistentIPs: true, + }, + kubevirt.FedoraVMWithPreconfiguredPrimaryUDNAttachment, + duplicateVM, + workloadNetworkConfig{ + preconfiguredMAC: "02:0A:0B:0C:0D:10", + }, + ), + ) }, Entry("NetworkAttachmentDefinitions", func(c networkAttachmentConfigParams) networkAttachmentConfig { netConfig := newNetworkAttachmentConfig(c) @@ -428,6 +527,14 @@ func obtainAddresses(virtClient *kubevirt.Client, vmName string) ([]string, erro return addressFromStatus(virtClient, vmName) } +func obtainMAC(virtClient *kubevirt.Client, vmName string) (string, error) { + macStr, err := virtClient.GetJSONPath("vmi", vmName, "{@.status.interfaces[0].mac}") + if err != nil { + return "", fmt.Errorf("failed to extract the MAC address from VM %q: %w", vmName, err) + } + return strings.ToUpper(macStr), nil +} + func restartVM(cli *kubevirt.Client, vmNamespace, vmName string) { GinkgoHelper() By(fmt.Sprintf("Restarting vmi %s/%s", vmNamespace, vmName)) @@ -442,6 +549,56 @@ func migrateVM(cli *kubevirt.Client, vmNamespace, vmName string) { waitForVMIMSuccess(cli, vmNamespace, vmName) } +func verifyVMMAC(virtClient *kubevirt.Client, vmName, expectedMAC string) { + GinkgoHelper() + var actualMAC string + Eventually(func(g Gomega) string { + GinkgoHelper() + + var err error + actualMAC, err = obtainMAC(virtClient, vmName) + g.Expect(err).NotTo(HaveOccurred(), "Failed to obtain MAC address for VM") + return actualMAC + }). + WithPolling(time.Second). + WithTimeout(5 * time.Minute). + Should(Equal(expectedMAC)) +} + +func duplicateVM(cli *kubevirt.Client, vmNamespace, vmName string) { + GinkgoHelper() + duplicateVMName := vmName + "-duplicate" + By(fmt.Sprintf("Duplicating VM %s/%s to %s/%s", vmNamespace, vmName, vmNamespace, duplicateVMName)) + + vmiSpecJSON, err := cli.GetJSONPath("vmi", vmName, "{.spec}") + Expect(err).NotTo(HaveOccurred()) + var vmiSpec map[string]interface{} + Expect(json.Unmarshal([]byte(vmiSpecJSON), &vmiSpec)).To(Succeed()) + + Expect(cli.CreateVMIFromSpec(vmNamespace, duplicateVMName, vmiSpec)).To(Succeed()) + waitForVMPodEventWithMessage(cli, vmNamespace, duplicateVMName, "is already allocated", 2*time.Minute) +} + +func waitForVMPodEventWithMessage(vmClient *kubevirt.Client, vmNamespace, vmName, expectedEventMessage string, timeout time.Duration) { + GinkgoHelper() + By(fmt.Sprintf("Waiting for event containing %q on VM %s/%s virt-launcher pod", expectedEventMessage, vmNamespace, vmName)) + + Eventually(func(g Gomega) []string { + const vmLabelKey = "vm.kubevirt.io/name" + podNames, err := vmClient.GetPodsByLabel(vmLabelKey, vmName) + g.Expect(err).NotTo(HaveOccurred(), "Failed to get pods by label %s=%s", vmLabelKey, vmName) + g.Expect(podNames).To(HaveLen(1), "Expected exactly one virt-launcher pod for VM %s/%s, but found %d pods: %v", vmNamespace, vmName, len(podNames), podNames) + + virtLauncherPodName := podNames[0] + eventMessages, err := vmClient.GetEventsForPod(virtLauncherPodName) + g.Expect(err).NotTo(HaveOccurred(), "Failed to get events for pod %s", virtLauncherPodName) + + return eventMessages + }).WithPolling(time.Second).WithTimeout(timeout).Should( + ContainElement(ContainSubstring(expectedEventMessage)), + fmt.Sprintf("Expected to find an event containing %q", expectedEventMessage)) +} + func waitForPodsCondition(fr *framework.Framework, pods []*corev1.Pod, conditionFn func(g Gomega, pod *corev1.Pod)) { for _, pod := range pods { Eventually(func(g Gomega) { @@ -627,3 +784,25 @@ func networkName(netSpecConfig string) string { Expect(json.Unmarshal([]byte(netSpecConfig), &nc)).To(Succeed()) return nc.Name } + +// formatAddressesAnnotation converts slice of IPs to the required JSON format for kubevirt addresses annotation +func formatAddressesAnnotation(preconfiguredIPs []string) (string, error) { + const primaryUDNNetworkName = "overlay" + if len(preconfiguredIPs) == 0 { + return "", nil + } + + ips := make([]string, len(preconfiguredIPs)) + for i, ip := range preconfiguredIPs { + ips[i] = strings.TrimSpace(ip) + } + + staticIPs, err := json.Marshal(map[string][]string{ + primaryUDNNetworkName: ips, + }) + if err != nil { + return "", fmt.Errorf("failed to marshal static IPs: %w", err) + } + + return string(staticIPs), nil +} diff --git a/test/extended/networking/network_segmentation.go b/test/extended/networking/network_segmentation.go index 56902022a266..95dad4521090 100644 --- a/test/extended/networking/network_segmentation.go +++ b/test/extended/networking/network_segmentation.go @@ -1890,6 +1890,12 @@ type networkAttachmentConfigParams struct { role string } +// workloadNetworkConfig contains workload-specific network customizations +type workloadNetworkConfig struct { + preconfiguredIPs []string + preconfiguredMAC string +} + type networkAttachmentConfig struct { networkAttachmentConfigParams }