From 042fe0a98fe5e4efe35e64571e1e21cbf908ed98 Mon Sep 17 00:00:00 2001 From: Amit Sahastrabuddhe <33931378+AmitSahastra@users.noreply.github.com> Date: Thu, 16 Apr 2026 22:27:24 +0530 Subject: [PATCH 1/4] PCP-6119: feat: add AUTO link mode and LinkSubnetWithMode for MAAS link_subnet (#318) * Fix wrong subnet in use existing VM flow * feat: add AUTO link mode and LinkSubnetWithMode for MAAS link_subnet * fix unit tests * change to use new maas client go * fix pkg/maas/lxd/host_maas_client_test.go * add unit test for resolveLinkMode --------- Co-authored-by: kun zhou Co-authored-by: Kun Zhou <156021375+Kun483@users.noreply.github.com> --- Makefile | 2 +- api/v1beta1/maasmachine_types.go | 6 + api/v1beta1/zz_generated.deepcopy.go | 12 + controllers/maasmachine_controller.go | 20 +- go.mod | 2 +- go.sum | 4 +- pkg/maas/client/mock/clienset_mock.go | 224 ++++++++++++ pkg/maas/dns/dns_test.go | 2 + pkg/maas/machine/machine.go | 75 +++- pkg/maas/machine/machine_test.go | 500 +++++++++++++++++++++++++- 10 files changed, 821 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 43e9621f6..5eb3a4597 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ endif # Image URL to use all building/pushing image targets IMAGE_NAME := cluster-api-provider-maas-controller REGISTRY ?= "us-east1-docker.pkg.dev/spectro-images/dev/${USER}/cluster-api" -SPECTRO_VERSION ?= 4.8.3-dev-12112025 +SPECTRO_VERSION ?= 4.8.3-dev-tmo-19022026 IMG_TAG ?= v0.6.1-spectro-${SPECTRO_VERSION} CONTROLLER_IMG ?= ${REGISTRY}/${IMAGE_NAME} diff --git a/api/v1beta1/maasmachine_types.go b/api/v1beta1/maasmachine_types.go index 8d29fa7c2..9d96a65dd 100644 --- a/api/v1beta1/maasmachine_types.go +++ b/api/v1beta1/maasmachine_types.go @@ -130,6 +130,12 @@ type VMConfig struct { // +optional Network string `json:"network,omitempty"` + // InterfaceLinkModes sets the MAAS link mode per interface (e.g. eth0, eth1, eth2). + // Keys are interface names ("eth0", "eth1", ...); values: "auto", "dhcp", "static", "link_up". + // When unset for an interface: eth0 defaults to "auto", others to "dhcp". Extensible for future interfaces. + // +optional + InterfaceLinkModes map[string]string `json:"interfaceLinkModes,omitempty"` + // AutoStart specifies whether the VM should automatically start // +kubebuilder:default=true // +optional diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 411d0db29..3c14aec5b 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -253,6 +253,11 @@ func (in *MaasMachineSpec) DeepCopyInto(out *MaasMachineSpec) { *out = new(string) **out = **in } + if in.Parent != nil { + in, out := &in.Parent, &out.Parent + *out = new(string) + **out = **in + } if in.ProviderID != nil { in, out := &in.ProviderID, &out.ProviderID *out = new(string) @@ -530,6 +535,13 @@ func (in *VMConfig) DeepCopyInto(out *VMConfig) { *out = new(int) **out = **in } + if in.InterfaceLinkModes != nil { + in, out := &in.InterfaceLinkModes, &out.InterfaceLinkModes + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.AutoStart != nil { in, out := &in.AutoStart, &out.AutoStart *out = new(bool) diff --git a/controllers/maasmachine_controller.go b/controllers/maasmachine_controller.go index 79d1261f8..971b984f9 100644 --- a/controllers/maasmachine_controller.go +++ b/controllers/maasmachine_controller.go @@ -439,12 +439,14 @@ func (r *MaasMachineReconciler) reconcileNormal(ctx context.Context, machineScop // TODO(saamalik) confirm that we'll never "recreate" a m; e.g: findMachine should always return err // if there used to be a m if m == nil || !(m.State == infrav1beta1.MachineStateDeployed || m.State == infrav1beta1.MachineStateDeploying) { - // If machine is in Ready state, verify network interfaces before deploying - // This ensures correct subnet assignment before deployment starts - if m != nil && m.State == infrav1beta1.MachineStateReady && machineScope.GetDynamicLXD() { - machineScope.Info("Machine is in Ready state, verifying network interfaces before deployment", "machineID", m.ID) + // If machine is in Ready or Allocated state, verify network interfaces before deploying. + // This ensures correct subnet assignment before deployment starts (avoids wrong eth0 subnet + // on 2nd VM when MAAS compose links eth0 to a different subnet than requested). + if m != nil && machineScope.GetDynamicLXD() && + (m.State == infrav1beta1.MachineStateReady || m.State == infrav1beta1.MachineStateAllocated) { + machineScope.Info("Verifying VM network interfaces before deployment", "machineID", m.ID, "state", m.State) if err := machineSvc.VerifyVMNetworkInterfaces(ctx, m.ID); err != nil { - machineScope.Error(err, "Failed to verify VM network interfaces in Ready state, requeuing", "machineID", m.ID) + machineScope.Error(err, "Failed to verify VM network interfaces before deploy, requeuing", "machineID", m.ID) conditions.MarkFalse(machineScope.MaasMachine, infrav1beta1.MachineDeployedCondition, infrav1beta1.MachineDeployingReason, clusterv1.ConditionSeverityWarning, "verifying network interfaces") return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } @@ -502,14 +504,14 @@ func (r *MaasMachineReconciler) reconcileNormal(ctx context.Context, machineScop } switch s := m.State; { - case s == infrav1beta1.MachineStateReady: + case s == infrav1beta1.MachineStateReady, s == infrav1beta1.MachineStateAllocated: machineScope.SetNotReady() conditions.MarkFalse(machineScope.MaasMachine, infrav1beta1.MachineDeployedCondition, infrav1beta1.MachineDeployingReason, clusterv1.ConditionSeverityWarning, "") // Note: Network interface verification happens before deployment is triggered (see above). - // This is a safety check in case the machine reached Ready state through a different path. + // This is a safety check in case the machine reached Ready/Allocated through a different path. if machineScope.GetDynamicLXD() { if err := machineSvc.VerifyVMNetworkInterfaces(ctx, m.ID); err != nil { - machineScope.Error(err, "Failed to verify VM network interfaces in Ready state (safety check)", "machineID", m.ID) + machineScope.Error(err, "Failed to verify VM network interfaces (safety check)", "machineID", m.ID, "state", s) // Requeue to retry verification - deployment should not proceed until interfaces are correct return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } @@ -534,7 +536,7 @@ func (r *MaasMachineReconciler) reconcileNormal(ctx context.Context, machineScop machineScope.SetNotReady() machineScope.Info("Machine is powered off!") conditions.MarkFalse(machineScope.MaasMachine, infrav1beta1.MachineDeployedCondition, infrav1beta1.MachinePoweredOffReason, clusterv1.ConditionSeverityWarning, "") - case s == infrav1beta1.MachineStateDeploying, s == infrav1beta1.MachineStateAllocated: + case s == infrav1beta1.MachineStateDeploying: machineScope.SetNotReady() conditions.MarkFalse(machineScope.MaasMachine, infrav1beta1.MachineDeployedCondition, infrav1beta1.MachineDeployingReason, clusterv1.ConditionSeverityWarning, "") case s == infrav1beta1.MachineStateDeployed: diff --git a/go.mod b/go.mod index 895e60924..67b7283eb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.36.3 github.com/pkg/errors v0.9.1 - github.com/spectrocloud/maas-client-go v0.1.4-beta1 + github.com/spectrocloud/maas-client-go v0.1.5-beta1 github.com/spf13/pflag v1.0.6 k8s.io/api v0.32.3 k8s.io/apiextensions-apiserver v0.32.3 diff --git a/go.sum b/go.sum index fe4a98dcf..100a0dd42 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/spectrocloud/maas-client-go v0.1.4-beta1 h1:adYeve0oaYjiPuNnogMzw8LN9VenQK/iwNOO74H4lKE= -github.com/spectrocloud/maas-client-go v0.1.4-beta1/go.mod h1:LSxLlmaNCmkaldtysbp7Beq/O2wptBb6qE5iKj+Y7Lw= +github.com/spectrocloud/maas-client-go v0.1.5-beta1 h1:OT16996QBW8krHTIctRuO0js+NhB6ODHoQTyiTLxoEU= +github.com/spectrocloud/maas-client-go v0.1.5-beta1/go.mod h1:LSxLlmaNCmkaldtysbp7Beq/O2wptBb6qE5iKj+Y7Lw= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= diff --git a/pkg/maas/client/mock/clienset_mock.go b/pkg/maas/client/mock/clienset_mock.go index 82ca8bdbd..97fc12765 100644 --- a/pkg/maas/client/mock/clienset_mock.go +++ b/pkg/maas/client/mock/clienset_mock.go @@ -191,6 +191,90 @@ func (mr *MockClientSetInterfaceMockRecorder) Zones() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Zones", reflect.TypeOf((*MockClientSetInterface)(nil).Zones)) } +// IPAddresses mocks base method. +func (m *MockClientSetInterface) IPAddresses() maasclient.IPAddresses { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IPAddresses") + ret0, _ := ret[0].(maasclient.IPAddresses) + return ret0 +} + +// IPAddresses indicates an expected call of IPAddresses. +func (mr *MockClientSetInterfaceMockRecorder) IPAddresses() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IPAddresses", reflect.TypeOf((*MockClientSetInterface)(nil).IPAddresses)) +} + +// Tags mocks base method. +func (m *MockClientSetInterface) Tags() maasclient.Tags { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tags") + ret0, _ := ret[0].(maasclient.Tags) + return ret0 +} + +// Tags indicates an expected call of Tags. +func (mr *MockClientSetInterfaceMockRecorder) Tags() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tags", reflect.TypeOf((*MockClientSetInterface)(nil).Tags)) +} + +// NetworkInterfaces mocks base method. +func (m *MockClientSetInterface) NetworkInterfaces() maasclient.NetworkInterfaces { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NetworkInterfaces") + ret0, _ := ret[0].(maasclient.NetworkInterfaces) + return ret0 +} + +// NetworkInterfaces indicates an expected call of NetworkInterfaces. +func (mr *MockClientSetInterfaceMockRecorder) NetworkInterfaces() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkInterfaces", reflect.TypeOf((*MockClientSetInterface)(nil).NetworkInterfaces)) +} + +// Subnets mocks base method. +func (m *MockClientSetInterface) Subnets() maasclient.Subnets { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Subnets") + ret0, _ := ret[0].(maasclient.Subnets) + return ret0 +} + +// Subnets indicates an expected call of Subnets. +func (mr *MockClientSetInterfaceMockRecorder) Subnets() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subnets", reflect.TypeOf((*MockClientSetInterface)(nil).Subnets)) +} + +// VMHosts mocks base method. +func (m *MockClientSetInterface) VMHosts() maasclient.VMHosts { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VMHosts") + ret0, _ := ret[0].(maasclient.VMHosts) + return ret0 +} + +// VMHosts indicates an expected call of VMHosts. +func (mr *MockClientSetInterfaceMockRecorder) VMHosts() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VMHosts", reflect.TypeOf((*MockClientSetInterface)(nil).VMHosts)) +} + +// IPRanges mocks base method. +func (m *MockClientSetInterface) IPRanges() maasclient.IPRanges { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IPRanges") + ret0, _ := ret[0].(maasclient.IPRanges) + return ret0 +} + +// IPRanges indicates an expected call of IPRanges. +func (mr *MockClientSetInterfaceMockRecorder) IPRanges() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IPRanges", reflect.TypeOf((*MockClientSetInterface)(nil).IPRanges)) +} + // MockMachines is a mock of Machines interface. type MockMachines struct { ctrl *gomock.Controller @@ -505,6 +589,146 @@ func (mr *MockMachineMockRecorder) Zone() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Zone", reflect.TypeOf((*MockMachine)(nil).Zone)) } +// PowerType mocks base method. +func (m *MockMachine) PowerType() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PowerType") + ret0, _ := ret[0].(string) + return ret0 +} + +// PowerType indicates an expected call of PowerType. +func (mr *MockMachineMockRecorder) PowerType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PowerType", reflect.TypeOf((*MockMachine)(nil).PowerType)) +} + +// DeployedInMemory mocks base method. +func (m *MockMachine) DeployedInMemory() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeployedInMemory") + ret0, _ := ret[0].(bool) + return ret0 +} + +// DeployedInMemory indicates an expected call of DeployedInMemory. +func (mr *MockMachineMockRecorder) DeployedInMemory() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeployedInMemory", reflect.TypeOf((*MockMachine)(nil).DeployedInMemory)) +} + +// BootInterfaceID mocks base method. +func (m *MockMachine) BootInterfaceID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BootInterfaceID") + ret0, _ := ret[0].(string) + return ret0 +} + +// BootInterfaceID indicates an expected call of BootInterfaceID. +func (mr *MockMachineMockRecorder) BootInterfaceID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BootInterfaceID", reflect.TypeOf((*MockMachine)(nil).BootInterfaceID)) +} + +// TotalStorageGB mocks base method. +func (m *MockMachine) TotalStorageGB() float64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TotalStorageGB") + ret0, _ := ret[0].(float64) + return ret0 +} + +// TotalStorageGB indicates an expected call of TotalStorageGB. +func (mr *MockMachineMockRecorder) TotalStorageGB() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TotalStorageGB", reflect.TypeOf((*MockMachine)(nil).TotalStorageGB)) +} + +// GetBootInterfaceType mocks base method. +func (m *MockMachine) GetBootInterfaceType() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBootInterfaceType") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetBootInterfaceType indicates an expected call of GetBootInterfaceType. +func (mr *MockMachineMockRecorder) GetBootInterfaceType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBootInterfaceType", reflect.TypeOf((*MockMachine)(nil).GetBootInterfaceType)) +} + +// ResourcePoolName mocks base method. +func (m *MockMachine) ResourcePoolName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourcePoolName") + ret0, _ := ret[0].(string) + return ret0 +} + +// ResourcePoolName indicates an expected call of ResourcePoolName. +func (mr *MockMachineMockRecorder) ResourcePoolName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourcePoolName", reflect.TypeOf((*MockMachine)(nil).ResourcePoolName)) +} + +// ZoneName mocks base method. +func (m *MockMachine) ZoneName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ZoneName") + ret0, _ := ret[0].(string) + return ret0 +} + +// ZoneName indicates an expected call of ZoneName. +func (mr *MockMachineMockRecorder) ZoneName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ZoneName", reflect.TypeOf((*MockMachine)(nil).ZoneName)) +} + +// BootInterfaceName mocks base method. +func (m *MockMachine) BootInterfaceName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BootInterfaceName") + ret0, _ := ret[0].(string) + return ret0 +} + +// BootInterfaceName indicates an expected call of BootInterfaceName. +func (mr *MockMachineMockRecorder) BootInterfaceName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BootInterfaceName", reflect.TypeOf((*MockMachine)(nil).BootInterfaceName)) +} + +// Tags mocks base method. +func (m *MockMachine) Tags() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Tags") + ret0, _ := ret[0].([]string) + return ret0 +} + +// Tags indicates an expected call of Tags. +func (mr *MockMachineMockRecorder) Tags() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Tags", reflect.TypeOf((*MockMachine)(nil).Tags)) +} + +// Parent mocks base method. +func (m *MockMachine) Parent() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Parent") + ret0, _ := ret[0].(string) + return ret0 +} + +// Parent indicates an expected call of Parent. +func (mr *MockMachineMockRecorder) Parent() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Parent", reflect.TypeOf((*MockMachine)(nil).Parent)) +} + // MockDNSResources is a mock of DNSResources interface. type MockDNSResources struct { ctrl *gomock.Controller diff --git a/pkg/maas/dns/dns_test.go b/pkg/maas/dns/dns_test.go index c398d0153..b4a80736d 100644 --- a/pkg/maas/dns/dns_test.go +++ b/pkg/maas/dns/dns_test.go @@ -34,6 +34,8 @@ func (f *fakeClientSet) Users() maasclient.Users { retur func (f *fakeClientSet) Zones() maasclient.Zones { return nil } func (f *fakeClientSet) SSHKeys() maasclient.SSHKeys { return nil } func (f *fakeClientSet) VMHosts() maasclient.VMHosts { return nil } +func (f *fakeClientSet) IPRanges() maasclient.IPRanges { return nil } +func (f *fakeClientSet) Subnets() maasclient.Subnets { return nil } // fakeIPAddress satisfies maasclient.IPAddress for tests type fakeIPAddress struct{ ip net.IP } diff --git a/pkg/maas/machine/machine.go b/pkg/maas/machine/machine.go index d8951d0b2..cf33dbfaf 100644 --- a/pkg/maas/machine/machine.go +++ b/pkg/maas/machine/machine.go @@ -1,6 +1,7 @@ package machine import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -287,6 +288,11 @@ func (s *Service) createVMViaMAAS(ctx context.Context, userDataB64 string) (*inf if err != nil { return nil, errors.Wrap(err, "failed to get existing VM by system-id") } + // Verify and fix network interfaces (eth0/eth1 subnets) before deploy so that even if MAAS + // selected the wrong subnet for eth0 after commissioning/acquire, we correct it before deploy. + if err := s.VerifyVMNetworkInterfaces(ctx, m.SystemID()); err != nil { + return nil, errors.Wrap(err, "failed to verify VM network interfaces before deploy") + } // Best-effort: set hostname and static IP before deploy machineName := s.scope.Machine.Name vmName := fmt.Sprintf("vm-%s", machineName) @@ -475,7 +481,10 @@ func (s *Service) PrepareLXDVM(ctx context.Context) (*infrav1beta1.Machine, erro if subnet0 == "" || subnet1 == "" { s.scope.Info("Skipping setting network interfaces due to empty subnet name(s)", "subnet0", subnet0, "subnet1", subnet1) } else { - // Build interfaces parameter: eth0 gets first subnet (MAAS management), eth1 gets second subnet + // Build interfaces parameter: eth0 = first subnet (typically PXE), eth1 = second subnet (typically management). + // eth0 is sent without ip= so it uses DHCP; eth1 gets static IP when configured. Using DHCP on eth0 + // can cause DHCP-provided routes (e.g. to MAAS controller) to be installed on eth0; if the controller + // is on the eth1 network, ensure PXE subnet DHCP does not push those routes, or use static IP for eth0. // Format: "eth0:subnet=;eth1:subnet=" or // "eth0:subnet=;eth1:subnet=,ip=" if static IP is configured interfacesParam := fmt.Sprintf("eth0:subnet=%s;eth1:subnet=%s", subnet0, subnet1) @@ -578,6 +587,24 @@ func (s *Service) getSubnetCIDR(subnet maasclient.Subnet) string { return subnet.CIDR() } +// resolveLinkMode returns the MAAS link mode for an interface: from VMConfig.InterfaceLinkModes, else default (eth0=auto, others=dhcp). +func (s *Service) resolveLinkMode(ifaceName string) string { + defaultMode := maasclient.ModeDHCP + if ifaceName == "eth0" { + defaultMode = maasclient.ModeAuto + } + mm := s.scope.MaasMachine + if mm.Spec.LXD == nil || mm.Spec.LXD.VMConfig == nil || len(mm.Spec.LXD.VMConfig.InterfaceLinkModes) == 0 { + return defaultMode + } + m := strings.TrimSpace(mm.Spec.LXD.VMConfig.InterfaceLinkModes[ifaceName]) + allowed := map[string]bool{maasclient.ModeAuto: true, maasclient.ModeDHCP: true, maasclient.ModeStatic: true, maasclient.ModeLinkUp: true} + if m != "" && allowed[m] { + return m + } + return defaultMode +} + // VerifyVMNetworkInterfaces verifies and fixes LXD VM network interfaces to have expected subnets after commissioning. func (s *Service) VerifyVMNetworkInterfaces(ctx context.Context, systemID string) error { mm := s.scope.MaasMachine @@ -608,7 +635,7 @@ func (s *Service) VerifyVMNetworkInterfaces(ctx context.Context, systemID string return nil } - s.scope.Info("Verifying VM network interfaces", "system-id", systemID, "state", machineState, "expected-subnets", fmt.Sprintf("%s,%s", expected0, expected1)) + s.scope.Info("Verifying VM network interfaces", "system-id", systemID, "state", machineState, "expected-subnets", fmt.Sprintf("%s,%s", expected0, expected1), "eth0-mode", s.resolveLinkMode("eth0"), "eth1-mode", s.resolveLinkMode("eth1")) // Fetch network interfaces - refetch if we encounter issues with subnet data interfaces, err := s.maasClient.NetworkInterfaces().Get(ctx, systemID) @@ -690,11 +717,29 @@ func (s *Service) VerifyVMNetworkInterfaces(ctx context.Context, systemID string } } + // subnetsMatch returns true if actual and expected represent the same subnet (CIDR or name). + subnetsMatch := func(actual, expected string) bool { + if actual == "" || expected == "" { + return false + } + if strings.EqualFold(actual, expected) { + return true + } + // If both parse as CIDR, compare by network equality (robust to formatting) + _, actualNet, err1 := net.ParseCIDR(actual) + _, expectedNet, err2 := net.ParseCIDR(expected) + if err1 == nil && err2 == nil && actualNet != nil && expectedNet != nil { + return actualNet.IP.Equal(expectedNet.IP) && actualNet.Mask != nil && expectedNet.Mask != nil && + bytes.Equal(actualNet.Mask, expectedNet.Mask) + } + return false + } + var aggErr error if eth0Iface != nil { - if eth0Subnet == "" || !strings.EqualFold(eth0Subnet, expected0) { + if eth0Subnet == "" || !subnetsMatch(eth0Subnet, expected0) { s.scope.Info("Fixing eth0 subnet mismatch", "system-id", systemID, "expected", expected0, "actual", eth0Subnet) - if err := s.fixInterfaceSubnet(ctx, systemID, eth0Iface, expected0, "eth0"); err != nil { + if err := s.fixInterfaceSubnet(ctx, systemID, eth0Iface, expected0, "eth0", s.resolveLinkMode("eth0")); err != nil { s.scope.Error(err, "Failed to fix eth0 subnet", "system-id", systemID, "expected", expected0, "actual", eth0Subnet) aggErr = errors.Wrap(err, "eth0 subnet correction failed") } else { @@ -708,9 +753,9 @@ func (s *Service) VerifyVMNetworkInterfaces(ctx context.Context, systemID string } if eth1Iface != nil { - if eth1Subnet == "" || !strings.EqualFold(eth1Subnet, expected1) { + if eth1Subnet == "" || !subnetsMatch(eth1Subnet, expected1) { s.scope.Info("Fixing eth1 subnet mismatch", "system-id", systemID, "expected", expected1, "actual", eth1Subnet) - if err := s.fixInterfaceSubnet(ctx, systemID, eth1Iface, expected1, "eth1"); err != nil { + if err := s.fixInterfaceSubnet(ctx, systemID, eth1Iface, expected1, "eth1", s.resolveLinkMode("eth1")); err != nil { s.scope.Error(err, "Failed to fix eth1 subnet", "system-id", systemID, "expected", expected1, "actual", eth1Subnet) if aggErr != nil { aggErr = errors.Wrap(aggErr, err.Error()) @@ -734,7 +779,7 @@ func (s *Service) VerifyVMNetworkInterfaces(ctx context.Context, systemID string return aggErr } -func (s *Service) fixInterfaceSubnet(ctx context.Context, systemID string, iface maasclient.NetworkInterface, expectedSubnetIdentifier, ifaceName string) error { +func (s *Service) fixInterfaceSubnet(ctx context.Context, systemID string, iface maasclient.NetworkInterface, expectedSubnetIdentifier, ifaceName string, linkMode string) error { interfaceID := iface.ID() ifaceClient := s.maasClient.NetworkInterfaces().Interface(systemID, interfaceID) @@ -747,13 +792,19 @@ func (s *Service) fixInterfaceSubnet(ctx context.Context, systemID string, iface } } - // Link the subnet with auto IP assignment (empty string means auto/DHCP) - // The MAAS API accepts subnet names, CIDRs, or IDs directly - if err := ifaceClient.LinkSubnet(ctx, expectedSubnetIdentifier, ""); err != nil { - return fmt.Errorf("failed to link subnet %s to %s: %w", expectedSubnetIdentifier, ifaceName, err) + // Link the subnet with the given mode (from VMConfig.Eth0LinkMode/Eth1LinkMode or defaults). + // When linkMode is empty we use LinkSubnet (client default: auto when no IP). + if linkMode != "" { + if err := ifaceClient.LinkSubnetWithMode(ctx, expectedSubnetIdentifier, linkMode, ""); err != nil { + return fmt.Errorf("failed to link subnet %s to %s (mode=%s): %w", expectedSubnetIdentifier, ifaceName, linkMode, err) + } + } else { + if err := ifaceClient.LinkSubnet(ctx, expectedSubnetIdentifier, ""); err != nil { + return fmt.Errorf("failed to link subnet %s to %s: %w", expectedSubnetIdentifier, ifaceName, err) + } } - s.scope.Info("Fixed subnet on interface", "system-id", systemID, "interface", ifaceName, "subnet", expectedSubnetIdentifier) + s.scope.Info("Fixed subnet on interface", "system-id", systemID, "interface", ifaceName, "subnet", expectedSubnetIdentifier, "link-mode", linkMode) return nil } diff --git a/pkg/maas/machine/machine_test.go b/pkg/maas/machine/machine_test.go index 161c03b52..d0e168264 100644 --- a/pkg/maas/machine/machine_test.go +++ b/pkg/maas/machine/machine_test.go @@ -2,6 +2,7 @@ package machine import ( "context" + "fmt" "net" "testing" @@ -11,8 +12,10 @@ import ( "k8s.io/klog/v2/klogr" "sigs.k8s.io/cluster-api/api/v1beta1" + infrav1beta1 "github.com/spectrocloud/cluster-api-provider-maas/api/v1beta1" mockclientset "github.com/spectrocloud/cluster-api-provider-maas/pkg/maas/client/mock" "github.com/spectrocloud/cluster-api-provider-maas/pkg/maas/scope" + "github.com/spectrocloud/maas-client-go/maasclient" ) func TestMachine(t *testing.T) { @@ -105,7 +108,7 @@ func TestMachine(t *testing.T) { mockClientSetInterface.EXPECT().Machines().Return(mockMachines) mockMachines.EXPECT().Machine("abc123").Return(mockMachine) mockMachine.EXPECT().Releaser().Return(mockMachineReleaser) - mockMachineReleaser.EXPECT().Release(context.Background()).Return(mockMachine, nil) + mockMachineReleaser.EXPECT().Release(gomock.Any()).Return(mockMachine, nil) err := s.ReleaseMachine("abc123") g.Expect(err).ToNot(HaveOccurred()) @@ -229,3 +232,498 @@ func TestMachine(t *testing.T) { // g.Expect(machine).To(BeNil()) //}) } + +// ---- Fakes for network interface tests ---- + +type fakeNetworkInterfaces struct { + ifaces []maasclient.NetworkInterface + fetchErr error + captured *captureNetworkInterface // returned by Interface() +} + +func (f *fakeNetworkInterfaces) Get(_ context.Context, _ string) ([]maasclient.NetworkInterface, error) { + return f.ifaces, f.fetchErr +} +func (f *fakeNetworkInterfaces) Interface(_, _ string) maasclient.NetworkInterface { + if f.captured != nil { + return f.captured + } + return &captureNetworkInterface{} +} +func (f *fakeNetworkInterfaces) SetBootInterfaceStaticIP(_ context.Context, _, _ string) error { return nil } +func (f *fakeNetworkInterfaces) SetStaticIPOnInterfaceID(_ context.Context, _, _, _ string) error { + return nil +} +func (f *fakeNetworkInterfaces) CreateBridge(_ context.Context, _, _, _ string) (maasclient.NetworkInterface, error) { + return nil, nil +} +func (f *fakeNetworkInterfaces) CreateBootInterfaceBridge(_ context.Context, _, _ string) (maasclient.NetworkInterface, error) { + return nil, nil +} + +// captureNetworkInterface records the mode passed to LinkSubnetWithMode so tests can assert on it. +type captureNetworkInterface struct { + capturedMode string + capturedSubnetID string + linkErr error +} + +func (c *captureNetworkInterface) Get(_ context.Context) (maasclient.NetworkInterface, error) { return c, nil } +func (c *captureNetworkInterface) Update(_ context.Context, _ maasclient.Params) error { return nil } +func (c *captureNetworkInterface) LinkSubnet(_ context.Context, _ string, _ string) error { return c.linkErr } +func (c *captureNetworkInterface) LinkSubnetWithMode(_ context.Context, subnetID, mode, _ string) error { + c.capturedSubnetID = subnetID + c.capturedMode = mode + return c.linkErr +} +func (c *captureNetworkInterface) LinkSubnetWithForce(_ context.Context, subnetID, mode, _ string) error { + c.capturedSubnetID = subnetID + c.capturedMode = mode + return c.linkErr +} +func (c *captureNetworkInterface) UnlinkSubnet(_ context.Context, _ string) error { return nil } +func (c *captureNetworkInterface) UpdateIPConfiguration(_ context.Context, _ maasclient.IPConfigurationUpdate) error { + return nil +} +func (c *captureNetworkInterface) SetStaticIP(_ context.Context, _ string) error { return nil } +func (c *captureNetworkInterface) SetDHCP(_ context.Context, _ string) error { return nil } +func (c *captureNetworkInterface) ID() string { return "" } +func (c *captureNetworkInterface) Name() string { return "" } +func (c *captureNetworkInterface) Type() string { return "" } +func (c *captureNetworkInterface) Enabled() bool { return true } +func (c *captureNetworkInterface) MACAddress() string { return "" } +func (c *captureNetworkInterface) Links() []maasclient.NetworkInterfaceLink { return nil } +func (c *captureNetworkInterface) Children() []string { return nil } +func (c *captureNetworkInterface) VLAN() maasclient.VLAN { return nil } + +// fakeNetworkInterface is an in-memory NetworkInterface with a configurable name, ID, and links. +type fakeNetworkInterface struct { + id string + name string + links []maasclient.NetworkInterfaceLink +} + +func (f *fakeNetworkInterface) Get(_ context.Context) (maasclient.NetworkInterface, error) { return f, nil } +func (f *fakeNetworkInterface) Update(_ context.Context, _ maasclient.Params) error { return nil } +func (f *fakeNetworkInterface) LinkSubnet(_ context.Context, _, _ string) error { return nil } +func (f *fakeNetworkInterface) LinkSubnetWithMode(_ context.Context, _, _, _ string) error { return nil } +func (f *fakeNetworkInterface) LinkSubnetWithForce(_ context.Context, _, _, _ string) error { return nil } +func (f *fakeNetworkInterface) UnlinkSubnet(_ context.Context, _ string) error { return nil } +func (f *fakeNetworkInterface) UpdateIPConfiguration(_ context.Context, _ maasclient.IPConfigurationUpdate) error { + return nil +} +func (f *fakeNetworkInterface) SetStaticIP(_ context.Context, _ string) error { return nil } +func (f *fakeNetworkInterface) SetDHCP(_ context.Context, _ string) error { return nil } +func (f *fakeNetworkInterface) ID() string { return f.id } +func (f *fakeNetworkInterface) Name() string { return f.name } +func (f *fakeNetworkInterface) Type() string { return "" } +func (f *fakeNetworkInterface) Enabled() bool { return true } +func (f *fakeNetworkInterface) MACAddress() string { return "" } +func (f *fakeNetworkInterface) Links() []maasclient.NetworkInterfaceLink { return f.links } +func (f *fakeNetworkInterface) Children() []string { return nil } +func (f *fakeNetworkInterface) VLAN() maasclient.VLAN { return nil } + +// fakeNetworkInterfaceLink is a minimal NetworkInterfaceLink with a configurable subnet. +type fakeNetworkInterfaceLink struct { + id string + subnet maasclient.Subnet +} + +func (l *fakeNetworkInterfaceLink) ID() string { return l.id } +func (l *fakeNetworkInterfaceLink) Mode() string { return "" } +func (l *fakeNetworkInterfaceLink) Subnet() maasclient.Subnet { return l.subnet } +func (l *fakeNetworkInterfaceLink) IPAddress() net.IP { return nil } + +// fakeSubnet is a minimal Subnet whose CIDR is used for subnet matching. +type fakeSubnet struct { + cidr string +} + +func (s *fakeSubnet) ID() int { return 0 } +func (s *fakeSubnet) Name() string { return s.cidr } +func (s *fakeSubnet) Space() string { return "" } +func (s *fakeSubnet) VLAN() maasclient.VLAN { return nil } +func (s *fakeSubnet) CIDR() string { return s.cidr } + +// ---- TestResolveLinkMode ---- + +func TestResolveLinkMode(t *testing.T) { + cases := []struct { + name string + lxd *infrav1beta1.MachineLXDConfig + ifaceName string + expected string + }{ + // Bullet 1: default mode selection + { + name: "default eth0 no lxd config", + ifaceName: "eth0", + expected: maasclient.ModeAuto, + }, + { + name: "default eth1 no lxd config", + ifaceName: "eth1", + expected: maasclient.ModeDHCP, + }, + { + name: "default eth2 no lxd config", + ifaceName: "eth2", + expected: maasclient.ModeDHCP, + }, + // Bullet 2: custom mode from InterfaceLinkModes map + { + name: "custom static on eth0", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth0": maasclient.ModeStatic}}}, + ifaceName: "eth0", + expected: maasclient.ModeStatic, + }, + { + name: "custom link_up on eth1", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth1": maasclient.ModeLinkUp}}}, + ifaceName: "eth1", + expected: maasclient.ModeLinkUp, + }, + { + name: "custom dhcp on eth0 overrides auto default", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth0": maasclient.ModeDHCP}}}, + ifaceName: "eth0", + expected: maasclient.ModeDHCP, + }, + // Bullet 3: all four allowed modes are accepted + { + name: "valid auto on eth1", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth1": maasclient.ModeAuto}}}, + ifaceName: "eth1", + expected: maasclient.ModeAuto, + }, + { + name: "valid dhcp on eth1", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth1": maasclient.ModeDHCP}}}, + ifaceName: "eth1", + expected: maasclient.ModeDHCP, + }, + { + name: "valid static on eth1", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth1": maasclient.ModeStatic}}}, + ifaceName: "eth1", + expected: maasclient.ModeStatic, + }, + // Bullet 4: invalid/empty modes fall back to the interface default + { + name: "invalid mode eth0 falls back to auto", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth0": "bogus"}}}, + ifaceName: "eth0", + expected: maasclient.ModeAuto, + }, + { + name: "invalid mode eth1 falls back to dhcp", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth1": "bogus"}}}, + ifaceName: "eth1", + expected: maasclient.ModeDHCP, + }, + { + name: "empty mode eth0 falls back to auto", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth0": ""}}}, + ifaceName: "eth0", + expected: maasclient.ModeAuto, + }, + { + name: "whitespace-only mode falls back to auto", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{"eth0": " "}}}, + ifaceName: "eth0", + expected: maasclient.ModeAuto, + }, + // Nil guards + { + name: "lxd present vmconfig nil", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: nil}, + ifaceName: "eth1", + expected: maasclient.ModeDHCP, + }, + { + name: "interface link modes empty map", + lxd: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{InterfaceLinkModes: map[string]string{}}}, + ifaceName: "eth0", + expected: maasclient.ModeAuto, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := &Service{ + scope: &scope.MachineScope{ + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{LXD: tc.lxd}, + }, + }, + } + got := s.resolveLinkMode(tc.ifaceName) + if got != tc.expected { + t.Errorf("resolveLinkMode(%q) = %q, want %q", tc.ifaceName, got, tc.expected) + } + }) + } +} + +// ---- TestVerifyVMNetworkInterfaces ---- + +func TestVerifyVMNetworkInterfaces(t *testing.T) { + log := klogr.New() + + // helpers to build fake interfaces with a single link pointing to a named subnet + eth0With := func(cidr string) *fakeNetworkInterface { + return &fakeNetworkInterface{ + id: "iface-0", + name: "eth0", + links: []maasclient.NetworkInterfaceLink{ + &fakeNetworkInterfaceLink{id: "link-0", subnet: &fakeSubnet{cidr: cidr}}, + }, + } + } + eth1With := func(cidr string) *fakeNetworkInterface { + return &fakeNetworkInterface{ + id: "iface-1", + name: "eth1", + links: []maasclient.NetworkInterfaceLink{ + &fakeNetworkInterfaceLink{id: "link-1", subnet: &fakeSubnet{cidr: cidr}}, + }, + } + } + + t.Run("lxd nil returns nil immediately", func(t *testing.T) { + g := NewGomegaWithT(t) + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{Spec: infrav1beta1.MaasMachineSpec{LXD: nil}}, + }, + } + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + }) + + t.Run("vmconfig nil returns nil immediately", func(t *testing.T) { + g := NewGomegaWithT(t) + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: nil}, + }, + }, + }, + } + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + }) + + t.Run("network empty returns nil immediately", func(t *testing.T) { + g := NewGomegaWithT(t) + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{Network: ""}}, + }, + }, + }, + } + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + }) + + t.Run("network with one subnet skips verification", func(t *testing.T) { + g := NewGomegaWithT(t) + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{Network: "only-one-subnet"}}, + }, + }, + }, + } + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + }) + + t.Run("machine in Deployed state is skipped", func(t *testing.T) { + g := NewGomegaWithT(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockCS := mockclientset.NewMockClientSetInterface(ctrl) + mockMachines := mockclientset.NewMockMachines(ctrl) + mockMachine := mockclientset.NewMockMachine(ctrl) + + mockCS.EXPECT().Machines().Return(mockMachines) + mockMachines.EXPECT().Machine("sys-1").Return(mockMachine) + mockMachine.EXPECT().Get(gomock.Any()).Return(mockMachine, nil) + mockMachine.EXPECT().State().Return("Deployed") + + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{Network: "sn0,sn1"}}, + }, + }, + }, + maasClient: mockCS, + } + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + }) + + t.Run("subnets already match no fixes applied", func(t *testing.T) { + g := NewGomegaWithT(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockCS := mockclientset.NewMockClientSetInterface(ctrl) + mockMachines := mockclientset.NewMockMachines(ctrl) + mockMachine := mockclientset.NewMockMachine(ctrl) + + mockCS.EXPECT().Machines().Return(mockMachines) + mockMachines.EXPECT().Machine("sys-1").Return(mockMachine) + mockMachine.EXPECT().Get(gomock.Any()).Return(mockMachine, nil) + mockMachine.EXPECT().State().Return("Ready") + mockCS.EXPECT().NetworkInterfaces().Return(&fakeNetworkInterfaces{ + ifaces: []maasclient.NetworkInterface{eth0With("sn0"), eth1With("sn1")}, + }) + + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{Network: "sn0,sn1"}}, + }, + }, + }, + maasClient: mockCS, + } + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + }) + + t.Run("eth0 subnet mismatch fixed with default auto mode", func(t *testing.T) { + g := NewGomegaWithT(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockCS := mockclientset.NewMockClientSetInterface(ctrl) + mockMachines := mockclientset.NewMockMachines(ctrl) + mockMachine := mockclientset.NewMockMachine(ctrl) + + captureIface := &captureNetworkInterface{} + fakeNI := &fakeNetworkInterfaces{ + ifaces: []maasclient.NetworkInterface{eth0With("10.0.0.0/24"), eth1With("sn1")}, + captured: captureIface, + } + + mockCS.EXPECT().Machines().Return(mockMachines) + mockMachines.EXPECT().Machine("sys-1").Return(mockMachine) + mockMachine.EXPECT().Get(gomock.Any()).Return(mockMachine, nil) + mockMachine.EXPECT().State().Return("New") + mockCS.EXPECT().NetworkInterfaces().Return(fakeNI).AnyTimes() + + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{Network: "sn0,sn1"}}, + }, + }, + }, + maasClient: mockCS, + } + + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + if captureIface.capturedMode != maasclient.ModeAuto { + t.Errorf("link mode = %q, want %q", captureIface.capturedMode, maasclient.ModeAuto) + } + if captureIface.capturedSubnetID != "sn0" { + t.Errorf("subnet ID = %q, want %q", captureIface.capturedSubnetID, "sn0") + } + }) + + t.Run("eth0 subnet mismatch fixed with custom static mode", func(t *testing.T) { + g := NewGomegaWithT(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockCS := mockclientset.NewMockClientSetInterface(ctrl) + mockMachines := mockclientset.NewMockMachines(ctrl) + mockMachine := mockclientset.NewMockMachine(ctrl) + + captureIface := &captureNetworkInterface{} + fakeNI := &fakeNetworkInterfaces{ + ifaces: []maasclient.NetworkInterface{eth0With("10.0.0.0/24"), eth1With("sn1")}, + captured: captureIface, + } + + mockCS.EXPECT().Machines().Return(mockMachines) + mockMachines.EXPECT().Machine("sys-1").Return(mockMachine) + mockMachine.EXPECT().Get(gomock.Any()).Return(mockMachine, nil) + mockMachine.EXPECT().State().Return("New") + mockCS.EXPECT().NetworkInterfaces().Return(fakeNI).AnyTimes() + + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{ + VMConfig: &infrav1beta1.VMConfig{ + Network: "sn0,sn1", + InterfaceLinkModes: map[string]string{"eth0": maasclient.ModeStatic}, + }, + }, + }, + }, + }, + maasClient: mockCS, + } + + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(Succeed()) + if captureIface.capturedMode != maasclient.ModeStatic { + t.Errorf("link mode = %q, want %q", captureIface.capturedMode, maasclient.ModeStatic) + } + }) + + t.Run("fixInterfaceSubnet error is propagated", func(t *testing.T) { + g := NewGomegaWithT(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockCS := mockclientset.NewMockClientSetInterface(ctrl) + mockMachines := mockclientset.NewMockMachines(ctrl) + mockMachine := mockclientset.NewMockMachine(ctrl) + + captureIface := &captureNetworkInterface{linkErr: fmt.Errorf("link failed")} + fakeNI := &fakeNetworkInterfaces{ + ifaces: []maasclient.NetworkInterface{eth0With("10.0.0.0/24"), eth1With("sn1")}, + captured: captureIface, + } + + mockCS.EXPECT().Machines().Return(mockMachines) + mockMachines.EXPECT().Machine("sys-1").Return(mockMachine) + mockMachine.EXPECT().Get(gomock.Any()).Return(mockMachine, nil) + mockMachine.EXPECT().State().Return("New") + mockCS.EXPECT().NetworkInterfaces().Return(fakeNI).AnyTimes() + + s := &Service{ + scope: &scope.MachineScope{ + Logger: log, + MaasMachine: &infrav1beta1.MaasMachine{ + Spec: infrav1beta1.MaasMachineSpec{ + LXD: &infrav1beta1.MachineLXDConfig{VMConfig: &infrav1beta1.VMConfig{Network: "sn0,sn1"}}, + }, + }, + }, + maasClient: mockCS, + } + + g.Expect(s.VerifyVMNetworkInterfaces(context.Background(), "sys-1")).To(HaveOccurred()) + }) +} From 56ad02a708820ee4070dff8774b88134ee598bff Mon Sep 17 00:00:00 2001 From: Kun Zhou <156021375+Kun483@users.noreply.github.com> Date: Thu, 7 May 2026 11:45:05 -0700 Subject: [PATCH 2/4] change to use laetst version of maas sdk to include storage overcommit prevention fix (#338) --- Makefile | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5eb3a4597..2589a7015 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ endif # Image URL to use all building/pushing image targets IMAGE_NAME := cluster-api-provider-maas-controller REGISTRY ?= "us-east1-docker.pkg.dev/spectro-images/dev/${USER}/cluster-api" -SPECTRO_VERSION ?= 4.8.3-dev-tmo-19022026 +SPECTRO_VERSION ?= storage-overcommit-prevention-20260506 IMG_TAG ?= v0.6.1-spectro-${SPECTRO_VERSION} CONTROLLER_IMG ?= ${REGISTRY}/${IMAGE_NAME} diff --git a/go.mod b/go.mod index 67b7283eb..13fb2350c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.36.3 github.com/pkg/errors v0.9.1 - github.com/spectrocloud/maas-client-go v0.1.5-beta1 + github.com/spectrocloud/maas-client-go v0.1.7-beta1 github.com/spf13/pflag v1.0.6 k8s.io/api v0.32.3 k8s.io/apiextensions-apiserver v0.32.3 diff --git a/go.sum b/go.sum index 100a0dd42..2c9ace5e4 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/spectrocloud/maas-client-go v0.1.5-beta1 h1:OT16996QBW8krHTIctRuO0js+NhB6ODHoQTyiTLxoEU= -github.com/spectrocloud/maas-client-go v0.1.5-beta1/go.mod h1:LSxLlmaNCmkaldtysbp7Beq/O2wptBb6qE5iKj+Y7Lw= +github.com/spectrocloud/maas-client-go v0.1.7-beta1 h1:j5NtaNPTxoh2PxhKTJYpZUeJukWy4vC7xT2yy+1syDk= +github.com/spectrocloud/maas-client-go v0.1.7-beta1/go.mod h1:LSxLlmaNCmkaldtysbp7Beq/O2wptBb6qE5iKj+Y7Lw= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= From 27104a0d19a1deee0f1f664752f38b91de367f98 Mon Sep 17 00:00:00 2001 From: Kun Zhou <156021375+Kun483@users.noreply.github.com> Date: Mon, 16 Mar 2026 15:53:01 -0700 Subject: [PATCH 3/4] Add MinDiskSizeGB option to SelectOptions and implement storage checks in host selection (#325) --- pkg/maas/lxd/host_maas_client.go | 31 +++++++++++++++++++++++++++++-- pkg/maas/machine/machine.go | 11 ++++++----- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pkg/maas/lxd/host_maas_client.go b/pkg/maas/lxd/host_maas_client.go index 08a57cc81..909cf4014 100644 --- a/pkg/maas/lxd/host_maas_client.go +++ b/pkg/maas/lxd/host_maas_client.go @@ -274,6 +274,7 @@ type SelectOptions struct { Tags []string // All required on VM host (host.Tags()); nil/empty = any MinCores int // Minimum available cores on host; 0 = no minimum MinMemory int // Minimum available memory (MB) on host; 0 = no minimum + MinDiskSizeGB int // Minimum uncommitted disk space (GB) on host; 0 = no minimum // ClusterID When set (non-empty), the selector counts how many control-plane VMs for this // cluster already exist on each eligible host. Hosts with fewer existing CP VMs @@ -377,7 +378,33 @@ func SelectLXDHostWithMaasClient(client lxdHostSelectorClient, hosts []maasclien continue } - // 5. Health check: backing machine must be powered on and Deployed + // 5. Storage filter: ensure at least one local storage pool has enough uncommitted space. + // MAAS allows over-commitment by counting pending (allocated but not yet used) storage + // as available. We subtract pending to prevent composing VMs onto a host that is + // already over-committed at the storage layer. + if opts.MinDiskSizeGB > 0 { + // MAAS reports storage in decimal bytes (1 GB = 1,000,000,000 bytes), + // matching what the MAAS UI displays — use 1000^3, not 1024^3. + requiredBytes := int64(opts.MinDiskSizeGB) * 1000 * 1000 * 1000 + hasStorage := false + for _, pool := range host.StoragePools() { + if pool.Remote { + continue + } + uncommitted := pool.Available - pool.Pending + if uncommitted >= requiredBytes { + hasStorage = true + break + } + } + if !hasStorage { + log.Info("Skipping host: insufficient uncommitted storage", "host", host.Name(), + "requiredGB", opts.MinDiskSizeGB, "pools", host.StoragePools()) + continue + } + } + + // 7. Health check: backing machine must be powered on and Deployed hostSystemID := host.HostSystemID() if hostSystemID == "" { continue @@ -393,7 +420,7 @@ func SelectLXDHostWithMaasClient(client lxdHostSelectorClient, hosts []maasclien continue } - // 6. Maintenance check + // 8. Maintenance check if hasMaintenanceTag(machine.Tags()) { log.Info("Skipping host under maintenance", "host", host.Name()) continue diff --git a/pkg/maas/machine/machine.go b/pkg/maas/machine/machine.go index cf33dbfaf..878b1036b 100644 --- a/pkg/maas/machine/machine.go +++ b/pkg/maas/machine/machine.go @@ -432,11 +432,12 @@ func (s *Service) PrepareLXDVM(ctx context.Context) (*infrav1beta1.Machine, erro // Build SelectOptions from MaasMachine spec selectOpts := lxd.SelectOptions{ - Zone: zone, - ResourcePool: resourcePool, - Tags: mm.Spec.Tags, - MinCores: cpu, - MinMemory: mem, + Zone: zone, + ResourcePool: resourcePool, + Tags: mm.Spec.Tags, + MinCores: cpu, + MinMemory: mem, + MinDiskSizeGB: diskSizeGB, } if s.scope.IsControlPlane() { From 70dde135859d83647bdba913e49b9d4d442b358f Mon Sep 17 00:00:00 2001 From: kun zhou Date: Fri, 15 May 2026 16:00:14 -0700 Subject: [PATCH 4/4] fix go vulnerabilities --- go.mod | 10 +++++----- go.sum | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 13fb2350c..909d8b33d 100644 --- a/go.mod +++ b/go.mod @@ -58,12 +58,12 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/otel v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.28.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/term v0.42.0 // indirect + golang.org/x/text v0.36.0 // indirect golang.org/x/time v0.8.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/protobuf v1.36.5 // indirect diff --git a/go.sum b/go.sum index 2c9ace5e4..44fa909e0 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -231,8 +231,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -240,8 +240,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -254,15 +254,15 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -271,8 +271,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=