diff --git a/api/v1beta1/azurecluster_validation.go b/api/v1beta1/azurecluster_validation.go
index 479e9cddd4b..9a3dc545358 100644
--- a/api/v1beta1/azurecluster_validation.go
+++ b/api/v1beta1/azurecluster_validation.go
@@ -433,12 +433,19 @@ func validateAPIServerLB(lb *LoadBalancerSpec, old *LoadBalancerSpec, cidrs []st
 			if err := validateInternalLBIPAddress(privateIP, cidrs, fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP")); err != nil {
 				allErrs = append(allErrs, err)
 			}
-		} else {
-			// API Server LB should not have a Private IP if APIServerILB feature is disabled.
-			if privateIPCount > 0 {
-				allErrs = append(allErrs, field.Forbidden(fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP"),
-					"Public Load Balancers cannot have a Private IP"))
+
+			if old != nil && len(old.FrontendIPs) > 0 && len(lb.FrontendIPs) > 0 {
+				oldIP := old.FrontendIPs[0].PrivateIPAddress
+				newIP := lb.FrontendIPs[0].PrivateIPAddress
+				if newIP != oldIP {
+					allErrs = append(allErrs, field.Forbidden(fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP"),
+						"field is immutable"))
+				}
 			}
+		} else if privateIPCount > 0 {
+			// API Server LB should not have a Private IP if APIServerILB feature is disabled.
+			allErrs = append(allErrs, field.Forbidden(fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP"),
+				"Public Load Balancers cannot have a Private IP"))
 		}
 	}
 
@@ -457,8 +464,13 @@ func validateAPIServerLB(lb *LoadBalancerSpec, old *LoadBalancerSpec, cidrs []st
 				allErrs = append(allErrs, err)
 			}
 
-			if old != nil && len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress {
-				allErrs = append(allErrs, field.Forbidden(fldPath.Child("name"), "API Server load balancer private IP should not be modified after AzureCluster creation."))
+			if old != nil && len(old.FrontendIPs) > 0 && len(lb.FrontendIPs) > 0 {
+				oldIP := old.FrontendIPs[0].PrivateIPAddress
+				newIP := lb.FrontendIPs[0].PrivateIPAddress
+				if newIP != oldIP {
+					allErrs = append(allErrs, field.Forbidden(fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP"),
+						"field is immutable"))
+				}
 			}
 		}
 	}
diff --git a/api/v1beta1/azurecluster_validation_test.go b/api/v1beta1/azurecluster_validation_test.go
index 5859f0aae01..1bd86fcd89a 100644
--- a/api/v1beta1/azurecluster_validation_test.go
+++ b/api/v1beta1/azurecluster_validation_test.go
@@ -1285,6 +1285,69 @@ func TestValidateAPIServerLB(t *testing.T) {
 				Detail:   "Internal LB IP address needs to be in control plane subnet range ([10.0.0.0/24 10.1.0.0/24])",
 			},
 		},
+		{
+			name:        "public LB + APIServerILB: changing private IP after creation is forbidden",
+			featureGate: feature.APIServerILB,
+			old: &LoadBalancerSpec{
+				LoadBalancerClassSpec: LoadBalancerClassSpec{Type: Public, SKU: SKUStandard},
+				Name:                  "my-public-lb",
+				FrontendIPs: []FrontendIP{
+					{
+						Name:            "ip-priv",
+						FrontendIPClass: FrontendIPClass{PrivateIPAddress: "10.0.0.10"},
+					},
+					{
+						Name:     "ip-pub",
+						PublicIP: &PublicIPSpec{Name: "pub", DNSName: "pub"},
+					},
+				},
+			},
+			lb: &LoadBalancerSpec{
+				LoadBalancerClassSpec: LoadBalancerClassSpec{Type: Public, SKU: SKUStandard},
+				Name:                  "my-public-lb",
+				FrontendIPs: []FrontendIP{
+					{
+						Name:            "ip-priv",
+						FrontendIPClass: FrontendIPClass{PrivateIPAddress: "10.0.0.11"},
+					},
+					{
+						Name:     "ip-pub",
+						PublicIP: &PublicIPSpec{Name: "pub", DNSName: "pub"},
+					},
+				},
+			},
+			cpCIDRS: []string{"10.0.0.0/24"},
+			wantErr: true,
+			expectedErr: field.Error{
+				Type:   "FieldValueForbidden",
+				Field:  "apiServerLB.frontendIPConfigs[0].privateIP",
+				Detail: "field is immutable",
+			},
+		},
+		{
+			name: "internal LB: changing private IP after creation is forbidden",
+			old: &LoadBalancerSpec{
+				LoadBalancerClassSpec: LoadBalancerClassSpec{Type: Internal, SKU: SKUStandard},
+				Name:                  "my-private-lb",
+				FrontendIPs: []FrontendIP{
+					{Name: "ip-1", FrontendIPClass: FrontendIPClass{PrivateIPAddress: "10.1.0.3"}},
+				},
+			},
+			lb: &LoadBalancerSpec{
+				LoadBalancerClassSpec: LoadBalancerClassSpec{Type: Internal, SKU: SKUStandard},
+				Name:                  "my-private-lb",
+				FrontendIPs: []FrontendIP{
+					{Name: "ip-1", FrontendIPClass: FrontendIPClass{PrivateIPAddress: "10.1.0.4"}},
+				},
+			},
+			cpCIDRS: []string{"10.1.0.0/24"},
+			wantErr: true,
+			expectedErr: field.Error{
+				Type:   "FieldValueForbidden",
+				Field:  "apiServerLB.frontendIPConfigs[0].privateIP",
+				Detail: "field is immutable",
+			},
+		},
 	}
 
 	for _, test := range testcases {