diff --git a/pkg/autopilot/checks/checks.go b/pkg/autopilot/checks/checks.go index 3cd8395d3392..e6604f652612 100644 --- a/pkg/autopilot/checks/checks.go +++ b/pkg/autopilot/checks/checks.go @@ -17,9 +17,11 @@ package checks import ( "context" "fmt" + "strings" "github.com/k0sproject/k0s/pkg/kubernetes" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/metadata" @@ -51,6 +53,12 @@ func CanUpdate(ctx context.Context, log logrus.FieldLogger, clientFactory kubern } for _, ar := range r.APIResources { + // Skip resources which don't have the same name and kind. This is to skip + // subresources such as FlowSchema/Status + if strings.Contains(ar.Name, "/") { + continue + } + gv := gv // Copy over the default GroupVersion from the list // Apply resource-specific overrides if ar.Group != "" { @@ -60,7 +68,7 @@ func CanUpdate(ctx context.Context, log logrus.FieldLogger, clientFactory kubern gv.Version = ar.Version } - removedInVersion := removedInVersion(gv.WithKind(ar.Kind)) + removedInVersion, currentVersion := removedInVersion(gv.WithKind(ar.Kind)) if removedInVersion == "" || semver.Compare(newVersion, removedInVersion) < 0 { continue } @@ -84,7 +92,28 @@ func CanUpdate(ctx context.Context, log logrus.FieldLogger, clientFactory kubern } if found := len(metas.Items); found > 0 { - return fmt.Errorf("%s.%s %s has been removed in Kubernetes %s, but there are %d such resources in the cluster", ar.Name, gv.Group, gv.Version, removedInVersion, found) + if currentVersion == "" { + return fmt.Errorf("%s.%s %s has been removed in Kubernetes %s, but there are %d such resources in the cluster", ar.Name, gv.Group, gv.Version, removedInVersion, found) + } + // If we find removed APIs, it could be because the APIserver is serving the same object with an older GVK + // for compatibility reasons while the current good API still works. + newGV := gv + newGV.Version = currentVersion + outdatedItems := []metav1.PartialObjectMetadata{} + for _, item := range metas.Items { + // Currently none of the deleted resources are namespaced, so we can skip the namespace check. + // However we keep it in the list so that it breaks if we add a namespaced resource. + _, err := metaClient.Resource(newGV.WithResource(ar.Name)). + Get(ctx, item.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + outdatedItems = append(outdatedItems, item) + } else if err != nil { + return err + } + } + if foundOutdated := len(outdatedItems); foundOutdated > 0 { + return fmt.Errorf("%s.%s %s has been removed in Kubernetes %s, but there are %d such resources in the cluster", ar.Name, gv.Group, gv.Version, removedInVersion, found) + } } } } diff --git a/pkg/autopilot/checks/removedapis.go b/pkg/autopilot/checks/removedapis.go index 4a5a4d7001c3..6c042ac66b35 100644 --- a/pkg/autopilot/checks/removedapis.go +++ b/pkg/autopilot/checks/removedapis.go @@ -22,11 +22,15 @@ import ( ) type removedAPI struct { - group, version, kind, removedInVersion string + group, version, kind, removedInK8sVersion string + // currentAPIVersion declares a version that is still supported for the Group Kind. + // If it's empty, it means that the Group Kind is removed in the removedInVersion. + currentAPIVersion string } -// Returns the Kubernetes version in which candidate has been removed, if any. -func removedInVersion(candidate schema.GroupVersionKind) string { +// If candidate has been removed, returns the kubernetes version in which it was removed +// and the current version for Group Kind. +func removedInVersion(candidate schema.GroupVersionKind) (string, string) { if idx, found := sort.Find(len(removedGVKs), func(i int) int { if cmp := cmp.Compare(candidate.Group, removedGVKs[i].group); cmp != 0 { return cmp @@ -36,77 +40,17 @@ func removedInVersion(candidate schema.GroupVersionKind) string { } return cmp.Compare(candidate.Kind, removedGVKs[i].kind) }); found { - return removedGVKs[idx].removedInVersion + return removedGVKs[idx].removedInK8sVersion, removedGVKs[idx].currentAPIVersion } - return "" + return "", "" } // Sorted array of removed APIs. -var removedGVKs = [65]removedAPI{ - {"admissionregistration.k8s.io", "v1beta1", "MutatingWebhookConfiguration", "v1.22.0"}, - {"admissionregistration.k8s.io", "v1beta1", "ValidatingWebhookConfiguration", "v1.22.0"}, - {"apiextensions.k8s.io", "v1beta1", "CustomResourceDefinition", "v1.22.0"}, - {"apiregistration.k8s.io", "v1beta1", "APIService", "v1.22.0"}, - {"apps", "v1beta1", "Deployment", "v1.16.0"}, - {"apps", "v1beta1", "ReplicaSet", "v1.16.0"}, - {"apps", "v1beta1", "StatefulSet", "v1.16.0"}, - {"apps", "v1beta2", "DaemonSet", "v1.16.0"}, - {"apps", "v1beta2", "Deployment", "v1.16.0"}, - {"apps", "v1beta2", "ReplicaSet", "v1.16.0"}, - {"apps", "v1beta2", "StatefulSet", "v1.16.0"}, - {"authentication.k8s.io", "v1beta1", "TokenReview", "v1.22.0"}, - {"autoscaling", "v2beta1", "HorizontalPodAutoscaler", "v1.25.0"}, - {"autoscaling", "v2beta1", "HorizontalPodAutoscalerList", "v1.25.0"}, - {"autoscaling", "v2beta2", "HorizontalPodAutoscaler", "v1.26.0"}, - {"autoscaling", "v2beta2", "HorizontalPodAutoscalerList", "v1.26.0"}, - {"batch", "v1beta1", "CronJob", "v1.25.0"}, - {"batch", "v1beta1", "CronJobList", "v1.25.0"}, - {"certificates.k8s.io", "v1beta1", "CertificateSigningRequest", "v1.22.0"}, - {"coordination.k8s.io", "v1beta1", "Lease", "v1.22.0"}, - {"discovery.k8s.io", "v1beta1", "EndpointSlice", "v1.25.0"}, - {"events.k8s.io", "v1beta1", "Event", "v1.25.0"}, - {"extensions", "v1beta1", "DaemonSet", "v1.16.0"}, - {"extensions", "v1beta1", "Deployment", "v1.16.0"}, - {"extensions", "v1beta1", "Ingress", "v1.22.0"}, - {"extensions", "v1beta1", "NetworkPolicy", "v1.16.0"}, - {"extensions", "v1beta1", "PodSecurityPolicy", "v1.16.0"}, - {"extensions", "v1beta1", "ReplicaSet", "v1.16.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta1", "FlowControl", "v1.26.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta1", "FlowSchema", "v1.26.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta1", "PriorityLevelConfiguration", "v1.26.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta2", "FlowSchema", "v1.29.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta2", "PriorityLevelConfiguration", "v1.29.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta3", "FlowSchema", "v1.32.0"}, - {"flowcontrol.apiserver.k8s.io", "v1beta3", "PriorityLevelConfiguration", "v1.32.0"}, - {"k0s.k0sproject.example.com", "v1beta1", "RemovedCRD", "v99.99.99"}, // This is a test entry - {"networking.k8s.io", "v1beta1", "Ingress", "v1.22.0"}, - {"networking.k8s.io", "v1beta1", "IngressClass", "v1.22.0"}, - {"policy", "v1beta1", "PodDisruptionBudget", "v1.25.0"}, - {"policy", "v1beta1", "PodDisruptionBudgetList", "v1.25.0"}, - {"policy", "v1beta1", "PodSecurityPolicy", "v1.25.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "ClusterRole", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "ClusterRoleBinding", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "ClusterRoleBindingList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "ClusterRoleList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "Role", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "RoleBinding", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "RoleBindingList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1alpha1", "RoleList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "ClusterRole", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "ClusterRoleBinding", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "ClusterRoleBindingList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "ClusterRoleList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "Role", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "RoleBinding", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "RoleBindingList", "v1.22.0"}, - {"rbac.authorization.k8s.io", "v1beta1", "RoleList", "v1.22.0"}, - {"scheduling.k8s.io", "v1alpha1", "PriorityClass", "v1.17.0"}, - {"scheduling.k8s.io", "v1beta1", "PriorityClass", "v1.22.0"}, - {"storage.k8s.io", "v1beta1", "CSIDriver", "v1.22.0"}, - {"storage.k8s.io", "v1beta1", "CSINode", "v1.22.0"}, - {"storage.k8s.io", "v1beta1", "CSIStorageCapacity", "v1.27.0"}, - {"storage.k8s.io", "v1beta1", "CSIStorageCapacity", "v1.27.0"}, - {"storage.k8s.io", "v1beta1", "StorageClass", "v1.22.0"}, - {"storage.k8s.io", "v1beta1", "VolumeAttachment", "v1.22.0"}, +var removedGVKs = [...]removedAPI{ + {"flowcontrol.apiserver.k8s.io", "v1beta2", "FlowSchema", "v1.29.0", "v1beta3"}, + {"flowcontrol.apiserver.k8s.io", "v1beta2", "PriorityLevelConfiguration", "v1.29.0", "v1"}, + {"flowcontrol.apiserver.k8s.io", "v1beta3", "FlowSchema", "v1.32.0", "v1"}, + {"flowcontrol.apiserver.k8s.io", "v1beta3", "PriorityLevelConfiguration", "v1.32.0", "v1"}, + {"k0s.k0sproject.example.com", "v1beta1", "RemovedCRD", "v99.99.99", ""}, // This is a test entry } diff --git a/pkg/autopilot/checks/removedapis_test.go b/pkg/autopilot/checks/removedapis_test.go index 46bacf83b569..e9bd4640f3d5 100644 --- a/pkg/autopilot/checks/removedapis_test.go +++ b/pkg/autopilot/checks/removedapis_test.go @@ -19,9 +19,8 @@ import ( "slices" "testing" - "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/runtime/schema" ) func TestRemovedGVKs(t *testing.T) { @@ -36,10 +35,21 @@ func TestRemovedGVKs(t *testing.T) { }), "removedGVKs needs to be sorted, so that it can be used for binary searches") // Test two random entries at the top and the bottom of the list - assert.Equal(t, "v1.22.0", removedInVersion(schema.GroupVersionKind{ - Group: "apiregistration.k8s.io", Version: "v1beta1", Kind: "APIService", - })) - assert.Equal(t, "v1.27.0", removedInVersion(schema.GroupVersionKind{ - Group: "storage.k8s.io", Version: "v1beta1", Kind: "CSIStorageCapacity", - })) + version, currentVersion := removedInVersion(schema.GroupVersionKind{ + Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema", + }) + assert.Equal(t, "v1.29.0", version) + assert.Equal(t, "v1beta3", currentVersion) + + version, currentVersion = removedInVersion(schema.GroupVersionKind{ + Group: "k0s.k0sproject.example.com", Version: "v1beta1", Kind: "RemovedCRD", + }) + assert.Equal(t, "v99.99.99", version) + assert.Equal(t, "", currentVersion) + + version, currentVersion = removedInVersion(schema.GroupVersionKind{ + Group: "k0s.k0sproject.example.com", Version: "v1beta1", Kind: "MustFail", + }) + assert.Equal(t, "", version) + assert.Equal(t, "", currentVersion) }