Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport release-1.32] Validate removed apis #5604

Merged
merged 2 commits into from
Feb 24, 2025
+64 −81
Merged

[Backport release-1.32] Validate removed apis #5604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4d32b6e
Remove ancient APIs from the Removed API list
juanluisvaladas Jan 24, 2025
3d9a87b
Removed APIs, ensure object is available
juanluisvaladas Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions pkg/autopilot/checks/checks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ package checks
import (
"context"
"fmt"
"strings"

"github.com/k0sproject/k0s/pkg/kubernetes"

apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/metadata"
Expand Down Expand Up @@ -51,6 +53,12 @@ func CanUpdate(ctx context.Context, log logrus.FieldLogger, clientFactory kubern
}

for _, ar := range r.APIResources {
// Skip resources which don't have the same name and kind. This is to skip
// subresources such as FlowSchema/Status
if strings.Contains(ar.Name, "/") {
continue
}

gv := gv // Copy over the default GroupVersion from the list
// Apply resource-specific overrides
if ar.Group != "" {
Expand All @@ -60,7 +68,7 @@ func CanUpdate(ctx context.Context, log logrus.FieldLogger, clientFactory kubern
gv.Version = ar.Version
}

removedInVersion := removedInVersion(gv.WithKind(ar.Kind))
removedInVersion, currentVersion := removedInVersion(gv.WithKind(ar.Kind))
if removedInVersion == "" || semver.Compare(newVersion, removedInVersion) < 0 {
continue
}
Expand All @@ -84,7 +92,28 @@ func CanUpdate(ctx context.Context, log logrus.FieldLogger, clientFactory kubern
}

if found := len(metas.Items); found > 0 {
return fmt.Errorf("%s.%s %s has been removed in Kubernetes %s, but there are %d such resources in the cluster", ar.Name, gv.Group, gv.Version, removedInVersion, found)
if currentVersion == "" {
return fmt.Errorf("%s.%s %s has been removed in Kubernetes %s, but there are %d such resources in the cluster", ar.Name, gv.Group, gv.Version, removedInVersion, found)
}
// If we find removed APIs, it could be because the APIserver is serving the same object with an older GVK
// for compatibility reasons while the current good API still works.
newGV := gv
newGV.Version = currentVersion
outdatedItems := []metav1.PartialObjectMetadata{}
for _, item := range metas.Items {
// Currently none of the deleted resources are namespaced, so we can skip the namespace check.
// However we keep it in the list so that it breaks if we add a namespaced resource.
_, err := metaClient.Resource(newGV.WithResource(ar.Name)).
Get(ctx, item.GetName(), metav1.GetOptions{})
if apierrors.IsNotFound(err) {
outdatedItems = append(outdatedItems, item)
} else if err != nil {
return err
}
}
if foundOutdated := len(outdatedItems); foundOutdated > 0 {
return fmt.Errorf("%s.%s %s has been removed in Kubernetes %s, but there are %d such resources in the cluster", ar.Name, gv.Group, gv.Version, removedInVersion, found)
}
}
}
}
Expand Down
86 changes: 15 additions & 71 deletions pkg/autopilot/checks/removedapis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,15 @@ import (
)

type removedAPI struct {
group, version, kind, removedInVersion string
group, version, kind, removedInK8sVersion string
// currentAPIVersion declares a version that is still supported for the Group Kind.
// If it's empty, it means that the Group Kind is removed in the removedInVersion.
currentAPIVersion string
}

// Returns the Kubernetes version in which candidate has been removed, if any.
func removedInVersion(candidate schema.GroupVersionKind) string {
// If candidate has been removed, returns the kubernetes version in which it was removed
// and the current version for Group Kind.
func removedInVersion(candidate schema.GroupVersionKind) (string, string) {
if idx, found := sort.Find(len(removedGVKs), func(i int) int {
if cmp := cmp.Compare(candidate.Group, removedGVKs[i].group); cmp != 0 {
return cmp
Expand All @@ -36,77 +40,17 @@ func removedInVersion(candidate schema.GroupVersionKind) string {
}
return cmp.Compare(candidate.Kind, removedGVKs[i].kind)
}); found {
return removedGVKs[idx].removedInVersion
return removedGVKs[idx].removedInK8sVersion, removedGVKs[idx].currentAPIVersion
}

return ""
return "", ""
}

// Sorted array of removed APIs.
var removedGVKs = [65]removedAPI{
{"admissionregistration.k8s.io", "v1beta1", "MutatingWebhookConfiguration", "v1.22.0"},
{"admissionregistration.k8s.io", "v1beta1", "ValidatingWebhookConfiguration", "v1.22.0"},
{"apiextensions.k8s.io", "v1beta1", "CustomResourceDefinition", "v1.22.0"},
{"apiregistration.k8s.io", "v1beta1", "APIService", "v1.22.0"},
{"apps", "v1beta1", "Deployment", "v1.16.0"},
{"apps", "v1beta1", "ReplicaSet", "v1.16.0"},
{"apps", "v1beta1", "StatefulSet", "v1.16.0"},
{"apps", "v1beta2", "DaemonSet", "v1.16.0"},
{"apps", "v1beta2", "Deployment", "v1.16.0"},
{"apps", "v1beta2", "ReplicaSet", "v1.16.0"},
{"apps", "v1beta2", "StatefulSet", "v1.16.0"},
{"authentication.k8s.io", "v1beta1", "TokenReview", "v1.22.0"},
{"autoscaling", "v2beta1", "HorizontalPodAutoscaler", "v1.25.0"},
{"autoscaling", "v2beta1", "HorizontalPodAutoscalerList", "v1.25.0"},
{"autoscaling", "v2beta2", "HorizontalPodAutoscaler", "v1.26.0"},
{"autoscaling", "v2beta2", "HorizontalPodAutoscalerList", "v1.26.0"},
{"batch", "v1beta1", "CronJob", "v1.25.0"},
{"batch", "v1beta1", "CronJobList", "v1.25.0"},
{"certificates.k8s.io", "v1beta1", "CertificateSigningRequest", "v1.22.0"},
{"coordination.k8s.io", "v1beta1", "Lease", "v1.22.0"},
{"discovery.k8s.io", "v1beta1", "EndpointSlice", "v1.25.0"},
{"events.k8s.io", "v1beta1", "Event", "v1.25.0"},
{"extensions", "v1beta1", "DaemonSet", "v1.16.0"},
{"extensions", "v1beta1", "Deployment", "v1.16.0"},
{"extensions", "v1beta1", "Ingress", "v1.22.0"},
{"extensions", "v1beta1", "NetworkPolicy", "v1.16.0"},
{"extensions", "v1beta1", "PodSecurityPolicy", "v1.16.0"},
{"extensions", "v1beta1", "ReplicaSet", "v1.16.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta1", "FlowControl", "v1.26.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta1", "FlowSchema", "v1.26.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta1", "PriorityLevelConfiguration", "v1.26.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta2", "FlowSchema", "v1.29.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta2", "PriorityLevelConfiguration", "v1.29.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta3", "FlowSchema", "v1.32.0"},
{"flowcontrol.apiserver.k8s.io", "v1beta3", "PriorityLevelConfiguration", "v1.32.0"},
{"k0s.k0sproject.example.com", "v1beta1", "RemovedCRD", "v99.99.99"}, // This is a test entry
{"networking.k8s.io", "v1beta1", "Ingress", "v1.22.0"},
{"networking.k8s.io", "v1beta1", "IngressClass", "v1.22.0"},
{"policy", "v1beta1", "PodDisruptionBudget", "v1.25.0"},
{"policy", "v1beta1", "PodDisruptionBudgetList", "v1.25.0"},
{"policy", "v1beta1", "PodSecurityPolicy", "v1.25.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "ClusterRole", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "ClusterRoleBinding", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "ClusterRoleBindingList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "ClusterRoleList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "Role", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "RoleBinding", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "RoleBindingList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1alpha1", "RoleList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "ClusterRole", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "ClusterRoleBinding", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "ClusterRoleBindingList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "ClusterRoleList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "Role", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "RoleBinding", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "RoleBindingList", "v1.22.0"},
{"rbac.authorization.k8s.io", "v1beta1", "RoleList", "v1.22.0"},
{"scheduling.k8s.io", "v1alpha1", "PriorityClass", "v1.17.0"},
{"scheduling.k8s.io", "v1beta1", "PriorityClass", "v1.22.0"},
{"storage.k8s.io", "v1beta1", "CSIDriver", "v1.22.0"},
{"storage.k8s.io", "v1beta1", "CSINode", "v1.22.0"},
{"storage.k8s.io", "v1beta1", "CSIStorageCapacity", "v1.27.0"},
{"storage.k8s.io", "v1beta1", "CSIStorageCapacity", "v1.27.0"},
{"storage.k8s.io", "v1beta1", "StorageClass", "v1.22.0"},
{"storage.k8s.io", "v1beta1", "VolumeAttachment", "v1.22.0"},
var removedGVKs = [...]removedAPI{
{"flowcontrol.apiserver.k8s.io", "v1beta2", "FlowSchema", "v1.29.0", "v1beta3"},
{"flowcontrol.apiserver.k8s.io", "v1beta2", "PriorityLevelConfiguration", "v1.29.0", "v1"},
{"flowcontrol.apiserver.k8s.io", "v1beta3", "FlowSchema", "v1.32.0", "v1"},
{"flowcontrol.apiserver.k8s.io", "v1beta3", "PriorityLevelConfiguration", "v1.32.0", "v1"},
{"k0s.k0sproject.example.com", "v1beta1", "RemovedCRD", "v99.99.99", ""}, // This is a test entry
}
26 changes: 18 additions & 8 deletions pkg/autopilot/checks/removedapis_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ import (
"slices"
"testing"

"k8s.io/apimachinery/pkg/runtime/schema"

"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/runtime/schema"
)

func TestRemovedGVKs(t *testing.T) {
Expand All @@ -36,10 +35,21 @@ func TestRemovedGVKs(t *testing.T) {
}), "removedGVKs needs to be sorted, so that it can be used for binary searches")

// Test two random entries at the top and the bottom of the list
assert.Equal(t, "v1.22.0", removedInVersion(schema.GroupVersionKind{
Group: "apiregistration.k8s.io", Version: "v1beta1", Kind: "APIService",
}))
assert.Equal(t, "v1.27.0", removedInVersion(schema.GroupVersionKind{
Group: "storage.k8s.io", Version: "v1beta1", Kind: "CSIStorageCapacity",
}))
version, currentVersion := removedInVersion(schema.GroupVersionKind{
Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema",
})
assert.Equal(t, "v1.29.0", version)
assert.Equal(t, "v1beta3", currentVersion)

version, currentVersion = removedInVersion(schema.GroupVersionKind{
Group: "k0s.k0sproject.example.com", Version: "v1beta1", Kind: "RemovedCRD",
})
assert.Equal(t, "v99.99.99", version)
assert.Equal(t, "", currentVersion)

version, currentVersion = removedInVersion(schema.GroupVersionKind{
Group: "k0s.k0sproject.example.com", Version: "v1beta1", Kind: "MustFail",
})
assert.Equal(t, "", version)
assert.Equal(t, "", currentVersion)
}
Loading