diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0106fdd..bc145cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,8 +28,35 @@ jobs: with: go-version: ${{ steps.go_version.outputs.version }} - - name: Determine golang-ci version - run: make test + - name: Run tests + run: | + GINKGO="go run github.com/onsi/ginkgo/v2/ginkgo --github-output" make test + + performance-tests: + name: Run performance tests + runs-on: ubuntu-24.04 + + steps: + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - uses: kubernetes-sigs/kwok@main + with: + command: 'kwokctl' + + - name: Set up kwokctl + uses: kubernetes-sigs/kwok@main + with: + command: kwokctl + + - name: Checkout Git Repository + uses: actions/checkout@v4 + + - name: Run performance tests + run: | + GINKGO="go run github.com/onsi/ginkgo/v2/ginkgo --github-output" make test-perf acceptance: name: Run acceptance tests diff --git a/Dockerfile b/Dockerfile index 051c761..4c6181f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG TARGETARCH WORKDIR /namespace-lister # Copy the Go Modules manifests -COPY go.mod go.sum . +COPY go.mod go.sum ./ # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download diff --git a/Makefile b/Makefile index 44b4b2d..dc8dc5a 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,14 @@ LOCALBIN := $(ROOT_DIR)/bin OUT_DIR := $(ROOT_DIR)/out +GINKGO ?= ginkgo GO ?= go +PERF_OUT_DIR := $(OUT_DIR)/perf +PERF_CLUSTER_PROVIDER ?= kwokctl +PERF_CLUSTER_KUBECONFIG ?= $(PERF_OUT_DIR)/namespace-lister-perf-test +PERF_CLUSTER_PROVIDER_FLAGS ?= --disable-qps-limits + GOLANG_CI ?= $(GO) run -modfile $(ROOT_DIR)/hack/tools/golang-ci/go.mod github.com/golangci/golangci-lint/cmd/golangci-lint IMG ?= namespace-lister:latest @@ -46,7 +52,15 @@ fmt: ## Run go fmt against code. .PHONY: test test: ## Run go test against code. - $(GO) test ./... + $(GINKGO) --label-filter='!perf' + +.PHONY: test-perf +test-perf: ## Run performance tests + -$(PERF_CLUSTER_PROVIDER) delete cluster --name namespace-lister-perf-test + KUBECONFIG=$(PERF_CLUSTER_KUBECONFIG) $(PERF_CLUSTER_PROVIDER) create cluster \ + --name namespace-lister-perf-test $(PERF_CLUSTER_PROVIDER_FLAGS) + KUBECONFIG=$(PERF_CLUSTER_KUBECONFIG) $(GINKGO) --label-filter='perf' \ + --keep-going --procs=1 --flake-attempts 2 --output-dir=$(PERF_OUT_DIR) .PHONY: image-build image-build: diff --git a/go.mod b/go.mod index 91e781d..5c2587b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/konflux-ci/namespace-lister -go 1.22.2 +go 1.22.9 require ( github.com/go-logr/logr v1.4.2 diff --git a/performance_test.go b/performance_test.go new file mode 100644 index 0000000..05a82f1 --- /dev/null +++ b/performance_test.go @@ -0,0 +1,287 @@ +package main + +import ( + "context" + "fmt" + "slices" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gmeasure" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + NamespaceTypeLabelKey string = "konflux-ci.dev/type" + NamespaceTypeUserLabelValue string = "user" +) + +var _ = Describe("Authorizing requests", func() { + BeforeEach(func() {}) + + It("efficiently authorize on a huge environment", Serial, Label("perf"), func(ctx context.Context) { + // new gomega experiment + experiment := gmeasure.NewExperiment("Authorizing Request") + + // Register the experiment as a ReportEntry - this will cause Ginkgo's reporter infrastructure + // to print out the experiment's report and to include the experiment in any generated reports + AddReportEntry(experiment.Name, experiment) + + // prepare scheme + s := runtime.NewScheme() + utilruntime.Must(corev1.AddToScheme(s)) + utilruntime.Must(rbacv1.AddToScheme(s)) + + // get kubernetes client config + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = 500 + restConfig.Burst = 500 + c, err := client.New(restConfig, client.Options{Scheme: s}) + utilruntime.Must(err) + + // create resources + username := "user" + err, ans, uns := createResources(ctx, c, username, 300, 800, 1200) + utilruntime.Must(err) + + // create cache + ls, err := labels.Parse(fmt.Sprintf("%s=%s", NamespaceTypeLabelKey, NamespaceTypeUserLabelValue)) + utilruntime.Must(err) + cacheConfig := cacheConfig{restConfig: restConfig, namespacesLabelSector: ls} + cache, err := BuildAndStartCache(ctx, &cacheConfig) + utilruntime.Must(err) + + // create authorizer and namespacelister + authzr := NewAuthorizer(ctx, cache) + nl := NewNamespaceLister(cache, authzr) + + // we sample a function repeatedly to get a statistically significant set of measurements + experiment.Sample(func(idx int) { + experiment.MeasureDuration("listing", func() { + nn, err := nl.ListNamespaces(ctx, username) + if err != nil { + panic(err) + } + if lnn := len(nn.Items); lnn != len(ans) { + panic(fmt.Errorf("expecting %d namespaces, received %d", len(ans), lnn)) + } + }) + }, gmeasure.SamplingConfig{N: 30, Duration: 2 * time.Minute}) + // we'll sample the function up to 30 times or up to 2 minutes, whichever comes first. + + // we sample a function repeatedly to get a statistically significant set of measurements + experiment.Sample(func(idx int) { + nsName := ans[0].GetName() + // measure how long it takes to allow a request and store the duration in a "authorization-allow" measurement + experiment.MeasureDuration("authorization-allow", func() { + d, _, err := authzr.Authorize(ctx, authorizer.AttributesRecord{ + User: &user.DefaultInfo{Name: username}, + Verb: "get", + Resource: "namespaces", + APIGroup: corev1.GroupName, + APIVersion: corev1.SchemeGroupVersion.Version, + Name: nsName, + Namespace: nsName, + ResourceRequest: true, + }) + if err != nil { + panic(err) + } + if d != authorizer.DecisionAllow { + panic(fmt.Sprintf("expected decision Allow, got %d (0 Deny, 1 Allowed, 2 NoOpinion)", d)) + } + }) + }, gmeasure.SamplingConfig{N: 30, Duration: 2 * time.Minute}) + // we'll sample the function up to 30 times or up to 2 minutes, whichever comes first. + + // we sample a function repeatedly to get a statistically significant set of measurements + experiment.Sample(func(idx int) { + nsName := uns[0].GetName() + // measure how long it takes to produce a NoOpinion decision to a request + // and store the duration in a "authorization-no-opinion" measurement + experiment.MeasureDuration("authorization-noopinion", func() { + d, _, err := authzr.Authorize(ctx, authorizer.AttributesRecord{ + User: &user.DefaultInfo{Name: username}, + Verb: "get", + Resource: "namespaces", + APIGroup: corev1.GroupName, + APIVersion: corev1.SchemeGroupVersion.Version, + Name: nsName, + Namespace: nsName, + ResourceRequest: true, + }) + if err != nil { + panic(err) + } + if d != authorizer.DecisionNoOpinion { + panic(fmt.Sprintf("expected decision NoOpinion, got %d (0 Deny, 1 Allowed, 2 NoOpinion)", d)) + } + }) + }, gmeasure.SamplingConfig{N: 30, Duration: 2 * time.Minute}) + // we'll sample the function up to 30 times or up to 2 minutes, whichever comes first. + + // we get the median listing duration from the experiment we just ran + repaginationStats := experiment.GetStats("listing") + medianDuration := repaginationStats.DurationFor(gmeasure.StatMedian) + + // and assert that it hasn't changed much from ~100ms + Expect(medianDuration).To(BeNumerically("~", 100*time.Millisecond, 70*time.Millisecond)) + }) +}) + +func createResources(ctx context.Context, cli client.Client, user string, numAllowedNamespaces, numUnallowedNamespaces, numNonMatchingClusterRoles int) (error, []client.Object, []client.Object) { + // cluster scoped resources + mcr, nmcr := matchingClusterRoles(1), nonMatchingClusterRoles(numNonMatchingClusterRoles) + ans, uns := namespaces("allowed-tenant-", numAllowedNamespaces), namespaces("unallowed-tenant-", numUnallowedNamespaces) + if err := create(ctx, cli, slices.Concat(mcr, nmcr, ans, uns)); err != nil { + return fmt.Errorf("could not create cluster scoped resources: %w", err), nil, nil + } + + // namespace scoped resources + atr := allowedTenants(user, ans, 10, "ClusterRole", mcr[0].GetName(), "ClusterRole", nmcr[0].GetName()) + utr := unallowedTenants(user, uns, 10, "ClusterRole", mcr[0].GetName(), "ClusterRole", nmcr[0].GetName()) + if err := create(ctx, cli, slices.Concat(atr, utr)); err != nil { + return fmt.Errorf("could not create namespaced scoped resources: %w", err), nil, nil + } + return nil, ans, uns +} + +func create(ctx context.Context, cli client.Client, rr []client.Object) error { + for _, r := range rr { + if err := cli.Create(ctx, r); client.IgnoreAlreadyExists(err) != nil { + return err + } + } + return nil +} + +func matchingClusterRoles(quantity int) []client.Object { + crr := make([]client.Object, quantity) + for i := range quantity { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("perf-cluster-role-matching-%d", i), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{corev1.GroupName}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + }, + } + crr[i] = cr + } + return crr +} + +func nonMatchingClusterRoles(quantity int) []client.Object { + crr := make([]client.Object, quantity) + for i := range quantity { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "perf-cluster-role-non-matching-", + }, + } + crr[i] = cr + } + return crr +} + +func namespaces(generateName string, quantity int) []client.Object { + rr := make([]client.Object, quantity) + for i := range quantity { + rr[i] = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: generateName, + Labels: map[string]string{ + NamespaceTypeLabelKey: NamespaceTypeUserLabelValue, + }, + }, + } + } + return rr +} + +func allowedTenants(user string, namespaces []client.Object, pollutingRoleBindings int, matchingRoleRefKind, matchingRoleRefName, nonMatchingRoleRefKind, nonMatchingRoleRefName string) []client.Object { + rr := make([]client.Object, 0, len(namespaces)*(pollutingRoleBindings+1)) + for _, n := range namespaces { + + // add access role binding + rr = append(rr, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "allowed-tenant-", + Namespace: n.GetName(), + }, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: user}}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: matchingRoleRefKind, + Name: matchingRoleRefName, + }, + }) + + // add pollution + for range pollutingRoleBindings { + rr = append(rr, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pollution-", + Namespace: n.GetName(), + }, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: user}}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: nonMatchingRoleRefKind, + Name: nonMatchingRoleRefName, + }, + }) + } + } + return rr +} + +func unallowedTenants(user string, namespaces []client.Object, pollutingRoleBindings int, matchingRoleRefKind, matchingRoleRefName, nonMatchingRoleRefKind, nonMatchingRoleRefName string) []client.Object { + rr := make([]client.Object, 0, len(namespaces)*(pollutingRoleBindings+1)) + for _, n := range namespaces { + // add access role binding + rr = append(rr, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "non-allowed-tenant-", + Namespace: n.GetName(), + }, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: "not-the-perf-test-user"}}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: matchingRoleRefKind, + Name: matchingRoleRefName, + }, + }) + + // add pollution + for range pollutingRoleBindings { + rr = append(rr, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pollution-", + Namespace: n.GetName(), + }, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: user}}, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: nonMatchingRoleRefKind, + Name: nonMatchingRoleRefName, + }, + }) + } + } + return rr +}