Skip to content

✨ Add e2e tests for frontproxy objects #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,12 @@ graph TB
classDef ca color:#F77
classDef cert color:orange
```

### Running E2E tests locally

In order to run the E2E tests locally, you will need to setup cert-manager with the sample clusterissuer:

```sh
helm upgrade --install --namespace cert-manager --create-namespace --version v1.16.2 --set crds.enabled=true cert-manager jetstack/cert-manager
kubectl apply -n cert-manager --filename hack/ci/testdata/clusterissuer.yaml
```
4 changes: 2 additions & 2 deletions internal/controller/kubeconfig_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,8 @@ func (r *KubeconfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)

clientCertIssuer = resources.GetRootShardCAName(&rootShard, operatorv1alpha1.FrontProxyClientCA)
serverCA = resources.GetRootShardCAName(&rootShard, operatorv1alpha1.ServerCA)
serverURL = fmt.Sprintf("https://%s:6443", rootShard.Spec.External.Hostname)
serverName = rootShard.Spec.External.Hostname
serverURL = fmt.Sprintf("https://%s:6443", frontProxy.Spec.ExternalHostname)
serverName = frontProxy.Spec.ExternalHostname
Comment on lines -130 to +131
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's up with this change? The hostname should be preferring the shard external hostname, because any interaction via kubectl plugins with use the external hostname exposed by the shard anyway. If I remember correctly we talked about this a while ago, in the current kcp way of doing things, this won't really work.


default:
return ctrl.Result{}, fmt.Errorf("no valid target for kubeconfig found")
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ var _ = BeforeSuite(func() {
// default path defined in controller-runtime which is /usr/local/kubebuilder/.
// Note that you must have the required binaries setup under the bin directory to perform
// the tests directly. When we run make test it will be setup and used automatically.
BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s",
BinaryAssetsDirectory: filepath.Join("..", "..", "_tools", "k8s",
fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)),
}

Expand Down
90 changes: 90 additions & 0 deletions test/e2e/frontproxies/frontproxies_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
//go:build e2e

/*
Copyright 2025 The KCP Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package frontproxies

import (
"context"
"fmt"
"testing"
"time"

"github.com/go-logr/logr"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrlruntime "sigs.k8s.io/controller-runtime"

operatorv1alpha1 "github.com/kcp-dev/kcp-operator/sdk/apis/operator/v1alpha1"
"github.com/kcp-dev/kcp-operator/test/utils"
)

func TestCreateFrontProxy(t *testing.T) {
fmt.Println()

ctrlruntime.SetLogger(logr.Discard())

client := utils.GetKubeClient(t)
ctx := context.Background()
namespace := "create-frontproxy"

utils.CreateSelfDestructingNamespace(t, ctx, client, namespace)

// deploy rootshard
rootShard := utils.DeployRootShard(ctx, t, client, namespace)

// deploy front-proxy
frontProxy := utils.DeployFrontProxy(ctx, t, client, namespace, rootShard.Name)

// create front-proxy kubeconfig
configSecretName := "kubeconfig-front-proxy-e2e"

fpConfig := operatorv1alpha1.Kubeconfig{}
fpConfig.Name = "front-proxy"
fpConfig.Namespace = namespace
fpConfig.Spec = operatorv1alpha1.KubeconfigSpec{
Target: operatorv1alpha1.KubeconfigTarget{
FrontProxyRef: &corev1.LocalObjectReference{
Name: frontProxy.Name,
},
},
Username: "e2e",
Validity: metav1.Duration{Duration: 2 * time.Hour},
SecretRef: corev1.LocalObjectReference{
Name: configSecretName,
},
Groups: []string{"system:masters"},
}

t.Log("Creating kubeconfig for FrontProxy…")
if err := client.Create(ctx, &fpConfig); err != nil {
t.Fatal(err)
}
utils.WaitForObject(t, ctx, client, &corev1.Secret{}, types.NamespacedName{Namespace: fpConfig.Namespace, Name: fpConfig.Spec.SecretRef.Name})

// verify that we can use frontproxy kubeconfig to access rootshard workspaces
t.Log("Connecting to FrontProxy…")
kcpClient := utils.ConnectWithKubeconfig(t, ctx, client, namespace, fpConfig.Name)
// proof of life: list something every logicalcluster in kcp has
t.Log("Should be able to list Secrets.")
secrets := &corev1.SecretList{}
if err := kcpClient.List(ctx, secrets); err != nil {
t.Fatalf("Failed to list secrets in kcp: %v", err)
}
}
41 changes: 41 additions & 0 deletions test/utils/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,44 @@ func DeployRootShard(ctx context.Context, t *testing.T, client ctrlruntimeclient

return rootShard
}

func DeployFrontProxy(ctx context.Context, t *testing.T, client ctrlruntimeclient.Client, namespace string, rootShardName string, patches ...func(*operatorv1alpha1.FrontProxy)) operatorv1alpha1.FrontProxy {
t.Helper()

frontProxy := operatorv1alpha1.FrontProxy{}
frontProxy.Name = "front-proxy"
frontProxy.Namespace = namespace

frontProxy.Spec = operatorv1alpha1.FrontProxySpec{
RootShard: operatorv1alpha1.RootShardConfig{
Reference: &corev1.LocalObjectReference{
Name: rootShardName,
},
},
ExternalHostname: fmt.Sprintf("%s.%s.svc.cluster.local", frontProxy.Name, frontProxy.Namespace),
Auth: &operatorv1alpha1.AuthSpec{
// we need to remove the default system:masters group in order to do our testing
DropGroups: []string{""},
},
}

for _, patch := range patches {
patch(&frontProxy)
}

t.Logf("Creating FrontProxy %s…", frontProxy.Name)
if err := client.Create(ctx, &frontProxy); err != nil {
t.Fatal(err)
}

opts := []ctrlruntimeclient.ListOption{
ctrlruntimeclient.InNamespace(frontProxy.Namespace),
ctrlruntimeclient.MatchingLabels{
"app.kubernetes.io/component": "front-proxy",
"app.kubernetes.io/instance": frontProxy.Name,
},
}
WaitForPods(t, ctx, client, opts...)

return frontProxy
}
5 changes: 5 additions & 0 deletions test/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ func ConnectWithKubeconfig(
parts := strings.Split(hostname, ".")
serviceName := parts[0]

// for frontproxies add required suffix
if config.Spec.Target.FrontProxyRef != nil {
serviceName += "-front-proxy"
}

portNum, err := strconv.ParseInt(portString, 10, 32)
if err != nil {
t.Fatalf("Failed to parse kubeconfig's port %q: %v", portString, err)
Expand Down
2 changes: 1 addition & 1 deletion test/utils/wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func WaitForPods(t *testing.T, ctx context.Context, client ctrlruntimeclient.Cli

t.Log("Waiting for pods to be available…")

err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 3*time.Minute, false, func(ctx context.Context) (done bool, err error) {
err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 5*time.Minute, false, func(ctx context.Context) (done bool, err error) {
pods := corev1.PodList{}
if err := client.List(ctx, &pods, listOpts...); err != nil {
return false, err
Expand Down