Skip to content

Commit bfd00bd

Browse files
authored
Merge pull request #57 from sttts/sttts-cluster-aware-rest-mapper
Make RESTMapper in client cluster-aware
2 parents a3df1ba + f5e9ba1 commit bfd00bd

14 files changed

+567
-106
lines changed

.golangci.yml

-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ linters:
1313
- exhaustive
1414
- exportloopref
1515
- ginkgolinter
16-
- goconst
1716
- gocritic
1817
- gocyclo
1918
- gofmt

examples/kcp/go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ require (
5050
github.com/google/uuid v1.3.1 // indirect
5151
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
5252
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
53+
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
5354
github.com/imdario/mergo v0.3.13 // indirect
5455
github.com/inconshreveable/mousetrap v1.1.0 // indirect
5556
github.com/josharian/intern v1.0.0 // indirect

examples/kcp/go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
9696
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
9797
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
9898
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
99+
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
100+
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
99101
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
100102
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
101103
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ require (
3131
)
3232

3333
require (
34+
github.com/hashicorp/golang-lru/v2 v2.0.7
3435
github.com/kcp-dev/apimachinery/v2 v2.0.0-alpha.0.0.20230926071920-57d168bcbe34
3536
github.com/kcp-dev/logicalcluster/v3 v3.0.5
3637
)

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
6565
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
6666
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
6767
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
68+
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
69+
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
6870
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
6971
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
7072
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=

pkg/client/client.go

+31-5
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import (
2323
"net/http"
2424
"strings"
2525

26+
lru "github.com/hashicorp/golang-lru/v2"
27+
"github.com/kcp-dev/logicalcluster/v3"
2628
"k8s.io/apimachinery/pkg/api/meta"
2729
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2830
"k8s.io/apimachinery/pkg/runtime"
@@ -47,6 +49,10 @@ type Options struct {
4749
// Mapper, if provided, will be used to map GroupVersionKinds to Resources
4850
Mapper meta.RESTMapper
4951

52+
// MapperWithContext, if provided, will be used to map GroupVersionKinds to Resources.
53+
// This overrides Mapper if set.
54+
MapperWithContext func(context.Context) (meta.RESTMapper, error)
55+
5056
// Cache, if provided, is used to read objects from the cache.
5157
Cache *CacheOptions
5258

@@ -56,6 +62,10 @@ type Options struct {
5662

5763
// DryRun instructs the client to only perform dry run requests.
5864
DryRun *bool
65+
66+
// KcpClusterDiscoveryCacheSize is the size of the cache for cluster discovery
67+
// information backing the client's REST mapper.
68+
KcpClusterDiscoveryCacheSize int
5969
}
6070

6171
// WarningHandlerOptions are options for configuring a
@@ -170,16 +180,27 @@ func newClient(config *rest.Config, options Options) (*client, error) {
170180
}
171181
}
172182

183+
if options.KcpClusterDiscoveryCacheSize == 0 {
184+
options.KcpClusterDiscoveryCacheSize = 1000
185+
}
186+
187+
// Init a MapperWithContext if none provided
188+
if options.MapperWithContext == nil {
189+
options.MapperWithContext = func(context.Context) (meta.RESTMapper, error) { return options.Mapper, nil }
190+
}
191+
173192
resources := &clientRestResources{
174193
httpClient: options.HTTPClient,
175194
config: config,
176195
scheme: options.Scheme,
177-
mapper: options.Mapper,
196+
mapper: options.MapperWithContext,
178197
codecs: serializer.NewCodecFactory(options.Scheme),
179-
180-
structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
181-
unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
182198
}
199+
cr, err := lru.New[logicalcluster.Path, clusterResources](options.KcpClusterDiscoveryCacheSize)
200+
if err != nil {
201+
return nil, err
202+
}
203+
resources.clusterResources = cr
183204

184205
rawMetaClient, err := metadata.NewForConfigAndClient(metadata.ConfigFor(config), options.HTTPClient)
185206
if err != nil {
@@ -197,11 +218,16 @@ func newClient(config *rest.Config, options Options) (*client, error) {
197218
},
198219
metadataClient: metadataClient{
199220
client: rawMetaClient,
200-
restMapper: options.Mapper,
221+
restMapper: options.MapperWithContext,
201222
},
202223
scheme: options.Scheme,
203224
mapper: options.Mapper,
204225
}
226+
mapperCache, err := lru.New[logicalcluster.Name, meta.RESTMapper](options.KcpClusterDiscoveryCacheSize)
227+
if err != nil {
228+
return nil, err
229+
}
230+
c.metadataClient.mapperCache = mapperCache
205231
if options.Cache == nil || options.Cache.Reader == nil {
206232
return c, nil
207233
}

pkg/client/client_rest_resources.go

+40-14
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,32 @@ limitations under the License.
1717
package client
1818

1919
import (
20+
"context"
2021
"net/http"
2122
"strings"
2223
"sync"
2324

25+
lru "github.com/hashicorp/golang-lru/v2"
26+
"github.com/kcp-dev/logicalcluster/v3"
2427
"k8s.io/apimachinery/pkg/api/meta"
2528
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2629
"k8s.io/apimachinery/pkg/runtime"
2730
"k8s.io/apimachinery/pkg/runtime/schema"
2831
"k8s.io/apimachinery/pkg/runtime/serializer"
2932
"k8s.io/client-go/rest"
3033
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
34+
"sigs.k8s.io/controller-runtime/pkg/kontext"
3135
)
3236

37+
type clusterResources struct {
38+
mapper meta.RESTMapper
39+
40+
// structuredResourceByType stores structured type metadata
41+
structuredResourceByType map[schema.GroupVersionKind]*resourceMeta
42+
// unstructuredResourceByType stores unstructured type metadata
43+
unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta
44+
}
45+
3346
// clientRestResources creates and stores rest clients and metadata for Kubernetes types.
3447
type clientRestResources struct {
3548
// httpClient is the http client to use for requests
@@ -42,21 +55,18 @@ type clientRestResources struct {
4255
scheme *runtime.Scheme
4356

4457
// mapper maps GroupVersionKinds to Resources
45-
mapper meta.RESTMapper
58+
mapper func(ctx context.Context) (meta.RESTMapper, error)
4659

4760
// codecs are used to create a REST client for a gvk
4861
codecs serializer.CodecFactory
4962

50-
// structuredResourceByType stores structured type metadata
51-
structuredResourceByType map[schema.GroupVersionKind]*resourceMeta
52-
// unstructuredResourceByType stores unstructured type metadata
53-
unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta
54-
mu sync.RWMutex
63+
clusterResources *lru.Cache[logicalcluster.Path, clusterResources]
64+
mu sync.RWMutex
5565
}
5666

5767
// newResource maps obj to a Kubernetes Resource and constructs a client for that Resource.
5868
// If the object is a list, the resource represents the item's type instead.
59-
func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) {
69+
func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool, mapper meta.RESTMapper) (*resourceMeta, error) {
6070
if strings.HasSuffix(gvk.Kind, "List") && isList {
6171
// if this was a list, treat it as a request for the item's resource
6272
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
@@ -66,7 +76,7 @@ func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, i
6676
if err != nil {
6777
return nil, err
6878
}
69-
mapping, err := c.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
79+
mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
7080
if err != nil {
7181
return nil, err
7282
}
@@ -75,7 +85,7 @@ func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, i
7585

7686
// getResource returns the resource meta information for the given type of object.
7787
// If the object is a list, the resource represents the item's type instead.
78-
func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, error) {
88+
func (c *clientRestResources) getResource(ctx context.Context, obj runtime.Object) (*resourceMeta, error) {
7989
gvk, err := apiutil.GVKForObject(obj, c.scheme)
8090
if err != nil {
8191
return nil, err
@@ -86,9 +96,25 @@ func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, er
8696
// It's better to do creation work twice than to not let multiple
8797
// people make requests at once
8898
c.mu.RLock()
89-
resourceByType := c.structuredResourceByType
99+
cluster, _ := kontext.ClusterFrom(ctx)
100+
cr, found := c.clusterResources.Get(cluster.Path())
101+
if !found {
102+
m, err := c.mapper(ctx)
103+
if err != nil {
104+
c.mu.RUnlock()
105+
return nil, err
106+
}
107+
cr = clusterResources{
108+
mapper: m,
109+
structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
110+
unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
111+
}
112+
c.clusterResources.Purge()
113+
c.clusterResources.Add(cluster.Path(), cr)
114+
}
115+
resourceByType := cr.structuredResourceByType
90116
if isUnstructured {
91-
resourceByType = c.unstructuredResourceByType
117+
resourceByType = cr.unstructuredResourceByType
92118
}
93119
r, known := resourceByType[gvk]
94120
c.mu.RUnlock()
@@ -100,7 +126,7 @@ func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, er
100126
// Initialize a new Client
101127
c.mu.Lock()
102128
defer c.mu.Unlock()
103-
r, err = c.newResource(gvk, meta.IsListType(obj), isUnstructured)
129+
r, err = c.newResource(gvk, meta.IsListType(obj), isUnstructured, cr.mapper)
104130
if err != nil {
105131
return nil, err
106132
}
@@ -109,8 +135,8 @@ func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, er
109135
}
110136

111137
// getObjMeta returns objMeta containing both type and object metadata and state.
112-
func (c *clientRestResources) getObjMeta(obj runtime.Object) (*objMeta, error) {
113-
r, err := c.getResource(obj)
138+
func (c *clientRestResources) getObjMeta(ctx context.Context, obj runtime.Object) (*objMeta, error) {
139+
r, err := c.getResource(ctx, obj)
114140
if err != nil {
115141
return nil, err
116142
}

pkg/client/metadata_client.go

+30-10
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,15 @@ import (
2020
"context"
2121
"fmt"
2222
"strings"
23+
"sync"
2324

25+
lru "github.com/hashicorp/golang-lru/v2"
26+
"github.com/kcp-dev/logicalcluster/v3"
2427
"k8s.io/apimachinery/pkg/api/meta"
2528
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2629
"k8s.io/apimachinery/pkg/runtime/schema"
2730
"k8s.io/client-go/metadata"
31+
"sigs.k8s.io/controller-runtime/pkg/kontext"
2832
)
2933

3034
// TODO(directxman12): we could rewrite this on top of the low-level REST
@@ -34,12 +38,28 @@ import (
3438

3539
// metadataClient is a client that reads & writes metadata-only requests to/from the API server.
3640
type metadataClient struct {
37-
client metadata.Interface
38-
restMapper meta.RESTMapper
41+
client metadata.Interface
42+
restMapper func(ctx context.Context) (meta.RESTMapper, error)
43+
mu sync.Mutex
44+
mapperCache *lru.Cache[logicalcluster.Name, meta.RESTMapper]
3945
}
4046

41-
func (mc *metadataClient) getResourceInterface(gvk schema.GroupVersionKind, ns string) (metadata.ResourceInterface, error) {
42-
mapping, err := mc.restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
47+
func (mc *metadataClient) getResourceInterface(ctx context.Context, gvk schema.GroupVersionKind, ns string) (metadata.ResourceInterface, error) {
48+
cluster, _ := kontext.ClusterFrom(ctx)
49+
mc.mu.Lock()
50+
mapper, _ := mc.mapperCache.Get(cluster)
51+
if mapper == nil {
52+
var err error
53+
mapper, err = mc.restMapper(ctx)
54+
if err != nil {
55+
mc.mu.Unlock()
56+
return nil, err
57+
}
58+
mc.mapperCache.Add(cluster, mapper)
59+
}
60+
mc.mu.Unlock()
61+
62+
mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
4363
if err != nil {
4464
return nil, err
4565
}
@@ -56,7 +76,7 @@ func (mc *metadataClient) Delete(ctx context.Context, obj Object, opts ...Delete
5676
return fmt.Errorf("metadata client did not understand object: %T", obj)
5777
}
5878

59-
resInt, err := mc.getResourceInterface(metadata.GroupVersionKind(), metadata.Namespace)
79+
resInt, err := mc.getResourceInterface(ctx, metadata.GroupVersionKind(), metadata.Namespace)
6080
if err != nil {
6181
return err
6282
}
@@ -77,7 +97,7 @@ func (mc *metadataClient) DeleteAllOf(ctx context.Context, obj Object, opts ...D
7797
deleteAllOfOpts := DeleteAllOfOptions{}
7898
deleteAllOfOpts.ApplyOptions(opts)
7999

80-
resInt, err := mc.getResourceInterface(metadata.GroupVersionKind(), deleteAllOfOpts.ListOptions.Namespace)
100+
resInt, err := mc.getResourceInterface(ctx, metadata.GroupVersionKind(), deleteAllOfOpts.ListOptions.Namespace)
81101
if err != nil {
82102
return err
83103
}
@@ -93,7 +113,7 @@ func (mc *metadataClient) Patch(ctx context.Context, obj Object, patch Patch, op
93113
}
94114

95115
gvk := metadata.GroupVersionKind()
96-
resInt, err := mc.getResourceInterface(gvk, metadata.Namespace)
116+
resInt, err := mc.getResourceInterface(ctx, gvk, metadata.Namespace)
97117
if err != nil {
98118
return err
99119
}
@@ -127,7 +147,7 @@ func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object, op
127147
getOpts := GetOptions{}
128148
getOpts.ApplyOptions(opts)
129149

130-
resInt, err := mc.getResourceInterface(gvk, key.Namespace)
150+
resInt, err := mc.getResourceInterface(ctx, gvk, key.Namespace)
131151
if err != nil {
132152
return err
133153
}
@@ -154,7 +174,7 @@ func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...List
154174
listOpts := ListOptions{}
155175
listOpts.ApplyOptions(opts)
156176

157-
resInt, err := mc.getResourceInterface(gvk, listOpts.Namespace)
177+
resInt, err := mc.getResourceInterface(ctx, gvk, listOpts.Namespace)
158178
if err != nil {
159179
return err
160180
}
@@ -175,7 +195,7 @@ func (mc *metadataClient) PatchSubResource(ctx context.Context, obj Object, subR
175195
}
176196

177197
gvk := metadata.GroupVersionKind()
178-
resInt, err := mc.getResourceInterface(gvk, metadata.Namespace)
198+
resInt, err := mc.getResourceInterface(ctx, gvk, metadata.Namespace)
179199
if err != nil {
180200
return err
181201
}

0 commit comments

Comments
 (0)