Skip to content

Commit 0687efd

Browse files
stanhustanhu
authored
fix: use keyset pagination only when it is supported (#755)
* fix: use keyset pagination only when it is supported #744 switched to keyset pagination for the `/api/v4/projects/:id/jobs` API endpoint, but this feature is only available in GitLab 15.9 and later (https://docs.gitlab.com/ee/api/jobs.html#list-project-jobs). This commit retrieves the instance metadata at startup to determine whether keyset pagination is available. If the metadata is not available or returns a version < 15.9, offset pagination will be used. * chore: Use golang.org/x/mod/semver for version parsing
1 parent c4d9649 commit 0687efd

File tree

10 files changed

+307
-59
lines changed

10 files changed

+307
-59
lines changed

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ require (
9393
go.opentelemetry.io/otel/metric v1.21.0 // indirect
9494
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
9595
golang.org/x/crypto v0.16.0 // indirect
96+
golang.org/x/mod v0.14.0 // indirect
9697
golang.org/x/net v0.19.0 // indirect
9798
golang.org/x/oauth2 v0.15.0 // indirect
9899
golang.org/x/sync v0.5.0 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,8 @@ golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq
227227
golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
228228
golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
229229
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
230+
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
231+
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
230232
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
231233
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
232234
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=

‎pkg/controller/metadata.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package controller
2+
3+
import (
4+
"context"
5+
6+
goGitlab "github.com/xanzy/go-gitlab"
7+
8+
"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab"
9+
)
10+
11+
func (c *Controller) GetGitLabMetadata(ctx context.Context) error {
12+
options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
13+
14+
metadata, _, err := c.Gitlab.Metadata.GetMetadata(options...)
15+
if err != nil {
16+
return err
17+
}
18+
19+
if metadata.Version != "" {
20+
c.Gitlab.UpdateVersion(gitlab.NewGitLabVersion(metadata.Version))
21+
}
22+
23+
return nil
24+
}

‎pkg/controller/metadata_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package controller
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
10+
"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config"
11+
"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab"
12+
)
13+
14+
func TestGetGitLabMetadataSuccess(t *testing.T) {
15+
tests := []struct {
16+
name string
17+
data string
18+
expectedVersion gitlab.GitLabVersion
19+
}{
20+
{
21+
name: "successful parse",
22+
data: `
23+
{
24+
"version":"16.7.0-pre",
25+
"revision":"3fe364fe754",
26+
"kas":{
27+
"enabled":true,
28+
"externalUrl":"wss://kas.gitlab.com",
29+
"version":"v16.7.0-rc2"
30+
},
31+
"enterprise":true
32+
}
33+
`,
34+
expectedVersion: gitlab.NewGitLabVersion("v16.7.0-pre"),
35+
},
36+
{
37+
name: "unsuccessful parse",
38+
data: `
39+
{
40+
"revision":"3fe364fe754"
41+
}
42+
`,
43+
expectedVersion: gitlab.NewGitLabVersion(""),
44+
},
45+
}
46+
47+
for _, tc := range tests {
48+
t.Run(tc.name, func(t *testing.T) {
49+
ctx, c, mux, srv := newTestController(config.Config{})
50+
defer srv.Close()
51+
52+
mux.HandleFunc("/api/v4/metadata",
53+
func(w http.ResponseWriter, r *http.Request) {
54+
fmt.Fprint(w, tc.data)
55+
})
56+
57+
assert.NoError(t, c.GetGitLabMetadata(ctx))
58+
assert.Equal(t, tc.expectedVersion, c.Gitlab.Version())
59+
})
60+
}
61+
}

‎pkg/controller/scheduler.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,10 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G
307307
ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule")
308308
defer span.End()
309309

310+
go func() {
311+
c.GetGitLabMetadata(ctx)
312+
}()
313+
310314
for tt, cfg := range map[schemas.TaskType]config.SchedulerConfig{
311315
schemas.TaskTypePullProjectsFromWildcards: config.SchedulerConfig(pull.ProjectsFromWildcards),
312316
schemas.TaskTypePullEnvironmentsFromProjects: config.SchedulerConfig(pull.EnvironmentsFromProjects),

‎pkg/gitlab/client.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"net/http"
88
"strconv"
9+
"sync"
910
"sync/atomic"
1011
"time"
1112

@@ -36,6 +37,9 @@ type Client struct {
3637
RequestsCounter atomic.Uint64
3738
RequestsLimit int
3839
RequestsRemaining int
40+
41+
version GitLabVersion
42+
mutex sync.RWMutex
3943
}
4044

4145
// ClientConfig ..
@@ -140,6 +144,19 @@ func (c *Client) rateLimit(ctx context.Context) {
140144
c.RequestsCounter.Add(1)
141145
}
142146

147+
func (c *Client) UpdateVersion(version GitLabVersion) {
148+
c.mutex.Lock()
149+
defer c.mutex.Unlock()
150+
c.version = version
151+
}
152+
153+
func (c *Client) Version() GitLabVersion {
154+
c.mutex.RLock()
155+
defer c.mutex.RUnlock()
156+
157+
return c.version
158+
}
159+
143160
func (c *Client) requestsRemaining(response *goGitlab.Response) {
144161
if response == nil {
145162
return

‎pkg/gitlab/jobs.go‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -231,13 +231,24 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
231231
var (
232232
foundJobs []*goGitlab.Job
233233
resp *goGitlab.Response
234+
opt *goGitlab.ListJobsOptions
234235
)
235236

236-
opt := &goGitlab.ListJobsOptions{
237-
ListOptions: goGitlab.ListOptions{
238-
Pagination: "keyset",
239-
PerPage: 100,
240-
},
237+
keysetPagination := c.Version().PipelineJobsKeysetPaginationSupported()
238+
if keysetPagination {
239+
opt = &goGitlab.ListJobsOptions{
240+
ListOptions: goGitlab.ListOptions{
241+
Pagination: "keyset",
242+
PerPage: 100,
243+
},
244+
}
245+
} else {
246+
opt = &goGitlab.ListJobsOptions{
247+
ListOptions: goGitlab.ListOptions{
248+
Page: 1,
249+
PerPage: 100,
250+
},
251+
}
241252
}
242253

243254
options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
@@ -274,7 +285,8 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
274285
}
275286
}
276287

277-
if resp.NextLink == "" {
288+
if keysetPagination && resp.NextLink == "" ||
289+
(!keysetPagination && resp.CurrentPage >= resp.NextPage) {
278290
var notFoundJobs []string
279291

280292
for k := range jobsToRefresh {
@@ -295,9 +307,11 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
295307
break
296308
}
297309

298-
options = []goGitlab.RequestOptionFunc{
299-
goGitlab.WithContext(ctx),
300-
goGitlab.WithKeysetPaginationParameters(resp.NextLink),
310+
if keysetPagination {
311+
options = []goGitlab.RequestOptionFunc{
312+
goGitlab.WithContext(ctx),
313+
goGitlab.WithKeysetPaginationParameters(resp.NextLink),
314+
}
301315
}
302316
}
303317

‎pkg/gitlab/jobs_test.go‎

Lines changed: 79 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -132,64 +132,93 @@ func TestListPipelineBridges(t *testing.T) {
132132
}
133133

134134
func TestListRefMostRecentJobs(t *testing.T) {
135-
ctx, mux, server, c := getMockedClient()
136-
defer server.Close()
137-
138-
ref := schemas.Ref{
139-
Project: schemas.NewProject("foo"),
140-
Name: "yay",
135+
tests := []struct {
136+
name string
137+
keysetPagination bool
138+
expectedQueryParams url.Values
139+
}{
140+
{
141+
name: "offset pagination",
142+
keysetPagination: false,
143+
expectedQueryParams: url.Values{
144+
"page": []string{"1"},
145+
"per_page": []string{"100"},
146+
},
147+
},
148+
{
149+
name: "keyset pagination",
150+
keysetPagination: true,
151+
expectedQueryParams: url.Values{
152+
"pagination": []string{"keyset"},
153+
"per_page": []string{"100"},
154+
},
155+
},
141156
}
142157

143-
jobs, err := c.ListRefMostRecentJobs(ctx, ref)
144-
assert.NoError(t, err)
145-
assert.Len(t, jobs, 0)
158+
for _, tc := range tests {
159+
t.Run(tc.name, func(t *testing.T) {
160+
ctx, mux, server, c := getMockedClient()
161+
defer server.Close()
146162

147-
mux.HandleFunc("/api/v4/projects/foo/jobs",
148-
func(w http.ResponseWriter, r *http.Request) {
149-
assert.Equal(t, "GET", r.Method)
150-
expectedQueryParams := url.Values{
151-
"pagination": []string{"keyset"},
152-
"per_page": []string{"100"},
163+
if tc.keysetPagination {
164+
c.UpdateVersion(NewGitLabVersion("16.0.0"))
165+
} else {
166+
c.UpdateVersion(NewGitLabVersion("15.0.0"))
153167
}
154-
assert.Equal(t, expectedQueryParams, r.URL.Query())
155-
fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)
156-
})
157168

158-
mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"),
159-
func(w http.ResponseWriter, r *http.Request) {
160-
w.WriteHeader(http.StatusNotFound)
161-
})
169+
ref := schemas.Ref{
170+
Project: schemas.NewProject("foo"),
171+
Name: "yay",
172+
}
162173

163-
ref.LatestJobs = schemas.Jobs{
164-
"foo": {
165-
ID: 1,
166-
Name: "foo",
167-
},
168-
"bar": {
169-
ID: 2,
170-
Name: "bar",
171-
},
172-
}
174+
jobs, err := c.ListRefMostRecentJobs(ctx, ref)
175+
assert.NoError(t, err)
176+
assert.Len(t, jobs, 0)
177+
178+
mux.HandleFunc("/api/v4/projects/foo/jobs",
179+
func(w http.ResponseWriter, r *http.Request) {
180+
assert.Equal(t, "GET", r.Method)
181+
assert.Equal(t, tc.expectedQueryParams, r.URL.Query())
182+
fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)
183+
})
184+
185+
mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"),
186+
func(w http.ResponseWriter, r *http.Request) {
187+
w.WriteHeader(http.StatusNotFound)
188+
})
189+
190+
ref.LatestJobs = schemas.Jobs{
191+
"foo": {
192+
ID: 1,
193+
Name: "foo",
194+
},
195+
"bar": {
196+
ID: 2,
197+
Name: "bar",
198+
},
199+
}
173200

174-
jobs, err = c.ListRefMostRecentJobs(ctx, ref)
175-
assert.NoError(t, err)
176-
assert.Len(t, jobs, 2)
177-
assert.Equal(t, 3, jobs[0].ID)
178-
assert.Equal(t, 4, jobs[1].ID)
201+
jobs, err = c.ListRefMostRecentJobs(ctx, ref)
202+
assert.NoError(t, err)
203+
assert.Len(t, jobs, 2)
204+
assert.Equal(t, 3, jobs[0].ID)
205+
assert.Equal(t, 4, jobs[1].ID)
179206

180-
ref.LatestJobs["baz"] = schemas.Job{
181-
ID: 5,
182-
Name: "baz",
183-
}
207+
ref.LatestJobs["baz"] = schemas.Job{
208+
ID: 5,
209+
Name: "baz",
210+
}
184211

185-
jobs, err = c.ListRefMostRecentJobs(ctx, ref)
186-
assert.NoError(t, err)
187-
assert.Len(t, jobs, 2)
188-
assert.Equal(t, 3, jobs[0].ID)
189-
assert.Equal(t, 4, jobs[1].ID)
212+
jobs, err = c.ListRefMostRecentJobs(ctx, ref)
213+
assert.NoError(t, err)
214+
assert.Len(t, jobs, 2)
215+
assert.Equal(t, 3, jobs[0].ID)
216+
assert.Equal(t, 4, jobs[1].ID)
190217

191-
// Test invalid project id
192-
ref.Project.Name = "bar"
193-
_, err = c.ListRefMostRecentJobs(ctx, ref)
194-
assert.Error(t, err)
218+
// Test invalid project id
219+
ref.Project.Name = "bar"
220+
_, err = c.ListRefMostRecentJobs(ctx, ref)
221+
assert.Error(t, err)
222+
})
223+
}
195224
}

‎pkg/gitlab/version.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package gitlab
2+
3+
import (
4+
"strings"
5+
6+
"golang.org/x/mod/semver"
7+
)
8+
9+
type GitLabVersion struct {
10+
Version string
11+
}
12+
13+
func NewGitLabVersion(version string) GitLabVersion {
14+
ver := ""
15+
if strings.HasPrefix(version, "v") {
16+
ver = version
17+
} else if version != "" {
18+
ver = "v" + version
19+
}
20+
21+
return GitLabVersion{Version: ver}
22+
}
23+
24+
// PipelineJobsKeysetPaginationSupported returns true if the GitLab instance
25+
// is running 15.9 or later.
26+
func (v GitLabVersion) PipelineJobsKeysetPaginationSupported() bool {
27+
if v.Version == "" {
28+
return false
29+
}
30+
31+
return semver.Compare(v.Version, "v15.9.0") >= 0
32+
}

0 commit comments

Comments
 (0)