Skip to content

Add scale sets #376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 46 commits into from
May 6, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
79b9a15
Add scaleset client
gabriel-samfira Apr 4, 2025
51c7d2a
Add scaleset types
gabriel-samfira Apr 6, 2025
d7d6d1e
Add mocks
gabriel-samfira Apr 6, 2025
e51f19a
Fix lint errors
gabriel-samfira Apr 6, 2025
5ba53ad
Switch to locking package
gabriel-samfira Apr 7, 2025
85eac36
Add ScaleSet models, functions and types
gabriel-samfira Apr 8, 2025
7e1a83c
Add API endpoint for some scaleset ops
gabriel-samfira Apr 11, 2025
7174e03
Add scaleset commands
gabriel-samfira Apr 11, 2025
6a5c309
Add some worker code
gabriel-samfira Apr 15, 2025
a2aeac7
Scale set workers properly come online
gabriel-samfira Apr 16, 2025
19ba210
Several fixes
gabriel-samfira Apr 16, 2025
12f40a5
Fix refresh session
gabriel-samfira Apr 17, 2025
c177c31
Add some message handling
gabriel-samfira Apr 17, 2025
fc4bd86
Add some db functions to handle scaleset instances
gabriel-samfira Apr 17, 2025
8d10dd4
Update garm-provider-common
gabriel-samfira Apr 17, 2025
94f264d
Handle JobStarted and JobCompleted
gabriel-samfira Apr 17, 2025
d949cec
Keep a cache of runners in the scaleset worker
gabriel-samfira Apr 17, 2025
8c62b6d
Obey enabled/disabled status
gabriel-samfira Apr 17, 2025
bc470c5
WiP
gabriel-samfira Apr 17, 2025
7376a5f
Fix scale set restart logic
gabriel-samfira Apr 19, 2025
020210d
Handle scale up and down; add provider worker
gabriel-samfira Apr 20, 2025
436fd77
WiP
gabriel-samfira Apr 23, 2025
004ad1f
Add provider worker code
gabriel-samfira Apr 24, 2025
f2ad7a3
Fix leftover instances and refactor
gabriel-samfira Apr 27, 2025
a4ac85a
Update CLI to show scale sets
gabriel-samfira Apr 27, 2025
884be62
Fix lint errors
gabriel-samfira Apr 27, 2025
55b4e74
Update mocks
gabriel-samfira Apr 27, 2025
4b1d51f
Fix nil pointer deref
gabriel-samfira Apr 27, 2025
64d1501
DeleteInstance should noop if error not found
gabriel-samfira Apr 27, 2025
22302fd
Add scaleset watcher to provider
gabriel-samfira Apr 28, 2025
fafe98e
Update go-github
gabriel-samfira Apr 28, 2025
73340da
Add RateLimit() function to gh client
gabriel-samfira Apr 28, 2025
059734f
Add runner periodic cleanup check
gabriel-samfira May 1, 2025
92d04c8
Add tests for cache and locking
gabriel-samfira May 2, 2025
c601f88
Add more tests
gabriel-samfira May 2, 2025
2a5e374
Remove unused field
gabriel-samfira May 2, 2025
ff383ea
Add db scaleset tests
gabriel-samfira May 2, 2025
77895d9
Add more tests
gabriel-samfira May 2, 2025
3b3095c
Scale sets are unique within a runner group
gabriel-samfira May 3, 2025
1d093cc
Slight refactor; add creds cache worker
gabriel-samfira May 5, 2025
9f64096
revert main
gabriel-samfira May 5, 2025
2f3c745
Add instance cache
gabriel-samfira May 5, 2025
0e1fa00
Add some more caching, record scaleset jobs
gabriel-samfira May 6, 2025
a80b900
Update dependencies
gabriel-samfira May 6, 2025
f7cd743
Add more tests
gabriel-samfira May 6, 2025
2f2ff62
Deduplicate code
gabriel-samfira May 6, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ $(LOCALBIN):
GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint

## Tool Versions
GOLANGCI_LINT_VERSION ?= v1.61.0
GOLANGCI_LINT_VERSION ?= v1.64.8

.PHONY: golangci-lint
golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. If wrong version is installed, it will be overwritten.
Expand Down
98 changes: 98 additions & 0 deletions apiserver/controllers/enterprises.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,62 @@ func (a *APIController) CreateEnterprisePoolHandler(w http.ResponseWriter, r *ht
}
}

// swagger:route POST /enterprises/{enterpriseID}/scalesets enterprises scalesets CreateEnterpriseScaleSet
//
// Create enterprise pool with the parameters given.
//
// Parameters:
// + name: enterpriseID
// description: Enterprise ID.
// type: string
// in: path
// required: true
//
// + name: Body
// description: Parameters used when creating the enterprise scale set.
// type: CreateScaleSetParams
// in: body
// required: true
//
// Responses:
// 200: ScaleSet
// default: APIErrorResponse
func (a *APIController) CreateEnterpriseScaleSetHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()

vars := mux.Vars(r)
enterpriseID, ok := vars["enterpriseID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No enterprise ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}

var scaleSetData runnerParams.CreateScaleSetParams
if err := json.NewDecoder(r.Body).Decode(&scaleSetData); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to decode")
handleError(ctx, w, gErrors.ErrBadRequest)
return
}

scaleSet, err := a.r.CreateEntityScaleSet(ctx, runnerParams.GithubEntityTypeEnterprise, enterpriseID, scaleSetData)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "error creating enterprise scale set")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(scaleSet); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /enterprises/{enterpriseID}/pools enterprises pools ListEnterprisePools
//
// List enterprise pools.
Expand Down Expand Up @@ -319,6 +375,48 @@ func (a *APIController) ListEnterprisePoolsHandler(w http.ResponseWriter, r *htt
}
}

// swagger:route GET /enterprises/{enterpriseID}/scalesets enterprises scalesets ListEnterpriseScaleSets
//
// List enterprise scale sets.
//
// Parameters:
// + name: enterpriseID
// description: Enterprise ID.
// type: string
// in: path
// required: true
//
// Responses:
// 200: ScaleSets
// default: APIErrorResponse
func (a *APIController) ListEnterpriseScaleSetsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
enterpriseID, ok := vars["enterpriseID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No enterprise ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}

scaleSets, err := a.r.ListEntityScaleSets(ctx, runnerParams.GithubEntityTypeEnterprise, enterpriseID)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "listing scale sets")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(scaleSets); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /enterprises/{enterpriseID}/pools/{poolID} enterprises pools GetEnterprisePool
//
// Get enterprise pool by ID.
Expand Down
48 changes: 48 additions & 0 deletions apiserver/controllers/instances.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,54 @@ func (a *APIController) ListPoolInstancesHandler(w http.ResponseWriter, r *http.
}
}

// swagger:route GET /scalesets/{scalesetID}/instances instances ListScaleSetInstances
//
// List runner instances in a scale set.
//
// Parameters:
// + name: scalesetID
// description: Runner scale set ID.
// type: string
// in: path
// required: true
//
// Responses:
// 200: Instances
// default: APIErrorResponse
func (a *APIController) ListScaleSetInstancesHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
scalesetID, ok := vars["scalesetID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No pool ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}
id, err := strconv.ParseUint(scalesetID, 10, 32)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to parse id")
handleError(ctx, w, gErrors.ErrBadRequest)
return
}

instances, err := a.r.ListScaleSetInstances(ctx, uint(id))
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "listing pool instances")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(instances); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /instances/{instanceName} instances GetInstance
//
// Get runner instance by name.
Expand Down
98 changes: 98 additions & 0 deletions apiserver/controllers/organizations.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,62 @@ func (a *APIController) CreateOrgPoolHandler(w http.ResponseWriter, r *http.Requ
}
}

// swagger:route POST /organizations/{orgID}/scalesets organizations scalesets CreateOrgScaleSet
//
// Create organization scale set with the parameters given.
//
// Parameters:
// + name: orgID
// description: Organization ID.
// type: string
// in: path
// required: true
//
// + name: Body
// description: Parameters used when creating the organization scale set.
// type: CreateScaleSetParams
// in: body
// required: true
//
// Responses:
// 200: ScaleSet
// default: APIErrorResponse
func (a *APIController) CreateOrgScaleSetHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()

vars := mux.Vars(r)
orgID, ok := vars["orgID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No org ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}

var scalesetData runnerParams.CreateScaleSetParams
if err := json.NewDecoder(r.Body).Decode(&scalesetData); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to decode")
handleError(ctx, w, gErrors.ErrBadRequest)
return
}

scaleSet, err := a.r.CreateEntityScaleSet(ctx, runnerParams.GithubEntityTypeOrganization, orgID, scalesetData)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "error creating organization scale set")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(scaleSet); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /organizations/{orgID}/pools organizations pools ListOrgPools
//
// List organization pools.
Expand Down Expand Up @@ -329,6 +385,48 @@ func (a *APIController) ListOrgPoolsHandler(w http.ResponseWriter, r *http.Reque
}
}

// swagger:route GET /organizations/{orgID}/scalesets organizations scalesets ListOrgScaleSets
//
// List organization scale sets.
//
// Parameters:
// + name: orgID
// description: Organization ID.
// type: string
// in: path
// required: true
//
// Responses:
// 200: ScaleSets
// default: APIErrorResponse
func (a *APIController) ListOrgScaleSetsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
orgID, ok := vars["orgID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No org ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}

scaleSets, err := a.r.ListEntityScaleSets(ctx, runnerParams.GithubEntityTypeOrganization, orgID)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "listing scale sets")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(scaleSets); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /organizations/{orgID}/pools/{poolID} organizations pools GetOrgPool
//
// Get organization pool by ID.
Expand Down
98 changes: 98 additions & 0 deletions apiserver/controllers/repositories.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,62 @@ func (a *APIController) CreateRepoPoolHandler(w http.ResponseWriter, r *http.Req
}
}

// swagger:route POST /repositories/{repoID}/scalesets repositories scalesets CreateRepoScaleSet
//
// Create repository scale set with the parameters given.
//
// Parameters:
// + name: repoID
// description: Repository ID.
// type: string
// in: path
// required: true
//
// + name: Body
// description: Parameters used when creating the repository scale set.
// type: CreateScaleSetParams
// in: body
// required: true
//
// Responses:
// 200: ScaleSet
// default: APIErrorResponse
func (a *APIController) CreateRepoScaleSetHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()

vars := mux.Vars(r)
repoID, ok := vars["repoID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No repo ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}

var scaleSetData runnerParams.CreateScaleSetParams
if err := json.NewDecoder(r.Body).Decode(&scaleSetData); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to decode")
handleError(ctx, w, gErrors.ErrBadRequest)
return
}

scaleSet, err := a.r.CreateEntityScaleSet(ctx, runnerParams.GithubEntityTypeRepository, repoID, scaleSetData)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "error creating repository scale set")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(scaleSet); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /repositories/{repoID}/pools repositories pools ListRepoPools
//
// List repository pools.
Expand Down Expand Up @@ -328,6 +384,48 @@ func (a *APIController) ListRepoPoolsHandler(w http.ResponseWriter, r *http.Requ
}
}

// swagger:route GET /repositories/{repoID}/scalesets repositories scalesets ListRepoScaleSets
//
// List repository scale sets.
//
// Parameters:
// + name: repoID
// description: Repository ID.
// type: string
// in: path
// required: true
//
// Responses:
// 200: ScaleSets
// default: APIErrorResponse
func (a *APIController) ListRepoScaleSetsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
repoID, ok := vars["repoID"]
if !ok {
w.WriteHeader(http.StatusBadRequest)
if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
Error: "Bad Request",
Details: "No repo ID specified",
}); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
return
}

scaleSets, err := a.r.ListEntityScaleSets(ctx, runnerParams.GithubEntityTypeRepository, repoID)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "listing scale sets")
handleError(ctx, w, err)
return
}

w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(scaleSets); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
}
}

// swagger:route GET /repositories/{repoID}/pools/{poolID} repositories pools GetRepoPool
//
// Get repository pool by ID.
Expand Down
Loading
Loading