Skip to content

Commit 79ea409

Browse files
adrianrioboadrianriobo
committed
feat: expand pool across full region to minimize capacity issues
Signed-off-by: Adrian Riobo <[email protected]>
1 parent c91e2da commit 79ea409

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+8745
-943
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ VERSION ?= 1.0.0-dev
22
CONTAINER_MANAGER ?= podman
33

44
# Image URL to use all building/pushing image targets
5-
IMG ?= quay.io/redhat-developer/mapt:v${VERSION}
6-
TKN_IMG ?= quay.io/redhat-developer/mapt:v${VERSION}-tkn
5+
IMG ?= ghcr.io/redhat-developer/mapt:pr-421
6+
TKN_IMG ?= ghcr.io/redhat-developer/mapt:pr-421-tkn
77

88
# Integrations
99
# renovate: datasource=github-releases depName=cirruslabs/cirrus-cli

‎cmd/mapt/cmd/aws/services/mac-pool.go

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const (
2020
cmdHousekeep = "house-keep"
2121
cmdHousekeepDesc = "house keeping for mac pool. Detroy old machines on over capacity and create new ones if capacity not meet"
2222

23+
// Pool params
2324
paramName = "name"
2425
paramNameDesc = "pool name it is a unique identifier for the pool. The name should be unique for the whole AWS account"
2526
paramOfferedCapacity = "offered-capacity"
@@ -28,6 +29,17 @@ const (
2829
paramMaxSize = "max-size"
2930
paramMaxSizeDesc = "max number of machines in the pool"
3031
paramMaxSizeDefault = 2
32+
// Machines in the pool params
33+
paramVPCID = "vpcid"
34+
paramVPCIDDesc = "VPC Id to setup mac machines"
35+
paramVPCIDDefault = ""
36+
paramSSHSGID = "ssh-sgid"
37+
paramSSHSGIDDesc = "Security group Id to securize ssh access to machines. SSH can only be used from instances with this SG."
38+
paramSSHSGIDDefault = ""
39+
// Request / Release
40+
paramTicket = "ticket"
41+
paramTicketDesc = "this is a unique identifier to tag the dedicated host meanwhile it is being locked to identify the request which locked it. It will be used on release to identify the machine being released"
42+
paramTicketDefault = ""
3143
)
3244

3345
func GetMacPoolCmd() *cobra.Command {
@@ -67,14 +79,13 @@ func createMP() *cobra.Command {
6779
DebugLevel: viper.GetUint(params.DebugLevel),
6880
Tags: viper.GetStringMapString(params.Tags),
6981
},
70-
&macpool.MacPoolRequestArgs{
82+
&macpool.PoolRequestArgs{
7183
Prefix: "main",
72-
PoolName: viper.GetString(paramName),
84+
Name: viper.GetString(paramName),
7385
Architecture: viper.GetString(awsParams.MACArch),
7486
OSVersion: viper.GetString(awsParams.MACOSVersion),
7587
OfferedCapacity: viper.GetInt(paramOfferedCapacity),
76-
MaxSize: viper.GetInt(paramMaxSize),
77-
FixedLocation: viper.IsSet(awsParams.MACFixedLocation)}); err != nil {
88+
MaxSize: viper.GetInt(paramMaxSize)}); err != nil {
7889
logging.Error(err)
7990
}
8091
return nil
@@ -89,7 +100,6 @@ func createMP() *cobra.Command {
89100
flagSet.StringP(awsParams.MACArch, "", awsParams.MACArchDefault, awsParams.MACArchDesc)
90101
flagSet.StringP(awsParams.MACOSVersion, "", awsParams.MACOSVersionDefault, awsParams.MACOSVersionDesc)
91102
flagSet.StringToStringP(params.Tags, "", nil, params.TagsDesc)
92-
flagSet.Bool(awsParams.MACFixedLocation, false, awsParams.MACFixedLocationDesc)
93103
c.PersistentFlags().AddFlagSet(flagSet)
94104
return c
95105
}
@@ -138,28 +148,35 @@ func houseKeep() *cobra.Command {
138148
DebugLevel: viper.GetUint(params.DebugLevel),
139149
Tags: viper.GetStringMapString(params.Tags),
140150
},
141-
&macpool.MacPoolRequestArgs{
142-
Prefix: "main",
143-
PoolName: viper.GetString(paramName),
144-
Architecture: viper.GetString(awsParams.MACArch),
145-
OSVersion: viper.GetString(awsParams.MACOSVersion),
146-
OfferedCapacity: viper.GetInt(paramOfferedCapacity),
147-
MaxSize: viper.GetInt(paramMaxSize),
148-
FixedLocation: viper.IsSet(awsParams.MACFixedLocation)}); err != nil {
151+
&macpool.HouseKeepRequestArgs{
152+
Pool: &macpool.PoolRequestArgs{
153+
Prefix: "main",
154+
Name: viper.GetString(paramName),
155+
Architecture: viper.GetString(awsParams.MACArch),
156+
OSVersion: viper.GetString(awsParams.MACOSVersion),
157+
OfferedCapacity: viper.GetInt(paramOfferedCapacity),
158+
MaxSize: viper.GetInt(paramMaxSize),
159+
},
160+
Machine: &macpool.MachineRequestArgs{
161+
VPCID: viper.GetString(paramVPCID),
162+
SSHSGID: viper.GetString(paramSSHSGID)},
163+
}); err != nil {
149164
logging.Error(err)
150165
}
151166
return nil
152167
},
153168
}
154169
flagSet := pflag.NewFlagSet(params.CreateCmdName, pflag.ExitOnError)
170+
params.AddCommonFlags(flagSet)
155171
flagSet.StringToStringP(params.Tags, "", nil, params.TagsDesc)
156172
flagSet.StringP(paramName, "", "", paramNameDesc)
157173
flagSet.Int(paramOfferedCapacity, paramOfferedCapacityDefault, paramOfferedCapacityDesc)
158174
flagSet.Int(paramMaxSize, paramMaxSizeDefault, paramMaxSizeDesc)
159175
flagSet.StringP(awsParams.MACArch, "", awsParams.MACArchDefault, awsParams.MACArchDesc)
160176
flagSet.StringP(awsParams.MACOSVersion, "", awsParams.MACOSVersion, awsParams.MACOSVersionDefault)
161-
flagSet.Bool(awsParams.MACFixedLocation, false, awsParams.MACFixedLocationDesc)
162177
flagSet.Bool(params.Serverless, false, params.ServerlessDesc)
178+
flagSet.StringP(paramVPCID, "", paramVPCIDDefault, paramVPCIDDesc)
179+
flagSet.StringP(paramSSHSGID, "", paramSSHSGIDDefault, paramSSHSGIDDesc)
163180
c.PersistentFlags().AddFlagSet(flagSet)
164181
return c
165182
}
@@ -209,7 +226,12 @@ func request() *cobra.Command {
209226
PoolName: viper.GetString(paramName),
210227
Architecture: viper.GetString(awsParams.MACArch),
211228
OSVersion: viper.GetString(awsParams.MACOSVersion),
212-
Timeout: viper.GetString(params.Timeout),
229+
Machine: &macpool.MachineRequestArgs{
230+
VPCID: viper.GetString(paramVPCID),
231+
SSHSGID: viper.GetString(paramSSHSGID),
232+
},
233+
Ticket: viper.GetString(paramTicket),
234+
Timeout: viper.GetString(params.Timeout),
213235
}); err != nil {
214236
logging.Error(err)
215237
}
@@ -223,6 +245,9 @@ func request() *cobra.Command {
223245
flagSet.StringP(awsParams.MACArch, "", awsParams.MACArchDefault, awsParams.MACArchDesc)
224246
flagSet.StringP(awsParams.MACOSVersion, "", awsParams.MACOSVersion, awsParams.MACOSVersionDefault)
225247
flagSet.StringP(params.Timeout, "", "", params.TimeoutDesc)
248+
flagSet.StringP(paramVPCID, "", paramVPCIDDefault, paramVPCIDDesc)
249+
flagSet.StringP(paramSSHSGID, "", paramSSHSGIDDefault, paramSSHSGIDDesc)
250+
flagSet.StringP(paramTicket, "", paramTicketDefault, paramTicketDesc)
226251
flagSet.Bool(params.Serverless, false, params.ServerlessDesc)
227252
flagSet.Bool(params.Remote, false, params.RemoteDesc)
228253
flagSet.AddFlagSet(params.GetGHActionsFlagset())
@@ -247,20 +272,22 @@ func release() *cobra.Command {
247272
Serverless: viper.IsSet(params.Serverless),
248273
Remote: viper.IsSet(params.Remote),
249274
},
250-
viper.GetString(awsParams.MACDHID)); err != nil {
275+
&macpool.MachineRequestArgs{
276+
VPCID: viper.GetString(paramVPCID),
277+
SSHSGID: viper.GetString(paramSSHSGID),
278+
},
279+
viper.GetString(paramTicket)); err != nil {
251280
logging.Error(err)
252281
}
253282
return nil
254283
},
255284
}
256285
flagSet := pflag.NewFlagSet(awsParams.MACReleaseCmd, pflag.ExitOnError)
257-
flagSet.StringP(awsParams.MACDHID, "", "", awsParams.MACDHIDDesc)
286+
flagSet.StringP(paramVPCID, "", paramVPCIDDefault, paramVPCIDDesc)
287+
flagSet.StringP(paramSSHSGID, "", paramSSHSGIDDefault, paramSSHSGIDDesc)
288+
flagSet.StringP(paramTicket, "", paramTicketDefault, paramTicketDesc)
258289
flagSet.Bool(params.Serverless, false, params.ServerlessDesc)
259290
flagSet.Bool(params.Remote, false, params.RemoteDesc)
260291
c.PersistentFlags().AddFlagSet(flagSet)
261-
err := c.MarkPersistentFlagRequired(awsParams.MACDHID)
262-
if err != nil {
263-
logging.Error(err)
264-
}
265292
return c
266293
}

‎go.mod

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,4 +174,3 @@ require (
174174
gopkg.in/warnings.v0 v0.1.2 // indirect
175175
lukechampine.com/frand v1.5.1 // indirect
176176
)
177-

‎pkg/integrations/cirrus/persistentworker.go

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func (args *PersistentWorkerArgs) GetUserDataValues() *integrations.UserDataValu
4343
CliURL: downloadURL(),
4444
Name: pwa.Name,
4545
Token: pwa.Token,
46-
Labels: getLabelsAsString(),
46+
Labels: GetLabelsAsString(),
4747
Port: cirrusPort,
4848
}
4949
}
@@ -74,18 +74,8 @@ func GetToken() string {
7474
"")
7575
}
7676

77-
// platform: darwin, linux, windows
78-
// arch: amd64, arm64
79-
func downloadURL() string {
80-
url := fmt.Sprintf(baseURL, version, *pwa.Platform, *pwa.Arch)
81-
if pwa.Platform == &Windows {
82-
url = fmt.Sprintf("%s.exe", url)
83-
}
84-
return url
85-
}
86-
8777
// Get labels in format
88-
func getLabelsAsString() string {
78+
func GetLabelsAsString() string {
8979
return util.IfNillable(pwa != nil,
9080
func() string {
9181
if len(pwa.Labels) > 0 {
@@ -101,3 +91,13 @@ func getLabelsAsString() string {
10191
},
10292
"")
10393
}
94+
95+
// platform: darwin, linux, windows
96+
// arch: amd64, arm64
97+
func downloadURL() string {
98+
url := fmt.Sprintf(baseURL, version, *pwa.Platform, *pwa.Arch)
99+
if pwa.Platform == &Windows {
100+
url = fmt.Sprintf("%s.exe", url)
101+
}
102+
return url
103+
}

‎pkg/manager/context/context.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,16 @@ type context struct {
6060
remote bool
6161
tags map[string]string
6262
tagsAsPulumiStringMap pulumi.StringMap
63+
// This will be set if we need specific customization on a specfici execution
64+
provider Provider
6365
}
6466

6567
// mapt context
6668
var mc *context
6769

6870
type Provider interface {
6971
Init(backedURL string) error
72+
Custom(ctx *pulumi.Context) (*pulumi.ProviderResource, error)
7073
}
7174

7275
func Init(ca *ContextArgs, provider Provider) error {
@@ -81,6 +84,7 @@ func Init(ca *ContextArgs, provider Provider) error {
8184
serverless: ca.Serverless,
8285
forceDestroy: ca.ForceDestroy,
8386
remote: ca.Remote,
87+
provider: provider,
8488
}
8589
addCommonTags()
8690
// Init provider
@@ -119,6 +123,18 @@ func IsForceDestroy() bool { return mc.forceDestroy }
119123

120124
func IsRemote() bool { return mc.remote }
121125

126+
func CommonOptions(ctx *pulumi.Context) (co []pulumi.ResourceOption) {
127+
// Check if provider requires customization
128+
cp, err := mc.provider.Custom(ctx)
129+
if cp != nil {
130+
co = append(co, pulumi.Provider(*cp))
131+
}
132+
if err != nil {
133+
logging.Errorf("Error registering custom provider %v", err)
134+
}
135+
return
136+
}
137+
122138
// It will create a runID
123139
// if context has been intialized it will set it as the runID for the context
124140
// otherwise it will return the value (one time value)

‎pkg/provider/aws/action/fedora/fedora.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,9 @@ func (r *Request) deploy(ctx *pulumi.Context) error {
255255
ctx.Export(fmt.Sprintf("%s-%s", r.Prefix, outputHost),
256256
c.GetHostIP(!r.Airgap))
257257
if len(r.Timeout) > 0 {
258-
if err = serverless.OneTimeDelayedTask(ctx,
258+
if err = serverless.OneTimeDelayedTask(
259+
ctx,
260+
fmt.Sprintf("fedora-timeout-%s", maptContext.RunID()),
259261
r.region, r.Prefix,
260262
awsFedoraDedicatedID,
261263
fmt.Sprintf("aws %s destroy --project-name %s --backed-url %s --serverless",

‎pkg/provider/aws/action/kind/kind.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ func (r *kindRequest) deploy(ctx *pulumi.Context) error {
195195
c.GetHostIP(true))
196196
if len(*r.timeout) > 0 {
197197
if err = serverless.OneTimeDelayedTask(ctx,
198+
fmt.Sprintf("kind-destroy-%s", maptContext.RunID()),
198199
*r.allocationData.Region, *r.prefix,
199200
awsKindID,
200201
fmt.Sprintf("aws %s destroy --project-name %s --backed-url %s --serverless --force-destroy",

0 commit comments

Comments
 (0)