From 03ef47253d04505a9a4e6126fd0d952d55b67afb Mon Sep 17 00:00:00 2001 From: James Ranson Date: Wed, 11 Dec 2024 07:58:42 -0700 Subject: [PATCH 1/9] support ENV via config file --- internal/commands/deployment.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/commands/deployment.go b/internal/commands/deployment.go index 316f95d..34b0b64 100644 --- a/internal/commands/deployment.go +++ b/internal/commands/deployment.go @@ -258,8 +258,11 @@ var ( } envVarsFlag = &cli.StringSliceFlag{ - Name: "env", - Sources: cli.EnvVars(deploymentEnvVar("ENV")), + Name: "env", + Sources: cli.NewValueSourceChain( + cli.EnvVar(deploymentEnvVar("ENV")), + altsrc.ConfigFile(configFlag.Name, "deployment.env"), + ), Usage: "`` formatted environment variables (use quotes for values with spaces or commas)", Category: "Deployment:", } From 4cec2458318ad6e229ed4fcf59052590ee30837f Mon Sep 17 00:00:00 2001 From: James Ranson Date: Wed, 11 Dec 2024 08:16:33 -0700 Subject: [PATCH 2/9] use consts for immutable values --- internal/commands/build.go | 4 ++-- internal/commands/deployment.go | 4 ++-- internal/commands/flags.go | 6 ++---- internal/commands/log.go | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/commands/build.go b/internal/commands/build.go index c0cedbe..c91d248 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -27,6 +27,8 @@ type etagPart struct { etag string } +const buildFlagEnvVarPrefix = globalFlagEnvVarPrefix + "BUILD_" + var Build = &cli.Command{ Name: "build", Usage: "options for builds", @@ -248,8 +250,6 @@ func buildFlagEnvVar(name string) string { } var ( - buildFlagEnvVarPrefix = globalFlagEnvVarPrefix + "BUILD_" - buildIDFlag = &cli.StringFlag{ Name: "build-id", Aliases: []string{"b"}, diff --git a/internal/commands/deployment.go b/internal/commands/deployment.go index 34b0b64..1c153b2 100644 --- a/internal/commands/deployment.go +++ b/internal/commands/deployment.go @@ -19,6 +19,8 @@ import ( "github.com/hathora/ci/internal/workaround" ) +const deploymentFlagEnvVarPrefix = globalFlagEnvVarPrefix + "DEPLOYMENT_" + var ( allowedTransportTypes = []string{"tcp", "udp", "tls"} minRoomsPerProcess = 1 @@ -192,8 +194,6 @@ func deploymentEnvVar(name string) string { } var ( - deploymentFlagEnvVarPrefix = fmt.Sprintf("%s%s", globalFlagEnvVarPrefix, "DEPLOYMENT_") - deploymentIDFlag = &cli.StringFlag{ Name: "deployment-id", Aliases: []string{"d"}, diff --git a/internal/commands/flags.go b/internal/commands/flags.go index c06115d..7a8170e 100644 --- a/internal/commands/flags.go +++ b/internal/commands/flags.go @@ -6,6 +6,8 @@ import ( "github.com/hathora/ci/internal/commands/altsrc" ) +const globalFlagEnvVarPrefix = "HATHORA_" + var ( outputFlag = &cli.StringFlag{ Name: "output", @@ -103,10 +105,6 @@ var ( } ) -var ( - globalFlagEnvVarPrefix = "HATHORA_" -) - func globalFlagEnvVar(name string) string { return globalFlagEnvVarPrefix + name } diff --git a/internal/commands/log.go b/internal/commands/log.go index 6d34f93..593e31c 100644 --- a/internal/commands/log.go +++ b/internal/commands/log.go @@ -16,6 +16,8 @@ import ( "github.com/hathora/ci/internal/workaround" ) +const logFlagEnvVarPrefix = globalFlagEnvVarPrefix + "LOG_" + var ( minTailLines = 1 maxTailLines = 5000 @@ -66,8 +68,6 @@ func logFlagEnvVar(name string) string { } var ( - logFlagEnvVarPrefix = globalFlagEnvVarPrefix + "LOG_" - followFlag = &cli.BoolFlag{ Name: "follow", Sources: cli.NewValueSourceChain( From 0dac8c083c0334c8ee6a81f150696ad9cdf4f2ea Mon Sep 17 00:00:00 2001 From: James Ranson Date: Wed, 11 Dec 2024 08:24:20 -0700 Subject: [PATCH 3/9] use consts for HTTP header elements --- internal/commands/build.go | 7 ++++--- internal/commands/build_test.go | 3 ++- internal/httputil/headers.go | 11 +++++++++-- internal/mock/hathora.go | 3 ++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/commands/build.go b/internal/commands/build.go index c91d248..ad63f19 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -15,6 +15,7 @@ import ( "github.com/hathora/ci/internal/archive" "github.com/hathora/ci/internal/commands/altsrc" + "github.com/hathora/ci/internal/httputil" "github.com/hathora/ci/internal/output" "github.com/hathora/ci/internal/sdk" "github.com/hathora/ci/internal/sdk/models/shared" @@ -187,7 +188,7 @@ func doBuildCreate(ctx context.Context, hathora *sdk.SDK, buildTag, buildId, fil return nil, fmt.Errorf("failed to upload parts: %w", err) } - resp, err := http.Post(createRes.CreatedBuildV3WithMultipartUrls.CompleteUploadPostRequestURL, "application/xml", bytes.NewBufferString(createEtagXML(etagParts...))) + resp, err := http.Post(createRes.CreatedBuildV3WithMultipartUrls.CompleteUploadPostRequestURL, httputil.ValueApplicationXML, bytes.NewBufferString(createEtagXML(etagParts...))) if err != nil { return nil, err } @@ -432,12 +433,12 @@ func uploadFileToS3(preSignedURL string, byteBuffer []byte, globalUploadProgress globalUploadProgress: globalUploadProgress, hideUploadProgress: hideUploadProgress, } - req, err := http.NewRequest("PUT", preSignedURL, progressReader) + req, err := http.NewRequest(http.MethodPut, preSignedURL, progressReader) if err != nil { os.Stderr.WriteString(fmt.Sprintf("failed to create request: %v", err)) return "", fmt.Errorf("failed to create request: %w", err) } - req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set(httputil.NameContentType, httputil.ValueApplicationOctetStream) req.ContentLength = int64(requestBody.Len()) client := &http.Client{} diff --git a/internal/commands/build_test.go b/internal/commands/build_test.go index 2eec04e..ac15c93 100644 --- a/internal/commands/build_test.go +++ b/internal/commands/build_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/hathora/ci/internal/commands" + "github.com/hathora/ci/internal/httputil" "github.com/hathora/ci/internal/mock" ) @@ -125,7 +126,7 @@ func Test_Integration_BuildCommands_Happy(t *testing.T) { expectRequest: func(t *testing.T, r *http.Request, requestBody *json.RawMessage) { assert.Equal(t, r.Method, http.MethodPost, "request method should be POST") assert.Equal(t, "/builds/v3/test-app-id/create", r.URL.Path, "request path should contain app id") - assert.Equal(t, "application/json", r.Header.Get("Content-Type"), "request should have a JSON content type") + assert.Equal(t, httputil.ValueApplicationJSON, r.Header.Get(httputil.NameContentType), "request should have a JSON content type") assert.NotNil(t, requestBody, "request body should not be nil") assert.Equal(t, `{"buildTag":"test-build-tag"}`, string(*requestBody), "request body should have supplied build tag") }, diff --git a/internal/httputil/headers.go b/internal/httputil/headers.go index b5fbe87..d8f685c 100644 --- a/internal/httputil/headers.go +++ b/internal/httputil/headers.go @@ -4,6 +4,13 @@ import ( "net/http" ) +const ( + NameContentType = "Content-Type" + ValueApplicationJSON = "application/json" + ValueApplicationOctetStream = "application/octet-stream" + ValueApplicationXML = "application/xml" +) + type contentTypeRoundTripper struct { underlying http.RoundTripper } @@ -19,8 +26,8 @@ func (c *contentTypeRoundTripper) RoundTrip(req *http.Request) (*http.Response, return res, err } - if res.Header.Get("content-type") == "" { - res.Header.Set("content-type", "application/octet-stream") + if res.Header.Get(NameContentType) == "" { + res.Header.Set(NameContentType, ValueApplicationOctetStream) } return res, err diff --git a/internal/mock/hathora.go b/internal/mock/hathora.go index b967137..46fa2b5 100644 --- a/internal/mock/hathora.go +++ b/internal/mock/hathora.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "testing" + "github.com/hathora/ci/internal/httputil" "github.com/hathora/ci/internal/sdk" "github.com/stretchr/testify/require" ) @@ -120,7 +121,7 @@ func (m *mockHathora) ServeHTTP(w http.ResponseWriter, r *http.Request) { jsonBody := json.RawMessage(body) m.capturedRequestBodies = append(m.capturedRequestBodies, &jsonBody) - w.Header().Set("Content-Type", "application/json") + w.Header().Set(httputil.NameContentType, httputil.ValueApplicationJSON) w.WriteHeader(m.cannedStatuses[m.nextResponseIndex]) _, err = w.Write(m.cannedResponses[m.nextResponseIndex]) From cc5d9c82ad55c1a1e6cea6b79447d027bbe2ffef Mon Sep 17 00:00:00 2001 From: James Ranson Date: Wed, 11 Dec 2024 12:24:53 -0700 Subject: [PATCH 4/9] update usage text --- internal/commands/deployment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/commands/deployment.go b/internal/commands/deployment.go index 1c153b2..8850d8c 100644 --- a/internal/commands/deployment.go +++ b/internal/commands/deployment.go @@ -292,7 +292,7 @@ var ( fromLatestFlag = &cli.BoolFlag{ Name: "from-latest", Sources: cli.EnvVars(deploymentEnvVar("FROM_LATEST")), - Usage: "whether to use settings from the latest deployment; if true other flags act as overrides", + Usage: "whether to use settings from the latest deployment; if true other flags and config file values act as overrides", Category: "Deployment:", } ) From 2509d8d42145dada24c0276e0868c1467d555abf Mon Sep 17 00:00:00 2001 From: James Ranson Date: Wed, 11 Dec 2024 13:46:09 -0700 Subject: [PATCH 5/9] update parser to work w/ env list from config file --- internal/commands/deployment.go | 11 +++++++++++ internal/shorthand/env_var.go | 29 +++++++++++++++++++++++++++-- internal/shorthand/env_var_test.go | 14 ++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/internal/commands/deployment.go b/internal/commands/deployment.go index 8850d8c..171a2c8 100644 --- a/internal/commands/deployment.go +++ b/internal/commands/deployment.go @@ -310,6 +310,17 @@ func parseContainerPorts(ports []string) ([]shared.ContainerPort, error) { } func parseEnvVars(envVars []string) ([]shared.DeploymentConfigV3Env, error) { + // Envs from a Config File are parsed from urfave/cli as a single-element + // string slice of all the values like: + // []string{`[KEY1=VAL1 KEY2=VAL2 KEY3="QUOTED VAL3"]`} + // This stanza parses those into a proper slice of one Env per slice element + if len(envVars) == 1 && strings.HasPrefix(envVars[0], "[") && strings.HasSuffix(envVars[0], "]") { + var err error + envVars, err = shorthand.ParseConfigFileVars(envVars[0]) + if err != nil { + return nil, err + } + } envVars = fixOverZealousCommaSplitting(envVars) output := make([]shared.DeploymentConfigV3Env, 0, len(envVars)) for _, envVar := range envVars { diff --git a/internal/shorthand/env_var.go b/internal/shorthand/env_var.go index 92e357f..bad436a 100644 --- a/internal/shorthand/env_var.go +++ b/internal/shorthand/env_var.go @@ -2,6 +2,7 @@ package shorthand import ( "fmt" + "regexp" "strings" "github.com/hathora/ci/internal/sdk/models/shared" @@ -14,15 +15,35 @@ func ParseDeploymentEnvVar(s string) (*shared.DeploymentConfigV3Env, error) { parts := strings.SplitN(s, "=", 2) if len(parts) != 2 { - return nil, fmt.Errorf("invalid env var format: %s", s) + return nil, ErrInvalidEnvVarFormat(s) + } + + parts[1] = strings.TrimSpace(parts[1]) + // remove any enclosing quotes + if strings.HasPrefix(parts[1], `"`) && strings.HasSuffix(parts[1], `"`) { + parts[1] = strings.TrimPrefix(strings.TrimSuffix(parts[1], `"`), `"`) } return &shared.DeploymentConfigV3Env{ Name: strings.TrimSpace(parts[0]), - Value: strings.TrimSpace(parts[1]), + Value: parts[1], }, nil } +var reEnvs = regexp.MustCompile(`\b[A-Za-z0-9_]+="[^"]*"|\b[A-Za-z0-9_]+=[^" ]+`) + +func ParseConfigFileVars(input string) ([]string, error) { + input = strings.TrimPrefix(strings.TrimSuffix(input, "]"), "[") + if len(input) == 0 { + return nil, nil + } + matches := reEnvs.FindAllString(input, -1) + if len(matches) == 0 { + return nil, ErrInvalidEnvVarFormat(input) + } + return matches, nil +} + func MapEnvToEnvConfig(input []shared.DeploymentV3Env) []shared.DeploymentConfigV3Env { output := make([]shared.DeploymentConfigV3Env, 0) @@ -32,3 +53,7 @@ func MapEnvToEnvConfig(input []shared.DeploymentV3Env) []shared.DeploymentConfig return output } + +func ErrInvalidEnvVarFormat(env string) error { + return fmt.Errorf("invalid env var format: %s", env) +} diff --git a/internal/shorthand/env_var_test.go b/internal/shorthand/env_var_test.go index 2da9eb0..e6dd5c3 100644 --- a/internal/shorthand/env_var_test.go +++ b/internal/shorthand/env_var_test.go @@ -67,3 +67,17 @@ func Test_DeploymentEnvVarShorthand(t *testing.T) { } } + +func TestParseConfigFileVars(t *testing.T) { + input := `[KEY1=VAL1 KEY3="VAL 3 WITH SPACES" KEY2=VAL2]` + out, err := shorthand.ParseConfigFileVars(input) + if err != nil { + t.Error(err) + } + assert.Equal(t, 3, len(out)) + input = `[KEY1 =SHOULD_ERROR]` + _, err = shorthand.ParseConfigFileVars(input) + if err == nil { + t.Error("expected error for invalid env var format") + } +} From 0d8918f0fb6ddb6453050897161d14cb12403869 Mon Sep 17 00:00:00 2001 From: James Ranson Date: Wed, 11 Dec 2024 14:11:12 -0700 Subject: [PATCH 6/9] rename func --- internal/shorthand/env_var.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/shorthand/env_var.go b/internal/shorthand/env_var.go index bad436a..d9f0d94 100644 --- a/internal/shorthand/env_var.go +++ b/internal/shorthand/env_var.go @@ -15,7 +15,7 @@ func ParseDeploymentEnvVar(s string) (*shared.DeploymentConfigV3Env, error) { parts := strings.SplitN(s, "=", 2) if len(parts) != 2 { - return nil, ErrInvalidEnvVarFormat(s) + return nil, NewErrInvalidEnvVarFormat(s) } parts[1] = strings.TrimSpace(parts[1]) @@ -39,7 +39,7 @@ func ParseConfigFileVars(input string) ([]string, error) { } matches := reEnvs.FindAllString(input, -1) if len(matches) == 0 { - return nil, ErrInvalidEnvVarFormat(input) + return nil, NewErrInvalidEnvVarFormat(input) } return matches, nil } @@ -54,6 +54,6 @@ func MapEnvToEnvConfig(input []shared.DeploymentV3Env) []shared.DeploymentConfig return output } -func ErrInvalidEnvVarFormat(env string) error { +func NewErrInvalidEnvVarFormat(env string) error { return fmt.Errorf("invalid env var format: %s", env) } From b2ec422718a1391ac84070b5eae2cb88f4f102e6 Mon Sep 17 00:00:00 2001 From: James Ranson Date: Thu, 12 Dec 2024 07:02:28 -0700 Subject: [PATCH 7/9] handle additional-container-ports from config file --- internal/commands/deployment.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/commands/deployment.go b/internal/commands/deployment.go index 171a2c8..8a88315 100644 --- a/internal/commands/deployment.go +++ b/internal/commands/deployment.go @@ -298,6 +298,10 @@ var ( ) func parseContainerPorts(ports []string) ([]shared.ContainerPort, error) { + // this converts a string representation of the slice from a config file into a real string slice + if len(ports) == 1 && strings.HasPrefix(ports[0], "[") && strings.HasSuffix(ports[0], "]") { + ports = strings.Split(strings.TrimPrefix(strings.TrimSuffix(ports[0], "]"), "["), " ") + } output := make([]shared.ContainerPort, 0, len(ports)) for _, port := range ports { p, err := shorthand.ParseContainerPort(port) From 12d6bc1eefb5e327679b5e744cd75e2f0555a0be Mon Sep 17 00:00:00 2001 From: James Ranson Date: Mon, 16 Dec 2024 12:03:07 -0700 Subject: [PATCH 8/9] don't sign binaries if cert secret is inaccessible --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8e44b67..4e7ec67 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -137,7 +137,7 @@ jobs: run: make build - name: Sign the Windows binary - if: matrix.platform.os == 'windows' + if: github.repository_owner == 'hathora' && matrix.platform.os == 'windows' run: | $decodedCertificate = [System.Convert]::FromBase64String("${{ secrets.SIGNING_CERTIFICATE_PFX }}") [System.IO.File]::WriteAllBytes("certificate.pfx", $decodedCertificate) From 73c3cf070a1ff9e8f50f139b94c3827de45f2774 Mon Sep 17 00:00:00 2001 From: James Ranson Date: Mon, 16 Dec 2024 15:00:41 -0700 Subject: [PATCH 9/9] don't sign binaries if cert secret is inaccessible --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e7ec67..9f2ca83 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,6 +119,8 @@ jobs: ] arch: [amd64, arm64] runs-on: ${{ matrix.platform.runner }} + env: + HAS_SECRET: ${{ secrets.SIGNING_CERTIFICATE_PFX }} steps: - name: Checkout uses: actions/checkout@v4 @@ -137,7 +139,7 @@ jobs: run: make build - name: Sign the Windows binary - if: github.repository_owner == 'hathora' && matrix.platform.os == 'windows' + if: env.HAS_SECRET && github.repository_owner == 'hathora' && matrix.platform.os == 'windows' run: | $decodedCertificate = [System.Convert]::FromBase64String("${{ secrets.SIGNING_CERTIFICATE_PFX }}") [System.IO.File]::WriteAllBytes("certificate.pfx", $decodedCertificate)