fix(server): bound request/statement time and raise ALB idle timeout

.env.example

@@ -34,6 +34,18 @@ TSIO_DATABASE_URL=postgres://tsio:tsio@localhost:6432/tsio?sslmode=disable
 # to run `tsioctl db migrate` out-of-band before the app starts.
 TSIO_DB_AUTO_MIGRATE=true
 
+# Max pgx pool connections per process. With multiple app tasks against one
+# Postgres, keep (tasks × this value) below the server's max_connections.
+TSIO_DB_MAX_CONNS=20
+
+# Per-statement timeout (ms) so a slow query can't hold a pool connection
+# until the load balancer times the request out. 0 disables.
+TSIO_DB_STATEMENT_TIMEOUT_MS=30000
+
+# Per-request timeout for public read (GET) endpoints; keep it below the load
+# balancer idle timeout. Write/upload/WebSocket routes are exempt.
+TSIO_READ_REQUEST_TIMEOUT=30s
+
 # ------------------------------------------------------------------------------
 # Object storage (S3 / MinIO)
 # ------------------------------------------------------------------------------

apps/server/cmd/tsio/main.go

@@ -64,7 +64,10 @@ func run() error {
 		}
 	}
 
-	pool, err := db.NewPool(ctx, cfg.DatabaseURL)
+	pool, err := db.NewPool(ctx, cfg.DatabaseURL,
+		db.WithMaxConns(cfg.DBMaxConns),
+		db.WithStatementTimeout(cfg.DBStatementTimeoutMs),
+	)
 	if err != nil {
 		return err
 	}
@@ -161,6 +164,7 @@ func run() error {
 		MaxUploadBytes:         cfg.MaxUploadBytes,
 		MaxArtifactBytes:       cfg.MaxArtifactBytes,
 		PresignTTL:             5 * time.Minute,
+		ReadRequestTimeout:     cfg.ReadRequestTimeout,
 	})
 
 	srv := &http.Server{

apps/server/internal/api/ws/events.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/coder/websocket"
 	"github.com/google/uuid"
@@ -29,6 +30,11 @@ import (
 	"github.com/mattermost/mattermost-test-system-io/apps/server/internal/events"
 )
 
+// wsPingInterval is how often the server sends a WebSocket ping. It must stay
+// well below any proxy/load-balancer idle timeout (the ALB is 120s) so an
+// otherwise-quiet subscription (no events for a while) isn't dropped as idle.
+const wsPingInterval = 30 * time.Second
+
 // hubAPI is the slice of *events.Hub the handler actually depends on. Defining
 // it as an interface lets unit tests substitute a recording fake without
 // pulling in the real Hub's broadcast semantics.
@@ -139,6 +145,31 @@ func (h *Handler) serve(ctx context.Context, conn *websocket.Conn, hub hubAPI) {
 	wg.Add(1)
 	go forwardChan(defaultCh)
 
+	// Keepalive: ping periodically so a connection with no events to forward
+	// isn't dropped by an idle proxy / load-balancer timeout before the next
+	// event arrives. coder/websocket's Ping is safe to call concurrently with
+	// the forwarder writes. A failed ping tears the connection down.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ticker := time.NewTicker(wsPingInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				pingCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+				err := conn.Ping(pingCtx)
+				cancel()
+				if err != nil {
+					cancelCtx()
+					return
+				}
+			}
+		}
+	}()
+
 	// Per-connection orchestration subscriptions. Keyed by composite identity
 	// so a duplicate subscribe.orchestration frame for the same run is a
 	// no-op (rather than leaking another subscriber).

apps/server/internal/config/config.go

@@ -31,6 +31,15 @@ type Config struct {
 	DBSSLMode     string `env:"TSIO_DB_SSLMODE" envDefault:"require"`
 	DBAutoMigrate bool   `env:"TSIO_DB_AUTO_MIGRATE" envDefault:"true"`
 
+	// DBMaxConns caps the pgx pool size per process. With multiple app tasks
+	// pointing at one Postgres, keep the aggregate (tasks × this value) under
+	// the server's max_connections.
+	DBMaxConns int `env:"TSIO_DB_MAX_CONNS" envDefault:"20"`
+	// DBStatementTimeoutMs bounds any single SQL statement server-side so a
+	// slow query cannot hold a pool connection until the load balancer times
+	// the request out. 0 disables the timeout.
+	DBStatementTimeoutMs int `env:"TSIO_DB_STATEMENT_TIMEOUT_MS" envDefault:"30000"`
+
 	// S3 credentials are optional: when running in ECS/EC2, the AWS SDK picks
 	// up credentials from the task role automatically. Explicit keys are used
 	// for local dev against MinIO or for S3 accounts that don't match the
@@ -80,6 +89,13 @@ type Config struct {
 
 	CORSAllowedOrigins []string `env:"TSIO_CORS_ALLOWED_ORIGINS" envSeparator:","`
 
+	// ReadRequestTimeout bounds how long a public read request (report and
+	// orchestration-status GETs) may run before its context is canceled and
+	// the in-flight DB query aborted. Write, upload, and WebSocket routes are
+	// intentionally exempt. Keep it comfortably below the load balancer idle
+	// timeout so slow reads surface as a fast error instead of a 504.
+	ReadRequestTimeout time.Duration `env:"TSIO_READ_REQUEST_TIMEOUT" envDefault:"30s"`
+
 	// Client-facing tunables surfaced via GET /api/v1/config.
 	UploadTimeoutMs int  `env:"TSIO_UPLOAD_TIMEOUT_MS" envDefault:"3600000"` // 1h
 	HTMLViewEnabled bool `env:"TSIO_HTML_VIEW_ENABLED" envDefault:"false"`

apps/server/internal/config/config_test.go

@@ -72,6 +72,38 @@ func TestLoad_readsEnv(t *testing.T) {
 	if !cfg.DBAutoMigrate {
 		t.Error("DBAutoMigrate should default to true")
 	}
+	if cfg.DBMaxConns != 20 {
+		t.Errorf("DBMaxConns default = %d, want 20", cfg.DBMaxConns)
+	}
+	if cfg.DBStatementTimeoutMs != 30000 {
+		t.Errorf("DBStatementTimeoutMs default = %d, want 30000", cfg.DBStatementTimeoutMs)
+	}
+	if cfg.ReadRequestTimeout != 30*time.Second {
+		t.Errorf("ReadRequestTimeout default = %v, want 30s", cfg.ReadRequestTimeout)
+	}
+}
+
+func TestLoad_readsGuardrailOverrides(t *testing.T) {
+	t.Setenv("TSIO_DATABASE_URL", "postgres://user:pass@localhost/db?sslmode=disable")
+	t.Setenv("TSIO_S3_BUCKET", "reports")
+	t.Setenv("TSIO_SESSION_SECRET", "test-secret")
+	t.Setenv("TSIO_DB_MAX_CONNS", "40")
+	t.Setenv("TSIO_DB_STATEMENT_TIMEOUT_MS", "15000")
+	t.Setenv("TSIO_READ_REQUEST_TIMEOUT", "10s")
+
+	cfg, err := Load()
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if cfg.DBMaxConns != 40 {
+		t.Errorf("DBMaxConns = %d, want 40", cfg.DBMaxConns)
+	}
+	if cfg.DBStatementTimeoutMs != 15000 {
+		t.Errorf("DBStatementTimeoutMs = %d, want 15000", cfg.DBStatementTimeoutMs)
+	}
+	if cfg.ReadRequestTimeout != 10*time.Second {
+		t.Errorf("ReadRequestTimeout = %v, want 10s", cfg.ReadRequestTimeout)
+	}
 }
 
 func TestLoad_assemblesDatabaseURLFromParts(t *testing.T) {

apps/server/internal/db/pool.go

@@ -3,13 +3,48 @@ package db
 import (
 	"context"
 	"fmt"
+	"math"
+	"strconv"
 	"time"
 
 	"github.com/jackc/pgx/v5/pgxpool"
 )
 
-// NewPool constructs a pgx connection pool with sane defaults.
-func NewPool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
+// PoolOption tweaks the pgxpool.Config before the pool is opened. Options are
+// variadic so existing callers (CLI tools, tests) keep the zero-config
+// defaults while the server can thread through operator-tunable knobs.
+type PoolOption func(*pgxpool.Config)
+
+// WithMaxConns overrides the pool's maximum connection count. Values outside
+// (0, math.MaxInt32] are ignored so a missing/invalid env knob falls back to
+// the default (and the int->int32 conversion is provably in range).
+func WithMaxConns(n int) PoolOption {
+	return func(cfg *pgxpool.Config) {
+		if n > 0 && n <= math.MaxInt32 {
+			cfg.MaxConns = int32(n)
+		}
+	}
+}
+
+// WithStatementTimeout sets a per-statement timeout (in milliseconds) on every
+// connection in the pool. This bounds any single query so a slow read cannot
+// hold a connection indefinitely and starve the pool — the query is aborted
+// server-side and the connection returns to the pool. A value of 0 explicitly
+// disables the timeout (Postgres statement_timeout=0), overriding any value
+// inherited from the connection string; a negative value leaves the connection
+// default untouched. Applies per-statement, so it does not affect long
+// multi-statement work like the background JSON extractor.
+func WithStatementTimeout(ms int) PoolOption {
+	return func(cfg *pgxpool.Config) {
+		if ms >= 0 {
+			cfg.ConnConfig.RuntimeParams["statement_timeout"] = strconv.Itoa(ms)
+		}
+	}
+}
+
+// NewPool constructs a pgx connection pool with sane defaults, applying any
+// supplied options after the defaults so callers can override them.
+func NewPool(ctx context.Context, databaseURL string, opts ...PoolOption) (*pgxpool.Pool, error) {
 	cfg, err := pgxpool.ParseConfig(databaseURL)
 	if err != nil {
 		return nil, fmt.Errorf("parse database url: %w", err)
@@ -19,6 +54,9 @@ func NewPool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
 	cfg.MaxConnIdleTime = 10 * time.Minute
 	cfg.MaxConnLifetime = 1 * time.Hour
 	cfg.ConnConfig.RuntimeParams["application_name"] = "tsio"
+	for _, opt := range opts {
+		opt(cfg)
+	}
 
 	pool, err := pgxpool.NewWithConfig(ctx, cfg)
 	if err != nil {

apps/server/internal/db/pool_test.go

@@ -0,0 +1,69 @@
+package db
+
+import (
+	"testing"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+const testDSN = "postgres://tsio:tsio@localhost:5432/tsio?sslmode=disable"
+
+func TestWithMaxConns(t *testing.T) {
+	cfg, err := pgxpool.ParseConfig(testDSN)
+	if err != nil {
+		t.Fatalf("parse config: %v", err)
+	}
+	WithMaxConns(50)(cfg)
+	if cfg.MaxConns != 50 {
+		t.Errorf("MaxConns = %d, want 50", cfg.MaxConns)
+	}
+}
+
+func TestWithMaxConns_ignoresNonPositive(t *testing.T) {
+	cfg, err := pgxpool.ParseConfig(testDSN)
+	if err != nil {
+		t.Fatalf("parse config: %v", err)
+	}
+	cfg.MaxConns = 20
+	WithMaxConns(0)(cfg)
+	WithMaxConns(-5)(cfg)
+	if cfg.MaxConns != 20 {
+		t.Errorf("MaxConns = %d, want unchanged 20", cfg.MaxConns)
+	}
+}
+
+func TestWithStatementTimeout(t *testing.T) {
+	cfg, err := pgxpool.ParseConfig(testDSN)
+	if err != nil {
+		t.Fatalf("parse config: %v", err)
+	}
+	WithStatementTimeout(15000)(cfg)
+	if got := cfg.ConnConfig.RuntimeParams["statement_timeout"]; got != "15000" {
+		t.Errorf("statement_timeout = %q, want \"15000\"", got)
+	}
+}
+
+func TestWithStatementTimeout_zeroDisablesExplicitly(t *testing.T) {
+	cfg, err := pgxpool.ParseConfig(testDSN)
+	if err != nil {
+		t.Fatalf("parse config: %v", err)
+	}
+	// Seed an inherited value to prove 0 overrides it with an explicit disable.
+	cfg.ConnConfig.RuntimeParams["statement_timeout"] = "5000"
+	WithStatementTimeout(0)(cfg)
+	if got := cfg.ConnConfig.RuntimeParams["statement_timeout"]; got != "0" {
+		t.Errorf("statement_timeout = %q, want \"0\" (explicit disable)", got)
+	}
+}
+
+func TestWithStatementTimeout_negativeLeavesInherited(t *testing.T) {
+	cfg, err := pgxpool.ParseConfig(testDSN)
+	if err != nil {
+		t.Fatalf("parse config: %v", err)
+	}
+	cfg.ConnConfig.RuntimeParams["statement_timeout"] = "5000"
+	WithStatementTimeout(-1)(cfg)
+	if got := cfg.ConnConfig.RuntimeParams["statement_timeout"]; got != "5000" {
+		t.Errorf("statement_timeout = %q, want inherited \"5000\" unchanged", got)
+	}
+}

apps/server/internal/server/server.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5"
+	chimw "github.com/go-chi/chi/v5/middleware"
 	"github.com/jackc/pgx/v5/pgxpool"
 
 	apiroot "github.com/mattermost/mattermost-test-system-io/apps/server/internal/api"
@@ -75,6 +76,10 @@ type Deps struct {
 	MaxUploadBytes   int64
 	MaxArtifactBytes int64
 	PresignTTL       time.Duration
+
+	// ReadRequestTimeout bounds public read (GET) handlers. Zero disables the
+	// per-request timeout. Upload, write, and WebSocket routes are exempt.
+	ReadRequestTimeout time.Duration
 }
 
 // Build constructs the chi router with the full feature set and returns it as
@@ -165,25 +170,11 @@ func Build(d Deps) chi.Router {
 		// Admin-key-gated bootstrap endpoint (X-Admin-Key header).
 		r.Post("/auth/oidc-policies", authH.CreateOIDCPolicy)
 
-		// --- Public: report reads ---
-		r.Get("/reports", reportsH.List)
-		r.Get("/reports/grouped", reportsH.Grouped)
-		r.Get("/reports/individual", reportsH.Individual)
-		r.Get("/reports/consolidated", reportsH.Consolidated)
-		r.Get("/reports/{id}", reportsH.Detail)
-		r.Get("/reports/{id}/suites", reportsH.Suites)
-		r.Get("/reports/{id}/suites/{suiteID}/specs", reportsH.SuiteSpecs)
-		r.Get("/reports/{id}/cases", reportsH.Cases)
-		r.Get("/reports/{id}/json", reportsH.JSONFile)
-		r.Get("/reports/{id}/search", reportsH.Search)
-
 		// --- Public: WebSocket (anonymous; the dashboard never attaches creds) ---
+		// Registered outside the read-timeout group below: the connection is
+		// long-lived and must not be canceled by a per-request deadline.
 		r.Get("/ws", wsH.Events)
 
-		// --- Public: orchestration status snapshot ---
-		// The dashboard fetches this without credentials, alongside the public
-		// report-reads above. Mutations (begin/checkout/complete/screenshots)
-		// stay in the protected group below.
 		publicOrchH := &orchapi.Handlers{
 			Pool:               d.Pool,
 			Store:              d.OrchestrationStore,
@@ -194,7 +185,31 @@ func Build(d Deps) chi.Router {
 			LeaseRetentionMs:   60_000,
 			MaxScreenshotBytes: 10 * 1024 * 1024,
 		}
-		orchapi.RegisterPublic(r, publicOrchH)
+
+		// --- Public reads: report views + orchestration status snapshot ---
+		// Grouped under a per-request timeout so a slow read query is canceled
+		// (freeing its pool connection) instead of hanging until the load
+		// balancer 504s it. WebSocket and write/upload routes are exempt.
+		r.Group(func(r chi.Router) {
+			if d.ReadRequestTimeout > 0 {
+				r.Use(chimw.Timeout(d.ReadRequestTimeout))
+			}
+			r.Get("/reports", reportsH.List)
+			r.Get("/reports/grouped", reportsH.Grouped)
+			r.Get("/reports/individual", reportsH.Individual)
+			r.Get("/reports/consolidated", reportsH.Consolidated)
+			r.Get("/reports/{id}", reportsH.Detail)
+			r.Get("/reports/{id}/suites", reportsH.Suites)
+			r.Get("/reports/{id}/suites/{suiteID}/specs", reportsH.SuiteSpecs)
+			r.Get("/reports/{id}/cases", reportsH.Cases)
+			r.Get("/reports/{id}/json", reportsH.JSONFile)
+			r.Get("/reports/{id}/search", reportsH.Search)
+
+			// The dashboard fetches this without credentials, alongside the
+			// public report-reads above. Mutations (begin/checkout/complete/
+			// screenshots) stay in the protected group below.
+			orchapi.RegisterPublic(r, publicOrchH)
+		})
 
 		// --- Protected: writes + admin-ish reads ---
 		r.Group(func(r chi.Router) {

infra/lib/constructs/networking.ts

@@ -130,6 +130,10 @@ export class Networking extends Construct {
       vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
       securityGroup: albSecurityGroup,
       dropInvalidHeaderFields: true,
+      // Above the AWS default of 60s so a slow-but-completing request isn't
+      // guillotined into a 504 while still holding a backend connection; the
+      // app's own per-request/statement timeouts (~30s) are the real bound.
+      idleTimeout: cdk.Duration.seconds(120),
     });
 
     this.alb.logAccessLogs(albAccessLogBucket, "alb-logs");