From 43efe04db6b968adc3ac91e7e94c5dcd930f80ee Mon Sep 17 00:00:00 2001 From: schen6318 <149955676+schen6318@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:43:42 -0400 Subject: [PATCH 1/5] chore(api): enhance HTTP server with timeouts for better reliability --- modules/api/main.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/api/main.go b/modules/api/main.go index f6bc5c230458..7a231708c461 100644 --- a/modules/api/main.go +++ b/modules/api/main.go @@ -93,7 +93,20 @@ func main() { func serve() { klog.V(1).InfoS("Listening and serving on", "address", args.InsecureAddress()) - go func() { klog.Fatal(http.ListenAndServe(args.InsecureAddress(), nil)) }() + + server := &http.Server{ + Addr: args.InsecureAddress(), + Handler: http.DefaultServeMux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + + go func() { + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + klog.Fatal(err) + } + }() } func serveTLS(certificates []tls.Certificate) { From 6966eaade5d9b1bd413ba338dd7dd284d8937c30 Mon Sep 17 00:00:00 2001 From: schen6318 <149955676+schen6318@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:47:11 -0400 Subject: [PATCH 2/5] chore(api): add timeouts to HTTP server for improved reliability --- modules/api/pkg/args/pprof.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/api/pkg/args/pprof.go b/modules/api/pkg/args/pprof.go index dc33d2c16ebe..fc70cd66daf1 100644 --- a/modules/api/pkg/args/pprof.go +++ b/modules/api/pkg/args/pprof.go @@ -19,6 +19,8 @@ import ( "net/http" "net/http/pprof" + "time" + "k8s.io/klog/v2" ) @@ -28,7 +30,14 @@ func initProfiler() { mux := http.NewServeMux() mux.HandleFunc(defaultProfilerPath, pprof.Index) go func() { - if err := http.ListenAndServe(fmt.Sprintf(":%d", defaultProfilerPort), mux); err != nil { + server := &http.Server{ + Addr: fmt.Sprintf(":%d", defaultPrometheusPort), + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { klog.Fatal(err) } }() From e0358ca8f2917c8f431f5b172e04703270721ef4 Mon Sep 17 00:00:00 2001 From: schen6318 <149955676+schen6318@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:49:27 -0400 Subject: [PATCH 3/5] chore(api): add timeouts to Prometheus HTTP server for improved reliability --- modules/api/pkg/args/prometheus.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/api/pkg/args/prometheus.go b/modules/api/pkg/args/prometheus.go index 7bb4bec87e15..9209a209ae0e 100644 --- a/modules/api/pkg/args/prometheus.go +++ b/modules/api/pkg/args/prometheus.go @@ -18,6 +18,8 @@ import ( "fmt" "net/http" + "time" + "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/klog/v2" ) @@ -28,7 +30,14 @@ func initPrometheus() { mux := http.NewServeMux() mux.Handle(defaultPrometheusPath, promhttp.Handler()) go func() { - if err := http.ListenAndServe(fmt.Sprintf(":%d", defaultPrometheusPort), mux); err != nil { + server := &http.Server{ + Addr: fmt.Sprintf(":%d", defaultPrometheusPort), + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { klog.Fatal(err) } }() From a3dde9326059b9b1beb4c6fa345d292014baf7e4 Mon Sep 17 00:00:00 2001 From: schen6318 <149955676+schen6318@users.noreply.github.com> Date: Tue, 1 Apr 2025 14:52:53 -0400 Subject: [PATCH 4/5] chore(api): add timeouts to metrics scraper HTTP server for improved reliability --- modules/metrics-scraper/main.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/metrics-scraper/main.go b/modules/metrics-scraper/main.go index 459529037fb1..89425b1b8dcf 100644 --- a/modules/metrics-scraper/main.go +++ b/modules/metrics-scraper/main.go @@ -75,7 +75,17 @@ func main() { api.Manager(r, db) // Bind to a port and pass our router in - klog.Fatal(http.ListenAndServe(":8000", handlers.CombinedLoggingHandler(os.Stdout, r))) + + server := &http.Server{ + Addr: ":8000", + Handler: handlers.CombinedLoggingHandler(os.Stdout, r), + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + klog.Fatal(err) + } }() // Start the machine. Scrape every metricResolution From fc7900b782fadaf0388f9a22a716a1ed7a05eded Mon Sep 17 00:00:00 2001 From: schen6318 <149955676+schen6318@users.noreply.github.com> Date: Tue, 1 Apr 2025 18:20:34 -0400 Subject: [PATCH 5/5] chore(api): add timeouts to TLS HTTP server for improved reliability --- modules/api/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/api/main.go b/modules/api/main.go index 7a231708c461..9880e2f21c7a 100644 --- a/modules/api/main.go +++ b/modules/api/main.go @@ -112,14 +112,21 @@ func serve() { func serveTLS(certificates []tls.Certificate) { klog.V(1).InfoS("Listening and serving on", "address", args.Address()) server := &http.Server{ - Addr: args.Address(), - Handler: http.DefaultServeMux, + Addr: args.Address(), + Handler: http.DefaultServeMux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, TLSConfig: &tls.Config{ Certificates: certificates, MinVersion: tls.VersionTLS12, }, } - go func() { klog.Fatal(server.ListenAndServeTLS("", "")) }() + go func() { + if err := server.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { + klog.Fatal(err) + } + }() } func ensureAPIServerConnectionOrDie() {