From f529edda2310446713e7e34f3f4d0333223cc43e Mon Sep 17 00:00:00 2001 From: Alex Bash Date: Tue, 23 Aug 2022 17:28:18 -0700 Subject: [PATCH 1/5] update backoff policy --- pkg/server/server.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 40cc82e..19d0560 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -10,7 +10,6 @@ import ( "github.com/zeriontech/sidecache/pkg/lock" "io" "io/ioutil" - "math" "net/http" "net/http/httputil" "net/url" @@ -252,9 +251,9 @@ func (server CacheServer) ReorderQueryString(url *url.URL) string { } func (server CacheServer) GetBackoff(attempt int) time.Duration { - multiplier := 1 - if attempt%2 != 0 { - multiplier = 5 + if attempt < 10 { + return 100 * time.Millisecond + } else { + return 500 * time.Millisecond } - return time.Duration(multiplier*int(math.Pow(10, float64(attempt/2+1)))) * time.Millisecond } From e1946b25549943ece4c77a7755cb52a234cced00 Mon Sep 17 00:00:00 2001 From: ivansanin Date: Wed, 24 Aug 2022 18:53:24 +0400 Subject: [PATCH 2/5] Added metrics for lock acquiring attempts --- pkg/server/prometheus_exporter.go | 36 +++++++++++++++++++++---------- pkg/server/server.go | 3 ++- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/server/prometheus_exporter.go b/pkg/server/prometheus_exporter.go index 70521fe..b41c462 100644 --- a/pkg/server/prometheus_exporter.go +++ b/pkg/server/prometheus_exporter.go @@ -1,11 +1,18 @@ package server import ( - "github.com/prometheus/client_golang/prometheus" "strings" + + "github.com/prometheus/client_golang/prometheus" ) var ( + buildInfoGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "sidecache_admission_build_info", + Help: "Build info for sidecache admission webhook", + }, []string{"version"}) + cacheHitCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "sidecache_" + ProjectName, @@ -13,29 +20,36 @@ var ( Help: "Cache hit count", }) + lockAcquiringAttemptsHistogram = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "sidecache_" + ProjectName, + Name: "lock_acquiring_attempts_histogram", + Help: "Lock acquiring attempts histogram", + Buckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100}, + }) + totalRequestCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "sidecache_" + ProjectName, Name: "all_request_hit_counter", Help: "All request hit counter", }) - - buildInfoGaugeVec = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "sidecache_admission_build_info", - Help: "Build info for sidecache admission webhook", - }, []string{"version"}) ) type Prometheus struct { - CacheHitCounter prometheus.Counter - TotalRequestCounter prometheus.Counter + CacheHitCounter prometheus.Counter + LockAcquiringAttemptsHistogram prometheus.Histogram + TotalRequestCounter prometheus.Counter } func NewPrometheusClient() *Prometheus { - prometheus.MustRegister(cacheHitCounter, totalRequestCounter, buildInfoGaugeVec) + prometheus.MustRegister(buildInfoGaugeVec, cacheHitCounter, lockAcquiringAttemptsHistogram, totalRequestCounter) - return &Prometheus{TotalRequestCounter: totalRequestCounter, CacheHitCounter: cacheHitCounter} + return &Prometheus{ + CacheHitCounter: cacheHitCounter, + LockAcquiringAttemptsHistogram: lockAcquiringAttemptsHistogram, + TotalRequestCounter: totalRequestCounter, + } } func BuildInfo(admission string) { diff --git a/pkg/server/server.go b/pkg/server/server.go index 19d0560..4cf7dff 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "github.com/zeriontech/sidecache/pkg/lock" "io" "io/ioutil" "net/http" @@ -19,6 +18,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/zeriontech/sidecache/pkg/cache" + "github.com/zeriontech/sidecache/pkg/lock" "go.uber.org/zap" ) @@ -160,6 +160,7 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) { // try to acquire the lock server.Logger.Info("acquiring the lock", zap.String("key", key)) if err := server.LockMgr.Acquire(key, LockTtl); err == nil { + server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt) + 1) server.Logger.Info("lock acquired", zap.String("key", key)) defer func() { // release the lock From cdcaf31adb124ba4639ac44311b94f9c3deaf4d6 Mon Sep 17 00:00:00 2001 From: ivansanin Date: Wed, 24 Aug 2022 21:04:54 +0400 Subject: [PATCH 3/5] Fixed start value for attempts --- pkg/server/server.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 4cf7dff..752f02f 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -148,10 +148,10 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) { resultKey := server.HashURL(server.ReorderQueryString(r.URL)) if UseLock { - attempt := 0 + attempt := 1 for { // check the cache - server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt+1)) + server.Logger.Info("checking the cache", zap.String("resultKey", resultKey), zap.Int("attempt", attempt)) if cachedDataBytes := server.CheckCache(resultKey); cachedDataBytes != nil { serveFromCache(cachedDataBytes, server, w, r) return @@ -160,8 +160,8 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) { // try to acquire the lock server.Logger.Info("acquiring the lock", zap.String("key", key)) if err := server.LockMgr.Acquire(key, LockTtl); err == nil { - server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt) + 1) - server.Logger.Info("lock acquired", zap.String("key", key)) + server.Prometheus.LockAcquiringAttemptsHistogram.Observe(float64(attempt)) + server.Logger.Info("lock acquired", zap.String("key", key), zap.Int("attempt", attempt)) defer func() { // release the lock if err := server.LockMgr.Release(key); err != nil { @@ -252,9 +252,8 @@ func (server CacheServer) ReorderQueryString(url *url.URL) string { } func (server CacheServer) GetBackoff(attempt int) time.Duration { - if attempt < 10 { + if attempt <= 10 { return 100 * time.Millisecond - } else { - return 500 * time.Millisecond } + return 500 * time.Millisecond } From daeefe63d999b265129a1d98872910d40a735362 Mon Sep 17 00:00:00 2001 From: ivansanin Date: Wed, 24 Aug 2022 21:51:53 +0400 Subject: [PATCH 4/5] Added min and max values for buckets --- pkg/server/prometheus_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/prometheus_exporter.go b/pkg/server/prometheus_exporter.go index b41c462..b2445db 100644 --- a/pkg/server/prometheus_exporter.go +++ b/pkg/server/prometheus_exporter.go @@ -25,7 +25,7 @@ var ( Namespace: "sidecache_" + ProjectName, Name: "lock_acquiring_attempts_histogram", Help: "Lock acquiring attempts histogram", - Buckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100}, + Buckets: []float64{0.999, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100, 1000}, }) totalRequestCounter = prometheus.NewCounter( From 264ead4c6ebe2e1a410f8334881885a42c790baf Mon Sep 17 00:00:00 2001 From: ivansanin Date: Fri, 26 Aug 2022 16:10:34 +0400 Subject: [PATCH 5/5] Fixed metrics, fixed lock key --- pkg/server/prometheus_exporter.go | 2 +- pkg/server/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/server/prometheus_exporter.go b/pkg/server/prometheus_exporter.go index b2445db..f46f13d 100644 --- a/pkg/server/prometheus_exporter.go +++ b/pkg/server/prometheus_exporter.go @@ -25,7 +25,7 @@ var ( Namespace: "sidecache_" + ProjectName, Name: "lock_acquiring_attempts_histogram", Help: "Lock acquiring attempts histogram", - Buckets: []float64{0.999, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100, 1000}, + Buckets: []float64{0.999, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 100, 200, 300, 500, 1000}, }) totalRequestCounter = prometheus.NewCounter( diff --git a/pkg/server/server.go b/pkg/server/server.go index 752f02f..61914e1 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -144,7 +144,7 @@ func (server CacheServer) CacheHandler(w http.ResponseWriter, r *http.Request) { }() path := strings.Split(r.URL.Path, "/") - key := "lock:" + path[1] + key := "lock:" + path[1] + "/" + path[2] resultKey := server.HashURL(server.ReorderQueryString(r.URL)) if UseLock {