proxyd: exemption based on X-Forwarded-For

Author: asymmetric

proxyd/config.go

@@ -56,14 +56,15 @@ type MetricsConfig struct {
 }
 
 type RateLimitConfig struct {
-	UseRedis         bool                                `toml:"use_redis"`
-	BaseRate         int                                 `toml:"base_rate"`
-	BaseInterval     TOMLDuration                        `toml:"base_interval"`
-	ExemptOrigins    []string                            `toml:"exempt_origins"`
-	ExemptUserAgents []string                            `toml:"exempt_user_agents"`
-	ErrorMessage     string                              `toml:"error_message"`
-	MethodOverrides  map[string]*RateLimitMethodOverride `toml:"method_overrides"`
-	IPHeaderOverride string                              `toml:"ip_header_override"`
+	UseRedis           bool                                `toml:"use_redis"`
+	BaseRate           int                                 `toml:"base_rate"`
+	BaseInterval       TOMLDuration                        `toml:"base_interval"`
+	ExemptProxyClients []string                            `toml:"exempt_proxy_clients"`
+	ExemptOrigins      []string                            `toml:"exempt_origins"`
+	ExemptUserAgents   []string                            `toml:"exempt_user_agents"`
+	ErrorMessage       string                              `toml:"error_message"`
+	MethodOverrides    map[string]*RateLimitMethodOverride `toml:"method_overrides"`
+	IPHeaderOverride   string                              `toml:"ip_header_override"`
 }
 
 type RateLimitMethodOverride struct {

proxyd/integration_tests/rate_limit_test.go

@@ -39,6 +39,14 @@ func TestFrontendMaxRPSLimit(t *testing.T) {
 		RequireEqualJSON(t, []byte(frontendOverLimitResponseWithID), limitedRes)
 	})
 
+	t.Run("exempt proxy client", func(t *testing.T) {
+		h := make(http.Header)
+		h.Set("X-Forwarded-For", "1.2.3.4")
+		client := NewProxydClientWithHeaders("http://127.0.0.1:8545", h)
+		_, codes := spamReqs(t, client, ethChainID, 429, 3)
+		require.Equal(t, 3, codes[200])
+	})
+
 	t.Run("exempt user agent over limit", func(t *testing.T) {
 		h := make(http.Header)
 		h.Set("User-Agent", "exempt_agent")

proxyd/integration_tests/testdata/frontend_rate_limit.toml

@@ -23,6 +23,7 @@ base_rate = 2
 base_interval = "1s"
 exempt_origins = ["exempt_origin"]
 exempt_user_agents = ["exempt_agent"]
+exempt_proxy_clients = [ "1.2.3.4" ]
 error_message = "over rate limit with special message"
 
 [rate_limit.method_overrides.eth_foobar]
@@ -32,4 +33,4 @@ interval = "1s"
 [rate_limit.method_overrides.eth_baz]
 limit = 1
 interval = "1s"
-global = true
+global = true

proxyd/server.go

@@ -72,6 +72,7 @@ type Server struct {
 	overrideLims            map[string]FrontendRateLimiter
 	senderLim               FrontendRateLimiter
 	allowedChainIds         []*big.Int
+	limExemptProxyClients   []*regexp.Regexp
 	limExemptOrigins        []*regexp.Regexp
 	limExemptUserAgents     []*regexp.Regexp
 	globallyLimitedMethods  map[string]bool
@@ -131,10 +132,18 @@ func NewServer(
 	}
 
 	var mainLim FrontendRateLimiter
+	limExemptProxyClients := make([]*regexp.Regexp, 0)
 	limExemptOrigins := make([]*regexp.Regexp, 0)
 	limExemptUserAgents := make([]*regexp.Regexp, 0)
 	if rateLimitConfig.BaseRate > 0 {
 		mainLim = limiterFactory(time.Duration(rateLimitConfig.BaseInterval), rateLimitConfig.BaseRate, "main")
+		for _, client := range rateLimitConfig.ExemptProxyClients {
+			pattern, err := regexp.Compile(client)
+			if err != nil {
+				return nil, err
+			}
+			limExemptProxyClients = append(limExemptProxyClients, pattern)
+		}
 		for _, origin := range rateLimitConfig.ExemptOrigins {
 			pattern, err := regexp.Compile(origin)
 			if err != nil {
@@ -194,6 +203,7 @@ func NewServer(
 		globallyLimitedMethods:  globalMethodLims,
 		senderLim:               senderLim,
 		allowedChainIds:         senderRateLimitConfig.AllowedChainIds,
+		limExemptProxyClients:   limExemptProxyClients,
 		limExemptOrigins:        limExemptOrigins,
 		limExemptUserAgents:     limExemptUserAgents,
 		rateLimitHeader:         rateLimitHeader,
@@ -269,6 +279,7 @@ func (s *Server) HandleRPC(w http.ResponseWriter, r *http.Request) {
 	userAgent := r.Header.Get("User-Agent")
 	// Use XFF in context since it will automatically be replaced by the remote IP
 	xff := stripXFF(GetXForwardedFor(ctx))
+	isUnlimitedProxyClient := s.isUnlimitedProxyClient(xff)
 	isUnlimitedOrigin := s.isUnlimitedOrigin(origin)
 	isUnlimitedUserAgent := s.isUnlimitedUserAgent(userAgent)
 
@@ -279,7 +290,7 @@ func (s *Server) HandleRPC(w http.ResponseWriter, r *http.Request) {
 
 	isLimited := func(method string) bool {
 		isGloballyLimitedMethod := s.isGlobalLimit(method)
-		if !isGloballyLimitedMethod && (isUnlimitedOrigin || isUnlimitedUserAgent) {
+		if !isGloballyLimitedMethod && (isUnlimitedProxyClient || isUnlimitedOrigin || isUnlimitedUserAgent) {
 			return false
 		}
 
@@ -802,6 +813,15 @@ func randStr(l int) string {
 	return hex.EncodeToString(b)
 }
 
+func (s *Server) isUnlimitedProxyClient(client string) bool {
+	for _, pat := range s.limExemptProxyClients {
+		if pat.MatchString(client) {
+			return true
+		}
+	}
+
+	return false
+}
 func (s *Server) isUnlimitedOrigin(origin string) bool {
 	for _, pat := range s.limExemptOrigins {
 		if pat.MatchString(origin) {