From 3bbf9986ec18c073a997fc9d44baf2e902987ad8 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 2 Oct 2025 10:43:23 +0200 Subject: [PATCH] fix: improve service throttling mechanism to reject clients instead of blocking The theory is that the connections are still backing up even while the service is sleeping. If we instead close connections agressively during the throttle period hopefully we'll be able to recover from overload periods more effectively --- src/DIRAC/Core/DISET/ServiceReactor.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/DIRAC/Core/DISET/ServiceReactor.py b/src/DIRAC/Core/DISET/ServiceReactor.py index 2f3bd76ced1..c3f7862be1a 100644 --- a/src/DIRAC/Core/DISET/ServiceReactor.py +++ b/src/DIRAC/Core/DISET/ServiceReactor.py @@ -200,6 +200,7 @@ def __acceptIncomingConnection(self, svcName=False): services at the same time """ sel = self.__getListeningSelector(svcName) + throttleExpires = None while self.__alive: clientTransport = None try: @@ -223,12 +224,19 @@ def __acceptIncomingConnection(self, svcName=False): gLogger.warn(f"Client connected from banned ip {clientIP}") clientTransport.close() continue + # Handle throttling + if self.__services[svcName].wantsThrottle and throttleExpires is None: + throttleExpires = time.time() + THROTTLE_SERVICE_SLEEP_SECONDS + if throttleExpires: + if time.time() > throttleExpires: + throttleExpires = None + else: + gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress())) + clientTransport.close() + continue # Handle connection self.__stats.connectionStablished() self.__services[svcName].handleConnection(clientTransport) - while self.__services[svcName].wantsThrottle: - gLogger.warn("Sleeping as service requested throttling", svcName) - time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS) # Renew context? now = time.time() renewed = False