fix: improve service throttling mechanism to reject clients instead of blocking

The theory is that the connections are still backing up even while the service is sleeping.
If we instead close connections agressively during the throttle period hopefully we'll be able
to recover from overload periods more effectively

Author: chrisburr

src/DIRAC/Core/DISET/ServiceReactor.py

@@ -200,6 +200,7 @@ def __acceptIncomingConnection(self, svcName=False):
                                   services at the same time
         """
         sel = self.__getListeningSelector(svcName)
+        throttleExpires = None
         while self.__alive:
             clientTransport = None
             try:
@@ -223,12 +224,19 @@ def __acceptIncomingConnection(self, svcName=False):
                 gLogger.warn(f"Client connected from banned ip {clientIP}")
                 clientTransport.close()
                 continue
+            # Handle throttling
+            if self.__services[svcName].wantsThrottle and throttleExpires is None:
+                throttleExpires = time.time() + THROTTLE_SERVICE_SLEEP_SECONDS
+            if throttleExpires:
+                if time.time() > throttleExpires:
+                    throttleExpires = None
+                else:
+                    gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress()))
+                    clientTransport.close()
+                    continue
             # Handle connection
             self.__stats.connectionStablished()
             self.__services[svcName].handleConnection(clientTransport)
-            while self.__services[svcName].wantsThrottle:
-                gLogger.warn("Sleeping as service requested throttling", svcName)
-                time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS)
             # Renew context?
             now = time.time()
             renewed = False