fix: Add random delay in Listener.cacheReloader

Triggering schema cache reload immediately upon receival of notification by the listener leads to thundering herd problem in PostgREST cluster.

This change adds random delay between 0 and 1 seconds between notification retrieval and schema cache reload triggering.

Author: mkleczek

postgrest.cabal

@@ -158,6 +158,7 @@ library
                     , stm-hamt                  >= 1.2 && < 2
                     , focus                     >= 1.0 && < 2
                     , some                      >= 1.0.4.1 && < 2
+                    , MonadRandom               >= 0.6.2 && < 0.7
                       -- -fno-spec-constr may help keep compile time memory use in check,
                       --   see https://gitlab.haskell.org/ghc/ghc/issues/16017#note_219304
                       -- -optP-Wno-nonportable-include-path

src/PostgREST/Listener.hs

@@ -18,6 +18,7 @@ import qualified PostgREST.AppState as AppState
 import qualified PostgREST.Config   as Config
 
 import           Control.Arrow              ((&&&))
+import           Control.Monad.Random
 import           Data.Bitraversable         (bisequence)
 import           Data.Either.Combinators    (whenRight)
 import qualified Database.PostgreSQL.LibPQ  as LibPQ
@@ -102,7 +103,13 @@ retryingListen appState = do
          | msg == "reload config" -> observer (DBListenerGotConfigMsg channel) >> AppState.readInDbConfig False appState
          | otherwise              -> pure () -- Do nothing if anything else than an empty message is sent
 
-    cacheReloader =
+      -- add a random delay between 0 and 1 second to avoid thundering herd problem
+      -- if multiple listeners receive the same notification at the same time
+      -- the cacheReloader is launched in a separate thread to avoid blocking the listener
+      -- during the random delay
+    cacheReloader = void $ forkIO $ do
+      delay <- getRandomR (0, 1000000)
+      threadDelay delay
       AppState.schemaCacheLoader appState
 
     releaseConnection = void . forkIO . handle (observer . DBListenerConnectionCleanupFail) . SQL.release

test/io/postgrest.py

@@ -18,7 +18,7 @@
 
 def sleep_until_postgrest_scache_reload():
     "Sleep until schema cache reload"
-    time.sleep(0.3)
+    time.sleep(1)
 
 
 def sleep_until_postgrest_config_reload():
@@ -28,7 +28,7 @@ def sleep_until_postgrest_config_reload():
 
 def sleep_until_postgrest_full_reload():
     "Sleep until schema cache plus config reload"
-    time.sleep(0.3)
+    time.sleep(1)
 
 
 class PostgrestTimedOut(Exception):