Retry Discord bot startup with backoff

Author: gbtami

server/pychess_global_app_state.py

@@ -11,6 +11,12 @@
 from aiohttp import web
 from aiohttp.web_ws import WebSocketResponse
 import aiohttp_jinja2
+from tenacity import (
+    AsyncRetrying,
+    before_sleep_log,
+    retry_if_exception_type,
+    wait_exponential_jitter,
+)
 
 from pythongettext.msgfmt import Msgfmt, PoSyntaxError
 from sortedcollections import ValueSortedDict
@@ -509,14 +515,32 @@ def __init_fishnet_monitor(self) -> dict:
     def __init_discord(self):
         if self.db is None:
             self.discord = FakeDiscordBot()
+            return
 
         # create Discord bot
         if DEV:
             self.discord = FakeDiscordBot()
         else:
+            if DISCORD_TOKEN == "":
+                log.warning("DISCORD_TOKEN is missing/empty; Discord bot disabled")
+                self.discord = FakeDiscordBot()
+                return
+
             bot = DiscordBot(self)
             self.discord = bot
-            asyncio.create_task(bot.start(DISCORD_TOKEN), name="Discord-BOT")
+            asyncio.create_task(self.__run_discord_bot(bot, DISCORD_TOKEN), name="Discord-BOT")
+
+    async def __run_discord_bot(self, bot: DiscordBot, token: str) -> None:
+        # Keep retrying startup/login on transient failures so relay can recover
+        # automatically after brief network/API hiccups during dyno boot.
+        async for attempt in AsyncRetrying(
+            retry=retry_if_exception_type(Exception),
+            wait=wait_exponential_jitter(initial=1, max=120),
+            reraise=True,
+            before_sleep=before_sleep_log(log, logging.WARNING),
+        ):
+            with attempt:
+                await bot.start(token)
 
     def __init_twitch(self) -> Twitch:
         result = Twitch(self.app)

tests/test_discord_startup.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import asyncio
+import unittest
+
+from pychess_global_app_state import PychessGlobalAppState
+
+
+class DiscordStartupTestCase(unittest.IsolatedAsyncioTestCase):
+    async def test_run_discord_bot_retries_after_error_until_cancelled(self):
+        class FlakyBot:
+            def __init__(self) -> None:
+                self.calls = 0
+
+            async def start(self, token: str) -> None:
+                self.calls += 1
+                if self.calls == 1:
+                    raise RuntimeError("transient startup failure")
+                raise asyncio.CancelledError()
+
+        holder = type("Holder", (), {})()
+        bot = FlakyBot()
+
+        run_discord_bot = PychessGlobalAppState._PychessGlobalAppState__run_discord_bot
+        with self.assertRaises(asyncio.CancelledError):
+            await run_discord_bot(holder, bot, "token-123")
+
+        self.assertEqual(bot.calls, 2)
+
+    async def test_run_discord_bot_returns_when_start_exits_cleanly(self):
+        class CleanBot:
+            def __init__(self) -> None:
+                self.calls = 0
+
+            async def start(self, token: str) -> None:
+                self.calls += 1
+                return
+
+        holder = type("Holder", (), {})()
+        bot = CleanBot()
+
+        run_discord_bot = PychessGlobalAppState._PychessGlobalAppState__run_discord_bot
+        await run_discord_bot(holder, bot, "token-123")
+        self.assertEqual(bot.calls, 1)
+
+
+if __name__ == "__main__":
+    unittest.main()