fix(blockchain): stop failover loop retrying forever when pool is dead

The loop decided it had tried every provider when the provider index got back to where it
started, but reconnection skips providers that refuse connections, so the index could step
over that value forever. Counting failed providers guarantees the loop stops.

Author: MoonBoi9001

src/models/blockchain_client.py

@@ -196,7 +196,7 @@ def _execute_rpc_call(self, func: Callable, *args: Any, **kwargs: Any) -> Any:
         Execute func(*args, **kwargs) with retries and provider failover, rotating
         through the RPC pool and raising ConnectionError once every provider has failed.
         """
-        initial_index = self.current_rpc_index
+        providers_tried = 0
         while True:
             try:
                 # Add retry logic with backoff for the specific function call
@@ -210,17 +210,19 @@ def do_call():
 
             # If we get an exception after all retries, log the error and switch to the next RPC provider
             except RPC_FAILOVER_EXCEPTIONS as e:
+                providers_tried += 1
                 current_provider = self.rpc_providers[self.current_rpc_index]
                 logger.warning(
                     f"RPC call failed with provider at index {self.current_rpc_index} ({current_provider}): {e}"
                 )
-                self._get_next_rpc_provider()
 
-                # If we have tried all RPC providers, log the error and raise an exception
-                if self.current_rpc_index == initial_index:
+                # Once every provider in the pool has failed, log the error and raise an exception
+                if providers_tried >= len(self.rpc_providers):
                     logger.error("All RPC providers failed. Cannot proceed.")
                     raise ConnectionError("All RPC providers are unreachable.") from e
 
+                self._get_next_rpc_provider()
+
             # If we get an unexpected exception, log the error and raise the exception
             except Exception as e:
                 logger.error(f"An unexpected error occurred during RPC call: {e}")

tests/test_blockchain_client.py

@@ -239,6 +239,33 @@ def test_execute_rpc_call_reraises_unexpected_exception(self, blockchain_client:
         blockchain_client.slack_notifier.send_info_notification.assert_not_called()
 
 
+    def test_execute_rpc_call_raises_after_trying_every_provider_even_if_index_skips(
+        self, blockchain_client: BlockchainClient, mocker: MockerFixture
+    ):
+        """
+        Tests the failover loop terminates once every provider has been tried, even when
+        reconnection skips over an unreachable provider so the RPC index never returns to
+        its starting value. Previously this exact situation looped forever.
+        """
+        # Arrange
+        mocker.patch("tenacity.nap.time")  # Skip retry backoff sleeps
+        mock_func = MagicMock(side_effect=requests.exceptions.ConnectionError("RPC down"))
+
+
+        def rotate_skipping_primary():
+            # Simulate _connect_to_rpc skipping the unreachable primary and landing on the backup
+            blockchain_client.current_rpc_index = 1
+
+        mocker.patch.object(blockchain_client, "_get_next_rpc_provider", side_effect=rotate_skipping_primary)
+
+        # Act & Assert
+        with pytest.raises(requests.exceptions.ConnectionError, match="All RPC providers are unreachable."):
+            blockchain_client._execute_rpc_call(mock_func)
+
+        # Each of the 2 providers gets 3 retry attempts before the pool is exhausted
+        assert mock_func.call_count == 6
+
+
     def test_init_fails_with_empty_rpc_list(self, mock_w3, mock_slack):
         """
         Tests that BlockchainClient raises an exception if initialized with an empty list of RPC providers.