[WIP] Fallback Chains

bittensor/core/async_subtensor.py

@@ -9,6 +9,7 @@
 import numpy as np
 import scalecodec
 from async_substrate_interface import AsyncSubstrateInterface
+async_substrate_interface.substrate_addons import RetrySubstrate
 from bittensor_commit_reveal import get_encrypted_commitment
 from bittensor_wallet.utils import SS58_FORMAT
 from numpy.typing import NDArray
@@ -90,6 +91,7 @@
     u16_normalized_float,
     u64_normalized_float,
     unlock_key,
+    determine_chain_endpoint_and_network,
 )
 from bittensor.utils.balance import (
     Balance,
@@ -115,6 +117,8 @@ def __init__(
         config: Optional["Config"] = None,
         _mock: bool = False,
         log_verbose: bool = False,
+        fallback_chains: Optional[list[str]] = None,
+        retry_forever: bool = False,
     ):
         """
         Initializes an instance of the AsyncSubtensor class.
@@ -124,17 +128,23 @@ def __init__(
             config (Optional[Config]): Configuration object for the AsyncSubtensor instance.
             _mock: Whether this is a mock instance. Mainly just for use in testing.
             log_verbose (bool): Enables or disables verbose logging.
+            fallback_chains: list of chain urls to try if the initial one fails
+            retry_forever: whether to continuously try the chains indefinitely if timeout failure
 
         Raises:
             Any exceptions raised during the setup, configuration, or connection process.
         """
+        fallback_chains_ = fallback_chains or []
         if config is None:
             config = AsyncSubtensor.config()
         self._config = copy.deepcopy(config)
         self.chain_endpoint, self.network = AsyncSubtensor.setup_config(
             network, self._config
         )
         self._mock = _mock
+        fallback_chain_urls = [
+            determine_chain_endpoint_and_network(x)[1] for x in fallback_chains_
+        ]
 
         self.log_verbose = log_verbose
         self._check_and_log_network_settings()
@@ -143,13 +153,16 @@ def __init__(
             f"Connecting to network: [blue]{self.network}[/blue], "
             f"chain_endpoint: [blue]{self.chain_endpoint}[/blue]..."
         )
-        self.substrate = AsyncSubstrateInterface(
-            url=self.chain_endpoint,
+        self.substrate = RetrySubstrate(
+            AsyncSubstrateInterface,
+            main_url=self.chain_endpoint,
             ss58_format=SS58_FORMAT,
             type_registry=TYPE_REGISTRY,
             use_remote_preset=True,
             chain_name="Bittensor",
             _mock=_mock,
+            fallback_chains=fallback_chain_urls,
+            retry_forever=retry_forever,
         )
         if self.log_verbose:
             logging.info(

bittensor/core/axon.py

@@ -504,9 +504,9 @@ def verify_custom(synapse: MyCustomSynapse):
             )
 
         param_class = first_param.annotation
-        assert issubclass(param_class, Synapse), (
-            "The first argument of forward_fn must inherit from bittensor.Synapse"
-        )
+        assert issubclass(
+            param_class, Synapse
+        ), "The first argument of forward_fn must inherit from bittensor.Synapse"
         request_name = param_class.__name__
 
         async def endpoint(*args, **kwargs):
@@ -580,19 +580,19 @@ async def endpoint(*args, **kwargs):
             blacklist_sig = Signature(
                 expected_params, return_annotation=Tuple[bool, str]
             )
-            assert signature(blacklist_fn) == blacklist_sig, (
-                f"The blacklist_fn function must have the signature: blacklist( synapse: {request_name} ) -> tuple[bool, str]"
-            )
+            assert (
+                signature(blacklist_fn) == blacklist_sig
+            ), f"The blacklist_fn function must have the signature: blacklist( synapse: {request_name} ) -> tuple[bool, str]"
         if priority_fn:
             priority_sig = Signature(expected_params, return_annotation=float)
-            assert signature(priority_fn) == priority_sig, (
-                f"The priority_fn function must have the signature: priority( synapse: {request_name} ) -> float"
-            )
+            assert (
+                signature(priority_fn) == priority_sig
+            ), f"The priority_fn function must have the signature: priority( synapse: {request_name} ) -> float"
         if verify_fn:
             verify_sig = Signature(expected_params, return_annotation=None)
-            assert signature(verify_fn) == verify_sig, (
-                f"The verify_fn function must have the signature: verify( synapse: {request_name} ) -> None"
-            )
+            assert (
+                signature(verify_fn) == verify_sig
+            ), f"The verify_fn function must have the signature: verify( synapse: {request_name} ) -> None"
 
         # Store functions in appropriate attribute dictionaries
         self.forward_class_types[request_name] = param_class
@@ -747,9 +747,9 @@ def check_config(cls, config: "Config"):
         Raises:
             AssertionError: If the axon or external ports are not in range [1024, 65535]
         """
-        assert 1024 < config.axon.port < 65535, (
-            "Axon port must be in range [1024, 65535]"
-        )
+        assert (
+            1024 < config.axon.port < 65535
+        ), "Axon port must be in range [1024, 65535]"
 
         assert config.axon.external_port is None or (
             1024 < config.axon.external_port < 65535

bittensor/core/subtensor.py

@@ -7,6 +7,7 @@
 import scalecodec
 from async_substrate_interface.errors import SubstrateRequestException
 from async_substrate_interface.sync_substrate import SubstrateInterface
+from async_substrate_interface.substrate_addons import RetrySubstrate
 from async_substrate_interface.types import ScaleObj
 from bittensor_commit_reveal import get_encrypted_commitment
 from numpy.typing import NDArray
@@ -92,6 +93,7 @@
     u16_normalized_float,
     u64_normalized_float,
     unlock_key,
+    determine_chain_endpoint_and_network,
 )
 from bittensor.utils.balance import (
     Balance,
@@ -117,6 +119,8 @@ def __init__(
         config: Optional["Config"] = None,
         _mock: bool = False,
         log_verbose: bool = False,
+        fallback_chains: Optional[list[str]] = None,
+        retry_forever: bool = False,
     ):
         """
         Initializes an instance of the Subtensor class.
@@ -126,10 +130,13 @@ def __init__(
             config (Optional[Config]): Configuration object for the AsyncSubtensor instance.
             _mock: Whether this is a mock instance. Mainly just for use in testing.
             log_verbose (bool): Enables or disables verbose logging.
+            fallback_chains: list of chain urls to try if the initial one fails
+            retry_forever: whether to continuously try the chains indefinitely if timeout failure
 
         Raises:
             Any exceptions raised during the setup, configuration, or connection process.
         """
+        fallback_chains_ = fallback_chains or []
         if config is None:
             config = self.config()
         self._config = copy.deepcopy(config)
@@ -143,13 +150,19 @@ def __init__(
             f"Connecting to network: [blue]{self.network}[/blue], "
             f"chain_endpoint: [blue]{self.chain_endpoint}[/blue]> ..."
         )
-        self.substrate = SubstrateInterface(
-            url=self.chain_endpoint,
+        fallback_chain_urls = [
+            determine_chain_endpoint_and_network(x)[1] for x in fallback_chains_
+        ]
+        self.substrate = RetrySubstrate(
+            substrate=SubstrateInterface,
+            main_url=self.chain_endpoint,
             ss58_format=SS58_FORMAT,
             type_registry=TYPE_REGISTRY,
             use_remote_preset=True,
             chain_name="Bittensor",
             _mock=_mock,
+            fallback_chains=fallback_chain_urls,
+            retry_forever=retry_forever,
         )
         if self.log_verbose:
             logging.info(

bittensor/core/types.py

@@ -2,6 +2,7 @@
 import argparse
 from typing import TypedDict, Optional
 
+
 from bittensor.utils import networking, Certificate
 from bittensor.utils.btlogging import logging
 from bittensor.core import settings