Allow higher rpcworkqueue limit conf (ordinals#3615)

Author: JeremyRubin

src/index/updater.rs

@@ -250,7 +250,7 @@ impl<'index> Updater<'index> {
     // Default rpcworkqueue in bitcoind is 16, meaning more than 16 concurrent requests will be rejected.
     // Since we are already requesting blocks on a separate thread, and we don't want to break if anything
     // else runs a request, we keep this to 12.
-    const PARALLEL_REQUESTS: usize = 12;
+    let parallel_requests: usize = settings.bitcoin_rpc_limit() as usize;
 
     thread::spawn(move || {
       let rt = tokio::runtime::Builder::new_multi_thread()
@@ -273,8 +273,8 @@ impl<'index> Updater<'index> {
             outpoints.push(outpoint);
           }
           // Break outpoints into chunks for parallel requests
-          let chunk_size = (outpoints.len() / PARALLEL_REQUESTS) + 1;
-          let mut futs = Vec::with_capacity(PARALLEL_REQUESTS);
+          let chunk_size = (outpoints.len() / parallel_requests) + 1;
+          let mut futs = Vec::with_capacity(parallel_requests);
           for chunk in outpoints.chunks(chunk_size) {
             let txids = chunk.iter().map(|outpoint| outpoint.txid).collect();
             let fut = fetcher.get_transactions(txids);

src/options.rs

@@ -21,6 +21,8 @@ pub struct Options {
     help = "Authenticate to Bitcoin Core RPC as <BITCOIN_RPC_USERNAME>."
   )]
   pub(crate) bitcoin_rpc_username: Option<String>,
+  #[arg(long, help = "Max <N> requests in flight. [default: 12]")]
+  pub(crate) bitcoin_rpc_limit: Option<u32>,
   #[arg(long = "chain", value_enum, help = "Use <CHAIN>. [default: mainnet]")]
   pub(crate) chain_argument: Option<Chain>,
   #[arg(

src/settings.rs

@@ -4,6 +4,7 @@ use {super::*, bitcoincore_rpc::Auth};
 #[serde(default, deny_unknown_fields)]
 pub struct Settings {
   bitcoin_data_dir: Option<PathBuf>,
+  bitcoin_rpc_limit: Option<u32>,
   bitcoin_rpc_password: Option<String>,
   bitcoin_rpc_url: Option<String>,
   bitcoin_rpc_username: Option<String>,
@@ -108,6 +109,7 @@ impl Settings {
   pub(crate) fn or(self, source: Settings) -> Self {
     Self {
       bitcoin_data_dir: self.bitcoin_data_dir.or(source.bitcoin_data_dir),
+      bitcoin_rpc_limit: self.bitcoin_rpc_limit.or(source.bitcoin_rpc_limit),
       bitcoin_rpc_password: self.bitcoin_rpc_password.or(source.bitcoin_rpc_password),
       bitcoin_rpc_url: self.bitcoin_rpc_url.or(source.bitcoin_rpc_url),
       bitcoin_rpc_username: self.bitcoin_rpc_username.or(source.bitcoin_rpc_username),
@@ -147,6 +149,7 @@ impl Settings {
   pub(crate) fn from_options(options: Options) -> Self {
     Self {
       bitcoin_data_dir: options.bitcoin_data_dir,
+      bitcoin_rpc_limit: options.bitcoin_rpc_limit,
       bitcoin_rpc_password: options.bitcoin_rpc_password,
       bitcoin_rpc_url: options.bitcoin_rpc_url,
       bitcoin_rpc_username: options.bitcoin_rpc_username,
@@ -230,6 +233,7 @@ impl Settings {
 
     Ok(Self {
       bitcoin_data_dir: get_path("BITCOIN_DATA_DIR"),
+      bitcoin_rpc_limit: get_u32("BITCOIN_RPC_LIMIT")?,
       bitcoin_rpc_password: get_string("BITCOIN_RPC_PASSWORD"),
       bitcoin_rpc_url: get_string("BITCOIN_RPC_URL"),
       bitcoin_rpc_username: get_string("BITCOIN_RPC_USERNAME"),
@@ -262,6 +266,7 @@ impl Settings {
       bitcoin_rpc_password: None,
       bitcoin_rpc_url: Some(rpc_url.into()),
       bitcoin_rpc_username: None,
+      bitcoin_rpc_limit: None,
       chain: Some(Chain::Regtest),
       commit_interval: None,
       config: None,
@@ -320,6 +325,7 @@ impl Settings {
 
     Ok(Self {
       bitcoin_data_dir: Some(bitcoin_data_dir),
+      bitcoin_rpc_limit: Some(self.bitcoin_rpc_limit.unwrap_or(12)),
       bitcoin_rpc_password: self.bitcoin_rpc_password,
       bitcoin_rpc_url: Some(
         self
@@ -550,6 +556,10 @@ impl Settings {
     }
   }
 
+  pub(crate) fn bitcoin_rpc_limit(&self) -> u32 {
+    self.bitcoin_rpc_limit.unwrap()
+  }
+
   pub(crate) fn server_url(&self) -> Option<&str> {
     self.server_url.as_deref()
   }
@@ -978,6 +988,7 @@ mod tests {
   fn from_env() {
     let env = vec![
       ("BITCOIN_DATA_DIR", "/bitcoin/data/dir"),
+      ("BITCOIN_RPC_LIMIT", "12"),
       ("BITCOIN_RPC_PASSWORD", "bitcoin password"),
       ("BITCOIN_RPC_URL", "url"),
       ("BITCOIN_RPC_USERNAME", "bitcoin username"),
@@ -1010,6 +1021,7 @@ mod tests {
       Settings::from_env(env).unwrap(),
       Settings {
         bitcoin_data_dir: Some("/bitcoin/data/dir".into()),
+        bitcoin_rpc_limit: Some(12),
         bitcoin_rpc_password: Some("bitcoin password".into()),
         bitcoin_rpc_url: Some("url".into()),
         bitcoin_rpc_username: Some("bitcoin username".into()),
@@ -1055,6 +1067,7 @@ mod tests {
         Options::try_parse_from([
           "ord",
           "--bitcoin-data-dir=/bitcoin/data/dir",
+          "--bitcoin-rpc-limit=12",
           "--bitcoin-rpc-password=bitcoin password",
           "--bitcoin-rpc-url=url",
           "--bitcoin-rpc-username=bitcoin username",
@@ -1081,6 +1094,7 @@ mod tests {
       ),
       Settings {
         bitcoin_data_dir: Some("/bitcoin/data/dir".into()),
+        bitcoin_rpc_limit: Some(12),
         bitcoin_rpc_password: Some("bitcoin password".into()),
         bitcoin_rpc_url: Some("url".into()),
         bitcoin_rpc_username: Some("bitcoin username".into()),

tests/settings.rs

@@ -7,6 +7,7 @@ fn default() {
     .stdout_regex(
       r#"\{
   "bitcoin_data_dir": ".*(Bitcoin|bitcoin)",
+  "bitcoin_rpc_limit": 12,
   "bitcoin_rpc_password": null,
   "bitcoin_rpc_url": "127.0.0.1:8332",
   "bitcoin_rpc_username": null,