retry crates.io requests when calling crate-search API

Author: syphar

src/bin/cratesfyi.rs

@@ -698,9 +698,9 @@ impl Context for BinContext {
             let config = self.config()?;
             let path = config.registry_index_path.clone();
             if let Some(registry_url) = config.registry_url.clone() {
-                Index::from_url(path, registry_url)
+                Index::from_url(path, registry_url, config.crates_io_api_call_retries)
             } else {
-                Index::new(path)
+                Index::new(path, config.crates_io_api_call_retries)
             }?
         };
         fn repository_stats_updater(self) -> RepositoryStatsUpdater = {

src/config.rs

@@ -38,6 +38,9 @@ pub struct Config {
     // Gitlab authentication
     pub(crate) gitlab_accesstoken: Option<String>,
 
+    // amount of retries for external API calls, mostly crates.io
+    pub crates_io_api_call_retries: u32,
+
     // request timeout in seconds
     pub(crate) request_timeout: Option<Duration>,
     pub(crate) report_request_timeouts: bool,
@@ -120,6 +123,8 @@ impl Config {
         Ok(Self {
             build_attempts: env("DOCSRS_BUILD_ATTEMPTS", 5)?,
 
+            crates_io_api_call_retries: env("DOCSRS_CRATESIO_API_CALL_RETRIES", 3)?,
+
             registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?,
             registry_url: maybe_env("REGISTRY_URL")?,
             prefix: prefix.clone(),

src/index/api.rs

@@ -15,6 +15,7 @@ const APP_USER_AGENT: &str = concat!(
 #[derive(Debug)]
 pub struct Api {
     api_base: Option<Url>,
+    max_retries: u32,
     client: reqwest::blocking::Client,
 }
 
@@ -47,7 +48,7 @@ pub struct CrateOwner {
 }
 
 impl Api {
-    pub(super) fn new(api_base: Option<Url>) -> Result<Self> {
+    pub(super) fn new(api_base: Option<Url>, max_retries: u32) -> Result<Self> {
         let headers = vec![
             (USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)),
             (ACCEPT, HeaderValue::from_static("application/json")),
@@ -59,7 +60,11 @@ impl Api {
             .default_headers(headers)
             .build()?;
 
-        Ok(Self { api_base, client })
+        Ok(Self {
+            api_base,
+            client,
+            max_retries,
+        })
     }
 
     fn api_base(&self) -> Result<Url> {
@@ -120,7 +125,7 @@ impl Api {
 
         let response: Response = retry(
             || Ok(self.client.get(url.clone()).send()?.error_for_status()?),
-            3,
+            self.max_retries,
         )?
         .json()?;
 
@@ -159,7 +164,7 @@ impl Api {
 
         let response: Response = retry(
             || Ok(self.client.get(url.clone()).send()?.error_for_status()?),
-            3,
+            self.max_retries,
         )?
         .json()?;
 

src/index/mod.rs

@@ -39,7 +39,7 @@ fn load_config(repo: &gix::Repository) -> Result<IndexConfig> {
 }
 
 impl Index {
-    pub fn from_url(path: PathBuf, url: String) -> Result<Self> {
+    pub fn from_url(path: PathBuf, url: String, max_api_call_retries: u32) -> Result<Self> {
         let diff = crates_index_diff::Index::from_path_or_cloned_with_options(
             &path,
             gix::progress::Discard,
@@ -49,21 +49,23 @@ impl Index {
         .context("initialising registry index repository")?;
 
         let config = load_config(diff.repository()).context("loading registry config")?;
-        let api = Api::new(config.api).context("initialising registry api client")?;
+        let api = Api::new(config.api, max_api_call_retries)
+            .context("initialising registry api client")?;
         Ok(Self {
             path,
             api,
             repository_url: Some(url),
         })
     }
 
-    pub fn new(path: PathBuf) -> Result<Self> {
+    pub fn new(path: PathBuf, max_api_call_retries: u32) -> Result<Self> {
         // This initializes the repository, then closes it afterwards to avoid leaking file descriptors.
         // See https://github.com/rust-lang/docs.rs/pull/847
         let diff = crates_index_diff::Index::from_path_or_cloned(&path)
             .context("initialising registry index repository")?;
         let config = load_config(diff.repository()).context("loading registry config")?;
-        let api = Api::new(config.api).context("initialising registry api client")?;
+        let api = Api::new(config.api, max_api_call_retries)
+            .context("initialising registry api client")?;
         Ok(Self {
             path,
             api,

src/test/mod.rs

@@ -372,8 +372,11 @@ impl TestEnvironment {
         self.index
             .get_or_init(|| {
                 Arc::new(
-                    Index::new(self.config().registry_index_path.clone())
-                        .expect("failed to initialize the index"),
+                    Index::new(
+                        self.config().registry_index_path.clone(),
+                        self.config().crates_io_api_call_retries,
+                    )
+                    .expect("failed to initialize the index"),
                 )
             })
             .clone()

src/utils/mod.rs

@@ -27,8 +27,7 @@ use serde::Serialize;
 use tracing::error;
 pub(crate) mod sized_buffer;
 
-use std::thread;
-use std::time::Duration;
+use std::{future::Future, thread, time::Duration};
 use tracing::warn;
 
 pub(crate) const APP_USER_AGENT: &str = concat!(
@@ -138,6 +137,30 @@ pub(crate) fn retry<T>(mut f: impl FnMut() -> Result<T>, max_attempts: u32) -> R
     unreachable!()
 }
 
+pub(crate) async fn retry_async<T, Fut, F: FnMut() -> Fut>(mut f: F, max_attempts: u32) -> Result<T>
+where
+    Fut: Future<Output = Result<T>>,
+{
+    for attempt in 1.. {
+        match f().await {
+            Ok(result) => return Ok(result),
+            Err(err) => {
+                if attempt > max_attempts {
+                    return Err(err);
+                } else {
+                    let sleep_for = 2u32.pow(attempt);
+                    warn!(
+                        "got error on attempt {}, will try again after {}s:\n{:?}",
+                        attempt, sleep_for, err
+                    );
+                    tokio::time::sleep(Duration::from_secs(sleep_for as u64)).await;
+                }
+            }
+        }
+    }
+    unreachable!();
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/web/releases.rs

@@ -5,7 +5,7 @@ use crate::{
     cdn,
     db::Pool,
     impl_axum_webpage,
-    utils::{report_error, spawn_blocking},
+    utils::{report_error, retry_async, spawn_blocking},
     web::{
         axum_parse_uri_with_params, axum_redirect, encode_url_path,
         error::{AxumNope, AxumResult},
@@ -128,7 +128,11 @@ struct SearchResult {
 /// Get the search results for a crate search query
 ///
 /// This delegates to the crates.io search API.
-async fn get_search_results(pool: Pool, query_params: &str) -> Result<SearchResult, anyhow::Error> {
+async fn get_search_results(
+    pool: Pool,
+    config: &Config,
+    query_params: &str,
+) -> Result<SearchResult, anyhow::Error> {
     #[derive(Deserialize)]
     struct CratesIoSearchResult {
         crates: Vec<CratesIoCrate>,
@@ -178,13 +182,19 @@ async fn get_search_results(pool: Pool, query_params: &str) -> Result<SearchResu
         }
     });
 
-    let releases: CratesIoSearchResult = HTTP_CLIENT
-        .get(url)
-        .send()
-        .await?
-        .error_for_status()?
-        .json()
-        .await?;
+    let releases: CratesIoSearchResult = retry_async(
+        || async {
+            Ok(HTTP_CLIENT
+                .get(url.clone())
+                .send()
+                .await?
+                .error_for_status()?)
+        },
+        config.crates_io_api_call_retries,
+    )
+    .await?
+    .json()
+    .await?;
 
     let names = Arc::new(
         releases
@@ -584,14 +594,14 @@ pub(crate) async fn search_handler(
             return Err(AxumNope::NoResults);
         }
 
-        get_search_results(pool, &query_params).await?
+        get_search_results(pool, &config, &query_params).await?
     } else if !query.is_empty() {
         let query_params: String = form_urlencoded::Serializer::new(String::new())
             .append_pair("q", &query)
             .append_pair("per_page", &RELEASES_IN_RELEASES.to_string())
             .finish();
 
-        get_search_results(pool, &format!("?{}", &query_params)).await?
+        get_search_results(pool, &config, &format!("?{}", &query_params)).await?
     } else {
         return Err(AxumNope::NoResults);
     };
@@ -968,6 +978,10 @@ mod tests {
     #[test_case(StatusCode::BAD_GATEWAY)]
     fn crates_io_errors_are_correctly_returned_and_we_dont_try_parsing(status: StatusCode) {
         wrapper(|env| {
+            env.override_config(|config| {
+                config.crates_io_api_call_retries = 0;
+            });
+
             let _m = mock("GET", "/api/v1/crates")
                 .match_query(Matcher::AllOf(vec![
                     Matcher::UrlEncoded("q".into(), "doesnt_matter_here".into()),