Add features to allow choosing between reqwest or curl for downloading

Cargo.toml

@@ -48,12 +48,13 @@ structopt = "0.3"
 error-chain = "^0.12"
 flate2 = { version = "^1.0", default-features = false, features = ["zlib"] }
 fs2 = "^0.4"
-headers = "^0.2"
 lazy_static = "^1.4"
 libc = "^0.2"
 tempfile = "^3.1"
 md-5 = "^0.8"
-reqwest = "^0.9"
+url = { version = "^1.7" }
+reqwest = { version = "^0.9", optional = true }
+curl = { version = "^0.4", optional = true}
 sha2 = "^0.8"
 serde = { version = "^1.0", features = ["derive"], optional = true }
 tectonic_xdv = { path = "xdv", version = "0.1.9-dev" }
@@ -62,7 +63,7 @@ toml = { version = "^0.5", optional = true }
 zip = { version = "^0.5", default-features = false, features = ["deflate"] }
 
 [features]
-default = ["serialization"]
+default = ["serialization", "reqwest"]
 # Note: we used to have this to couple "serde" and "serde-derive", but we've
 # adopted the newer scheme to avoid having to depend on both -- should maybe
 # just get rid of this feature:

dist/travis.sh

@@ -228,7 +228,9 @@ if $is_main_build ; then
     travis_fold_end cargo_fmt
     travis_fold_start cargo_clippy "cargo clippy" verbose
     travis_retry rustup component add clippy
-    cargo clippy --all --all-targets --all-features -- --deny warnings
+    cargo clippy --all --all-targets -- --deny warnings
+    cargo clippy --all --all-targets --no-default-features --features serialization -- --deny warnings
+    cargo clippy --all --all-targets --no-default-features --features curl -- --deny warnings
     travis_fold_end cargo_clippy
 fi
 
@@ -239,6 +241,10 @@ export RUST_BACKTRACE=1
 travis_fold_start cargo_build_no_default_features "cargo build --no-default-features" verbose
 cargo build --no-default-features --verbose
 travis_fold_end cargo_build_no_default_features
+travis_fold_start cargo_build_curl \
+    "cargo build --no-default-features --features serialization,curl" verbose
+cargo build --no-default-features --features serialization,curl
+travis_fold_end cargo_build_curl
 travis_fold_start cargo_build "cargo build" verbose
 cargo build --verbose
 travis_fold_end cargo_build

src/config.rs

@@ -12,7 +12,6 @@
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
-use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 
@@ -59,6 +58,7 @@ impl PersistentConfig {
     /// false, the default configuration is returned and the filesystem is not
     /// modified.
     pub fn open(auto_create_config_file: bool) -> Result<PersistentConfig> {
+        use std::fs::File;
         use std::io::ErrorKind as IoErrorKind;
         use std::io::{Read, Write};
         let mut cfg_path = if auto_create_config_file {
@@ -135,8 +135,8 @@ impl PersistentConfig {
         only_cached: bool,
         status: &mut dyn StatusBackend,
     ) -> Result<Box<dyn Bundle>> {
-        use reqwest::Url;
         use std::io;
+        use url::Url;
 
         if CONFIG_TEST_MODE_ACTIVATED.load(Ordering::SeqCst) {
             return Ok(Box::new(crate::test_util::TestBundle::default()));

src/errors.rs

@@ -10,9 +10,9 @@
 // hopefully show up in a future version.
 #![allow(deprecated)]
 
+use crate::io::download;
 use app_dirs;
 use error_chain::error_chain;
-use reqwest::StatusCode;
 use std::io::Write;
 use std::result::Result as StdResult;
 use std::{convert, ffi, io, num, str};
@@ -63,7 +63,7 @@ error_chain! {
         Nul(ffi::NulError);
         ParseInt(num::ParseIntError);
         Persist(tempfile::PersistError);
-        Reqwest(reqwest::Error);
+        Download(download::Error);
         ConfigRead(ReadError);
         ConfigWrite(WriteError);
         Utf8(str::Utf8Error);
@@ -97,7 +97,7 @@ error_chain! {
             display("the {} engine had an unrecoverable error", engine)
         }
 
-        UnexpectedHttpResponse(url: String, status: StatusCode) {
+        UnexpectedHttpResponse(url: String, status: download::StatusCode) {
             description("unexpected HTTP response to URL")
             display("unexpected HTTP response to URL {}: {}", url, status)
         }

src/io/cached_itarbundle.rs

@@ -5,7 +5,6 @@
 use error_chain::bail;
 use flate2::read::GzDecoder;
 use fs2::FileExt;
-use reqwest::{header::HeaderMap, Client, RedirectPolicy, Response, StatusCode};
 use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::fs::{self, File};
@@ -21,13 +20,14 @@ use crate::errors::{Error, ErrorKind, Result, ResultExt};
 use crate::status::StatusBackend;
 use crate::{ctry, tt_note, tt_warning};
 
-const MAX_HTTP_REDIRECTS_ALLOWED: usize = 10;
+use crate::io::download::{self, Client, Response, StatusCode};
+
 const MAX_HTTP_ATTEMPTS: usize = 4;
 
 /// A simple way to read chunks out of a big seekable byte stream. You could
 /// implement this for io::File pretty trivially but that's not currently
 /// needed.
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct HttpRangeReader {
     url: String,
     client: Client,
@@ -44,13 +44,8 @@ impl HttpRangeReader {
 
 impl HttpRangeReader {
     fn read_range(&mut self, offset: u64, length: usize) -> Result<Response> {
-        let end_inclusive = offset + length as u64 - 1;
-
-        let mut headers = HeaderMap::new();
-        use headers::HeaderMapExt;
-        headers.typed_insert(headers::Range::bytes(offset..=end_inclusive).unwrap());
-
-        let res = self.client.get(&self.url).headers(headers).send()?;
+        let end = offset + length as u64 - 1;
+        let res = download::get_range_inclusive(&mut self.client, &self.url, offset, end)?;
 
         if res.status() != StatusCode::PARTIAL_CONTENT {
             return Err(Error::from(ErrorKind::UnexpectedHttpResponse(
@@ -81,7 +76,7 @@ fn get_index(url: &str, status: &mut dyn StatusBackend) -> Result<GzDecoder<Resp
 
     tt_note!(status, "downloading index {}", index_url);
 
-    let res = Client::new().get(&index_url).send()?;
+    let res = download::get(&index_url)?;
     if !res.status().is_success() {
         return Err(Error::from(ErrorKind::UnexpectedHttpResponse(
             index_url,
@@ -93,59 +88,6 @@ fn get_index(url: &str, status: &mut dyn StatusBackend) -> Result<GzDecoder<Resp
     Ok(GzDecoder::new(res))
 }
 
-fn resolve_url(url: &str, status: &mut dyn StatusBackend) -> Result<String> {
-    tt_note!(status, "connecting to {}", url);
-
-    // First, we actually do a HEAD request on the URL for the data file.
-    // If it's redirected, we update our URL to follow the redirects. If
-    // we didn't do this separately, the index file would have to be the
-    // one with the redirect setup, which would be confusing and annoying.
-
-    let redirect_policy = RedirectPolicy::custom(|attempt| {
-        // In the process of resolving the file url it might be neccesary
-        // to stop at a certain level of redirection. This might be required
-        // because some hosts might redirect to a version of the url where
-        // it isn't possible to select the index file by appending .index.gz.
-        // (This mostly happens because CDNs redirect to the file hash.)
-        if attempt.previous().len() >= MAX_HTTP_REDIRECTS_ALLOWED {
-            attempt.too_many_redirects()
-        } else if let Some(segments) = attempt.url().clone().path_segments() {
-            let follow = segments
-                .last()
-                .map(|file| file.contains('.'))
-                .unwrap_or(true);
-            if follow {
-                attempt.follow()
-            } else {
-                attempt.stop()
-            }
-        } else {
-            attempt.follow()
-        }
-    });
-    let res = Client::builder()
-        .redirect(redirect_policy)
-        .build()?
-        .head(url)
-        .send()?;
-
-    if !(res.status().is_success() || res.status() == StatusCode::FOUND) {
-        return Err(Error::from(ErrorKind::UnexpectedHttpResponse(
-            url.to_string(),
-            res.status(),
-        )))
-        .chain_err(|| "couldn\'t probe".to_string());
-    }
-
-    let final_url = res.url().clone().into_string();
-
-    if final_url != url {
-        tt_note!(status, "resolved to {}", final_url);
-    }
-
-    Ok(final_url)
-}
-
 /// Attempts to download a file from the bundle.
 fn get_file(
     data: &mut HttpRangeReader,
@@ -224,7 +166,7 @@ fn parse_index_line(line: &str) -> Result<Option<(String, FileInfo)>> {
 
 /// Attempts to find the redirected url, download the index and digest.
 fn get_everything(url: &str, status: &mut dyn StatusBackend) -> Result<(String, String, String)> {
-    let url = resolve_url(url, status)?;
+    let url = download::resolve_url(url, status)?;
 
     let index = {
         let mut index = String::new();
@@ -325,7 +267,7 @@ fn make_txt_path(base: &Path, digest_text: &str) -> PathBuf {
 }
 
 /// Bundle provided by an indexed tar file over http with a local cache.
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct CachedITarBundle {
     url: String,
     redirect_url: String,

src/io/download_curl.rs

@@ -0,0 +1,102 @@
+use std::fmt;
+use std::io;
+
+use crate::errors::Result;
+use crate::status::StatusBackend;
+use curl::easy::Easy;
+
+pub use curl::Error;
+
+const MAX_HTTP_REDIRECTS_ALLOWED: u32 = 10;
+
+#[derive(Debug)]
+pub struct Client {
+    handle: Easy,
+}
+
+pub struct Response {
+    data: io::Cursor<Vec<u8>>,
+    status: StatusCode,
+}
+
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct StatusCode(u32);
+
+impl Default for Client {
+    fn default() -> Client {
+        Client {
+            handle: Easy::new(),
+        }
+    }
+}
+
+impl Client {
+    pub fn new() -> Client {
+        Client::default()
+    }
+
+    fn get(&mut self, url: &str, range: Option<(u64, u64)>) -> Result<Response> {
+        let handle = &mut self.handle;
+        handle.url(url)?;
+        handle.follow_location(true)?;
+        handle.max_redirections(MAX_HTTP_REDIRECTS_ALLOWED)?;
+        if let Some((start, end)) = range {
+            handle.range(&format!("{}-{}", start, end))?;
+        }
+        let mut buf = Vec::new();
+        {
+            let mut transfer = handle.transfer();
+            transfer.write_function(|data| {
+                buf.extend_from_slice(data);
+                Ok(data.len())
+            })?;
+            transfer.perform()?;
+        }
+        let data = io::Cursor::new(buf);
+        let status = StatusCode(handle.response_code()?);
+        Ok(Response { data, status })
+    }
+}
+
+impl Response {
+    pub fn status(&self) -> StatusCode {
+        self.status
+    }
+}
+
+impl io::Read for Response {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.data.read(buf)
+    }
+}
+
+impl StatusCode {
+    pub const PARTIAL_CONTENT: StatusCode = StatusCode(206);
+
+    pub fn is_success(self) -> bool {
+        200 <= self.0 && self.0 <= 299
+    }
+}
+
+impl fmt::Display for StatusCode {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(fmt, "StatusCode({})", self.0)
+    }
+}
+
+pub fn get(url: &str) -> Result<Response> {
+    Client::new().get(url, None)
+}
+
+pub fn get_range_inclusive(
+    client: &mut Client,
+    url: &str,
+    start: u64,
+    end: u64,
+) -> Result<Response> {
+    client.get(url, Some((start, end)))
+}
+
+pub fn resolve_url(url: &str, _status: &mut dyn StatusBackend) -> Result<String> {
+    Ok(url.into())
+}