Allow choice of reqwest or curl for HTTP backend

Cargo.toml

@@ -36,6 +36,7 @@ members = [
   "crates/cfg_support",
   "crates/dep_support",
   "crates/errors",
+  "crates/geturl",
   "crates/io_base",
   "crates/status_base",
   "crates/xdv",
@@ -63,12 +64,10 @@ cfg-if = "1.0"
 error-chain = "^0.12"
 flate2 = { version = "^1.0.19", default-features = false, features = ["zlib"] }
 fs2 = "^0.4"
-headers = "^0.2"
 lazy_static = "^1.4"
 libc = "^0.2"
 md-5 = "^0.9"
 open = "1.4.0"
-reqwest = "^0.9"
 serde = { version = "^1.0", features = ["derive"], optional = true }
 sha2 = "^0.9"
 structopt = "0.3"
@@ -78,23 +77,29 @@ tectonic_bridge_graphite2 = { path = "crates/bridge_graphite2", version = "0.0.0
 tectonic_bridge_harfbuzz = { path = "crates/bridge_harfbuzz", version = "0.0.0-dev.0" }
 tectonic_bridge_icu = { path = "crates/bridge_icu", version = "0.0.0-dev.0" }
 tectonic_errors = { path = "crates/errors", version = "0.0.0-dev.0" }
+tectonic_geturl = { path = "crates/geturl", version = "0.0.0-dev.0", default-features = false }
 tectonic_io_base = { path = "crates/io_base", version = "0.0.0-dev.0" }
 tectonic_status_base = { path = "crates/status_base", version = "0.0.0-dev.0" }
 tectonic_xdv = { path = "crates/xdv", version = "0.0.0-dev.0" }
 tempfile = "^3.1"
 termcolor = "^1.1"
 toml = { version = "^0.5", optional = true }
+url = "^2.0"
 zip = { version = "^0.5", default-features = false, features = ["deflate"] }
 
 [features]
-default = ["serialization"]
+default = ["geturl-reqwest", "serialization"]
+
 # Note: we used to have this to couple "serde" and "serde-derive", but we've
 # adopted the newer scheme to avoid having to depend on both -- should maybe
 # just get rid of this feature:
 serialization = ["serde", "toml"]
 
 external-harfbuzz = ["tectonic_bridge_harfbuzz/external-harfbuzz"]
 
+geturl-curl = ["tectonic_geturl/curl"]
+geturl-reqwest = ["tectonic_geturl/reqwest"]
+
 # developer feature to compile with the necessary flags for profiling tectonic.
 profile = []
 
@@ -125,6 +130,7 @@ tectonic_bridge_icu = "thiscommit:2021-01-09:diehoh1E"
 tectonic_cfg_support = "thiscommit:aeRoo7oa"
 tectonic_dep_support = "5faf4205bdd3d31101b749fc32857dd746f9e5bc"
 tectonic_errors = "317ae79ceaa2593fb56090e37bf1f5cc24213dd9"
+tectonic_geturl = "thiscommit:2021-01-10:eiv5ohMe"
 tectonic_io_base = "317ae79ceaa2593fb56090e37bf1f5cc24213dd9"
 tectonic_status_base = "317ae79ceaa2593fb56090e37bf1f5cc24213dd9"
 tectonic_xdv = "c91f2ef37858d1a0a724a5c3ddc2f7ea46373c77"

README.md

@@ -87,6 +87,21 @@ external library can be detected with either [pkg-config] or [vcpkg]. See the
 [vcpkg]: https://vcpkg.readthedocs.io/
 [howto-build]: https://tectonic-typesetting.github.io/book/latest/#update-link-when-published
 
+##### `geturl-curl`
+
+Use the [curl] crate to implement HTTP requests. In order for this to take
+effect, you must use `--no-default-features` because `geturl-reqwest` is a
+default feature and it takes precedence.
+
+[reqwest]: https://docs.rs/curl/
+
+##### `geturl-reqwest` (enabled by default)
+
+Use the [reqwest] crate to implement HTTP requests. This is the default
+selection.
+
+[reqwest]: https://docs.rs/reqwest/
+
 ##### `serialization` (enabled by default)
 
 This feature enables (de)serialization using the [serde](https://serde.rs/)

crates/geturl/CHANGELOG.md

@@ -0,0 +1,8 @@
+# See elsewhere for changelog
+
+This project’s release notes are curated from the Git history of its main
+branch. You can find them by looking at [the version of this file on the
+`release` branch][branch] or the [GitHub release history][gh-releases].
+
+[branch]: https://github.com/tectonic-typesetting/tectonic/blob/release/crates/geturl/CHANGELOG.md
+[gh-releases]: https://github.com/tectonic-typesetting/tectonic/releases

crates/geturl/Cargo.toml

@@ -0,0 +1,30 @@
+# Copyright 2020 the Tectonic Project
+# Licensed under the MIT License.
+
+[package]
+name = "tectonic_geturl"
+version = "0.0.0-dev.0"  # assigned with cranko (see README)
+authors = ["Peter Williams <[email protected]>"]
+description = """
+A generic interface for HTTP GETs and byte-range requests, with pluggable backends.
+"""
+homepage = "https://tectonic-typesetting.github.io/"
+documentation = "https://docs.rs/tectonic_geturl"
+repository = "https://github.com/tectonic-typesetting/tectonic/"
+readme = "README.md"
+license = "MIT"
+edition = "2018"
+
+[dependencies]
+cfg-if = "^1.0"
+curl = { version = "^0.4", optional = true }
+reqwest = { version = "^0.11", optional = true, features = ["blocking"] }
+tectonic_errors = { path = "../errors", version = "0.0.0-dev.0" }
+tectonic_status_base = { path = "../status_base", version = "0.0.0-dev.0" }
+
+[features]
+default = ["reqwest"]
+
+[package.metadata.internal_dep_versions]
+tectonic_errors = "e04798bcd9b1c1d68cc0a318a710bb30230a0300"
+tectonic_status_base = "401387acfd98113133db6981c301426431f55ea3"

crates/geturl/README.md

@@ -0,0 +1,31 @@
+# The `tectonic_geturl` create
+
+This crate is part of [the Tectonic
+project](https://tectonic-typesetting.github.io/en-US/). It provides an
+interface for fetching URLs using one of several HTTP backends.
+
+[![](http://meritbadge.herokuapp.com/tectonic_geturl)](https://crates.io/crates/tectonic_geturl)
+
+- [API documentation](https://docs.rs/tectonic_geturl/).
+- [Main Git repository](https://github.com/tectonic-typesetting/tectonic/).
+
+
+## Cargo features
+
+This crate provides the following [Cargo features][features]:
+
+[features]: https://doc.rust-lang.org/cargo/reference/features.html
+
+- **`curl`**: use the [curl] crate as a backend for performing URL gets.
+- **`reqwest`** (enabled by default): use the [reqwest] crate as a backend for
+  performing URL gets.
+
+[curl]: https://docs.rs/curl/
+[reqwest]: https://docs.rs/reqwest/
+
+There is always a “null” backend available, which will always return errors. If
+more than one backend is enabled, their prioritization is:
+
+- `reqwest` (most preferred)
+- `curl`
+- `null` (least preferred)

crates/geturl/src/curl.rs

@@ -0,0 +1,105 @@
+// Copyright 2020 the Tectonic Project
+// Licensed under the MIT License.
+
+//! A URL-get backend based on the `curl` crate.
+
+use curl::easy::Easy;
+use std::io::Cursor;
+use tectonic_errors::{anyhow::bail, Result};
+use tectonic_status_base::StatusBackend;
+
+use crate::{GetUrlBackend, RangeReader};
+
+const MAX_HTTP_REDIRECTS_ALLOWED: u32 = 10;
+
+fn get_url_generic(
+    handle: &mut Easy,
+    url: &str,
+    range: Option<(u64, usize)>,
+) -> Result<Cursor<Vec<u8>>> {
+    handle.url(url)?;
+    handle.follow_location(true)?;
+    handle.max_redirections(MAX_HTTP_REDIRECTS_ALLOWED)?;
+
+    if let Some((start, length)) = range {
+        let end = start + length as u64 - 1;
+        handle.range(&format!("{}-{}", start, end))?;
+    }
+
+    let mut buf = Vec::new();
+    {
+        let mut transfer = handle.transfer();
+        transfer.write_function(|data| {
+            buf.extend_from_slice(data);
+            Ok(data.len())
+        })?;
+        transfer.perform()?;
+    }
+
+    let code = handle.response_code()?;
+
+    if !(200..300).contains(&code) {
+        bail!(
+            "unsuccessful HTTP GET status code {} for url `{}`",
+            code,
+            url
+        );
+    }
+
+    Ok(Cursor::new(buf))
+}
+
+/// URL-get backend implemented using the `curl` crate.
+#[derive(Debug)]
+pub struct CurlBackend {
+    handle: Easy,
+}
+
+impl Default for CurlBackend {
+    fn default() -> Self {
+        CurlBackend {
+            handle: Easy::new(),
+        }
+    }
+}
+
+impl GetUrlBackend for CurlBackend {
+    type Response = Cursor<Vec<u8>>;
+    type RangeReader = CurlRangeReader;
+
+    fn get_url(&mut self, url: &str, _status: &mut dyn StatusBackend) -> Result<Self::Response> {
+        get_url_generic(&mut self.handle, url, None)
+    }
+
+    fn resolve_url(&mut self, url: &str, _status: &mut dyn StatusBackend) -> Result<String> {
+        Ok(url.into())
+    }
+
+    fn open_range_reader(&self, url: &str) -> Self::RangeReader {
+        CurlRangeReader::new(url)
+    }
+}
+
+/// Curl-based byte-range reader.
+#[derive(Debug)]
+pub struct CurlRangeReader {
+    url: String,
+    handle: Easy,
+}
+
+impl CurlRangeReader {
+    fn new(url: &str) -> CurlRangeReader {
+        CurlRangeReader {
+            url: url.to_owned(),
+            handle: Easy::new(),
+        }
+    }
+}
+
+impl RangeReader for CurlRangeReader {
+    type Response = Cursor<Vec<u8>>;
+
+    fn read_range(&mut self, offset: u64, length: usize) -> Result<Self::Response> {
+        get_url_generic(&mut self.handle, &self.url, Some((offset, length)))
+    }
+}

crates/geturl/src/lib.rs

@@ -0,0 +1,67 @@
+// Copyright 2020 the Tectonic Project.
+// Licensed under the MIT License.
+
+#![deny(missing_docs)]
+
+//! A simple, pluggable interface for HTTP GETs and range requests.
+//!
+//! The default interface is intentionally exposed as a concrete type, so that
+//! crates relying on this one need not use a lot of dyns and impl Traits. It is
+//! intended that the choice of HTTP backend is a build-time one, not a runtime
+//! one.
+
+use cfg_if::cfg_if;
+use std::io::Read;
+use tectonic_errors::Result;
+use tectonic_status_base::StatusBackend;
+
+/// A trait for reading byte ranges from an HTTP resource.
+pub trait RangeReader {
+    /// The readable type returned by the range request.
+    type Response: Read;
+
+    /// Read the specified range of bytes from this HTTP resource.
+    fn read_range(&mut self, offset: u64, length: usize) -> Result<Self::Response>;
+}
+
+/// A trait for simple HTTP operations needed by the Tectonic backends.
+pub trait GetUrlBackend: Default {
+    /// The readable type returned by URL get requests.
+    type Response: Read;
+
+    /// The range-reader type for URLs that will undergo byte-range reads.
+    type RangeReader: RangeReader;
+
+    /// Starting with an input URL, follow redirections to get a final URL.
+    ///
+    /// But we attempt to detect redirects into CDNs/S3/etc and *stop* following
+    /// before we get that deep.
+    fn resolve_url(&mut self, url: &str, status: &mut dyn StatusBackend) -> Result<String>;
+
+    /// Perform an HTTP GET on a URL, returning a readable result.
+    fn get_url(&mut self, url: &str, status: &mut dyn StatusBackend) -> Result<Self::Response>;
+
+    /// Open a range reader that can perform byte-range reads on the specified URL.
+    fn open_range_reader(&self, url: &str) -> Self::RangeReader;
+}
+
+pub mod null;
+
+#[cfg(feature = "curl")]
+pub mod curl;
+
+#[cfg(feature = "reqwest")]
+pub mod reqwest;
+
+cfg_if! {
+    if #[cfg(feature = "reqwest")] {
+        pub use crate::reqwest::ReqwestBackend as DefaultBackend;
+    } else if #[cfg(feature = "curl")] {
+        pub use crate::curl::CurlBackend as DefaultBackend;
+    } else {
+        pub use null::NullBackend as DefaultBackend;
+    }
+}
+
+/// The range-reader type exposed by the default URL-get backend (for convenience).
+pub type DefaultRangeReader = <DefaultBackend as GetUrlBackend>::RangeReader;

crates/geturl/src/null.rs

@@ -0,0 +1,60 @@
+// Copyright 2020 the Tectonic Project
+// Licensed under the MIT License.
+
+//! A geturl backend that always fails.
+
+use std::{
+    error::Error,
+    fmt::{Display, Error as FmtError, Formatter},
+    io::Empty,
+    result::Result as StdResult,
+};
+use tectonic_errors::Result;
+use tectonic_status_base::StatusBackend;
+
+use crate::{GetUrlBackend, RangeReader};
+
+/// The error type for the always-failing geturl backend.
+#[derive(Debug)]
+pub struct NoGetUrlBackendError {}
+
+impl Display for NoGetUrlBackendError {
+    fn fmt(&self, f: &mut Formatter) -> StdResult<(), FmtError> {
+        write!(f, "no get-URL backend was enabled")
+    }
+}
+
+impl Error for NoGetUrlBackendError {}
+
+/// The "null" URL-get backend, which always fails.
+#[derive(Debug, Default)]
+pub struct NullBackend {}
+
+impl GetUrlBackend for NullBackend {
+    type Response = Empty;
+    type RangeReader = NullRangeReader;
+
+    fn get_url(&mut self, _url: &str, _status: &mut dyn StatusBackend) -> Result<Empty> {
+        Err((NoGetUrlBackendError {}).into())
+    }
+
+    fn resolve_url(&mut self, _url: &str, _status: &mut dyn StatusBackend) -> Result<String> {
+        Err((NoGetUrlBackendError {}).into())
+    }
+
+    fn open_range_reader(&self, _url: &str) -> Self::RangeReader {
+        NullRangeReader {}
+    }
+}
+
+/// The "null" URL-get range reader, which always fails.
+#[derive(Debug)]
+pub struct NullRangeReader {}
+
+impl RangeReader for NullRangeReader {
+    type Response = Empty;
+
+    fn read_range(&mut self, _offset: u64, _length: usize) -> Result<Empty> {
+        Err((NoGetUrlBackendError {}).into())
+    }
+}