Merge pull request #29 from ghimiresdp/feature/max-chunk-size

Allow passing custom chunk size to download files through config

Author: ghimiresdp

README.md

@@ -9,6 +9,11 @@ furl is a high-performance command-line tool designed to download files faster
 by utilizing multiple threads to fetch chunks of data concurrently. Inspired by
 the simplicity of cURL and the robustness of wget.
 
+> [!Warning]
+> This branch is usually ahead of the latest release. If something does not
+> work as expected, consider checking out a specific release tag, or install
+> a stable version via cargo or winget. For more, see the [Installation](#installation) section.
+
 ![Example image](res/images/example.png)
 
 ## ✨ Features
@@ -122,8 +127,8 @@ async fn main(){
 }
 ```
 
-For a runnable workspace example that embeds `furl-cli` as a library,
-check [examples/embedded-minimal](examples/embedded-minimal).
+For runnable workspace examples that embed `furl-cli` as a library,
+check the [examples/](examples) directory (e.g. [embedded-minimal](examples/embedded-minimal), [no-indicator](examples/no-indicator)).
 
 ## 🏗 Architecture Decisions
 
@@ -145,14 +150,14 @@ directory. These are standalone workspace crates that show how to use
 - [x] Real-time progress bars
 - [x] Smart Threading (Completely ignore threading for files smaller than 1 MB).
 - [x] Package manager support (Windows: WinGet)
+- [x] Examples
+- [ ] Config file support (furl.toml)
+- [ ] Support for Proxy and Basic Auth
+- [ ] Resume interrupted downloads (Checkpoints)
 - [ ] Package manager support (Linux: APT)
 - [ ] Package manager support (Linux: Flatpak)
 - [ ] Package manager support (Linux: Snap)
 - [ ] Package manager support (macOS: Homebrew)
-- [ ] Examples
-- [ ] Resume interrupted downloads (Checkpoints)
-- [ ] Support for Proxy and Basic Auth
-- [ ] Config file support (furl.toml)
 
 ## 🤝 Contributing
 

crates/furl_cli/src/engine.rs

@@ -55,13 +55,37 @@ impl Chunk {
     }
 }
 
+#[derive(Clone)]
+pub struct DownloadConfig {
+    pub max_chunk_size: u64,
+}
+
+impl DownloadConfig {
+    pub fn new() -> Self {
+        Self {
+            max_chunk_size: _10MB,
+        }
+    }
+    pub fn set_max_chunk_size(mut self, size: u64) -> Self {
+        self.max_chunk_size = size;
+        self
+    }
+}
+
+impl Default for DownloadConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 pub struct Downloader {
     url: String,
     headers: HeaderMap,
     file_size: Option<u64>,
     filename: Option<String>,
     chunks: Arc<Mutex<Vec<Chunk>>>, // this stores downloaded chunk size
     reporter: Arc<dyn ProgressReporter + Send + Sync>,
+    config: Arc<DownloadConfig>,
 }
 
 pub trait HeaderUtils {
@@ -141,9 +165,15 @@ impl Downloader {
             filename: None,
             chunks: Arc::new(Mutex::new(Vec::new())),
             reporter: Arc::new(NoopReporter),
+            config: Arc::new(DownloadConfig::default()),
         }
     }
 
+    pub fn with_config(mut self, config: DownloadConfig) -> Self {
+        self.config = Arc::new(config);
+        self
+    }
+
     pub fn with_reporter<R: ProgressReporter + Send + Sync + 'static>(
         mut self,
         reporter: R,
@@ -265,24 +295,18 @@ impl Downloader {
             file.lock().await.set_len(file_size).await?;
 
             let mut start = 0;
-            let thread_size = file_size / threads;
-            let mut byte_size = thread_size;
-
-            //ignore threads if the file is less than a MB.
-            if file_size < _1MB {
+            // Determine chunk size: default to per-thread slice, cap at 10 MB for memory,
+            // or use full file if smaller than 1 MB (no threading benefit).
+            let chunk_size = if file_size < _1MB {
                 println!("ℹ️ The file is smaller than 1 MB, so skipping threads.");
-                byte_size = file_size;
-            }
-
-            // if the byte size is larger than 10 MB, split into 10 MB chunks
-            // so that memory consumption is less.
-            if thread_size > _10MB {
-                byte_size = _10MB
-            }
+                file_size
+            } else {
+                (file_size / threads).min(self.config.max_chunk_size)
+            };
 
             // split chunks to download
             while start < file_size {
-                let end = min(start + byte_size, file_size);
+                let end = min(start + chunk_size, file_size);
                 self.chunks.lock().await.push(Chunk::new(start, end));
                 start = end + 1;
             }
@@ -308,6 +332,7 @@ impl Downloader {
                 let url = self.url.clone();
                 let index_clone = Arc::clone(&index);
                 let reporter_clone = Arc::clone(&self.reporter);
+                let config = Arc::clone(&self.config);
 
                 let task = tokio::spawn(async move {
                     let mut worker_total: u64 = 0;
@@ -332,6 +357,7 @@ impl Downloader {
                             filename: None,
                             chunks: Arc::clone(&chunks),
                             reporter: Arc::clone(&reporter_clone),
+                            config: Arc::clone(&config),
                         };
 
                         // Download the chunk and accumulate the bytes downloaded by this worker
@@ -440,4 +466,15 @@ mod tests {
             assert_eq!(downloader.file_size, Some(0));
         });
     }
+    #[test]
+    fn test_custom_download_config() {
+        let config = DownloadConfig::new().set_max_chunk_size(5 * 1024 * 1024);
+        assert_eq!(config.max_chunk_size, 5 * 1024 * 1024);
+    }
+
+    #[test]
+    fn test_default_download_config() {
+        let config = DownloadConfig::new();
+        assert_eq!(config.max_chunk_size, 10 * 1024 * 1024);
+    }
 }

crates/furl_cli/src/features/cli.rs

@@ -15,7 +15,11 @@ pub struct FurlCliArgs {
     #[arg(short, long)]
     pub filename: Option<String>,
 
-    /// Number of threads, defaults to 8, maximum allowed 255
-    #[arg(short, long, default_value_t = 8)]
+    /// Number of threads, maximum allowed 255
+    #[arg(short, long, default_value_t = 8, value_parser = clap::value_parser!(u8).range(1..=255))]
     pub threads: u8,
+
+    /// Number of chunks in MB, maximum allowed 100
+    #[arg(short, long, default_value_t = 10, value_parser = clap::value_parser!(u8).range(1..=100))]
+    pub chunksize: u8,
 }

crates/furl_cli/src/lib.rs

@@ -20,7 +20,7 @@
 //! async fn main() {
 //!     let url = "https://raw.githubusercontent.com/ghimiresdp/furl-cli/refs/heads/main/res/images/example.png";
 //!     let mut downloader = Downloader::new(url);
-//!     if let Ok(_) = downloader.download(".", None, Some(4)).await {
+//!     if let Ok(_) = downloader.download("./downloads/", None, Some(4)).await {
 //!         println!("Download completed successfully!");
 //!     } else {
 //!         println!("Download failed.");

crates/furl_cli/src/main.rs

@@ -10,6 +10,7 @@
 //!
 
 use clap::Parser;
+use furl_core::engine::DownloadConfig;
 use furl_core::{Downloader, FurlCliArgs, GraphicalProgressReporter};
 use regex::Regex;
 use std::process::exit;
@@ -22,6 +23,7 @@ async fn main() {
     let path = Path::new(&args.out);
     let threads = args.threads;
     let filename = args.filename;
+    let chunk_size = args.chunksize;
 
     if !path.exists() {
         println!("The destination path does not exist");
@@ -31,8 +33,13 @@ async fn main() {
     // TODO: add extensive url pattern matcher
     let re = Regex::new(r"https?://[^\s/$.?#].[^\s]*").unwrap();
     if re.captures(&args.url).is_some() {
-        let mut downloader =
-            Downloader::new(&args.url).with_reporter(GraphicalProgressReporter::new());
+        // use config
+        let download_config =
+            DownloadConfig::new().set_max_chunk_size(chunk_size as u64 * 1024 * 1024);
+
+        let mut downloader = Downloader::new(&args.url)
+            .with_config(download_config)
+            .with_reporter(GraphicalProgressReporter::new());
         if downloader
             .download(&args.out, filename, Some(threads))
             .await