Repo export

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["icechunk", "icechunk-python", "icechunk-macros"]
+members = ["icechunk", "icechunk-python", "icechunk-macros", "icechunk-export"]
 default-members = ["icechunk"]
 resolver = "2"
 

icechunk-export/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "icechunk-export"
+version = "0.1.0"
+
+description = "Library to export Icechunk repositories"
+repository = "https://github.com/earth-mover/icechunk"
+homepage = "https://icechunk.io"
+license = "Apache-2.0"
+keywords = ["zarr", "xarray", "database"]
+categories = ["database", "science", "science::geo"]
+authors = ["Earthmover PBC"]
+edition = "2024"
+
+[dependencies]
+icechunk = { path = "../icechunk", version = "0.3.8" }
+tokio = { version = "1.47.1", features = ["rt-multi-thread", "macros"] }
+bytes = { version = "1.10.1" }
+err-into = "1.0.1"
+itertools = "0.14.0"
+futures = "0.3.31"
+chrono = "0.4.42"
+indicatif = "0.18.0"
+thiserror = "2.0.16"
+tokio-util = { version = "0.7.16", features = ["rt"] }
+async-trait = "0.1.89"
+
+[lints]
+workspace = true

icechunk-export/examples/export.rs

@@ -0,0 +1,41 @@
+use std::{path::Path, sync::Arc};
+
+use icechunk::{
+    Repository, Storage, config::S3Credentials, new_local_filesystem_storage,
+    new_s3_storage,
+};
+use icechunk_export::{ProgressBars, export};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let repo_path = Path::new("./icechunk-python/tests/data/test-repo-v2");
+    let source_storage = new_local_filesystem_storage(repo_path).await?;
+    // let source_storage = new_s3_storage(
+    //     icechunk::config::S3Options {
+    //         region: Some("us-east-1".to_string()),
+    //         endpoint_url: None,
+    //         anonymous: false,
+    //         allow_http: false,
+    //         force_path_style: false,
+    //         network_stream_timeout_seconds: None,
+    //     },
+    //     "icechunk-public-data".to_string(),
+    //     Some("v1/glad".to_string()),
+    //     Some(S3Credentials::FromEnv),
+    // )?;
+
+    let source = Repository::open(None, source_storage, Default::default()).await?;
+    let destination_path = Path::new("/tmp/test-export");
+    let destination = new_local_filesystem_storage(destination_path).await?;
+    export(
+        &source,
+        destination,
+        &icechunk_export::VersionSelection::AllHistory,
+        Arc::new(ProgressBars::new()),
+        100,
+    )
+    .await?;
+    println!("done");
+
+    Ok(())
+}

icechunk-export/examples/saturate_chunks.rs

@@ -0,0 +1,13 @@
+use std::path::Path;
+
+use icechunk_export::test_saturation;
+
+#[tokio::main]
+async fn main() {
+    //let path = Path::new("/tmp/chunk_ids.txt");
+    let args: Vec<String> = std::env::args().collect();
+    let path = Path::new(args[1].as_str());
+    let max_concurrent = args[2].parse().unwrap();
+    let copy_chunks = args[3].parse().unwrap();
+    test_saturation(max_concurrent, copy_chunks, path).await.unwrap();
+}