Work toward wasm-wasi compat

icechunk/Cargo.toml

@@ -17,21 +17,13 @@ async-trait = "0.1.88"
 bytes = { version = "1.10.1", features = ["serde"] }
 base64 = "0.22.1"
 futures = "0.3.31"
+getrandom = {version = "0.2.15", features = ["js"]}
 itertools = "0.14.0"
-object_store = { version = "0.12.3", features = [
-  "aws",
-  "gcp",
-  "azure",
-  "http",
-] }
 rand = "0.9.2"
 thiserror = "2.0.12"
 serde_json = "1.0.142"
 serde = { version = "1.0.219", features = ["derive", "rc"] }
 serde_with = { version = "3.14.0", features = ["hex"] }
-tokio = { version = "1.47.1", features = ["rt-multi-thread", "macros"] }
-test-strategy = "0.4.3"
-proptest = "1.7.0"
 quick_cache = "0.6.16"
 base32 = "0.5.1"
 chrono = { version = "0.4.41", features = ["serde"] }
@@ -40,17 +32,9 @@ rmp-serde = "1.3.0"
 url = "2.5.4"
 async-stream = "0.3.6"
 rmpv = { version = "1.3.0", features = ["serde", "with-serde"] }
-aws-sdk-s3 = "=1.78.0"
-aws-config = "=1.5.18"
-aws-credential-types = "1.2.4"
 typed-path = "0.11.0"
-aws-smithy-types-convert = { version = "0.60.9", features = [
-  "convert-chrono",
-  "convert-streams",
-] }
 typetag = "0.2.20"
 zstd = "0.13.3"
-tokio-util = { version = "0.7.16", features = ["compat", "io-util"] }
 serde_bytes = "0.11.17"
 regex = "1.11.1"
 tracing-error = "0.2.1"
@@ -67,9 +51,31 @@ dirs = { version = "6.0.0", optional = true }
 assert_fs = { version = "1.1.3", optional = true }
 flatbuffers = "25.2.10"
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+aws-sdk-s3 = "=1.78.0"
+aws-config = "=1.5.18"
+aws-credential-types = "1.2.4"
+aws-smithy-types-convert = { version = "0.60.9", features = [
+  "convert-chrono",
+  "convert-streams",
+] }
+object_store = { version = "0.12.3", features = [
+  "aws",
+  "gcp",
+  "azure",
+  "http",
+] }
+tokio = { version = "1.47.1", features = ["rt-multi-thread", "macros"] }
+tokio-util = { version = "0.7.16", features = ["compat", "io-util"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+tokio = { version = "1.47.1", features = ["io-util", "rt", "macros", "sync"] }
+
 [dev-dependencies]
 icechunk-macros = { path = "../icechunk-macros", version = "0.1.0" }
 pretty_assertions = "1.4.1"
+test-strategy = "0.4.3"
+proptest = "1.7.0"
 proptest-state-machine = "0.4.0"
 tempfile = "3.20.0"
 test-log = { version = "0.2.18", default-features = false, features = [

icechunk/src/config.rs

@@ -9,6 +9,7 @@ use std::{
 use async_trait::async_trait;
 use chrono::{DateTime, Utc};
 use itertools::Either;
+#[cfg(not(target_arch = "wasm32"))]
 pub use object_store::gcp::GcpCredential;
 use regex::bytes::Regex;
 use serde::{Deserialize, Serialize};
@@ -519,6 +520,7 @@ pub struct GcsBearerCredential {
     pub expires_after: Option<DateTime<Utc>>,
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 impl From<&GcsBearerCredential> for GcpCredential {
     fn from(value: &GcsBearerCredential) -> Self {
         GcpCredential { bearer: value.bearer.clone() }

icechunk/src/lib.rs

@@ -38,10 +38,10 @@ pub mod virtual_chunks;
 
 pub use config::{ObjectStoreConfig, RepositoryConfig};
 pub use repository::Repository;
-pub use storage::{
-    ObjectStorage, Storage, StorageError, new_in_memory_storage,
-    new_local_filesystem_storage, new_s3_storage,
-};
+pub use storage::{Storage, StorageError, new_in_memory_storage};
+
+#[cfg(not(target_arch = "wasm32"))]
+pub use storage::{ObjectStorage, new_local_filesystem_storage, new_s3_storage};
 pub use store::Store;
 
 mod private {

icechunk/src/storage/backends/mod.rs

@@ -0,0 +1,10 @@
+#[cfg(not(target_arch = "wasm32"))]
+pub mod object_store;
+#[cfg(not(target_arch = "wasm32"))]
+pub mod s3;
+
+// Re-export implementations conditionally
+#[cfg(not(target_arch = "wasm32"))]
+pub use object_store::ObjectStorage;
+#[cfg(not(target_arch = "wasm32"))]
+pub use s3::S3Storage;

icechunk/src/storage/object_store.rs → icechunk/src/storage/backends/object_store.rs

@@ -43,7 +43,7 @@ use tokio::{
 use tokio_util::compat::FuturesAsyncReadCompatExt;
 use tracing::instrument;
 
-use super::{
+use crate::storage::{
     CHUNK_PREFIX, CONFIG_PATH, ConcurrencySettings, DeleteObjectsResult, ETag,
     FetchConfigResult, Generation, GetRefResult, ListInfo, MANIFEST_PREFIX, REF_PREFIX,
     Reader, RetriesSettings, SNAPSHOT_PREFIX, Settings, Storage, StorageError,

icechunk/src/storage/s3.rs → icechunk/src/storage/backends/s3.rs

@@ -8,10 +8,10 @@ use std::{
 };
 
 use crate::{
-    Storage, StorageError,
     config::{S3Credentials, S3CredentialsFetcher, S3Options},
     format::{ChunkId, ChunkOffset, FileTypeTag, ManifestId, ObjectId, SnapshotId},
     private,
+    storage::{Storage, StorageError},
 };
 use async_trait::async_trait;
 use aws_config::{
@@ -42,7 +42,7 @@ use serde::{Deserialize, Serialize};
 use tokio::{io::AsyncRead, sync::OnceCell};
 use tracing::{error, instrument};
 
-use super::{
+use crate::storage::{
     CHUNK_PREFIX, CONFIG_PATH, DeleteObjectsResult, FetchConfigResult, GetRefResult,
     ListInfo, MANIFEST_PREFIX, REF_PREFIX, Reader, SNAPSHOT_PREFIX, Settings,
     StorageErrorKind, StorageResult, TRANSACTION_PREFIX, UpdateConfigResult, VersionInfo,

icechunk/src/storage/errors.rs

@@ -0,0 +1,79 @@
+use std::ffi::OsString;
+use thiserror::Error;
+
+use crate::error::ICError;
+
+#[cfg(not(target_arch = "wasm32"))]
+use aws_sdk_s3::{
+    config::http::HttpResponse,
+    error::SdkError,
+    operation::{
+        complete_multipart_upload::CompleteMultipartUploadError,
+        create_multipart_upload::CreateMultipartUploadError,
+        delete_objects::DeleteObjectsError, get_object::GetObjectError,
+        head_object::HeadObjectError, list_objects_v2::ListObjectsV2Error,
+        put_object::PutObjectError, upload_part::UploadPartError,
+    },
+    primitives::ByteStreamError,
+};
+
+#[derive(Debug, Error)]
+pub enum StorageErrorKind {
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("object store error {0}")]
+    ObjectStore(#[from] Box<::object_store::Error>),
+    #[error("bad object store prefix {0:?}")]
+    BadPrefix(OsString),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error getting object from object store {0}")]
+    S3GetObjectError(#[from] Box<SdkError<GetObjectError, HttpResponse>>),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error writing object to object store {0}")]
+    S3PutObjectError(#[from] Box<SdkError<PutObjectError, HttpResponse>>),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error creating multipart upload {0}")]
+    S3CreateMultipartUploadError(
+        #[from] Box<SdkError<CreateMultipartUploadError, HttpResponse>>,
+    ),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error uploading multipart part {0}")]
+    S3UploadPartError(#[from] Box<SdkError<UploadPartError, HttpResponse>>),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error completing multipart upload {0}")]
+    S3CompleteMultipartUploadError(
+        #[from] Box<SdkError<CompleteMultipartUploadError, HttpResponse>>,
+    ),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error getting object metadata from object store {0}")]
+    S3HeadObjectError(#[from] Box<SdkError<HeadObjectError, HttpResponse>>),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error listing objects in object store {0}")]
+    S3ListObjectError(#[from] Box<SdkError<ListObjectsV2Error, HttpResponse>>),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error deleting objects in object store {0}")]
+    S3DeleteObjectError(#[from] Box<SdkError<DeleteObjectsError, HttpResponse>>),
+    #[cfg(not(target_arch = "wasm32"))]
+    #[error("error streaming bytes from object store {0}")]
+    S3StreamError(#[from] Box<ByteStreamError>),
+    #[error("I/O error: {0}")]
+    IOError(#[from] std::io::Error),
+    #[error("storage configuration error: {0}")]
+    R2ConfigurationError(String),
+    #[error("storage error: {0}")]
+    Other(String),
+}
+
+pub type StorageError = ICError<StorageErrorKind>;
+
+// it would be great to define this impl in error.rs, but it conflicts with the blanket
+// `impl From<T> for T`
+impl<E> From<E> for StorageError
+where
+    E: Into<StorageErrorKind>,
+{
+    fn from(value: E) -> Self {
+        Self::new(value.into())
+    }
+}
+
+pub type StorageResult<A> = Result<A, StorageError>;