Revise object_store support

CHANGELOG.md

@@ -26,6 +26,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  - **Breaking** Remove `http` and `zip` from default features
  - Locking functionality for arrays is moved into stores
  - Improved `Array` documentation
+ - Add `object_store` feature and revise `object_store` support
+   - Add an `ObjectStore` store, which wraps any `object_store`-based store implementing `object_store::ObjectStore`.
+   - **Breaking** Removes the explicit `object_store`-based stores (e.g. `AsyncAmazonS3Store`, `AsyncHTTPStore`)
+   - **Breaking** Removes the `object_store_impl` macro
+   - **Breaking** Removes the `s3`, `gcp`, and `azure` crate features
 
 ## [0.7.3] - 2023-12-22
 

Cargo.toml

@@ -22,13 +22,11 @@ sharding = [] # Enable the sharding codec
 transpose = ["dep:ndarray"] # Enable the transpose codec
 zfp = ["dep:zfp-sys"] # Enable the experimental zfp codec
 zstd = ["dep:zstd"] # Enable the zstd codec
-http = ["dep:reqwest", "dep:url", "object_store/http"] # Enable the HTTP store (including the async variant if `async` is enabled)
-s3 = ["async", "dep:object_store", "object_store/aws"] # Enable the Amazon S3 store
-gcp = ["async", "dep:object_store", "object_store/gcp"] # Enable the Google Cloud Storage store
-azure = ["async", "dep:object_store", "object_store/azure"]  # Enable the Microsoft Azure Blob Storage store
+http = ["dep:reqwest", "dep:url"] # Enable the sync HTTP store
 zip = ["dep:zip"] # Enable the zip storage adapter
 ndarray = ["dep:ndarray"] # Adds ndarray utility functions to Array
-async = ["dep:object_store", "dep:async-trait", "dep:async-recursion", "dep:async-lock", "dep:futures"] # Enable experimental async API (async stores and array/group operations)
+async = ["dep:async-trait", "dep:async-recursion", "dep:async-lock", "dep:futures"] # Enable experimental async API
+object_store = ["dep:object_store"] # Enable object_store asynchronous stores support
 
 [package.metadata.docs.rs]
 all-features = true
@@ -80,11 +78,11 @@ required-features = ["ndarray"]
 
 [[example]]
 name = "async_array_write_read"
-required-features = ["ndarray", "async"]
+required-features = ["ndarray", "async", "object_store"]
 
 [[example]]
 name = "async_http_array_read"
-required-features = ["ndarray", "async", "http"]
+required-features = ["ndarray", "async", "object_store/http"]
 
 [[example]]
 name = "http_array_read"

examples/async_array_write_read.rs

@@ -14,7 +14,9 @@ async fn async_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
     // let store = Arc::new(store::AsyncFilesystemStore::new(
     //     "tests/data/array_write_read.zarr",
     // )?);
-    let store = Arc::new(store::AsyncMemoryStore::default());
+    let store = Arc::new(store::AsyncObjectStore::new(
+        object_store::memory::InMemory::new(),
+    ));
 
     // Create a group and write metadata to filesystem
     let group_path = "/group";

examples/async_http_array_read.rs

@@ -14,7 +14,11 @@ async fn http_array_read() -> Result<(), Box<dyn std::error::Error>> {
     const ARRAY_PATH: &str = "/group/array";
 
     // Create a HTTP store
-    let store = Arc::new(store::AsyncHTTPStore::new(HTTP_URL)?);
+    let store = Arc::new(store::AsyncObjectStore::new(
+        object_store::http::HttpBuilder::new()
+            .with_url(HTTP_URL)
+            .build()?,
+    ));
     let log_writer = Arc::new(std::sync::Mutex::new(
         // std::io::BufWriter::new(
         std::io::stdout(),

src/group.rs

@@ -314,7 +314,9 @@ mod tests {
     #[cfg(feature = "async")]
     #[tokio::test]
     async fn group_metadata_write_read_async() {
-        let store = std::sync::Arc::new(crate::storage::store::AsyncMemoryStore::new());
+        let store = std::sync::Arc::new(crate::storage::store::AsyncObjectStore::new(
+            object_store::memory::InMemory::new(),
+        ));
         let group_path = "/group";
         let group = GroupBuilder::new()
             .build(store.clone(), group_path)

src/lib.rs

@@ -17,7 +17,7 @@
 //! - [x] [ZEP0003 - Variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html) ([draft](https://github.com/orgs/zarr-developers/discussions/52)).
 //! - [x] Stores:
 //!   - Sync: [filesystem](crate::storage::store::FilesystemStore), [in memory](crate::storage::store::MemoryStore), [HTTP](crate::storage::store::HTTPStore), [ZIP](crate::storage::storage_adapter::ZipStorageAdapter).
-//!   - Async: [filesystem](crate::storage::store::AsyncFilesystemStore), [in memory](crate::storage::store::AsyncMemoryStore), [HTTP](crate::storage::store::AsyncHTTPStore), [Google Cloud Storage](crate::storage::store::AsyncGoogleCloudStore), [Amazon S3](crate::storage::store::AsyncAmazonS3Store), [Microsoft Azure Storage](crate::storage::store::AsyncMicrosoftAzureStore).
+//!   - Async: [`AsyncObjectStore`](crate::storage::store::AsyncObjectStore) (supports all [`object_store::ObjectStore`] stores, e.g. in-memory, HTTP, Google Cloud Storage, Amazon S3, Microsoft Azure Storage, etc.).
 //! - [x] Data types: [core data types](crate::array::data_type::DataType), [raw bits](crate::array::data_type::DataType::RawBits), [float16](crate::array::data_type::DataType::Float16), [bfloat16](crate::array::data_type::DataType::BFloat16) [(spec issue)](https://github.com/zarr-developers/zarr-specs/issues/130).
 //! - [x] Chunk grids: [regular](crate::array::chunk_grid::RegularChunkGrid), [rectangular](crate::array::chunk_grid::RectangularChunkGrid) ([draft](https://github.com/orgs/zarr-developers/discussions/52)).
 //! - [x] Chunk key encoding: [default](crate::array::chunk_key_encoding::DefaultChunkKeyEncoding), [v2](crate::array::chunk_key_encoding::V2ChunkKeyEncoding).

src/storage/store.rs

@@ -10,27 +10,14 @@ mod store_async;
 mod store_sync;
 // mod store_plugin;
 
-#[cfg(feature = "async")]
-pub use store_async::filesystem_store::AsyncFilesystemStore;
-#[cfg(feature = "async")]
-pub use store_async::memory_store::AsyncMemoryStore;
-
 pub use store_sync::filesystem_store::{FilesystemStore, FilesystemStoreCreateError};
 pub use store_sync::memory_store::MemoryStore;
 
-#[cfg(all(feature = "async", feature = "http"))]
-pub use store_async::http_store::AsyncHTTPStore;
 #[cfg(feature = "http")]
 pub use store_sync::http_store::{HTTPStore, HTTPStoreCreateError};
 
-#[cfg(all(feature = "async", feature = "s3"))]
-pub use store_async::amazon_s3_store::AsyncAmazonS3Store;
-
-#[cfg(all(feature = "async", feature = "gcp"))]
-pub use store_async::google_cloud_store::AsyncGoogleCloudStore;
-
-#[cfg(all(feature = "async", feature = "azure"))]
-pub use store_async::microsoft_azure_store::AsyncMicrosoftAzureStore;
+#[cfg(all(feature = "async", feature = "object_store"))]
+pub use store_async::object_store::AsyncObjectStore;
 
 // pub use store_plugin::{StorePlugin, StorePluginCreateError}; // Currently disabled.
 

src/storage/store/store_async.rs

@@ -1,12 +1,2 @@
-pub mod filesystem_store;
-pub mod memory_store;
+#[cfg(feature = "object_store")]
 pub mod object_store;
-
-#[cfg(feature = "s3")]
-pub mod amazon_s3_store;
-#[cfg(feature = "gcp")]
-pub mod google_cloud_store;
-#[cfg(feature = "http")]
-pub mod http_store;
-#[cfg(feature = "azure")]
-pub mod microsoft_azure_store;