diff --git a/CHANGELOG.md b/CHANGELOG.md index c640492b..f66486a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `{Array,Chunk}Representation::dimensionality()` - Add `ArraySubset::new_empty()` and `ArraySubset::is_empty()` - Add missing `IncompatibleArraySubsetAndShapeError::new()` + - Add `--usage-log` argument to examples to use `UsageLog` storage transformer - Add more tests for `Array`, codecs, store locks, and more ### Changed diff --git a/examples/array_write_read.rs b/examples/array_write_read.rs index f15b1503..d8afc666 100644 --- a/examples/array_write_read.rs +++ b/examples/array_write_read.rs @@ -1,4 +1,8 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use zarrs::storage::{ + storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer}, + ReadableWritableListableStorage, +}; fn array_write_read() -> Result<(), Box> { use std::sync::Arc; @@ -11,12 +15,26 @@ fn array_write_read() -> Result<(), Box> { // Create a store // let path = tempfile::TempDir::new()?; - // let store = Arc::new(store::FilesystemStore::new(path.path())?); - // let store = Arc::new(store::FilesystemStore::new( + // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(path.path())?); + // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new( // "tests/data/array_write_read.zarr", // )?); - // let store: zarrs::storage::ReadableWritableListableStorage = Arc::new(store::MemoryStore::new()); - let store = Arc::new(store::MemoryStore::new()); + let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new()); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log + .clone() + .create_readable_writable_listable_transformer(store); + } + } // Create a group let group_path = "/group"; diff --git a/examples/async_array_write_read.rs b/examples/async_array_write_read.rs index 56ace1c0..740c996c 100644 --- a/examples/async_array_write_read.rs +++ b/examples/async_array_write_read.rs @@ -1,3 +1,8 @@ +use zarrs::storage::{ + storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer}, + AsyncReadableWritableListableStorage, +}; + async fn async_array_write_read() -> Result<(), Box> { use futures::{stream::FuturesUnordered, StreamExt}; use std::sync::Arc; @@ -10,13 +15,28 @@ async fn async_array_write_read() -> Result<(), Box> { // Create a store // let path = tempfile::TempDir::new()?; - // let store = Arc::new(store::AsyncFilesystemStore::new(path.path())?); - // let store = Arc::new(store::AsyncFilesystemStore::new( + // let mut store: ReadableWritableListableStorage = Arc::new(store::AsyncFilesystemStore::new(path.path())?); + // let mut store: ReadableWritableListableStorage = Arc::new(store::AsyncFilesystemStore::new( // "tests/data/array_write_read.zarr", // )?); - let store = Arc::new(store::AsyncObjectStore::new( + let mut store: AsyncReadableWritableListableStorage = Arc::new(store::AsyncObjectStore::new( object_store::memory::InMemory::new(), )); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log + .clone() + .create_async_readable_writable_listable_transformer(store); + } + } // Create a group let group_path = "/group"; diff --git a/examples/async_http_array_read.rs b/examples/async_http_array_read.rs index 64588b76..6f1bd471 100644 --- a/examples/async_http_array_read.rs +++ b/examples/async_http_array_read.rs @@ -14,20 +14,26 @@ async fn http_array_read() -> Result<(), Box> { const ARRAY_PATH: &str = "/group/array"; // Create a HTTP store - let store = Arc::new(store::AsyncObjectStore::new( + let mut store: AsyncReadableWritableListableStorage = Arc::new(store::AsyncObjectStore::new( object_store::http::HttpBuilder::new() .with_url(HTTP_URL) .build()?, )); - let log_writer = Arc::new(std::sync::Mutex::new( - // std::io::BufWriter::new( - std::io::stdout(), - // ) - )); - let usage_log = UsageLogStorageTransformer::new(log_writer, || { - chrono::Utc::now().format("[%T%.3f] ").to_string() - }); - let store = usage_log.create_async_readable_transformer(store); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log + .clone() + .create_async_readable_writable_listable_transformer(store); + } + } // Init the existing array, reading metadata let array = Array::async_new(store, ARRAY_PATH).await?; diff --git a/examples/http_array_read.rs b/examples/http_array_read.rs index 5e79e0b6..c768abca 100644 --- a/examples/http_array_read.rs +++ b/examples/http_array_read.rs @@ -1,3 +1,5 @@ +use zarrs::storage::ReadableStorage; + fn http_array_read() -> Result<(), Box> { use std::sync::Arc; use zarrs::{ @@ -14,16 +16,20 @@ fn http_array_read() -> Result<(), Box> { const ARRAY_PATH: &str = "/group/array"; // Create a HTTP store - let store = Arc::new(store::HTTPStore::new(HTTP_URL)?); - let log_writer = Arc::new(std::sync::Mutex::new( - // std::io::BufWriter::new( - std::io::stdout(), - // ) - )); - let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { - chrono::Utc::now().format("[%T%.3f] ").to_string() - })); - let store = usage_log.create_readable_transformer(store); + let mut store: ReadableStorage = Arc::new(store::HTTPStore::new(HTTP_URL)?); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log.clone().create_readable_transformer(store); + } + } // Init the existing array, reading metadata let array = Array::new(store, ARRAY_PATH)?; diff --git a/examples/rectangular_array_write_read.rs b/examples/rectangular_array_write_read.rs index fa1315b1..12c8258d 100644 --- a/examples/rectangular_array_write_read.rs +++ b/examples/rectangular_array_write_read.rs @@ -1,3 +1,10 @@ +use std::sync::Arc; + +use zarrs::storage::{ + storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer}, + ReadableWritableListableStorage, +}; + fn rectangular_array_write_read() -> Result<(), Box> { use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use zarrs::array::ChunkGrid; @@ -13,8 +20,23 @@ fn rectangular_array_write_read() -> Result<(), Box> { // Create a store // let path = tempfile::TempDir::new()?; - // let store = Arc::new(store::FilesystemStore::new(path.path())?); - let store = std::sync::Arc::new(store::MemoryStore::new()); + // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(path.path())?); + let mut store: ReadableWritableListableStorage = std::sync::Arc::new(store::MemoryStore::new()); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log + .clone() + .create_readable_writable_listable_transformer(store); + } + } // Create a group let group_path = "/group"; diff --git a/examples/sharded_array_write_read.rs b/examples/sharded_array_write_read.rs index c9fdbac1..a621d3b4 100644 --- a/examples/sharded_array_write_read.rs +++ b/examples/sharded_array_write_read.rs @@ -1,5 +1,8 @@ use itertools::Itertools; -use zarrs::storage::ListableStorageTraits; +use zarrs::storage::{ + storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer}, + ReadableWritableListableStorage, +}; fn sharded_array_write_read() -> Result<(), Box> { use zarrs::{ @@ -17,21 +20,24 @@ fn sharded_array_write_read() -> Result<(), Box> { // Create a store // let path = tempfile::TempDir::new()?; - // let store = Arc::new(store::FilesystemStore::new(path.path())?); - // let store = Arc::new(store::FilesystemStore::new("tests/data/sharded_array_write_read.zarr")?); - let store = Arc::new(store::MemoryStore::new()); - - // let log_writer = Arc::new(std::sync::Mutex::new( - // // std::io::BufWriter::new( - // std::io::stdout(), - // // ) - // )); - // let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { - // chrono::Utc::now().format("[%T%.3f] ").to_string() - // })); - // let store = usage_log - // .clone() - // .create_readable_writable_transformer(store); + // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(path.path())?); + // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new("tests/data/sharded_array_write_read.zarr")?); + let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new()); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log + .clone() + .create_readable_writable_listable_transformer(store); + } + } // Create a group let group_path = "/group"; diff --git a/examples/zip_array_write_read.rs b/examples/zip_array_write_read.rs index fbbbf3b5..2ca61581 100644 --- a/examples/zip_array_write_read.rs +++ b/examples/zip_array_write_read.rs @@ -8,13 +8,17 @@ use std::{ use zarrs::{ array::{Array, ZARR_NAN_F32}, array_subset::ArraySubset, - storage::{ReadableStorageTraits, ReadableWritableStorageTraits, StoreKey}, + storage::{ + storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer}, + ReadableStorageTraits, ReadableWritableListableStorage, ReadableWritableStorageTraits, + StoreKey, + }, }; // const ARRAY_PATH: &'static str = "/array"; const ARRAY_PATH: &str = "/"; -fn write_array_to_storage( +fn write_array_to_storage( storage: Arc, ) -> Result, Box> { use zarrs::array::{chunk_grid::ChunkGridTraits, codec, DataType, FillValue}; @@ -150,7 +154,25 @@ fn zip_array_write_read() -> Result<(), Box> { let path = tempfile::TempDir::new()?; let mut zarr_dir = path.path().to_path_buf(); zarr_dir.push("hierarchy.zarr"); - let store = Arc::new(store::FilesystemStore::new(&zarr_dir)?); + let mut store: ReadableWritableListableStorage = + Arc::new(store::FilesystemStore::new(&zarr_dir)?); + if let Some(arg1) = std::env::args().collect::>().get(1) { + if arg1 == "--usage-log" { + let log_writer = Arc::new(std::sync::Mutex::new( + // std::io::BufWriter::new( + std::io::stdout(), + // ) + )); + let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || { + chrono::Utc::now().format("[%T%.3f] ").to_string() + })); + store = usage_log + .clone() + .create_readable_writable_listable_transformer(store); + } + } + + // Write the array to the store write_array_to_storage(store.clone())?; // Write the store to zip diff --git a/src/lib.rs b/src/lib.rs index a2f0cbea..f00b1cef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,9 @@ //! //! ## Examples //! Examples can be run with `cargo run --example `. +//! +//! Add `-- --usage-log` to see storage API calls during example execution. +//! //! Some examples require non-default features, which can be enabled with `--features `. //! //! #### Sync API