Add --usage-log argument to examples to use UsageLog storage tran…

…sformer
LDeakin · Feb 19, 2024 · 04195e7 · 04195e7
1 parent 66ab378
commit 04195e7
Show file tree

Hide file tree

Showing 9 changed files with 152 additions and 48 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  - Add `{Array,Chunk}Representation::dimensionality()`
  - Add `ArraySubset::new_empty()` and `ArraySubset::is_empty()`
  - Add missing `IncompatibleArraySubsetAndShapeError::new()`
+ - Add `--usage-log` argument to examples to use `UsageLog` storage transformer
  - Add more tests for `Array`, codecs, store locks, and more
 
 ### Changed

diff --git a/examples/array_write_read.rs b/examples/array_write_read.rs
@@ -1,4 +1,8 @@
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use zarrs::storage::{
+    storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
+    ReadableWritableListableStorage,
+};
 
 fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
     use std::sync::Arc;
@@ -11,12 +15,26 @@ fn array_write_read() -> Result<(), Box<dyn std::error::Error>> {
 
     // Create a store
     // let path = tempfile::TempDir::new()?;
-    // let store = Arc::new(store::FilesystemStore::new(path.path())?);
-    // let store = Arc::new(store::FilesystemStore::new(
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(path.path())?);
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(
     //     "tests/data/array_write_read.zarr",
     // )?);
-    // let store: zarrs::storage::ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
-    let store = Arc::new(store::MemoryStore::new());
+    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log
+                .clone()
+                .create_readable_writable_listable_transformer(store);
+        }
+    }
 
     // Create a group
     let group_path = "/group";

diff --git a/examples/async_array_write_read.rs b/examples/async_array_write_read.rs
@@ -1,3 +1,8 @@
+use zarrs::storage::{
+    storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
+    AsyncReadableWritableListableStorage,
+};
+
 async fn async_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
     use futures::{stream::FuturesUnordered, StreamExt};
     use std::sync::Arc;
@@ -10,13 +15,28 @@ async fn async_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
 
     // Create a store
     // let path = tempfile::TempDir::new()?;
-    // let store = Arc::new(store::AsyncFilesystemStore::new(path.path())?);
-    // let store = Arc::new(store::AsyncFilesystemStore::new(
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::AsyncFilesystemStore::new(path.path())?);
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::AsyncFilesystemStore::new(
     //     "tests/data/array_write_read.zarr",
     // )?);
-    let store = Arc::new(store::AsyncObjectStore::new(
+    let mut store: AsyncReadableWritableListableStorage = Arc::new(store::AsyncObjectStore::new(
         object_store::memory::InMemory::new(),
     ));
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log
+                .clone()
+                .create_async_readable_writable_listable_transformer(store);
+        }
+    }
 
     // Create a group
     let group_path = "/group";

diff --git a/examples/async_http_array_read.rs b/examples/async_http_array_read.rs
@@ -14,20 +14,26 @@ async fn http_array_read() -> Result<(), Box<dyn std::error::Error>> {
     const ARRAY_PATH: &str = "/group/array";
 
     // Create a HTTP store
-    let store = Arc::new(store::AsyncObjectStore::new(
+    let mut store: AsyncReadableWritableListableStorage = Arc::new(store::AsyncObjectStore::new(
         object_store::http::HttpBuilder::new()
             .with_url(HTTP_URL)
             .build()?,
     ));
-    let log_writer = Arc::new(std::sync::Mutex::new(
-        // std::io::BufWriter::new(
-        std::io::stdout(),
-        //    )
-    ));
-    let usage_log = UsageLogStorageTransformer::new(log_writer, || {
-        chrono::Utc::now().format("[%T%.3f] ").to_string()
-    });
-    let store = usage_log.create_async_readable_transformer(store);
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log
+                .clone()
+                .create_async_readable_writable_listable_transformer(store);
+        }
+    }
 
     // Init the existing array, reading metadata
     let array = Array::async_new(store, ARRAY_PATH).await?;

diff --git a/examples/http_array_read.rs b/examples/http_array_read.rs
@@ -1,3 +1,5 @@
+use zarrs::storage::ReadableStorage;
+
 fn http_array_read() -> Result<(), Box<dyn std::error::Error>> {
     use std::sync::Arc;
     use zarrs::{
@@ -14,16 +16,20 @@ fn http_array_read() -> Result<(), Box<dyn std::error::Error>> {
     const ARRAY_PATH: &str = "/group/array";
 
     // Create a HTTP store
-    let store = Arc::new(store::HTTPStore::new(HTTP_URL)?);
-    let log_writer = Arc::new(std::sync::Mutex::new(
-        // std::io::BufWriter::new(
-        std::io::stdout(),
-        //    )
-    ));
-    let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
-        chrono::Utc::now().format("[%T%.3f] ").to_string()
-    }));
-    let store = usage_log.create_readable_transformer(store);
+    let mut store: ReadableStorage = Arc::new(store::HTTPStore::new(HTTP_URL)?);
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log.clone().create_readable_transformer(store);
+        }
+    }
 
     // Init the existing array, reading metadata
     let array = Array::new(store, ARRAY_PATH)?;

diff --git a/examples/rectangular_array_write_read.rs b/examples/rectangular_array_write_read.rs
@@ -1,3 +1,10 @@
+use std::sync::Arc;
+
+use zarrs::storage::{
+    storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
+    ReadableWritableListableStorage,
+};
+
 fn rectangular_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
     use rayon::prelude::{IntoParallelIterator, ParallelIterator};
     use zarrs::array::ChunkGrid;
@@ -13,8 +20,23 @@ fn rectangular_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
 
     // Create a store
     // let path = tempfile::TempDir::new()?;
-    // let store = Arc::new(store::FilesystemStore::new(path.path())?);
-    let store = std::sync::Arc::new(store::MemoryStore::new());
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(path.path())?);
+    let mut store: ReadableWritableListableStorage = std::sync::Arc::new(store::MemoryStore::new());
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log
+                .clone()
+                .create_readable_writable_listable_transformer(store);
+        }
+    }
 
     // Create a group
     let group_path = "/group";

diff --git a/examples/sharded_array_write_read.rs b/examples/sharded_array_write_read.rs
@@ -1,5 +1,8 @@
 use itertools::Itertools;
-use zarrs::storage::ListableStorageTraits;
+use zarrs::storage::{
+    storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
+    ReadableWritableListableStorage,
+};
 
 fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
     use zarrs::{
@@ -17,21 +20,24 @@ fn sharded_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
 
     // Create a store
     // let path = tempfile::TempDir::new()?;
-    // let store = Arc::new(store::FilesystemStore::new(path.path())?);
-    // let store = Arc::new(store::FilesystemStore::new("tests/data/sharded_array_write_read.zarr")?);
-    let store = Arc::new(store::MemoryStore::new());
-
-    // let log_writer = Arc::new(std::sync::Mutex::new(
-    //     // std::io::BufWriter::new(
-    //     std::io::stdout(),
-    //     //    )
-    // ));
-    // let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
-    //     chrono::Utc::now().format("[%T%.3f] ").to_string()
-    // }));
-    // let store = usage_log
-    //     .clone()
-    //     .create_readable_writable_transformer(store);
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new(path.path())?);
+    // let mut store: ReadableWritableListableStorage = Arc::new(store::FilesystemStore::new("tests/data/sharded_array_write_read.zarr")?);
+    let mut store: ReadableWritableListableStorage = Arc::new(store::MemoryStore::new());
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log
+                .clone()
+                .create_readable_writable_listable_transformer(store);
+        }
+    }
 
     // Create a group
     let group_path = "/group";

diff --git a/examples/zip_array_write_read.rs b/examples/zip_array_write_read.rs
@@ -8,13 +8,17 @@ use std::{
 use zarrs::{
     array::{Array, ZARR_NAN_F32},
     array_subset::ArraySubset,
-    storage::{ReadableStorageTraits, ReadableWritableStorageTraits, StoreKey},
+    storage::{
+        storage_transformer::{StorageTransformerExtension, UsageLogStorageTransformer},
+        ReadableStorageTraits, ReadableWritableListableStorage, ReadableWritableStorageTraits,
+        StoreKey,
+    },
 };
 
 // const ARRAY_PATH: &'static str = "/array";
 const ARRAY_PATH: &str = "/";
 
-fn write_array_to_storage<TStorage: ReadableWritableStorageTraits + 'static>(
+fn write_array_to_storage<TStorage: ReadableWritableStorageTraits + ?Sized + 'static>(
     storage: Arc<TStorage>,
 ) -> Result<Array<TStorage>, Box<dyn std::error::Error>> {
     use zarrs::array::{chunk_grid::ChunkGridTraits, codec, DataType, FillValue};
@@ -150,7 +154,25 @@ fn zip_array_write_read() -> Result<(), Box<dyn std::error::Error>> {
     let path = tempfile::TempDir::new()?;
     let mut zarr_dir = path.path().to_path_buf();
     zarr_dir.push("hierarchy.zarr");
-    let store = Arc::new(store::FilesystemStore::new(&zarr_dir)?);
+    let mut store: ReadableWritableListableStorage =
+        Arc::new(store::FilesystemStore::new(&zarr_dir)?);
+    if let Some(arg1) = std::env::args().collect::<Vec<_>>().get(1) {
+        if arg1 == "--usage-log" {
+            let log_writer = Arc::new(std::sync::Mutex::new(
+                // std::io::BufWriter::new(
+                std::io::stdout(),
+                //    )
+            ));
+            let usage_log = Arc::new(UsageLogStorageTransformer::new(log_writer, || {
+                chrono::Utc::now().format("[%T%.3f] ").to_string()
+            }));
+            store = usage_log
+                .clone()
+                .create_readable_writable_listable_transformer(store);
+        }
+    }
+
+    // Write the array to the store
     write_array_to_storage(store.clone())?;
 
     // Write the store to zip

diff --git a/src/lib.rs b/src/lib.rs
@@ -51,6 +51,9 @@
 //!
 //! ## Examples
 //! Examples can be run with `cargo run --example <EXAMPLE_NAME>`.
+//!
+//! Add `-- --usage-log` to see storage API calls during example execution.
+//!
 //! Some examples require non-default features, which can be enabled with `--features <FEATURES>`.
 //!
 //! #### Sync API