Add store testing utility functions and fix `MemoryStore::get_partial…

…_values_key`

CHANGELOG.md

@@ -31,6 +31,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
    - **Breaking** Removes the explicit `object_store`-based stores (e.g. `AsyncAmazonS3Store`, `AsyncHTTPStore`)
    - **Breaking** Removes the `object_store_impl` macro
    - **Breaking** Removes the `s3`, `gcp`, and `azure` crate features
+ - Add store testing utility functions for unified store testing
+
+### Fixed
+ - Fixed `MemoryStore::get_partial_values_key` if given an invalid byte range, now returns `InvalidByteRangeError` instead of panicking
 
 ## [0.7.3] - 2023-12-22
 

src/storage/store/store_async.rs

@@ -1,2 +1,146 @@
 #[cfg(feature = "object_store")]
 pub mod object_store;
+
+#[cfg(test)]
+mod test_util {
+    use std::error::Error;
+
+    use crate::{
+        byte_range::ByteRange,
+        storage::{
+            AsyncListableStorageTraits, AsyncReadableStorageTraits, AsyncWritableStorageTraits,
+            StoreKeyRange, StoreKeyStartValue, StorePrefix,
+        },
+    };
+
+    /// Create a store with the following data
+    /// - a/
+    ///   - b [0, 1, 2]
+    ///   - c [0]
+    ///   - d/
+    ///     - e
+    ///   - f/
+    ///     - g
+    ///     - h
+    /// - i/
+    ///   - j/
+    ///     - k [0, 1]
+    pub async fn store_write<T: AsyncWritableStorageTraits>(
+        store: &T,
+    ) -> Result<(), Box<dyn Error>> {
+        store.erase_prefix(&StorePrefix::root()).await?;
+
+        store.set(&"a/b".try_into()?, &[0, 0, 0]).await?;
+        store
+            .set_partial_values(&[StoreKeyStartValue::new("a/b".try_into()?, 1, &[1, 2])])
+            .await?;
+
+        store.set(&"a/c".try_into()?, &[0]).await?;
+        store.set(&"a/d/e".try_into()?, &[]).await?;
+        store.set(&"a/f/g".try_into()?, &[]).await?;
+        store.set(&"a/f/h".try_into()?, &[]).await?;
+        store.set(&"i/j/k".try_into()?, &[0, 1]).await?;
+
+        store.set(&"erase".try_into()?, &[]).await?;
+        store.erase(&"erase".try_into()?).await?;
+
+        store.set(&"erase_values_0".try_into()?, &[]).await?;
+        store.set(&"erase_values_1".try_into()?, &[]).await?;
+        store
+            .erase_values(&["erase_values_0".try_into()?, "erase_values_1".try_into()?])
+            .await?;
+
+        store.set(&"erase_prefix/0".try_into()?, &[]).await?;
+        store.set(&"erase_prefix/1".try_into()?, &[]).await?;
+        store.erase_prefix(&"erase_prefix/".try_into()?).await?;
+
+        Ok(())
+    }
+
+    pub async fn store_read<T: AsyncReadableStorageTraits>(
+        store: &T,
+    ) -> Result<(), Box<dyn Error>> {
+        assert!(store.get(&"notfound".try_into()?).await?.is_none());
+        assert!(store.size_key(&"notfound".try_into()?).await?.is_none());
+        assert_eq!(store.get(&"a/b".try_into()?).await?, Some(vec![0, 1, 2]));
+        assert_eq!(store.size_key(&"a/b".try_into()?).await?, Some(3));
+        assert_eq!(store.size_key(&"a/c".try_into()?).await?, Some(1));
+        assert_eq!(store.size_key(&"i/j/k".try_into()?).await?, Some(2));
+        assert_eq!(
+            store
+                .get_partial_values_key(
+                    &"a/b".try_into()?,
+                    &[
+                        ByteRange::FromStart(1, Some(1)),
+                        ByteRange::FromEnd(0, Some(1))
+                    ]
+                )
+                .await?,
+            Some(vec![vec![1], vec![2]])
+        );
+        assert_eq!(
+            store
+                .get_partial_values(&[
+                    StoreKeyRange::new("a/b".try_into()?, ByteRange::FromStart(1, None)),
+                    StoreKeyRange::new("a/b".try_into()?, ByteRange::FromEnd(1, Some(2))),
+                    StoreKeyRange::new("i/j/k".try_into()?, ByteRange::FromStart(1, Some(1))),
+                ])
+                .await?,
+            vec![Some(vec![1, 2]), Some(vec![0, 1]), Some(vec![1])]
+        );
+        assert!(store
+            .get_partial_values(&[StoreKeyRange::new(
+                "a/b".try_into()?,
+                ByteRange::FromStart(1, Some(10))
+            ),])
+            .await
+            .is_err());
+
+        assert_eq!(store.size().await?, 6);
+        assert_eq!(store.size_prefix(&"a/".try_into()?).await?, 4);
+        assert_eq!(store.size_prefix(&"i/".try_into()?).await?, 2);
+
+        Ok(())
+    }
+
+    pub async fn store_list<T: AsyncListableStorageTraits>(
+        store: &T,
+    ) -> Result<(), Box<dyn Error>> {
+        assert_eq!(
+            store.list().await?,
+            &[
+                "a/b".try_into()?,
+                "a/c".try_into()?,
+                "a/d/e".try_into()?,
+                "a/f/g".try_into()?,
+                "a/f/h".try_into()?,
+                "i/j/k".try_into()?
+            ]
+        );
+
+        assert_eq!(
+            store.list_prefix(&"a/".try_into()?).await?,
+            &[
+                "a/b".try_into()?,
+                "a/c".try_into()?,
+                "a/d/e".try_into()?,
+                "a/f/g".try_into()?,
+                "a/f/h".try_into()?
+            ]
+        );
+        assert_eq!(
+            store.list_prefix(&"i/".try_into()?).await?,
+            &["i/j/k".try_into()?]
+        );
+
+        {
+            let list_dir = store.list_dir(&"a/".try_into()?).await?;
+            assert_eq!(list_dir.keys(), &["a/b".try_into()?, "a/c".try_into()?,]);
+            assert_eq!(
+                list_dir.prefixes(),
+                &["a/d/".try_into()?, "a/f/".try_into()?,]
+            );
+        }
+        Ok(())
+    }
+}

src/storage/store/store_async/object_store.rs

@@ -249,148 +249,27 @@ impl<T: object_store::ObjectStore> AsyncListableStorageTraits for AsyncObjectSto
 
 #[cfg(test)]
 mod tests {
-    use crate::storage::{
-        AsyncListableStorageTraits, AsyncReadableStorageTraits, AsyncWritableStorageTraits,
-        StoreKeyStartValue, StorePrefix,
-    };
-
     use super::*;
     use std::error::Error;
 
     #[tokio::test]
-    async fn memory_set() -> Result<(), Box<dyn Error>> {
-        let store = AsyncObjectStore::new(object_store::memory::InMemory::new());
-        let key = "a/b".try_into()?;
-        store.set(&key, &[0, 1, 2]).await?;
-        assert_eq!(store.get(&key).await?.unwrap(), &[0, 1, 2]);
-        store
-            .set_partial_values(&[StoreKeyStartValue::new(key.clone(), 1, &[3, 4])])
-            .await?;
-        assert_eq!(store.get(&key).await?.unwrap(), &[0, 3, 4]);
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn memory_list() -> Result<(), Box<dyn Error>> {
-        let store = AsyncObjectStore::new(object_store::memory::InMemory::new());
-
-        store.set(&"a/b".try_into()?, &[]).await?;
-        store.set(&"a/c".try_into()?, &[]).await?;
-        store.set(&"a/d/e".try_into()?, &[]).await?;
-        store.set(&"a/d/f".try_into()?, &[]).await?;
-        store.erase(&"a/d/e".try_into()?).await?;
-        assert_eq!(
-            store.list().await?,
-            &["a/b".try_into()?, "a/c".try_into()?, "a/d/f".try_into()?]
-        );
-        assert_eq!(
-            store.list_prefix(&"a/".try_into()?).await?,
-            &["a/b".try_into()?, "a/c".try_into()?, "a/d/f".try_into()?]
-        );
-        assert_eq!(
-            store.list_prefix(&"a/d/".try_into()?).await?,
-            &["a/d/f".try_into()?]
-        );
-        assert_eq!(
-            store.list_prefix(&"".try_into()?).await?,
-            &["a/b".try_into()?, "a/c".try_into()?, "a/d/f".try_into()?]
-        );
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn memory_list_dir() -> Result<(), Box<dyn Error>> {
+    async fn memory() -> Result<(), Box<dyn Error>> {
         let store = AsyncObjectStore::new(object_store::memory::InMemory::new());
-        store.set(&"a/b".try_into()?, &[]).await?;
-        store.set(&"a/c".try_into()?, &[]).await?;
-        store.set(&"a/d/e".try_into()?, &[]).await?;
-        store.set(&"a/f/g".try_into()?, &[]).await?;
-        store.set(&"a/f/h".try_into()?, &[]).await?;
-        store.set(&"b/c/d".try_into()?, &[]).await?;
-
-        let list_dir = store.list_dir(&"a/".try_into()?).await?;
-
-        assert_eq!(list_dir.keys(), &["a/b".try_into()?, "a/c".try_into()?,]);
-        assert_eq!(
-            list_dir.prefixes(),
-            &["a/d/".try_into()?, "a/f/".try_into()?,]
-        );
+        super::super::test_util::store_write(&store).await?;
+        super::super::test_util::store_read(&store).await?;
+        super::super::test_util::store_list(&store).await?;
         Ok(())
     }
 
     #[tokio::test]
-    async fn filesystem_set() -> Result<(), Box<dyn Error>> {
+    async fn filesystem() -> Result<(), Box<dyn Error>> {
         let path = tempfile::TempDir::new()?;
         let store = AsyncObjectStore::new(object_store::local::LocalFileSystem::new_with_prefix(
             path.path(),
         )?);
-
-        let key = "a/b".try_into()?;
-        store.set(&key, &[0, 1, 2]).await?;
-        assert_eq!(store.get(&key).await?.unwrap(), &[0, 1, 2]);
-        store
-            .set_partial_values(&[StoreKeyStartValue::new(key.clone(), 1, &[3, 4])])
-            .await?;
-        assert_eq!(store.get(&key).await?.unwrap(), &[0, 3, 4]);
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn filesystem_list() -> Result<(), Box<dyn Error>> {
-        let path = tempfile::TempDir::new()?;
-        let store = AsyncObjectStore::new(object_store::local::LocalFileSystem::new_with_prefix(
-            path.path(),
-        )?);
-
-        store.set(&"a/b".try_into()?, &[]).await?;
-        store.set(&"a/c".try_into()?, &[]).await?;
-        store.set(&"a/d/e".try_into()?, &[]).await?;
-        store.set(&"a/d/f".try_into()?, &[]).await?;
-        store.erase(&"a/d/e".try_into()?).await?;
-        assert_eq!(
-            store.list().await?,
-            &["a/b".try_into()?, "a/c".try_into()?, "a/d/f".try_into()?]
-        );
-        assert_eq!(
-            store.list_prefix(&"a/".try_into()?).await?,
-            &["a/b".try_into()?, "a/c".try_into()?, "a/d/f".try_into()?]
-        );
-        assert_eq!(
-            store.list_prefix(&"a/d/".try_into()?).await?,
-            &["a/d/f".try_into()?]
-        );
-        assert_eq!(
-            store.list_prefix(&"".try_into()?).await?,
-            &["a/b".try_into()?, "a/c".try_into()?, "a/d/f".try_into()?]
-        );
-
-        // assert!(crate::storage::node_exists(&store, &"/a/b".try_into()?).await?);
-        // assert!(crate::storage::node_exists_listable(&store, &"/a/b".try_into()?).await?);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn filesystem_list_dir() -> Result<(), Box<dyn Error>> {
-        let path = tempfile::TempDir::new()?;
-        let store = AsyncObjectStore::new(object_store::local::LocalFileSystem::new_with_prefix(
-            path.path(),
-        )?);
-
-        store.set(&"a/b".try_into()?, &[]).await?;
-        store.set(&"a/c".try_into()?, &[]).await?;
-        store.set(&"a/d/e".try_into()?, &[]).await?;
-        store.set(&"a/f/g".try_into()?, &[]).await?;
-        store.set(&"a/f/h".try_into()?, &[]).await?;
-        store.set(&"b/c/d".try_into()?, &[]).await?;
-
-        let list_dir = store.list_dir(&StorePrefix::new("a/")?).await?;
-
-        assert_eq!(list_dir.keys(), &["a/b".try_into()?, "a/c".try_into()?,]);
-        assert_eq!(
-            list_dir.prefixes(),
-            &["a/d/".try_into()?, "a/f/".try_into()?,]
-        );
+        super::super::test_util::store_write(&store).await?;
+        super::super::test_util::store_read(&store).await?;
+        super::super::test_util::store_list(&store).await?;
         Ok(())
     }
 }