Add missing partial decoding tests for codecs

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
    - **Breaking** added `chunk_shape_u64_unchecked` to `ChunkGridTraits` which must be implemented by chunk grids
  - Added `Array::retrieve_chunk_if_exists` and variants (`async_`, `_elements`, `_ndarray`)
  - Implement `AsyncBytesPartialDecoderTraits` for `std::io::Cursor<{&[u8],Vec<u8>}>`
+ - Added missing partial decoding tests for codecs
 
 ### Changed
  - Dependency bumps

src/array/codec/array_to_array/bitround.rs

@@ -276,4 +276,47 @@ mod tests {
         let answer: &[Vec<f32>] = &[vec![3.0, 4.0], vec![16.0, 16.0, 20.0, 20.0]];
         assert_eq!(answer, decoded_partial_chunk);
     }
+
+    #[cfg(feature = "async")]
+    #[tokio::test]
+    async fn codec_bitround_async_partial_decode() {
+        const JSON: &'static str = r#"{ "keepbits": 2 }"#;
+        let codec_configuration: BitroundCodecConfiguration = serde_json::from_str(JSON).unwrap();
+        let codec = BitroundCodec::new_with_configuration(&codec_configuration);
+
+        let elements: Vec<f32> = (0..32).map(|i| i as f32).collect();
+        let chunk_representation = ChunkRepresentation::new(
+            vec![(elements.len() as u64).try_into().unwrap()],
+            DataType::Float32,
+            0.0f32.into(),
+        )
+        .unwrap();
+        let bytes = crate::array::transmute_to_bytes_vec(elements);
+
+        let encoded = codec.encode(bytes.clone(), &chunk_representation).unwrap();
+        let decoded_regions = [
+            ArraySubset::new_with_ranges(&[3..5]),
+            ArraySubset::new_with_ranges(&[17..21]),
+        ];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let bytes_codec = BytesCodec::default();
+        let input_handle = bytes_codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let partial_decoder = codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder
+            .partial_decode(&decoded_regions)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = decoded_partial_chunk
+            .into_iter()
+            .map(|bytes| crate::array::transmute_from_bytes_vec::<f32>(bytes))
+            .collect_vec();
+        let answer: &[Vec<f32>] = &[vec![3.0, 4.0], vec![16.0, 16.0, 20.0, 20.0]];
+        assert_eq!(answer, decoded_partial_chunk);
+    }
 }

src/array/codec/array_to_array/transpose.rs

@@ -103,7 +103,15 @@ fn permute<T: Copy>(v: &[T], order: &TransposeOrder) -> Vec<T> {
 mod tests {
     use std::num::NonZeroU64;
 
-    use crate::array::{codec::ArrayCodecTraits, ChunkRepresentation, DataType, FillValue};
+    use crate::{
+        array::{
+            codec::{
+                ArrayCodecTraits, ArrayToArrayCodecTraits, ArrayToBytesCodecTraits, BytesCodec,
+            },
+            ChunkRepresentation, DataType, FillValue,
+        },
+        array_subset::ArraySubset,
+    };
 
     use super::*;
 
@@ -154,4 +162,86 @@ mod tests {
         }"#;
         codec_transpose_round_trip_impl(JSON, DataType::UInt16, FillValue::from(0u16));
     }
+
+    #[test]
+    fn codec_transpose_partial_decode() {
+        let codec = TransposeCodec::new(TransposeOrder::new(&[1, 0]).unwrap());
+
+        let elements: Vec<f32> = (0..16).map(|i| i as f32).collect();
+        let chunk_representation = ChunkRepresentation::new(
+            vec![NonZeroU64::new(4).unwrap(), NonZeroU64::new(4).unwrap()],
+            DataType::Float32,
+            0.0f32.into(),
+        )
+        .unwrap();
+        let bytes = crate::array::transmute_to_bytes_vec(elements);
+
+        let encoded = codec.encode(bytes.clone(), &chunk_representation).unwrap();
+        let decoded_regions = [
+            ArraySubset::new_with_ranges(&[1..3, 1..4]),
+            ArraySubset::new_with_ranges(&[2..4, 0..2]),
+        ];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let bytes_codec = BytesCodec::default();
+        let input_handle = bytes_codec
+            .partial_decoder(input_handle, &chunk_representation)
+            .unwrap();
+        let partial_decoder = codec
+            .partial_decoder(input_handle, &chunk_representation)
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder.partial_decode(&decoded_regions).unwrap();
+        let decoded_partial_chunk = decoded_partial_chunk
+            .into_iter()
+            .map(|bytes| crate::array::transmute_from_bytes_vec::<f32>(bytes))
+            .collect::<Vec<_>>();
+        let answer: &[Vec<f32>] = &[
+            vec![5.0, 9.0, 6.0, 10.0, 7.0, 11.0],
+            vec![8.0, 12.0, 9.0, 13.0],
+        ];
+        assert_eq!(answer, decoded_partial_chunk);
+    }
+
+    #[cfg(feature = "async")]
+    #[tokio::test]
+    async fn codec_transpose_async_partial_decode() {
+        let codec = TransposeCodec::new(TransposeOrder::new(&[1, 0]).unwrap());
+
+        let elements: Vec<f32> = (0..16).map(|i| i as f32).collect();
+        let chunk_representation = ChunkRepresentation::new(
+            vec![NonZeroU64::new(4).unwrap(), NonZeroU64::new(4).unwrap()],
+            DataType::Float32,
+            0.0f32.into(),
+        )
+        .unwrap();
+        let bytes = crate::array::transmute_to_bytes_vec(elements);
+
+        let encoded = codec.encode(bytes.clone(), &chunk_representation).unwrap();
+        let decoded_regions = [
+            ArraySubset::new_with_ranges(&[1..3, 1..4]),
+            ArraySubset::new_with_ranges(&[2..4, 0..2]),
+        ];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let bytes_codec = BytesCodec::default();
+        let input_handle = bytes_codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let partial_decoder = codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder
+            .partial_decode(&decoded_regions)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = decoded_partial_chunk
+            .into_iter()
+            .map(|bytes| crate::array::transmute_from_bytes_vec::<f32>(bytes))
+            .collect::<Vec<_>>();
+        let answer: &[Vec<f32>] = &[
+            vec![5.0, 9.0, 6.0, 10.0, 7.0, 11.0],
+            vec![8.0, 12.0, 9.0, 13.0],
+        ];
+        assert_eq!(answer, decoded_partial_chunk);
+    }
 }

src/array/codec/array_to_bytes/bytes.rs

@@ -120,9 +120,12 @@ fn reverse_endianness(v: &mut [u8], data_type: &DataType) {
 mod tests {
     use std::num::NonZeroU64;
 
-    use crate::array::{
-        codec::{ArrayCodecTraits, CodecTraits},
-        ChunkRepresentation, DataType, FillValue,
+    use crate::{
+        array::{
+            codec::{ArrayCodecTraits, ArrayToBytesCodecTraits, CodecTraits},
+            ChunkRepresentation, ChunkShape, DataType, FillValue,
+        },
+        array_subset::ArraySubset,
     };
 
     use super::*;
@@ -289,4 +292,69 @@ mod tests {
         )
         .unwrap();
     }
+
+    #[test]
+    fn codec_bytes_partial_decode() {
+        let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
+        let chunk_representation =
+            ChunkRepresentation::new(chunk_shape.to_vec(), DataType::UInt8, FillValue::from(0u8))
+                .unwrap();
+        let elements: Vec<u8> = (0..chunk_representation.num_elements() as u8).collect();
+        let bytes = elements;
+
+        let codec = BytesCodec::new(None);
+
+        let encoded = codec.encode(bytes, &chunk_representation).unwrap();
+        let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let partial_decoder = codec
+            .partial_decoder(input_handle, &chunk_representation)
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder.partial_decode(&decoded_regions).unwrap();
+
+        let decoded_partial_chunk: Vec<u8> = decoded_partial_chunk
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>()
+            .chunks(std::mem::size_of::<u8>())
+            .map(|b| u8::from_ne_bytes(b.try_into().unwrap()))
+            .collect();
+        let answer: Vec<u8> = vec![4, 8];
+        assert_eq!(answer, decoded_partial_chunk);
+    }
+
+    #[cfg(feature = "async")]
+    #[tokio::test]
+    async fn codec_bytes_async_partial_decode() {
+        let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
+        let chunk_representation =
+            ChunkRepresentation::new(chunk_shape.to_vec(), DataType::UInt8, FillValue::from(0u8))
+                .unwrap();
+        let elements: Vec<u8> = (0..chunk_representation.num_elements() as u8).collect();
+        let bytes = elements;
+
+        let codec = BytesCodec::new(None);
+
+        let encoded = codec.encode(bytes, &chunk_representation).unwrap();
+        let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let partial_decoder = codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder
+            .partial_decode(&decoded_regions)
+            .await
+            .unwrap();
+
+        let decoded_partial_chunk: Vec<u8> = decoded_partial_chunk
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>()
+            .chunks(std::mem::size_of::<u8>())
+            .map(|b| u8::from_ne_bytes(b.try_into().unwrap()))
+            .collect();
+        let answer: Vec<u8> = vec![4, 8];
+        assert_eq!(answer, decoded_partial_chunk);
+    }
 }

src/array/codec/array_to_bytes/pcodec.rs

@@ -192,7 +192,13 @@ impl PcodecDeltaEncodingOrder {
 mod tests {
     use std::num::NonZeroU64;
 
-    use crate::array::{codec::ArrayCodecTraits, ChunkRepresentation, DataType, FillValue};
+    use crate::{
+        array::{
+            codec::{ArrayCodecTraits, ArrayToBytesCodecTraits},
+            transmute_to_bytes_vec, ChunkRepresentation, ChunkShape, DataType, FillValue,
+        },
+        array_subset::ArraySubset,
+    };
 
     use super::*;
 
@@ -324,4 +330,75 @@ mod tests {
         )
         .is_err());
     }
+
+    #[test]
+    fn codec_pcodec_partial_decode() {
+        let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
+        let chunk_representation = ChunkRepresentation::new(
+            chunk_shape.to_vec(),
+            DataType::UInt32,
+            FillValue::from(0u32),
+        )
+        .unwrap();
+        let elements: Vec<u32> = (0..chunk_representation.num_elements() as u32).collect();
+        let bytes = transmute_to_bytes_vec(elements);
+
+        let codec = PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap());
+
+        let encoded = codec.encode(bytes, &chunk_representation).unwrap();
+        let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let partial_decoder = codec
+            .partial_decoder(input_handle, &chunk_representation)
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder.partial_decode(&decoded_regions).unwrap();
+
+        let decoded_partial_chunk: Vec<u8> = decoded_partial_chunk
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>()
+            .chunks(std::mem::size_of::<u8>())
+            .map(|b| u8::from_ne_bytes(b.try_into().unwrap()))
+            .collect();
+        let answer: Vec<u32> = vec![4, 8];
+        assert_eq!(transmute_to_bytes_vec(answer), decoded_partial_chunk);
+    }
+
+    #[cfg(feature = "async")]
+    #[tokio::test]
+    async fn codec_pcodec_async_partial_decode() {
+        let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
+        let chunk_representation = ChunkRepresentation::new(
+            chunk_shape.to_vec(),
+            DataType::UInt32,
+            FillValue::from(0u32),
+        )
+        .unwrap();
+        let elements: Vec<u32> = (0..chunk_representation.num_elements() as u32).collect();
+        let bytes = transmute_to_bytes_vec(elements);
+
+        let codec = PcodecCodec::new_with_configuration(&serde_json::from_str(JSON_VALID).unwrap());
+
+        let encoded = codec.encode(bytes, &chunk_representation).unwrap();
+        let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let partial_decoder = codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder
+            .partial_decode(&decoded_regions)
+            .await
+            .unwrap();
+
+        let decoded_partial_chunk: Vec<u8> = decoded_partial_chunk
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>()
+            .chunks(std::mem::size_of::<u8>())
+            .map(|b| u8::from_ne_bytes(b.try_into().unwrap()))
+            .collect();
+        let answer: Vec<u32> = vec![4, 8];
+        assert_eq!(transmute_to_bytes_vec(answer), decoded_partial_chunk);
+    }
 }

src/array/codec/array_to_bytes/sharding.rs

@@ -334,6 +334,43 @@ mod tests {
         assert_eq!(answer, decoded_partial_chunk);
     }
 
+    #[cfg(feature = "async")]
+    #[tokio::test]
+    async fn codec_sharding_async_partial_decode1() {
+        let chunk_shape: ChunkShape = vec![4, 4].try_into().unwrap();
+        let chunk_representation =
+            ChunkRepresentation::new(chunk_shape.to_vec(), DataType::UInt8, FillValue::from(0u8))
+                .unwrap();
+        let elements: Vec<u8> = (0..chunk_representation.num_elements() as u8).collect();
+        let bytes = elements;
+
+        let codec_configuration: ShardingCodecConfiguration =
+            serde_json::from_str(JSON_VALID1).unwrap();
+        let codec = ShardingCodec::new_with_configuration(&codec_configuration).unwrap();
+
+        let encoded = codec.encode(bytes, &chunk_representation).unwrap();
+        let decoded_regions = [ArraySubset::new_with_ranges(&[1..3, 0..1])];
+        let input_handle = Box::new(std::io::Cursor::new(encoded));
+        let partial_decoder = codec
+            .async_partial_decoder(input_handle, &chunk_representation)
+            .await
+            .unwrap();
+        let decoded_partial_chunk = partial_decoder
+            .partial_decode(&decoded_regions)
+            .await
+            .unwrap();
+
+        let decoded_partial_chunk: Vec<u8> = decoded_partial_chunk
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>()
+            .chunks(std::mem::size_of::<u8>())
+            .map(|b| u8::from_ne_bytes(b.try_into().unwrap()))
+            .collect();
+        let answer: Vec<u8> = vec![4, 8];
+        assert_eq!(answer, decoded_partial_chunk);
+    }
+
     #[cfg(feature = "gzip")]
     #[cfg(feature = "crc32c")]
     #[test]