Improve compatibility with other messagepack implementations, macro/functions for binary, no nightly

msgpacker-derive/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "msgpacker-derive"
-version = "0.3.1"
+version = "0.4.0"
 authors = ["Victor Lopez <[email protected]>"]
 categories = ["compression", "encoding", "parser-implementations"]
 edition = "2021"

msgpacker-derive/src/lib.rs

@@ -36,20 +36,62 @@ fn contains_attribute(field: &Field, name: &str) -> bool {
 
 fn impl_fields_named(name: Ident, f: FieldsNamed) -> impl Into<TokenStream> {
     let mut values: Punctuated<FieldValue, Token![,]> = Punctuated::new();
+    let field_len = f.named.len();
     let block_packable: Block = parse_quote! {
         {
             let mut n = 0;
+            n += ::msgpacker::derive_util::get_array_info(buf, #field_len);
         }
     };
     let block_unpackable: Block = parse_quote! {
         {
             let mut n = 0;
+            let expected_len = #field_len;
+
+            let format = ::msgpacker::derive_util::take_byte(&mut buf)?;
+
+            let (header_bytes, actual_len) = match format {
+                0x90..=0x9f => (1, (format & 0x0f) as usize),
+                ::msgpacker::derive_util::Format::ARRAY16 => {
+                    let len = ::msgpacker::derive_util::take_num(&mut buf, u16::from_be_bytes)? as usize;
+                    (3, len)
+                }
+                ::msgpacker::derive_util::Format::ARRAY32 => {
+                    let len = ::msgpacker::derive_util::take_num(&mut buf, u32::from_be_bytes)? as usize;
+                    (5, len)
+                }
+                _ => return Err(::msgpacker::Error::UnexpectedFormatTag.into()),
+            };
+
+            if actual_len != expected_len {
+                return Err(::msgpacker::Error::UnexpectedStructLength.into());
+            }
+
+            n += header_bytes;
         }
     };
     let block_unpackable_iter: Block = parse_quote! {
         {
             let mut bytes = bytes.into_iter();
             let mut n = 0;
+            let expected_len = #field_len;
+            let format = ::msgpacker::derive_util::take_byte_iter(bytes.by_ref())?;
+            let (header_bytes, actual_len) = match format {
+                0x90..=0x9f => (1, (format & 0x0f) as usize),
+                ::msgpacker::derive_util::Format::ARRAY16 => {
+                    let len = ::msgpacker::derive_util::take_num_iter(&mut bytes, u16::from_be_bytes)? as usize;
+                    (3, len)
+                }
+                ::msgpacker::derive_util::Format::ARRAY32 => {
+                    let len = ::msgpacker::derive_util::take_num_iter(&mut bytes, u16::from_be_bytes)? as usize;
+                    (5, len)
+                }
+                _ => return Err(::msgpacker::Error::UnexpectedFormatTag.into()),
+            };
+            if actual_len != expected_len {
+                return Err(::msgpacker::Error::UnexpectedStructLength.into());
+            }
+            n += header_bytes;
         }
     };
 
@@ -103,7 +145,26 @@ fn impl_fields_named(name: Ident, f: FieldsNamed) -> impl Into<TokenStream> {
                             t
                         })?;
                     });
-                } else if contains_attribute(&field, "array") || is_vec && !is_vec_u8 {
+                } else if contains_attribute(&field, "binary") && is_vec_u8 {
+                    block_packable.stmts.push(parse_quote! {
+                        n += ::msgpacker::pack_binary(buf, &self.#ident);
+                    });
+
+                    block_unpackable.stmts.push(parse_quote! {
+                        let #ident = ::msgpacker::unpack_bytes(buf).map(|(nv, t)| {
+                            n += nv;
+                            buf = &buf[nv..];
+                            t.to_vec()
+                        })?;
+                    });
+
+                    block_unpackable_iter.stmts.push(parse_quote! {
+                        let #ident = ::msgpacker::unpack_bytes_iter(bytes.by_ref()).map(|(nv, t)| {
+                            n += nv;
+                            t
+                        })?;
+                    });
+                } else if contains_attribute(&field, "array") || is_vec {
                     block_packable.stmts.push(parse_quote! {
                         n += ::msgpacker::pack_array(buf, &self.#ident);
                     });
@@ -298,26 +359,37 @@ fn impl_fields_unnamed(name: Ident, f: FieldsUnnamed) -> impl Into<TokenStream>
 fn impl_fields_unit(name: Ident) -> impl Into<TokenStream> {
     quote! {
         impl ::msgpacker::Packable for #name {
-            fn pack<T>(&self, _buf: &mut T) -> usize
+            fn pack<T>(&self, buf: &mut T) -> usize
             where
                 T: Extend<u8>,
             {
-                0
+                ::msgpacker::derive_util::get_array_info(buf, 0)
             }
         }
 
         impl ::msgpacker::Unpackable for #name {
             type Error = ::msgpacker::Error;
 
             fn unpack(mut buf: &[u8]) -> Result<(usize, Self), Self::Error> {
-                Ok((0, Self))
+                let format = ::msgpacker::derive_util::take_byte(&mut buf)?;
+                let (_, len) = match format {
+                    0x90 => (1, 0),
+                    _ => return Err(Error::UnexpectedFormatTag.into()),
+                };
+                Ok((1, Self))
             }
 
             fn unpack_iter<I>(bytes: I) -> Result<(usize, Self), Self::Error>
             where
                 I: IntoIterator<Item = u8>,
             {
-                Ok((0, Self))
+                let mut bytes = bytes.into_iter();
+                let format = ::msgpacker::derive_util::take_byte_iter(bytes.by_ref())?;
+                let (_, len) = match format {
+                    0x90 => (1, 0),
+                    _ => return Err(Error::UnexpectedFormatTag.into()),
+                };
+                Ok((1, Self))
             }
         }
     }

msgpacker/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "msgpacker"
-version = "0.4.5"
+version = "0.5.0"
 authors = ["Victor Lopez <[email protected]>"]
 categories = ["compression", "encoding", "parser-implementations"]
 edition = "2021"
@@ -11,7 +11,7 @@ repository = "https://github.com/codx-dev/msgpacker"
 description = "MessagePack protocol implementation for Rust."
 
 [dependencies]
-msgpacker-derive = { version = "0.3", optional = true }
+msgpacker-derive = { path = "../msgpacker-derive", optional = true }
 
 [dev-dependencies]
 proptest = "1.2"

msgpacker/src/error.rs

@@ -15,6 +15,10 @@ pub enum Error {
     UnexpectedFormatTag,
     /// The provided bin length is not valid.
     UnexpectedBinLength,
+    /// The struct we're targeting does not match the data.
+    UnexpectedStructLength,
+    /// The array we're targeting does not match the data.
+    UnexpectedArrayLength,
 }
 
 impl fmt::Display for Error {

msgpacker/src/format.rs

@@ -1,40 +1,73 @@
+/// A container for the Format constants
 pub struct Format {}
 
 impl Format {
+    /// Nil format stores nil in 1 byte.
     pub const NIL: u8 = 0xc0;
+    /// Bool format family stores false or true in 1 byte.
     pub const TRUE: u8 = 0xc3;
+    /// Bool format family stores false or true in 1 byte.
     pub const FALSE: u8 = 0xc2;
+    /// Positive fixint stores 7-bit positive integer
     pub const POSITIVE_FIXINT: u8 = 0x7f;
+    /// Uint 8 stores a 8-bit unsigned integer
     pub const UINT8: u8 = 0xcc;
+    /// Uint 16 stores a 16-bit big-endian unsigned integer
     pub const UINT16: u8 = 0xcd;
+    /// Uint 32 stores a 32-bit big-endian unsigned integer
     pub const UINT32: u8 = 0xce;
+    /// Uint 64 stores a 64-bit big-endian unsigned integer
     pub const UINT64: u8 = 0xcf;
+    /// Int 8 stores a 8-bit signed integer
     pub const INT8: u8 = 0xd0;
+    /// Int 16 stores a 16-bit big-endian signed integer
     pub const INT16: u8 = 0xd1;
+    /// Int 32 stores a 32-bit big-endian signed integer
     pub const INT32: u8 = 0xd2;
+    /// Int 64 stores a 64-bit big-endian signed integer
     pub const INT64: u8 = 0xd3;
+    /// Float 32 stores a floating point number in IEEE 754 single precision floating point number
     pub const FLOAT32: u8 = 0xca;
+    /// Float 64 stores a floating point number in IEEE 754 double precision floating point number
     pub const FLOAT64: u8 = 0xcb;
+    /// Bin 8 stores a byte array whose length is upto (2^8)-1 bytes
     pub const BIN8: u8 = 0xc4;
+    /// Bin 16 stores a byte array whose length is upto (2^16)-1 bytes
     pub const BIN16: u8 = 0xc5;
+    /// Bin 32 stores a byte array whose length is upto (2^32)-1 bytes
     pub const BIN32: u8 = 0xc6;
+    /// Str 8 stores a byte array whose length is upto (2^8)-1 bytes
     pub const STR8: u8 = 0xd9;
+    /// Str 16 stores a byte array whose length is upto (2^16)-1 bytes
     pub const STR16: u8 = 0xda;
+    /// Str 32 stores a byte array whose length is upto (2^32)-1 bytes
     pub const STR32: u8 = 0xdb;
+    /// Array 16 stores an array whose length is upto (2^16)-1 elements
     pub const ARRAY16: u8 = 0xdc;
+    /// Array 32 stores an array whose length is upto (2^32)-1 elements
     pub const ARRAY32: u8 = 0xdd;
+    /// Map 16 stores a map whose length is upto (2^16)-1 elements
     pub const MAP16: u8 = 0xde;
+    /// Map 32 stores a map whose length is upto (2^32)-1 elements
     pub const MAP32: u8 = 0xdf;
 }
 
 #[cfg(feature = "alloc")]
 impl Format {
+    /// Fixext 1 stores an integer and a byte array whose length is 1 byte
     pub const FIXEXT1: u8 = 0xd4;
+    /// Fixext 2 stores an integer and a byte array whose length is 2 byte
     pub const FIXEXT2: u8 = 0xd5;
+    /// Fixext 4 stores an integer and a byte array whose length is 4 byte
     pub const FIXEXT4: u8 = 0xd6;
+    /// Fixext 8 stores an integer and a byte array whose length is 8 byte
     pub const FIXEXT8: u8 = 0xd7;
+    /// Fixext 16 stores an integer and a byte array whose length is 16 byte
     pub const FIXEXT16: u8 = 0xd8;
+    /// Ext 8 stores an integer and a byte array whose length is upto (2^8)-1 bytes
     pub const EXT8: u8 = 0xc7;
+    /// Ext 16 stores an integer and a byte array whose length is upto (2^16)-1 bytes
     pub const EXT16: u8 = 0xc8;
+    /// Ext 32 stores an integer and a byte array whose length is upto (2^32)-1 bytes
     pub const EXT32: u8 = 0xc9;
 }

msgpacker/src/helpers.rs

@@ -1,12 +1,6 @@
 use super::Error;
 
-pub fn take_byte_iter<I>(mut bytes: I) -> Result<u8, Error>
-where
-    I: Iterator<Item = u8>,
-{
-    bytes.next().ok_or(Error::BufferTooShort)
-}
-
+/// Take one byte off the provided buffer, advance the pointer, or error.
 pub fn take_byte(buf: &mut &[u8]) -> Result<u8, Error> {
     if buf.is_empty() {
         return Err(Error::BufferTooShort);
@@ -16,6 +10,15 @@ pub fn take_byte(buf: &mut &[u8]) -> Result<u8, Error> {
     Ok(l[0])
 }
 
+/// Take one byte from the iterator or error.
+pub fn take_byte_iter<I>(mut bytes: I) -> Result<u8, Error>
+where
+    I: Iterator<Item = u8>,
+{
+    bytes.next().ok_or(Error::BufferTooShort)
+}
+
+/// Read bytes off the buffer, using the provided function, or error.
 pub fn take_num<V, const N: usize>(buf: &mut &[u8], f: fn([u8; N]) -> V) -> Result<V, Error> {
     if buf.len() < N {
         return Err(Error::BufferTooShort);
@@ -27,6 +30,18 @@ pub fn take_num<V, const N: usize>(buf: &mut &[u8], f: fn([u8; N]) -> V) -> Resu
     Ok(f(val))
 }
 
+/// Read a number off the iterator, using the provided function, or error.
+pub fn take_num_iter<I, V, const N: usize>(mut bytes: I, f: fn([u8; N]) -> V) -> Result<V, Error>
+where
+    I: Iterator<Item = u8>,
+{
+    let mut array = [0u8; N]; // Initialize with zeroes
+    for byte in array.iter_mut() {
+        *byte = bytes.next().ok_or(Error::BufferTooShort)?;
+    }
+    Ok(f(array))
+}
+
 #[cfg(feature = "alloc")]
 pub fn take_buffer<'a>(buf: &mut &'a [u8], len: usize) -> Result<&'a [u8], Error> {
     if buf.len() < len {
@@ -37,17 +52,6 @@ pub fn take_buffer<'a>(buf: &mut &'a [u8], len: usize) -> Result<&'a [u8], Error
     Ok(l)
 }
 
-pub fn take_num_iter<I, V, const N: usize>(bytes: I, f: fn([u8; N]) -> V) -> Result<V, Error>
-where
-    I: Iterator<Item = u8>,
-{
-    bytes
-        .array_chunks()
-        .next()
-        .ok_or(Error::BufferTooShort)
-        .map(f)
-}
-
 #[cfg(feature = "alloc")]
 pub fn take_buffer_iter<I>(bytes: I, len: usize) -> Result<alloc::vec::Vec<u8>, Error>
 where

msgpacker/src/lib.rs

@@ -1,9 +1,5 @@
 #![cfg_attr(not(feature = "std"), no_std)]
-#![feature(
-    iter_array_chunks,
-    maybe_uninit_array_assume_init,
-    maybe_uninit_uninit_array
-)]
+// #![feature(iter_array_chunks, maybe_uninit_array_assume_init)]
 #![warn(missing_docs)]
 #![doc = include_str!("../README.md")]
 
@@ -21,8 +17,18 @@ mod unpack;
 
 pub use error::Error;
 use format::Format;
-pub use pack::{pack_array, pack_map};
-pub use unpack::{unpack_array, unpack_array_iter, unpack_map, unpack_map_iter};
+pub use pack::{pack_array, pack_binary, pack_map};
+pub use unpack::{
+    unpack_array, unpack_array_iter, unpack_binary, unpack_binary_iter, unpack_map, unpack_map_iter,
+};
+
+/// This module exposes some utility variables and functions for msgpacker-derive
+#[cfg(feature = "derive")]
+pub mod derive_util {
+    pub use crate::format::Format;
+    pub use crate::helpers::{take_byte, take_byte_iter, take_num, take_num_iter};
+    pub use crate::pack::get_array_info;
+}
 
 #[cfg(feature = "alloc")]
 pub use extension::Extension;