From bdb716dd6872bea31547ef85ff61e7cfa08a4ddd Mon Sep 17 00:00:00 2001 From: weekday Date: Fri, 10 Jan 2025 06:14:22 +0000 Subject: [PATCH] Optimize hexadecimal conversions in serde_utils to speed up JSON serialization of byte collections Slow serialization to JSON was claimed to be the problem that led to 2e92e00246890ae37322c9df05a3cd344e50a57e. BlobSidecar serialization is now around 15.4 times faster. BlobSidecar deserialization is now around 12 times faster. SignedBeaconBlock serialization is now up to 1.5 times faster. SignedBeaconBlock deserialization is now up to 1.1 times faster. faster-hex might be faster than const-hex in some cases, but faster-hex does not provide the API we require. See https://crates.io/crates/const-hex/1.14.0 for benchmark results comparing various hexadecimal conversion crates. --- Cargo.lock | 46 ++++++++++++++++++- Cargo.toml | 2 +- serde_utils/Cargo.toml | 2 +- .../src/prefixed_hex_or_bytes_array.rs | 2 +- serde_utils/src/prefixed_hex_or_bytes_cow.rs | 2 +- .../prefixed_hex_or_bytes_generic_array.rs | 2 +- .../src/prefixed_hex_or_bytes_slice.rs | 3 +- 7 files changed, 51 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e668568..dc63de34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1300,6 +1300,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-hex" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b0485bab839b018a8f1723fc5391819fea5f8f0f32288ef8a735fd096b6160c" +dependencies = [ + "cfg-if", + "cpufeatures", + "hex", + "proptest", + "serde", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -6085,6 +6098,22 @@ dependencies = [ "types", ] +[[package]] +name = "proptest" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" +dependencies = [ + "bitflags 2.6.0", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "unarray", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -6267,6 +6296,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -7018,9 +7056,9 @@ name = "serde_utils" version = "0.0.0" dependencies = [ "bincode", + "const-hex", "generic-array", "hex", - "hex_fmt", "itertools 0.13.0", "num-traits", "serde", @@ -8250,6 +8288,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unescape" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 92fd5f56..8246db26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -300,6 +300,7 @@ bytesize = { version = '1', features = ['serde'] } cached = '0.53' chrono = '0.4' clap = { version = '4', features = ['derive'] } +const-hex = '1.14' const_format = '0.2' constant_time_eq = '0.3' conv = '0.3' @@ -339,7 +340,6 @@ hash_hasher = '2' hashlink = '0.9' hex = { version = '0.4', features = ['serde'] } hex-literal = '0.4' -hex_fmt = '0.3' hmac = '0.12' http = '1' http-body-util = '0.1' diff --git a/serde_utils/Cargo.toml b/serde_utils/Cargo.toml index 36e62599..ff22d80d 100644 --- a/serde_utils/Cargo.toml +++ b/serde_utils/Cargo.toml @@ -7,9 +7,9 @@ authors = ["Grandine "] workspace = true [dependencies] +const-hex = { workspace = true } generic-array = { workspace = true } hex = { workspace = true } -hex_fmt = { workspace = true } itertools = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } diff --git a/serde_utils/src/prefixed_hex_or_bytes_array.rs b/serde_utils/src/prefixed_hex_or_bytes_array.rs index eec39b63..feb0c72a 100644 --- a/serde_utils/src/prefixed_hex_or_bytes_array.rs +++ b/serde_utils/src/prefixed_hex_or_bytes_array.rs @@ -29,7 +29,7 @@ pub fn deserialize<'de, D: Deserializer<'de>, const N: usize>( let digits = shared::strip_hex_prefix(string)?; let mut bytes = [0; N]; - hex::decode_to_slice(digits, &mut bytes).map_err(E::custom)?; + const_hex::decode_to_slice(digits, &mut bytes).map_err(E::custom)?; Ok(bytes) } diff --git a/serde_utils/src/prefixed_hex_or_bytes_cow.rs b/serde_utils/src/prefixed_hex_or_bytes_cow.rs index c2ea6e08..7e7142c8 100644 --- a/serde_utils/src/prefixed_hex_or_bytes_cow.rs +++ b/serde_utils/src/prefixed_hex_or_bytes_cow.rs @@ -34,7 +34,7 @@ pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result(self, string: &str) -> Result { let digits = shared::strip_hex_prefix(string)?; - let bytes = hex::decode(digits).map_err(E::custom)?; + let bytes = const_hex::decode(digits).map_err(E::custom)?; Ok(Cow::Owned(bytes)) } } diff --git a/serde_utils/src/prefixed_hex_or_bytes_generic_array.rs b/serde_utils/src/prefixed_hex_or_bytes_generic_array.rs index b43bbff3..9cc21f21 100644 --- a/serde_utils/src/prefixed_hex_or_bytes_generic_array.rs +++ b/serde_utils/src/prefixed_hex_or_bytes_generic_array.rs @@ -43,7 +43,7 @@ pub fn deserialize<'de, D: Deserializer<'de>, N: ArrayLength>( let digits = shared::strip_hex_prefix(string)?; let mut bytes = GenericArray::default(); - hex::decode_to_slice(digits, &mut bytes).map_err(E::custom)?; + const_hex::decode_to_slice(digits, &mut bytes).map_err(E::custom)?; Ok(bytes) } diff --git a/serde_utils/src/prefixed_hex_or_bytes_slice.rs b/serde_utils/src/prefixed_hex_or_bytes_slice.rs index 9a40f069..cc5853e2 100644 --- a/serde_utils/src/prefixed_hex_or_bytes_slice.rs +++ b/serde_utils/src/prefixed_hex_or_bytes_slice.rs @@ -1,9 +1,8 @@ -use hex_fmt::HexFmt; use serde::Serializer; pub fn serialize(bytes: impl AsRef<[u8]>, serializer: S) -> Result { if serializer.is_human_readable() { - serializer.collect_str(&format_args!("0x{}", HexFmt(bytes))) + serializer.serialize_str(const_hex::encode_prefixed(bytes).as_str()) } else { serializer.serialize_bytes(bytes.as_ref()) }