diff --git a/include/kota/codec/fbs/decode.h b/include/kota/codec/fbs/decode.h index 7e5c46ee..b1c24aa5 100644 --- a/include/kota/codec/fbs/decode.h +++ b/include/kota/codec/fbs/decode.h @@ -485,6 +485,394 @@ bool MapReader::visit_entry(KF&& key_fn, VF&& val_fn) { } // namespace decode_detail +// Schema-driven deep verification: walks the wire layout implied by T's +// reflection schema and bounds-checks every table, vector, string and scalar +// with flatbuffers::Verifier before any of it is dereferenced. This is what +// makes from_flatbuffer safe on untrusted or corrupted buffers, and it is +// exposed standalone (verify_flatbuffer) for zero-copy readers (table_view) +// that never run the decoder. +// +// Limitation: a field whose layout is fully delegated to an opaque adapter +// (behavior::with<> or a serialize_visit specialization without a `wire_type` +// typedef) cannot be described statically; such subtrees are skipped. +namespace verify_detail { + +using decode_detail::has_wire_type_v; +using decode_detail::wire_type_of_t; +using decode_detail::needs_wrapper_in_vector; + +template +struct unwrap_indirection { + using type = T; +}; + +template +struct unwrap_indirection> { + using type = typename unwrap_indirection::type; +}; + +template +struct unwrap_indirection> { + using type = typename unwrap_indirection::type; +}; + +template +struct unwrap_indirection> { + using type = typename unwrap_indirection::type; +}; + +template +struct unwrap_indirection> { + using type = typename unwrap_indirection::type; +}; + +template +using unwrap_indirection_t = typename unwrap_indirection::type; + +template +bool verify_field(verifier_t& v, const Table* tbl, voffset_t slot); + +template +bool verify_struct_slots(verifier_t& v, const Table* tbl); + +// Verify the offset stored at `slot` (absent is fine), then run `f` on the +// child table between VerifyTableStart/EndTable so verifier depth limits hold. +template +bool verify_child_table(verifier_t& v, const Table* tbl, voffset_t slot, F&& f) { + if(tbl->GetOptionalFieldOffset(slot) == 0) { + return true; + } + if(!tbl->VerifyOffset(v, slot)) { + return false; + } + const auto* child = tbl->GetPointer(slot); + if(!child->VerifyTableStart(v)) { + return false; + } + if(!f(child)) { + return false; + } + return v.EndTable(); +} + +// Verify one already-bounds-checked element of a Vector>. +template +bool verify_elem_table(verifier_t& v, const Table* child, F&& f) { + if(!child->VerifyTableStart(v)) { + return false; + } + if(!f(child)) { + return false; + } + return v.EndTable(); +} + +template +bool verify_variant_slots(verifier_t& v, const Table* var_table) { + return [&](std::type_identity>) { + if(!var_table->template VerifyField( + v, detail::first_field, alignof(std::uint32_t))) { + return false; + } + const auto tag = var_table->template GetField(detail::first_field, 0); + const auto index = static_cast(tag); + if(index >= sizeof...(Ts)) { + return false; + } + const auto payload_slot = static_cast( + detail::first_field + detail::field_step * static_cast(index + 1)); + bool ok = true; + std::size_t i = 0; + [[maybe_unused]] bool matched = + ((i++ == index ? (ok = verify_field, Config>(v, + var_table, + payload_slot), + true) + : false) || + ...); + return ok; + }(std::type_identity{}); +} + +template +bool verify_tuple_slots(verifier_t& v, const Table* tbl) { + return [&](std::index_sequence) { + return (verify_field>, Config>( + v, + tbl, + static_cast(detail::first_field + + detail::field_step * static_cast(Is))) && + ...); + }(std::make_index_sequence>{}); +} + +template +bool verify_map_entries(verifier_t& v, const Table* tbl, voffset_t slot) { + if(tbl->GetOptionalFieldOffset(slot) == 0) { + return true; + } + if(!tbl->VerifyOffset(v, slot)) { + return false; + } + const auto* vec = tbl->GetPointer*>(slot); + if(!v.VerifyVector(vec)) { + return false; + } + for(uoffset_t i = 0; i < vec->size(); ++i) { + const auto* entry = vec->template GetAs(i); + bool ok = verify_elem_table(v, entry, [&](const Table* e) { + return verify_field(v, e, detail::first_field) && + verify_field( + v, + e, + static_cast(detail::first_field + detail::field_step)); + }); + if(!ok) { + return false; + } + } + return true; +} + +template +bool verify_vector_at(verifier_t& v, const Table* tbl, voffset_t slot) { + if(tbl->GetOptionalFieldOffset(slot) == 0) { + return true; + } + if(!tbl->VerifyOffset(v, slot)) { + return false; + } + + using raw_E = std::remove_cvref_t; + using clean_E = codec::detail::clean_t; + using wire_E = + std::conditional_t, wire_type_of_t, clean_E>; + + if constexpr(needs_wrapper_in_vector()) { + // optional/pointer/nested-container elements are boxed in wrapper + // tables holding the payload at the first field slot. + const auto* vec = tbl->GetPointer*>(slot); + if(!v.VerifyVector(vec)) { + return false; + } + for(uoffset_t i = 0; i < vec->size(); ++i) { + const auto* child = vec->template GetAs
(i); + bool ok = verify_elem_table(v, child, [&](const Table* c) { + return verify_field(v, c, detail::first_field); + }); + if(!ok) { + return false; + } + } + return true; + } else if constexpr(proxy_detail::is_scalar_v || std::same_as) { + using storage_t = proxy_detail::scalar_storage_t; + return v.VerifyVector(tbl->GetPointer*>(slot)); + } else if constexpr(proxy_detail::is_string_like_v) { + const auto* vec = tbl->GetPointer*>(slot); + // VerifyVector bounds/alignment-checks the offset array itself; + // VerifyVectorOfStrings assumes that already happened. + return v.VerifyVector(vec) && v.VerifyVectorOfStrings(vec); + } else if constexpr(meta::bytes_like) { + return v.VerifyVector(tbl->GetPointer*>(slot)); + } else if constexpr(can_inline_struct_v && !proxy_detail::is_tuple_like_v) { + return v.VerifyVector(tbl->GetPointer*>(slot)); + } else { + // table-shaped elements: tuples, variants, reflectable structs + const auto* vec = tbl->GetPointer*>(slot); + if(!v.VerifyVector(vec)) { + return false; + } + for(uoffset_t i = 0; i < vec->size(); ++i) { + const auto* child = vec->template GetAs
(i); + bool ok; + if constexpr(is_specialization_of) { + ok = verify_elem_table(v, child, [&](const Table* c) { + return verify_variant_slots(v, c); + }); + } else if constexpr(proxy_detail::is_tuple_like_v) { + ok = verify_elem_table(v, child, [&](const Table* c) { + return verify_tuple_slots(v, c); + }); + } else if constexpr(meta::reflectable_class) { + ok = verify_elem_table(v, child, [&](const Table* c) { + return verify_struct_slots(v, c); + }); + } else { + // unknown element layout (e.g. monostate boxes): bounds-check only + ok = verify_elem_table(v, child, [](const Table*) { return true; }); + } + if(!ok) { + return false; + } + } + return true; + } +} + +template +bool verify_field(verifier_t& v, const Table* tbl, voffset_t slot) { + using T0 = std::remove_cv_t; + + if constexpr(meta::annotated_type) { + using attrs_t = typename T0::attrs; + using inner_t = std::remove_cvref_t; + if constexpr(tuple_has_spec_v) { + using adapter = typename tuple_find_spec_t::adapter; + if constexpr(requires { typename adapter::wire_type; }) { + return verify_field(v, tbl, slot); + } else { + return true; // opaque adapter: layout unknown, skip + } + } else if constexpr(tuple_has_spec_v) { + using target = typename tuple_find_spec_t::target; + return verify_field(v, tbl, slot); + } else if constexpr(tuple_has_spec_v) { + if(tbl->GetOptionalFieldOffset(slot) == 0) { + return true; + } + return tbl->VerifyOffset(v, slot) && + v.VerifyString(tbl->GetPointer(slot)); + } else { + return verify_field(v, tbl, slot); + } + } else if constexpr(std::same_as) { + return verify_child_table(v, tbl, slot, [](const Table*) { return true; }); + } else if constexpr(has_wire_type_v) { + return verify_field, Config>(v, tbl, slot); + } else { + constexpr auto kind = meta::kind_of(); + using enum meta::type_kind; + + if constexpr(kind == optional || kind == pointer) { + return verify_field, Config>(v, tbl, slot); + } else if constexpr(kind == boolean) { + return tbl->template VerifyField(v, slot, alignof(std::uint8_t)); + } else if constexpr(proxy_detail::is_scalar_v || std::same_as) { + using storage_t = proxy_detail::scalar_storage_t; + return tbl->template VerifyField(v, slot, alignof(storage_t)); + } else if constexpr(meta::str_like) { + if(tbl->GetOptionalFieldOffset(slot) == 0) { + return true; + } + return tbl->VerifyOffset(v, slot) && + v.VerifyString(tbl->GetPointer(slot)); + } else if constexpr(kind == bytes) { + if(tbl->GetOptionalFieldOffset(slot) == 0) { + return true; + } + return tbl->VerifyOffset(v, slot) && + v.VerifyVector(tbl->GetPointer*>(slot)); + } else if constexpr(can_inline_struct_v && !proxy_detail::is_tuple_like_v) { + const auto fo = tbl->GetOptionalFieldOffset(slot); + return fo == 0 || v.VerifyFieldStruct(reinterpret_cast(tbl), + fo, + sizeof(T0), + alignof(T0)); + } else if constexpr(is_specialization_of) { + return verify_child_table(v, tbl, slot, [&](const Table* c) { + return verify_variant_slots(v, c); + }); + } else if constexpr(proxy_detail::is_tuple_like_v) { + return verify_child_table(v, tbl, slot, [&](const Table* c) { + return verify_tuple_slots(v, c); + }); + } else if constexpr(proxy_detail::is_map_range_v) { + using entry_t = std::ranges::range_value_t; + using key_t = kota::map_entry_key_t; + using mapped_t = kota::map_entry_mapped_t; + return verify_map_entries(v, tbl, slot); + } else if constexpr(proxy_detail::is_range_like_v) { + using element_t = std::ranges::range_value_t; + return verify_vector_at(v, tbl, slot); + } else if constexpr(meta::reflectable_class) { + return verify_child_table(v, tbl, slot, [&](const Table* c) { + return verify_struct_slots(v, c); + }); + } else { + return true; // unknown layout: leave to the decoder's null handling + } + } +} + +template +bool verify_struct_slots(verifier_t& v, const Table* tbl) { + using schema = meta::virtual_schema; + using slots = typename schema::slots; + + return [&](std::index_sequence) { + return (verify_field::raw_type>, + Config>( + v, + tbl, + static_cast(detail::first_field + + detail::field_step * static_cast(Is))) && + ...); + }(std::make_index_sequence>{}); +} + +template +bool verify_root(verifier_t& v, const Table* root) { + using T0 = std::remove_cv_t; + + if constexpr(meta::annotated_type) { + return verify_root, Config>(v, root); + } else { + if(!root->VerifyTableStart(v)) { + return false; + } + bool ok; + constexpr auto kind = meta::kind_of(); + using enum meta::type_kind; + if constexpr(std::same_as || kind == null) { + ok = true; + } else if constexpr(kind == optional || kind == pointer) { + return v.EndTable() && + verify_root, Config>(v, root); + } else if constexpr(is_specialization_of) { + ok = verify_variant_slots(v, root); + } else if constexpr(proxy_detail::is_tuple_like_v) { + ok = verify_tuple_slots(v, root); + } else if constexpr(meta::reflectable_class && !proxy_detail::is_map_range_v && + !proxy_detail::is_range_like_v && !meta::str_like) { + ok = verify_struct_slots(v, root); + } else { + // scalars, strings, bytes, sequences and maps are boxed into a + // synthetic root table holding the value at the first field slot. + ok = verify_field(v, root, detail::first_field); + } + return ok && v.EndTable(); + } +} + +} // namespace verify_detail + +/// Deep-verify that `buf` is a structurally valid flatbuffer for type T. +/// All offsets, tables, vectors and strings reachable through T's schema are +/// bounds-checked; returns false on any inconsistency. +template +auto verify_flatbuffer(std::span buf) -> bool { + if(buf.size() < 2 * sizeof(uoffset_t)) { + return false; + } + const auto* data = reinterpret_cast(buf.data()); + if(!::flatbuffers::BufferHasIdentifier(data, detail::buffer_identifier)) { + return false; + } + verifier_t verifier(data, buf.size()); + // GetRoot only reads the root uoffset (guarded by the size check above); + // verify_root's VerifyTableStart bounds-checks the table it points at. + const auto* root = ::flatbuffers::GetRoot
(data); + return verify_detail::verify_root, default_config>(verifier, + root); +} + +template +auto verify_flatbuffer(std::span buf) -> bool { + return verify_flatbuffer( + std::span(reinterpret_cast(buf.data()), buf.size())); +} + template auto from_flatbuffer(std::span buf, T& out) -> std::expected { if(buf.empty()) { @@ -494,20 +882,17 @@ auto from_flatbuffer(std::span buf, T& out) -> std::expected(buf.data()); auto size = buf.size(); - if(!::flatbuffers::BufferHasIdentifier(data, detail::buffer_identifier)) { + if(size < 2 * sizeof(uoffset_t) || + !::flatbuffers::BufferHasIdentifier(data, detail::buffer_identifier)) { return std::unexpected(rich_error("invalid buffer identifier")); } - const auto* root = ::flatbuffers::GetRoot
(data); - if(root == nullptr) { - return std::unexpected(rich_error("null root table")); - } - - verifier_t verifier(data, size); - if(!root->VerifyTableStart(verifier) || !verifier.EndTable()) { + if(!verify_flatbuffer(buf)) { return std::unexpected(rich_error("buffer verification failed")); } + const auto* root = ::flatbuffers::GetRoot
(data); + rich_error err; scoped_context guard(err); @@ -528,7 +913,7 @@ auto from_flatbuffer(std::span buf, T& out) -> std::expected template requires std::default_initializable auto from_flatbuffer(std::span buf) -> std::expected { - T value{}; + auto value = T(); auto result = from_flatbuffer(buf, value); if(!result) { return std::unexpected(result.error()); @@ -539,7 +924,7 @@ auto from_flatbuffer(std::span buf) -> std::expected requires std::default_initializable auto from_flatbuffer(std::span buf) -> std::expected { - T value{}; + auto value = T(); auto result = from_flatbuffer(buf, value); if(!result) { return std::unexpected(result.error()); diff --git a/include/kota/codec/fbs/encode.h b/include/kota/codec/fbs/encode.h index 867f8de9..f2d8ad38 100644 --- a/include/kota/codec/fbs/encode.h +++ b/include/kota/codec/fbs/encode.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include +#include "kota/meta/compare.h" #include "kota/meta/type_kind.h" #include "kota/codec/fbs/type.h" #include "kota/codec/visit/config.h" @@ -52,12 +54,11 @@ struct boxed_table_collector; struct byte_collector; struct map_entry_collector; -struct key_capture_visitor; struct root_visitor; template -inline auto two_pass(builder_t& fbb, Body&& body) -> table_offset_t; +inline auto two_pass(builder_t& fbb, Body&& body) -> std::optional; struct alloc_field_visitor { builder_t& fbb; @@ -187,7 +188,12 @@ struct write_field_visitor { template bool visit_float(T v) { - fbb.AddElement(sid, v); + // Decoders read non-float/double floats as double; keep widths in sync. + if constexpr(std::same_as || std::same_as) { + fbb.AddElement(sid, v); + } else { + fbb.AddElement(sid, static_cast(v)); + } return true; } @@ -495,7 +501,10 @@ struct boxed_table_collector { auto start = fbb.StartTable(); write_field_visitor wv{fbb, detail::first_field, av.stored_offset}; - KOTA_CODEC_TRY(writer(wv)); + if(!writer(wv)) { + fbb.EndTable(start); + return false; + } table_offsets.push_back(table_offset_t(fbb.EndTable(start))); return true; } @@ -546,51 +555,9 @@ struct byte_collector { } }; -struct key_capture_visitor { - std::string captured; - - using error_type = rich_error; - constexpr static bool human_readable = false; - - bool visit_bool(bool v) { - captured = v ? "true" : "false"; - return true; - } - - template - bool visit_int(T v) { - captured = std::to_string(static_cast(v)); - return true; - } - - template - bool visit_uint(T v) { - captured = std::to_string(static_cast(v)); - return true; - } - - template - bool visit_float(T v) { - captured = std::to_string(static_cast(v)); - return true; - } - - template - bool visit_str(const T& v) { - captured = std::string(std::string_view(v)); - return true; - } - - template - bool visit_char(T v) { - captured = std::string(1, static_cast(v)); - return true; - } -}; - struct map_entry_collector { builder_t& fbb; - std::vector> entries; + std::vector entries; template inline bool visit_entry(KF&& key_fn, VF&& value_fn); @@ -619,7 +586,11 @@ struct root_visitor { template bool visit_float(T v) { - return box_root_scalar(v); + if constexpr(std::same_as || std::same_as) { + return box_root_scalar(v); + } else { + return box_root_scalar(static_cast(v)); + } } template @@ -680,35 +651,77 @@ struct root_visitor { }; template -auto two_pass(builder_t& fbb, Body&& body) -> table_offset_t { +auto two_pass(builder_t& fbb, Body&& body) -> std::optional { alloc_table_visitor av{fbb, {}, 0}; if(!body(av)) - return table_offset_t{0}; + return std::nullopt; auto start = fbb.StartTable(); write_table_visitor wv{fbb, av.offsets, 0}; if(!body(wv)) { fbb.EndTable(start); - return table_offset_t{0}; + return std::nullopt; } return table_offset_t(fbb.EndTable(start)); } -template -inline bool encode_sorted_map(builder_t& fbb, Body&& body, uoffset_t& out_offset) { +// Reorder collected entry offsets so that wire order follows the keys' +// ordering under meta::lt — operator< when the key defines one, otherwise the +// reflection-synthesized field-by-field order. map_view::find binary-searches +// with the same comparison. +template +inline void sort_entries_by_key(std::vector& offsets, std::vector& keys) { + if(keys.size() != offsets.size()) { + return; + } + std::vector order(offsets.size()); + std::iota(order.begin(), order.end(), 0U); + std::stable_sort(order.begin(), order.end(), [&](std::uint32_t a, std::uint32_t b) { + return meta::lt(keys[a], keys[b]); + }); + std::vector sorted; + sorted.reserve(offsets.size()); + for(auto i: order) { + sorted.push_back(offsets[i]); + } + offsets = std::move(sorted); +} + +template +inline bool encode_sorted_map(builder_t& fbb, + const Container& m, + Body&& body, + uoffset_t& out_offset) { map_entry_collector coll{fbb, {}}; KOTA_CODEC_TRY(body(coll)); - std::sort(coll.entries.begin(), coll.entries.end(), [](const auto& a, const auto& b) { - return a.first < b.first; - }); + auto offsets = std::move(coll.entries); - std::vector sorted_offsets; - sorted_offsets.reserve(coll.entries.size()); - for(auto& entry: coll.entries) { - sorted_offsets.push_back(entry.second); + using entry_t = std::ranges::range_value_t; + using key_t = kota::map_entry_key_t; + + if constexpr(std::convertible_to) { + // String-like keys compare lexicographically; views stay valid because + // they point into key storage owned by the container. + std::vector keys; + keys.reserve(offsets.size()); + for(const auto& entry: m) { + keys.emplace_back(std::string_view(kota::detail::map_entry_key(entry))); + } + sort_entries_by_key(offsets, keys); + } else if constexpr(std::copy_constructible && + meta::synthesized_lt_with) { + std::vector keys; + keys.reserve(offsets.size()); + for(const auto& entry: m) { + keys.emplace_back(kota::detail::map_entry_key(entry)); + } + sort_entries_by_key(offsets, keys); } - out_offset = fbb.CreateVector(sorted_offsets.data(), sorted_offsets.size()).o; + // Keys that meta::lt cannot order keep iteration order; map_view rejects + // lookups on such key types at compile time. + + out_offset = fbb.CreateVector(offsets.data(), offsets.size()).o; return true; } @@ -724,7 +737,10 @@ inline bool auto start = fbb.StartTable(); fbb.AddElement(detail::first_field, static_cast(index)); write_field_visitor payload_write{fbb, payload_slot, payload_alloc.stored_offset}; - body(payload_write); + if(!body(payload_write)) { + fbb.EndTable(start); + return false; + } out_offset = fbb.EndTable(start); return true; } @@ -769,7 +785,14 @@ bool seq_encode_impl(builder_t& fbb, const Container& c, Body&& body, uoffset_t& } } else if constexpr(meta::int_like || meta::uint_like || meta::floating_like || meta::char_like) { - using wire_t = std::conditional_t, std::int8_t, element_t>; + using wire_t = std::conditional_t< + meta::char_like, + std::int8_t, + std::conditional_t && + !std::same_as && + !std::same_as, + double, + element_t>>; if constexpr(std::ranges::contiguous_range && std::ranges::sized_range && std::same_as) { auto data = std::ranges::data(c); @@ -868,7 +891,10 @@ bool alloc_field_visitor::visit_struct(const T&, Body&& body) { return true; } else { auto off = two_pass(fbb, std::forward(body)); - stored_offset = off.o; + if(!off) { + return false; + } + stored_offset = off->o; return true; } } @@ -881,13 +907,16 @@ bool alloc_field_visitor::visit_seq(const Container& c, Body&& body) { template bool alloc_field_visitor::visit_tuple(const T&, Body&& body) { auto off = two_pass(fbb, std::forward(body)); - stored_offset = off.o; + if(!off) { + return false; + } + stored_offset = off->o; return true; } template -bool alloc_field_visitor::visit_map(const Container&, Body&& body) { - return encode_sorted_map(fbb, std::forward(body), stored_offset); +bool alloc_field_visitor::visit_map(const Container& m, Body&& body) { + return encode_sorted_map(fbb, m, std::forward(body), stored_offset); } template @@ -898,14 +927,20 @@ bool alloc_field_visitor::visit_variant(std::size_t index, Body&& body) { template bool table_elem_visitor::visit_struct(const T&, Body&& body) { auto off = two_pass(fbb, std::forward(body)); - table_offsets.push_back(off); + if(!off) { + return false; + } + table_offsets.push_back(*off); return true; } template bool table_elem_visitor::visit_tuple(const T&, Body&& body) { auto off = two_pass(fbb, std::forward(body)); - table_offsets.push_back(off); + if(!off) { + return false; + } + table_offsets.push_back(*off); return true; } @@ -929,9 +964,9 @@ bool table_elem_visitor::visit_seq(const Container& c, Body&& body) { } template -bool table_elem_visitor::visit_map(const Container&, Body&& body) { +bool table_elem_visitor::visit_map(const Container& m, Body&& body) { uoffset_t vec_off = 0; - KOTA_CODEC_TRY(encode_sorted_map(fbb, std::forward(body), vec_off)); + KOTA_CODEC_TRY(encode_sorted_map(fbb, m, std::forward(body), vec_off)); auto start = fbb.StartTable(); fbb.AddOffset(detail::first_field, offset_t(vec_off)); @@ -941,9 +976,6 @@ bool table_elem_visitor::visit_map(const Container&, Body&& body) { template bool map_entry_collector::visit_entry(KF&& key_fn, VF&& value_fn) { - key_capture_visitor capture; - KOTA_CODEC_TRY(key_fn(capture)); - auto table_off = two_pass(fbb, [&](auto& sv) -> bool { KOTA_CODEC_TRY(sv.visit_field(std::integral_constant{}, std::string_view{"key"}, @@ -953,14 +985,20 @@ bool map_entry_collector::visit_entry(KF&& key_fn, VF&& value_fn) { [&](auto& vv) -> bool { return value_fn(vv); })); return true; }); - - entries.emplace_back(std::move(capture.captured), table_off); + if(!table_off) { + return false; + } + entries.push_back(*table_off); return true; } template bool root_visitor::visit_struct(const T&, Body&& body) { - root_off = two_pass(fbb, std::forward(body)); + auto off = two_pass(fbb, std::forward(body)); + if(!off) { + return false; + } + root_off = *off; return true; } @@ -977,14 +1015,18 @@ bool root_visitor::visit_seq(const Container& c, Body&& body) { template bool root_visitor::visit_tuple(const T&, Body&& body) { - root_off = two_pass(fbb, std::forward(body)); + auto off = two_pass(fbb, std::forward(body)); + if(!off) { + return false; + } + root_off = *off; return true; } template -bool root_visitor::visit_map(const Container&, Body&& body) { +bool root_visitor::visit_map(const Container& m, Body&& body) { uoffset_t vec_off = 0; - KOTA_CODEC_TRY(encode_sorted_map(fbb, std::forward(body), vec_off)); + KOTA_CODEC_TRY(encode_sorted_map(fbb, m, std::forward(body), vec_off)); auto start = fbb.StartTable(); fbb.AddOffset(detail::first_field, offset_t(vec_off)); diff --git a/include/kota/codec/fbs/proxy.h b/include/kota/codec/fbs/proxy.h index 52166040..a00ba647 100644 --- a/include/kota/codec/fbs/proxy.h +++ b/include/kota/codec/fbs/proxy.h @@ -16,6 +16,7 @@ #include #include +#include "kota/meta/compare.h" #include "kota/meta/schema.h" #include "kota/codec/fbs/type.h" #include "kota/codec/visit/common.h" @@ -642,9 +643,9 @@ class map_view { } template - requires std::totally_ordered_with< + requires meta::synthesized_lt_with< proxy_detail::field_return_type_t>, - const U&> + U> auto operator[](const U& key) const -> proxy_detail::field_return_type_t> { using clean_v = proxy_detail::deep_clean_t; @@ -658,9 +659,9 @@ class map_view { } template - requires std::totally_ordered_with< + requires meta::synthesized_lt_with< proxy_detail::field_return_type_t>, - const U&> + U> auto find(const U& key) const -> std::optional> { auto entry = find_entry(key); if(!entry.valid()) { @@ -670,9 +671,9 @@ class map_view { } template - requires std::totally_ordered_with< + requires meta::synthesized_lt_with< proxy_detail::field_return_type_t>, - const U&> + U> auto contains(const U& key) const -> bool { return find_entry(key).valid(); } @@ -682,6 +683,9 @@ class map_view { } private: + // Binary search under meta::lt — the exact ordering the encoder sorted + // entries with (operator< when the key defines one, otherwise the + // reflection-synthesized field-by-field order). template auto find_entry(const U& key) const -> proxy_detail::table_view_type { using clean_k = proxy_detail::deep_clean_t; @@ -697,7 +701,7 @@ class map_view { const auto* entry = vector->template GetAs
(static_cast(mid)); auto entry_key = proxy_detail::read_field(proxy_detail::table_view_type(entry), proxy_detail::field_slot(0)); - if(entry_key < key) { + if(meta::lt(entry_key, key)) { lo = mid + 1; } else { hi = mid; @@ -711,7 +715,7 @@ class map_view { const auto* entry = vector->template GetAs
(static_cast(lo)); auto entry_view = proxy_detail::table_view_type(entry); auto entry_key = proxy_detail::read_field(entry_view, proxy_detail::field_slot(0)); - if(entry_key == key) { + if(!meta::lt(entry_key, key) && !meta::lt(key, entry_key)) { return entry_view; } return {}; diff --git a/include/kota/codec/fbs/type.h b/include/kota/codec/fbs/type.h index e34dda87..aa3ad337 100644 --- a/include/kota/codec/fbs/type.h +++ b/include/kota/codec/fbs/type.h @@ -100,7 +100,12 @@ struct schema_struct_trait { } } - constexpr static bool value = meta::reflectable_class && std::is_trivial_v && + // FlatBuffers inline structs are read and written by memcpy, so trivial + // copyability and a fixed layout are what matters; default member + // initializers (which break std::is_trivial) are fine. + constexpr static bool value = meta::reflectable_class && + std::is_trivially_copyable_v && + std::is_default_constructible_v && std::is_standard_layout_v && fields_supported(); }; diff --git a/include/kota/codec/visit/decode.h b/include/kota/codec/visit/decode.h index 71ecadf9..987c9b05 100644 --- a/include/kota/codec/visit/decode.h +++ b/include/kota/codec/visit/decode.h @@ -31,6 +31,11 @@ namespace kota::codec { template , typename = void> struct deserialize_visit {}; +/// Declared in visit/encode.h; value-mode specializations (wire_type + +/// to_wire + from_wire) also drive decoding, see decode_value. +template +struct serialize_visit; + template bool decode_value(Vis& vis, T& out); @@ -724,6 +729,17 @@ bool decode_value(Vis& vis, T& out) { if constexpr(requires(Vis& v, V& val) { deserialize_visit::visit(v, val); }) { return deserialize_visit::visit(vis, out); + } else if constexpr(requires { + serialize_visit::from_wire( + std::declval< + typename serialize_visit::wire_type>()); + }) { + // Value-mode serialize_visit specialization: decode the declared + // wire_type, then convert back through from_wire. + auto wire = typename serialize_visit::wire_type(); + KOTA_CODEC_TRY(decode_value(vis, wire)); + out = serialize_visit::from_wire(std::move(wire)); + return true; } else if constexpr(meta::annotated_type) { using attrs_t = typename V::attrs; auto&& inner = meta::annotated_value(out); @@ -864,7 +880,7 @@ bool decode_value(Vis& vis, T& out) { } std::size_t idx = 0; return vis.visit_seq([&](auto& ev) -> bool { - element_t item{}; + auto item = element_t(); bool ok = decode_value(ev, item); if(!ok) { if constexpr(Config::detailed_error) { @@ -884,7 +900,7 @@ bool decode_value(Vis& vis, T& out) { } std::size_t idx = 0; while(sv.has_element()) { - element_t item{}; + auto item = element_t(); bool ok = sv.visit_element( [&](auto& ev) -> bool { return decode_value(ev, item); }); if(!ok) { @@ -957,15 +973,22 @@ bool decode_value(Vis& vis, T& out) { } } else if constexpr(kind == map) { using kv_t = std::ranges::range_value_t; - using key_t = std::remove_const_t; - using mapped_t = typename kv_t::second_type; + // Decode keys into owning storage: entry protocols that hand out + // views (string_view, llvm::StringRef, ...) would otherwise decode + // into a dangling view before insertion copies it. + using raw_key_t = kota::map_entry_key_t; + using key_t = std::conditional_t && + !std::same_as, + std::string, + raw_key_t>; + using mapped_t = kota::map_entry_mapped_t; if constexpr(detail::data_driven) { if constexpr(requires { out.clear(); }) { out.clear(); } std::size_t idx = 0; return vis.visit_map([&](auto& kv, auto& vv) -> bool { - key_t key{}; + auto key = key_t(); bool ok = decode_value(kv, key); if(!ok) { if constexpr(Config::detailed_error) { @@ -974,7 +997,7 @@ bool decode_value(Vis& vis, T& out) { } return false; } - mapped_t val{}; + auto val = mapped_t(); ok = decode_value(vv, val); if(!ok) { if constexpr(Config::detailed_error) { @@ -994,8 +1017,8 @@ bool decode_value(Vis& vis, T& out) { } std::size_t idx = 0; while(sv.has_entry()) { - key_t key{}; - mapped_t val{}; + auto key = key_t(); + auto val = mapped_t(); bool ok = sv.visit_entry( [&](auto& kv) -> bool { return decode_value(kv, key); }, [&](auto& vv) -> bool { return decode_value(vv, val); }); diff --git a/include/kota/codec/visit/encode.h b/include/kota/codec/visit/encode.h index 40dfd6e1..492c3c26 100644 --- a/include/kota/codec/visit/encode.h +++ b/include/kota/codec/visit/encode.h @@ -172,6 +172,11 @@ bool encode_value(Vis& vis, const T& value) { serialize_visit::visit(v, val); }) { return serialize_visit::visit(vis, value); + } else if constexpr(requires(const V& val) { serialize_visit::to_wire(val); }) { + // Value-mode specialization: `wire_type` declares the on-wire layout, + // to_wire/from_wire convert. One specialization serves both + // directions (decode_value picks up from_wire). + return encode_value(vis, serialize_visit::to_wire(value)); } else if constexpr(meta::annotated_type) { using attrs_t = typename V::attrs; auto&& inner = meta::annotated_value(value); @@ -333,7 +338,9 @@ bool encode_value(Vis& vis, const T& value) { } else if constexpr(kind == map) { return vis.visit_map(value, [&](auto& mv) -> bool { std::size_t idx = 0; - for(const auto& [k, v]: value) { + for(const auto& entry: value) { + auto&& k = kota::detail::map_entry_key(entry); + auto&& v = kota::detail::map_entry_value(entry); bool ok = mv.visit_entry( [&](auto& kv) -> bool { return encode_value(kv, k); }, [&](auto& vv) -> bool { return encode_value(vv, v); }); diff --git a/include/kota/meta/compare.h b/include/kota/meta/compare.h index 41750022..23e1e7c8 100644 --- a/include/kota/meta/compare.h +++ b/include/kota/meta/compare.h @@ -554,6 +554,15 @@ constexpr bool compare_ge(const L& lhs, const R& rhs) { namespace kota::meta { +/// SFINAE-friendly approximation of "meta::lt can order L and R": either an +/// operator< exists, or both sides are reflectable aggregates whose ordering +/// meta::lt synthesizes field by field. (meta::lt itself hard-errors on +/// incomparable operands, so callers that need to *probe* use this concept.) +template +concept synthesized_lt_with = + lt_comparable_with || detail::reflectable_pair, + std::remove_cvref_t>; + struct eq_t { using is_transparent = void; diff --git a/include/kota/meta/type_info.h b/include/kota/meta/type_info.h index e55f9854..d3f55955 100644 --- a/include/kota/meta/type_info.h +++ b/include/kota/meta/type_info.h @@ -498,8 +498,8 @@ struct type_instance_impl { template struct type_instance_impl { using kv_t = std::ranges::range_value_t; - using key_t = std::remove_const_t; - using mapped_t = typename kv_t::second_type; + using key_t = kota::map_entry_key_t; + using mapped_t = kota::map_entry_mapped_t; constexpr inline static map_type_info value = { {type_kind::map, meta::type_name()}, diff --git a/include/kota/support/ranges.h b/include/kota/support/ranges.h index fe52927c..04404c3a 100644 --- a/include/kota/support/ranges.h +++ b/include/kota/support/ranges.h @@ -12,11 +12,12 @@ namespace kota { namespace detail { -// A type counts as a "map value" if either: -// (a) it is tuple-like with exactly two elements (std::pair, std::tuple, -// llvm::StringMapEntry, ...), or +// A type counts as a "map value" if one of the following holds: +// (a) it is tuple-like with exactly two elements (std::pair, std::tuple, ...), // (b) it exposes `.first` / `.second` members — covers types that derive from -// std::pair without re-specializing std::tuple_size (e.g. llvm::detail::DenseMapPair). +// std::pair without re-specializing std::tuple_size (e.g. llvm::detail::DenseMapPair), +// (c) it exposes `getKey()` / `getValue()` accessors (e.g. llvm::StringMapEntry), or +// (d) it is a plain two-field aggregate, destructurable via structured bindings. template concept map_entry_tuple_like = requires { { std::tuple_size::value } -> std::convertible_to; @@ -30,7 +31,69 @@ concept map_entry_pair_like = requires(T& t) { }; template -concept map_entry_like = map_entry_tuple_like || map_entry_pair_like; +concept map_entry_keyed_like = requires(T& t) { + t.getKey(); + t.getValue(); +}; + +// Implicitly converts to anything but T itself, so aggregate-arity probing +// below never matches T's copy/move constructor. +template +struct entry_probe_arg { + template + requires (!std::same_as, T>) + operator U(); +}; + +// A two-field aggregate: brace-initializable from exactly two elements. +template +concept map_entry_aggregate_like = + std::is_aggregate_v && !std::is_array_v && !map_entry_tuple_like && + !map_entry_pair_like && !map_entry_keyed_like && + requires { T{entry_probe_arg{}, entry_probe_arg{}}; } && + !requires { T{entry_probe_arg{}, entry_probe_arg{}, entry_probe_arg{}}; }; + +template +concept map_entry_like = map_entry_tuple_like || map_entry_pair_like || + map_entry_keyed_like || map_entry_aggregate_like; + +/// Access the key of a map entry, regardless of which entry protocol it uses. +/// Returns a reference for member-based protocols and a value for accessor-based +/// ones that return by value (e.g. llvm::StringMapEntry::getKey()). +template + requires map_entry_like> +constexpr decltype(auto) map_entry_key(E&& e) { + using T = std::remove_cvref_t; + if constexpr(map_entry_tuple_like) { + using std::get; + return get<0>(std::forward(e)); + } else if constexpr(map_entry_pair_like) { + return (std::forward(e).first); + } else if constexpr(map_entry_keyed_like) { + return e.getKey(); + } else { + auto&& [k, v] = e; + return (k); + } +} + +/// Access the mapped value of a map entry; see map_entry_key. +template + requires map_entry_like> +constexpr decltype(auto) map_entry_value(E&& e) { + using T = std::remove_cvref_t; + if constexpr(map_entry_tuple_like) { + using std::get; + return get<1>(std::forward(e)); + } else if constexpr(map_entry_pair_like) { + return (std::forward(e).second); + } else if constexpr(map_entry_keyed_like) { + return e.getValue(); + } else { + auto&& [k, v] = e; + return (v); + } +} } // namespace detail @@ -44,15 +107,9 @@ template struct map_entry_types; template -struct map_entry_types>> { - using key_type = std::remove_cvref_t>; - using mapped_type = std::remove_cvref_t>; -}; - -template -struct map_entry_types && map_entry_pair_like>> { - using key_type = std::remove_cvref_t().first)>; - using mapped_type = std::remove_cvref_t().second)>; +struct map_entry_types>> { + using key_type = std::remove_cvref_t()))>; + using mapped_type = std::remove_cvref_t()))>; }; } // namespace detail diff --git a/tests/unit/codec/fbs/flatbuffers_map_protocol_tests.cpp b/tests/unit/codec/fbs/flatbuffers_map_protocol_tests.cpp new file mode 100644 index 00000000..3bed8d78 --- /dev/null +++ b/tests/unit/codec/fbs/flatbuffers_map_protocol_tests.cpp @@ -0,0 +1,466 @@ +#if __has_include() + +#include +#include +#include +#include +#include +#include +#include + +#include "kota/zest/zest.h" +#include "kota/meta/attrs.h" +#include "kota/codec/fbs/fbs.h" +#include "flatbuffers/flatbuffers.h" + +namespace kota::codec { + +namespace { + +using fbs::from_flatbuffer; +using fbs::map_view; +using fbs::table_view; +using fbs::to_flatbuffer; + +// A minimal map with deterministic, deliberately unsorted iteration order. +// Entries are plain pairs, so the container classifies as a map via the +// pair-like entry protocol. +template +struct scrambled_map { + using key_type = K; + using mapped_type = V; + + std::vector> entries; + + auto begin() const { + return entries.begin(); + } + + auto end() const { + return entries.end(); + } + + auto begin() { + return entries.begin(); + } + + auto end() { + return entries.end(); + } + + void clear() { + entries.clear(); + } + + void insert_or_assign(K key, V value) { + for(auto& [k, v]: entries) { + if(k == key) { + v = std::move(value); + return; + } + } + entries.emplace_back(std::move(key), std::move(value)); + } +}; + +// Mimics llvm::StringMap: entries expose getKey()/getValue() instead of +// first/second, key storage is owned by the container, and key_type is a +// non-owning const char* the decoder must not decode into directly. +template +struct mock_string_map { + struct entry_type { + std::string key_storage; + V value_storage; + + std::string_view getKey() const { + return key_storage; + } + + const V& getValue() const { + return value_storage; + } + + V& getValue() { + return value_storage; + } + }; + + using key_type = const char*; + using mapped_type = V; + + std::vector entries; + + auto begin() const { + return entries.begin(); + } + + auto end() const { + return entries.end(); + } + + auto begin() { + return entries.begin(); + } + + auto end() { + return entries.end(); + } + + void clear() { + entries.clear(); + } + + void insert_or_assign(std::string_view key, V value) { + for(auto& e: entries) { + if(e.key_storage == key) { + e.value_storage = std::move(value); + return; + } + } + entries.push_back(entry_type{std::string(key), std::move(value)}); + } + + const V* find(std::string_view key) const { + for(const auto& e: entries) { + if(e.key_storage == key) { + return &e.value_storage; + } + } + return nullptr; + } +}; + +struct int_key_holder { + scrambled_map table; +}; + +struct u64_key_holder { + scrambled_map table; +}; + +struct span_key { + std::uint32_t begin; + std::uint32_t end; + + constexpr auto operator<=>(const span_key&) const = default; +}; + +struct span_key_holder { + scrambled_map table; +}; + +// Deliberately no operator< / <=> : ordering must be synthesized field by +// field via reflection (meta::lt), on both the encode sort and map_view's +// binary search. +struct bare_key { + std::uint16_t group; + std::uint32_t item; + + constexpr bool operator==(const bare_key&) const = default; +}; + +struct bare_key_holder { + scrambled_map table; +}; + +struct string_map_holder { + mock_string_map table; +}; + +// Entries that are plain two-field aggregates: no first/second, no tuple +// protocol, no getKey/getValue — only destructurable via structured bindings. +struct aggregate_entry_map { + struct entry_type { + std::uint32_t id; + std::string label; + }; + + using key_type = std::uint32_t; + using mapped_type = std::string; + + std::vector entries; + + auto begin() const { + return entries.begin(); + } + + auto end() const { + return entries.end(); + } + + auto begin() { + return entries.begin(); + } + + auto end() { + return entries.end(); + } + + void clear() { + entries.clear(); + } + + void insert_or_assign(std::uint32_t key, std::string value) { + for(auto& e: entries) { + if(e.id == key) { + e.label = std::move(value); + return; + } + } + entries.push_back(entry_type{key, std::move(value)}); + } +}; + +struct aggregate_entry_holder { + aggregate_entry_map table; +}; + +TEST_SUITE(serde_flatbuffers_map_protocol) { + +// Multi-digit integer keys used to be sorted by their decimal string +// representation ("10" < "9"), breaking map_view's numeric binary search. +TEST_CASE(integer_keys_sort_numerically_for_binary_search) { + int_key_holder input; + for(std::uint32_t key: {17U, 3U, 100U, 9U, 25U, 1U, 42U, 10U, 7U, 88U, 2U, 56U}) { + input.table.insert_or_assign(key, "v" + std::to_string(key)); + } + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + ASSERT_TRUE(root.valid()); + + auto view = root[&int_key_holder::table]; + ASSERT_TRUE(view.valid()); + ASSERT_EQ(view.size(), input.table.entries.size()); + + for(const auto& [key, value]: input.table.entries) { + ASSERT_TRUE(view.contains(key)); + EXPECT_EQ(view[key], std::string_view(value)); + } + EXPECT_FALSE(view.contains(11U)); + + // wire order must follow numeric key order + std::uint32_t last = 0; + for(std::size_t i = 0; i < view.size(); ++i) { + auto key = view.at(i).template get<0>(); + if(i > 0) { + EXPECT_TRUE(last < key); + } + last = key; + } +} + +TEST_CASE(u64_keys_survive_round_trip_and_lookup) { + u64_key_holder input; + const std::uint64_t keys[] = { + 0xdeadbeefcafebabeULL, + 7ULL, + 0x10000000000ULL, + 0xffffffffffffffffULL, + 1000ULL, + }; + std::int32_t v = 1; + for(auto key: keys) { + input.table.insert_or_assign(key, v++); + } + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + auto view = root[&u64_key_holder::table]; + ASSERT_EQ(view.size(), 5U); + for(const auto& [key, value]: input.table.entries) { + ASSERT_TRUE(view.contains(key)); + EXPECT_EQ(view[key], value); + } + + u64_key_holder decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(decoded.table.entries.size(), 5U); + for(const auto& [key, value]: input.table.entries) { + bool found = false; + for(const auto& [dk, dv]: decoded.table.entries) { + if(dk == key) { + EXPECT_EQ(dv, value); + found = true; + } + } + EXPECT_TRUE(found); + } +} + +// Struct keys previously failed to compile (the encoder stringified keys for +// sorting). They now sort by the key's own ordering, which map_view reuses. +TEST_CASE(inline_struct_keys_sort_and_lookup) { + span_key_holder input; + input.table.insert_or_assign(span_key{30, 40}, 3); + input.table.insert_or_assign(span_key{10, 20}, 1); + input.table.insert_or_assign(span_key{10, 15}, 0); + input.table.insert_or_assign(span_key{50, 60}, 5); + input.table.insert_or_assign(span_key{20, 25}, 2); + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + auto view = root[&span_key_holder::table]; + ASSERT_EQ(view.size(), 5U); + + for(const auto& [key, value]: input.table.entries) { + ASSERT_TRUE(view.contains(key)); + EXPECT_EQ(view[key], value); + } + EXPECT_FALSE(view.contains(span_key{10, 21})); + + span_key last{}; + for(std::size_t i = 0; i < view.size(); ++i) { + auto key = view.at(i).template get<0>(); + if(i > 0) { + EXPECT_TRUE(last < key); + } + last = key; + } + + span_key_holder decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(decoded.table.entries.size(), 5U); + for(const auto& [key, value]: input.table.entries) { + bool found = false; + for(const auto& [dk, dv]: decoded.table.entries) { + if(dk == key) { + EXPECT_EQ(dv, value); + found = true; + } + } + EXPECT_TRUE(found); + } +} + +// Keys without any comparison operator: meta::lt synthesizes the ordering +// from reflection, and map_view searches with the same synthesized order. +TEST_CASE(comparison_free_struct_keys_sort_and_lookup) { + bare_key_holder input; + input.table.insert_or_assign(bare_key{2, 7}, 27); + input.table.insert_or_assign(bare_key{1, 100}, 110); + input.table.insert_or_assign(bare_key{2, 3}, 23); + input.table.insert_or_assign(bare_key{1, 5}, 15); + input.table.insert_or_assign(bare_key{3, 1}, 31); + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + auto view = root[&bare_key_holder::table]; + ASSERT_EQ(view.size(), 5U); + + for(const auto& [key, value]: input.table.entries) { + ASSERT_TRUE(view.contains(key)); + EXPECT_EQ(view[key], value); + } + EXPECT_FALSE(view.contains(bare_key{2, 4})); + + // wire order follows the synthesized (group, item) lexicographic order + bare_key last{}; + for(std::size_t i = 0; i < view.size(); ++i) { + auto key = view.at(i).template get<0>(); + if(i > 0) { + EXPECT_TRUE(kota::meta::lt(last, key)); + } + last = key; + } + + bare_key_holder decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(decoded.table.entries.size(), 5U); +} + +// llvm::StringMap-shaped containers: getKey()/getValue() entries, view-typed +// keys, container-owned key storage. +TEST_CASE(keyed_entry_container_encodes_as_map) { + string_map_holder input; + input.table.insert_or_assign("banana", 2); + input.table.insert_or_assign("apple", 1); + input.table.insert_or_assign("cherry", 3); + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + auto view = root[&string_map_holder::table]; + ASSERT_TRUE(view.valid()); + ASSERT_EQ(view.size(), 3U); + + EXPECT_EQ(view[std::string_view("apple")], 1); + EXPECT_EQ(view[std::string_view("banana")], 2); + EXPECT_EQ(view[std::string_view("cherry")], 3); + EXPECT_FALSE(view.contains(std::string_view("durian"))); + + // wire order is lexicographic regardless of insertion order + EXPECT_EQ(view.at(0).template get<0>(), "apple"); + EXPECT_EQ(view.at(1).template get<0>(), "banana"); + EXPECT_EQ(view.at(2).template get<0>(), "cherry"); + + string_map_holder decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(decoded.table.entries.size(), 3U); + const auto* apple = decoded.table.find("apple"); + ASSERT_TRUE(apple != nullptr); + EXPECT_EQ(*apple, 1); + const auto* cherry = decoded.table.find("cherry"); + ASSERT_TRUE(cherry != nullptr); + EXPECT_EQ(*cherry, 3); +} + +TEST_CASE(aggregate_entry_container_encodes_as_map) { + aggregate_entry_holder input; + input.table.insert_or_assign(12U, "twelve"); + input.table.insert_or_assign(3U, "three"); + input.table.insert_or_assign(101U, "hundred-one"); + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + auto view = root[&aggregate_entry_holder::table]; + ASSERT_TRUE(view.valid()); + ASSERT_EQ(view.size(), 3U); + EXPECT_EQ(view[3U], std::string_view("three")); + EXPECT_EQ(view[12U], std::string_view("twelve")); + EXPECT_EQ(view[101U], std::string_view("hundred-one")); + + aggregate_entry_holder decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(decoded.table.entries.size(), 3U); +} + +TEST_CASE(std_map_round_trip_still_works) { + std::map> input{ + {"a", {1, 2}}, + {"b", {}}, + {"c", {3}}, + }; + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + std::map> decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + EXPECT_TRUE(decoded == input); +} + +}; // TEST_SUITE(serde_flatbuffers_map_protocol) + +} // namespace + +} // namespace kota::codec + +#endif diff --git a/tests/unit/codec/fbs/flatbuffers_robustness_tests.cpp b/tests/unit/codec/fbs/flatbuffers_robustness_tests.cpp new file mode 100644 index 00000000..fd211533 --- /dev/null +++ b/tests/unit/codec/fbs/flatbuffers_robustness_tests.cpp @@ -0,0 +1,199 @@ +#if __has_include() + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kota/zest/zest.h" +#include "kota/meta/attrs.h" +#include "kota/meta/annotation.h" +#include "kota/codec/fbs/fbs.h" +#include "flatbuffers/flatbuffers.h" + +namespace kota::codec { + +namespace { + +using fbs::from_flatbuffer; +using fbs::to_flatbuffer; +using fbs::verify_flatbuffer; + +struct inner_node { + std::string name; + std::vector values; +}; + +struct kitchen_sink { + std::int32_t id = 0; + std::string title; + std::vector blob; + std::vector tags; + std::map counters; + std::map nodes; + std::optional maybe; + std::variant either = 0; + std::vector children; + std::vector> sparse; + std::tuple pair_like{0, ""}; +}; + +kitchen_sink make_sample() { + kitchen_sink value; + value.id = 42; + value.title = "sample"; + value.blob = {std::byte{0x01}, std::byte{0xFF}, std::byte{0x7E}}; + value.tags = {"alpha", "beta"}; + value.counters = {{"x", 1}, {"y", 2}}; + value.nodes[7] = inner_node{"seven", {1, 2, 3}}; + value.nodes[100] = inner_node{"hundred", {}}; + value.maybe = -5; + value.either = std::string("payload"); + value.children = {inner_node{"c0", {9}}, inner_node{"c1", {8, 7}}}; + value.sparse = {std::uint16_t{4}, std::nullopt, std::uint16_t{6}}; + value.pair_like = {12, "tail"}; + return value; +} + +struct failing_adapter { + template + static bool serialize(Vis&, const std::int32_t&) { + return scoped_context::fail(rich_error("adapter refused")); + } + + template + static bool deserialize(Vis&, std::int32_t&) { + return scoped_context::fail(rich_error("adapter refused")); + } +}; + +struct failing_inner { + meta::annotation> field; +}; + +struct failing_outer { + failing_inner child; + std::string tail; +}; + +struct long_double_holder { + long double value = 0; + std::vector values; +}; + +TEST_SUITE(serde_flatbuffers_robustness) { + +TEST_CASE(verify_accepts_valid_buffer) { + auto encoded = to_flatbuffer(make_sample()); + ASSERT_TRUE(encoded.has_value()); + EXPECT_TRUE(verify_flatbuffer(std::span(*encoded))); + + kitchen_sink decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(decoded.id, 42); + EXPECT_EQ(decoded.title, "sample"); + EXPECT_EQ(decoded.tags.size(), 2U); + EXPECT_TRUE(decoded.counters == make_sample().counters); + ASSERT_EQ(decoded.nodes.size(), 2U); + EXPECT_EQ(decoded.nodes[7].name, "seven"); + EXPECT_TRUE(decoded.maybe.has_value()); + ASSERT_EQ(decoded.either.index(), 1U); + EXPECT_EQ(std::get<1>(decoded.either), "payload"); + ASSERT_EQ(decoded.sparse.size(), 3U); + EXPECT_FALSE(decoded.sparse[1].has_value()); +} + +TEST_CASE(verify_rejects_wrong_root_type_gracefully) { + // A buffer encoded for one type read as a completely different type must + // never read out of bounds; it may decode to defaults or fail, but the + // verifier has to keep every access inside the buffer. + struct other_shape { + std::vector a; + std::map> b; + std::string c; + }; + + auto encoded = to_flatbuffer(make_sample()); + ASSERT_TRUE(encoded.has_value()); + + other_shape decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + (void)result; // either outcome is fine — the run must be clean under ASan +} + +TEST_CASE(truncated_buffers_never_crash) { + auto encoded = to_flatbuffer(make_sample()); + ASSERT_TRUE(encoded.has_value()); + + std::size_t rejected = 0; + for(std::size_t len = 0; len < encoded->size(); ++len) { + std::vector truncated(encoded->begin(), + encoded->begin() + static_cast(len)); + kitchen_sink decoded; + auto result = from_flatbuffer(std::span(truncated), decoded); + if(!result.has_value()) { + ++rejected; + } + if(len < 8) { + // Too short to even hold the root offset and identifier. + EXPECT_FALSE(result.has_value()); + } + } + // Chopping into real data must be caught; only trailing alignment padding + // may still verify, so the overwhelming majority of prefixes must fail. + EXPECT_TRUE(rejected + 16 >= encoded->size()); +} + +TEST_CASE(bitflipped_buffers_never_crash) { + auto encoded = to_flatbuffer(make_sample()); + ASSERT_TRUE(encoded.has_value()); + + for(std::size_t i = 0; i < encoded->size(); ++i) { + for(std::uint8_t pattern: {std::uint8_t{0xFF}, std::uint8_t{0x80}, std::uint8_t{0x01}}) { + auto copy = *encoded; + copy[i] ^= pattern; + kitchen_sink decoded; + auto result = from_flatbuffer(std::span(copy), decoded); + (void)result; // corrupt scalars may still decode; OOB must not happen + } + } +} + +TEST_CASE(encode_failure_in_nested_table_propagates) { + failing_outer input; + input.tail = "t"; + + auto encoded = to_flatbuffer(input); + EXPECT_FALSE(encoded.has_value()); +} + +TEST_CASE(long_double_round_trip) { + long_double_holder input; + input.value = 3.5L; + input.values = {1.25L, -2.5L, 1024.0L}; + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + long_double_holder decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(static_cast(decoded.value), 3.5); + ASSERT_EQ(decoded.values.size(), 3U); + EXPECT_EQ(static_cast(decoded.values[0]), 1.25); + EXPECT_EQ(static_cast(decoded.values[1]), -2.5); + EXPECT_EQ(static_cast(decoded.values[2]), 1024.0); +} + +}; // TEST_SUITE(serde_flatbuffers_robustness) + +} // namespace + +} // namespace kota::codec + +#endif diff --git a/tests/unit/codec/fbs/flatbuffers_wire_adapter_tests.cpp b/tests/unit/codec/fbs/flatbuffers_wire_adapter_tests.cpp new file mode 100644 index 00000000..c577f0a7 --- /dev/null +++ b/tests/unit/codec/fbs/flatbuffers_wire_adapter_tests.cpp @@ -0,0 +1,139 @@ +#if __has_include() + +#include +#include +#include +#include +#include +#include + +#include "kota/zest/zest.h" +#include "kota/codec/fbs/fbs.h" +#include "flatbuffers/flatbuffers.h" + +namespace kota::codec { + +namespace { + +/// An opaque value type the reflection framework cannot walk: private state, +/// no aggregate structure. +class ticket { +public: + ticket() = default; + + explicit ticket(std::uint32_t id) : id_(id) {} + + std::uint32_t id() const { + return id_; + } + + bool operator==(const ticket&) const = default; + +private: + std::uint32_t id_ = 0; +}; + +/// Bridged through an owning wire type while to_wire hands out a view. +class label { +public: + label() = default; + + explicit label(std::string text) : text_(std::move(text)) {} + + const std::string& text() const { + return text_; + } + + bool operator==(const label&) const = default; + +private: + std::string text_; +}; + +} // namespace + +/// Value-mode serialize_visit specializations: no visit() — wire_type +/// declares the on-wire layout, to_wire/from_wire convert, and one +/// specialization serves both encoding and decoding on every backend. +template +struct serialize_visit { + using wire_type = std::uint32_t; + + static wire_type to_wire(const ticket& value) { + return value.id(); + } + + static ticket from_wire(wire_type id) { + return ticket(id); + } +}; + +template +struct serialize_visit { + using wire_type = std::string; + + static std::string_view to_wire(const label& value) { + return value.text(); + } + + static label from_wire(wire_type text) { + return label(std::move(text)); + } +}; + +namespace { + +using fbs::from_flatbuffer; +using fbs::table_view; +using fbs::to_flatbuffer; + +struct order { + ticket id; + label name; + std::vector history; + std::map notes; +}; + +TEST_SUITE(serde_flatbuffers_wire_adapter) { + +TEST_CASE(adapted_types_round_trip_everywhere) { + order input; + input.id = ticket(42); + input.name = label("answer"); + input.history = {ticket(1), ticket(2), ticket(3)}; + input.notes = {{7, label("seven")}, {9, label("nine")}}; + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + order decoded; + auto result = from_flatbuffer(std::span(*encoded), decoded); + ASSERT_TRUE(result.has_value()); + EXPECT_TRUE(decoded.id == input.id); + EXPECT_TRUE(decoded.name == input.name); + EXPECT_TRUE(decoded.history == input.history); + EXPECT_TRUE(decoded.notes == input.notes); +} + +TEST_CASE(proxy_reads_wire_representation) { + order input; + input.id = ticket(42); + input.name = label("answer"); + + auto encoded = to_flatbuffer(input); + ASSERT_TRUE(encoded.has_value()); + + auto root = table_view::from_bytes(*encoded); + ASSERT_TRUE(root.valid()); + // Adapted fields surface as their wire representation. + EXPECT_EQ(root[&order::id], 42U); + EXPECT_EQ(root[&order::name], "answer"); +} + +}; // TEST_SUITE(serde_flatbuffers_wire_adapter) + +} // namespace + +} // namespace kota::codec + +#endif