Merge branch 'issue/gh-161'

* issue/gh-161:
  Integrate review feedback
  Fix regression in the new decoding logic
  Fix build on FreeBSD
  Replace CAF deserializers with Broker decoders
  Implement a decoder with inspect API support
  Use Broker's own serialization over CAF serializer
  Use the new binary decoder for Broker's variant
  Iterate on the new decoding API
  Implement new decoding logic for the binary format

CHANGES

@@ -1,3 +1,23 @@
+2.8.0-dev.131 | 2024-10-18 16:57:55 -0700
+
+  * Integrate review feedback (Dominik Charousset)
+
+  * Fix regression in the new decoding logic (Dominik Charousset)
+
+  * Fix build on FreeBSD (Dominik Charousset, Corelight)
+
+  * Replace CAF deserializers with Broker decoders (Dominik Charousset, Corelight)
+
+  * Implement a decoder with inspect API support (Dominik Charousset, Corelight)
+
+  * Use Broker's own serialization over CAF serializer (Dominik Charousset, Corelight)
+
+  * Use the new binary decoder for Broker's variant (Dominik Charousset, Corelight)
+
+  * Iterate on the new decoding API (Dominik Charousset, Corelight)
+
+  * Implement new decoding logic for the binary format (Dominik Charousset, Corelight)
+
 2.8.0-dev.121 | 2024-10-18 10:12:55 -0700
 
   * Fix bugprone and modernize clang-tidy findings (Dominik Charousset)

VERSION

@@ -1 +1 @@
-2.8.0-dev.121
+2.8.0-dev.131

libbroker/broker/alm/multipath.hh

@@ -229,8 +229,6 @@ private:
         }
         if (!down_.emplace(child)) {
           child->shallow_delete();
-          f.field_invariant_check_failed(
-            "a multipath may not contain duplicates");
           return false;
         }
       }

libbroker/broker/alm/multipath.test.cc

@@ -1,12 +1,10 @@
 #include "broker/alm/multipath.hh"
 
 #include "broker/broker-test.test.hh"
+#include "broker/format/bin.hh"
 
 #include <random>
 
-#include <caf/binary_deserializer.hpp>
-#include <caf/binary_serializer.hpp>
-
 #include "broker/alm/routing_table.hh"
 
 using broker::alm::multipath;
@@ -128,16 +126,16 @@ TEST(multipaths are serializable) {
     emplace(ab, 'G');
   }
   auto path = multipath{tptr};
-  caf::binary_serializer::container_type buf;
+  std::vector<std::byte> buf;
   MESSAGE("serializer the path into a buffer");
   {
-    caf::binary_serializer sink{nullptr, buf};
+    format::bin::v1::encoder sink{std::back_inserter(buf)};
     CHECK(sink.apply(path));
   }
   multipath copy;
   MESSAGE("deserializers a copy from the path from the buffer");
   {
-    caf::binary_deserializer source{nullptr, buf};
+    format::bin::v1::decoder source{buf.data(), buf.size()};
     CHECK(source.apply(copy));
   }
   MESSAGE("after a serialization roundtrip, the path is equal to its copy");

libbroker/broker/command_envelope.cc

@@ -3,12 +3,11 @@
 #include "broker/endpoint_id.hh"
 #include "broker/error.hh"
 #include "broker/expected.hh"
+#include "broker/format/bin.hh"
 #include "broker/internal/type_id.hh"
 #include "broker/internal_command.hh"
 #include "broker/topic.hh"
 
-#include <caf/binary_deserializer.hpp>
-#include <caf/binary_serializer.hpp>
 #include <caf/byte_buffer.hpp>
 #include <caf/deep_to_string.hpp>
 
@@ -65,14 +64,16 @@ class default_command_envelope : public command_envelope {
       receiver_(receiver),
       topic_(std::move(topic_str)),
       value_(std::move(cmd)) {
-    caf::binary_serializer sink{nullptr, buf_};
+    auto out = std::back_inserter(buf_);
+    format::bin::v1::encoder sink{out};
     if (!sink.apply(value_))
       throw std::logic_error("failed to serialize command");
   }
 
   default_command_envelope(std::string&& topic_str, internal_command&& cmd)
     : topic_(topic_str), value_(std::move(cmd)) {
-    caf::binary_serializer sink{nullptr, buf_};
+    auto out = std::back_inserter(buf_);
+    format::bin::v1::encoder sink{out};
     if (!sink.apply(value_))
       throw std::logic_error("failed to serialize command");
   }
@@ -143,7 +144,7 @@ class deserialized_command_envelope
 
   error parse() {
     auto [data, data_size] = this->raw_bytes();
-    caf::binary_deserializer src{nullptr, data, data_size};
+    format::bin::v1::decoder src{data, data_size};
     if (!src.apply(value_))
       return make_error(ec::invalid_data);
     return error{};

libbroker/broker/data.cc

@@ -5,6 +5,7 @@
 
 #include "broker/convert.hh"
 #include "broker/expected.hh"
+#include "broker/format/bin.hh"
 #include "broker/format/txt.hh"
 #include "broker/internal/native.hh"
 #include "broker/internal/type_id.hh"
@@ -106,6 +107,231 @@ const vector& data::to_list() const {
 
 namespace {
 
+// Assigns a value to a data object, converting as necessary.
+template <class T>
+void do_assign(data& dst, const T& arg) {
+  if constexpr (std::is_same_v<T, std::string_view>) {
+    dst = std::string{arg};
+  } else if constexpr (std::is_same_v<T, enum_value_view>) {
+    dst = enum_value{std::string{arg.name}};
+  } else {
+    dst = arg;
+  }
+}
+
+struct decoder_handler_value;
+
+struct decoder_handler_list;
+
+struct decoder_handler_set;
+
+struct decoder_handler_table;
+
+// Consumes events from a decoder and produces a data object.
+struct decoder_handler_value {
+  data result;
+
+  template <class T>
+  bool value(const T& arg) {
+    do_assign(result, arg);
+    return true;
+  }
+
+  decoder_handler_list begin_list();
+
+  bool end_list(decoder_handler_list&);
+
+  decoder_handler_set begin_set();
+
+  bool end_set(decoder_handler_set&);
+
+  decoder_handler_table begin_table();
+
+  bool end_table(decoder_handler_table&);
+};
+
+// Consumes events from a decoder and produces a list of data objects.
+struct decoder_handler_list {
+  vector result;
+
+  template <class T>
+  bool value(const T& arg) {
+    do_assign(result.emplace_back(), arg);
+    return true;
+  }
+
+  decoder_handler_list begin_list() {
+    return {};
+  }
+
+  bool end_list(decoder_handler_list& other) {
+    result.emplace_back() = std::move(other.result);
+    return true;
+  }
+
+  decoder_handler_set begin_set();
+
+  bool end_set(decoder_handler_set&);
+
+  decoder_handler_table begin_table();
+
+  bool end_table(decoder_handler_table&);
+};
+
+// Consumes events from a decoder and produces a set of data objects.
+struct decoder_handler_set {
+  set result;
+
+  template <class T>
+  bool value(const T& arg) {
+    data item;
+    do_assign(item, arg);
+    return result.insert(std::move(item)).second;
+  }
+
+  decoder_handler_list begin_list() {
+    return {};
+  }
+
+  bool end_list(decoder_handler_list& other) {
+    return result.insert(data{std::move(other.result)}).second;
+  }
+
+  decoder_handler_set begin_set() {
+    return {};
+  }
+
+  bool end_set(decoder_handler_set& other) {
+    return result.insert(data{std::move(other.result)}).second;
+  }
+
+  decoder_handler_table begin_table();
+
+  bool end_table(decoder_handler_table&);
+};
+
+struct decoder_handler_table {
+  table result;
+  std::optional<data> key;
+
+  bool add(data&& arg) {
+    if (!key) {
+      key.emplace(std::move(arg));
+      return true;
+    }
+    auto res = result.emplace(std::move(*key), std::move(arg)).second;
+    key.reset();
+    return res;
+  }
+
+  template <class T>
+  bool value(const T& arg) {
+    data val;
+    do_assign(val, arg);
+    return add(std::move(val));
+  }
+
+  decoder_handler_list begin_list() {
+    return {};
+  }
+
+  bool end_list(decoder_handler_list& other) {
+    return add(data{std::move(other.result)});
+  }
+
+  decoder_handler_set begin_set() {
+    return {};
+  }
+
+  bool end_set(decoder_handler_set& other) {
+    return add(data{std::move(other.result)});
+  }
+
+  decoder_handler_table begin_table() {
+    return {};
+  }
+
+  bool end_table(decoder_handler_table& other) {
+    return add(data{std::move(other.result)});
+  }
+
+  void begin_key_value_pair() {
+    // nop
+  }
+
+  void end_key_value_pair() {
+    // nop
+  }
+};
+
+decoder_handler_list decoder_handler_value::begin_list() {
+  return {};
+}
+
+bool decoder_handler_value::end_list(decoder_handler_list& other) {
+  result = std::move(other.result);
+  return true;
+}
+
+decoder_handler_set decoder_handler_value::begin_set() {
+  return {};
+}
+
+bool decoder_handler_value::end_set(decoder_handler_set& other) {
+  result = std::move(other.result);
+  return true;
+}
+
+decoder_handler_table decoder_handler_value::begin_table() {
+  return {};
+}
+
+bool decoder_handler_value::end_table(decoder_handler_table& other) {
+  result = std::move(other.result);
+  return true;
+}
+
+decoder_handler_set decoder_handler_list::begin_set() {
+  return {};
+}
+
+bool decoder_handler_list::end_set(decoder_handler_set& other) {
+  result.emplace_back(std::move(other.result));
+  return true;
+}
+
+decoder_handler_table decoder_handler_list::begin_table() {
+  return {};
+}
+
+bool decoder_handler_list::end_table(decoder_handler_table& other) {
+  result.emplace_back(std::move(other.result));
+  return true;
+}
+
+decoder_handler_table decoder_handler_set::begin_table() {
+  return {};
+}
+
+bool decoder_handler_set::end_table(decoder_handler_table& other) {
+  return result.insert(data{std::move(other.result)}).second;
+}
+
+} // namespace
+
+bool data::deserialize(const std::byte* payload, size_t payload_size) {
+  decoder_handler_value handler;
+  auto payload_end = payload + payload_size;
+  auto [ok, pos] = format::bin::v1::decode(payload, payload_end, handler);
+  if (!ok || pos != payload_end) {
+    return false;
+  }
+  *this = std::move(handler.result);
+  return true;
+}
+
+namespace {
+
 template <class Container>
 void container_convert(Container& c, std::string& str, char left, char right) {
   constexpr auto* delim = ", ";

libbroker/broker/data.hh

@@ -325,6 +325,9 @@ public:
     return data_;
   }
 
+  /// Deserializes the data from a binary representation.
+  bool deserialize(const std::byte* payload, size_t payload_size);
+
 private:
   data_variant data_;
 };