feat: refactor domain objects deserialization

conanfile.py

@@ -59,7 +59,7 @@ class KnuthDatabaseConan(KnuthConanFileV2):
 
     def build_requirements(self):
         if self.options.tests:
-            self.test_requires("catch2/3.6.0")
+            self.test_requires("catch2/3.7.1")
 
     def requirements(self):
         self.requires("domain/0.37.0", transitive_headers=True, transitive_libs=True)

include/kth/database/databases/block_database.ipp

@@ -106,7 +106,7 @@ domain::chain::block internal_database_basis<Clock>::get_block(uint32_t height,
         }
 
         auto data = db_value_to_data_chunk(value);
-        auto res = domain::create<domain::chain::block>(data);
+        auto res = domain::create_old<domain::chain::block>(data);
         return res;
     }
     // db_mode_ == db_mode_type::pruned {

include/kth/database/databases/header_abla_entry.hpp

@@ -30,28 +30,8 @@ void to_data_with_abla_state(W& sink, domain::chain::block const& block) {
     }
 }
 
-std::optional<header_with_abla_state_t> get_header_and_abla_state_from_data(data_chunk const& data);
-std::optional<header_with_abla_state_t> get_header_and_abla_state_from_data(std::istream& stream);
+expect<header_with_abla_state_t> get_header_and_abla_state_from_data(byte_reader& reader);
 
-template <typename R, KTH_IS_READER(R)>
-std::optional<header_with_abla_state_t> get_header_and_abla_state_from_data(R& source) {
-    domain::chain::header header;
-    header.from_data(source, true);
-
-    if ( ! source) {
-        return {};
-    }
-
-    uint64_t block_size = source.read_8_bytes_little_endian();
-    uint64_t control_block_size = source.read_8_bytes_little_endian();
-    uint64_t elastic_buffer_size = source.read_8_bytes_little_endian();
-
-    if ( ! source) {
-        return std::make_tuple(std::move(header), 0, 0, 0);
-    }
-
-    return std::make_tuple(std::move(header), block_size, control_block_size, elastic_buffer_size);
-}
 
 } // namespace kth::database
 

include/kth/database/databases/header_database.ipp

@@ -57,7 +57,8 @@ domain::chain::header internal_database_basis<Clock>::get_header(uint32_t height
     }
 
     auto data = db_value_to_data_chunk(value);
-    auto opt = get_header_and_abla_state_from_data(data);
+    byte_reader reader(data);
+    auto opt = get_header_and_abla_state_from_data(reader);
     if ( ! opt) {
         return {};
     }
@@ -74,7 +75,8 @@ std::optional<header_with_abla_state_t> internal_database_basis<Clock>::get_head
     }
 
     auto data = db_value_to_data_chunk(value);
-    auto opt = get_header_and_abla_state_from_data(data);
+    byte_reader reader(data);
+    auto opt = get_header_and_abla_state_from_data(reader);
     if ( ! opt) {
         return {};
     }

include/kth/database/databases/history_database.ipp

@@ -161,7 +161,7 @@ domain::chain::history_compact::list internal_database_basis<Clock>::get_history
     if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) {
 
         auto data = db_value_to_data_chunk(value);
-        auto entry = domain::create<history_entry>(data);
+        auto entry = domain::create_old<history_entry>(data);
 
         if (from_height == 0 || entry.height() >= from_height) {
             result.push_back(history_entry_to_history_compact(entry));
@@ -174,7 +174,7 @@ domain::chain::history_compact::list internal_database_basis<Clock>::get_history
             }
 
             auto data = db_value_to_data_chunk(value);
-            auto entry = domain::create<history_entry>(data);
+            auto entry = domain::create_old<history_entry>(data);
 
             if (from_height == 0 || entry.height() >= from_height) {
                 result.push_back(history_entry_to_history_compact(entry));
@@ -220,7 +220,7 @@ std::vector<hash_digest> internal_database_basis<Clock>::get_history_txns(short_
     if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) {
 
         auto data = db_value_to_data_chunk(value);
-        auto entry = domain::create<history_entry>(data);
+        auto entry = domain::create_old<history_entry>(data);
 
         if (from_height == 0 || entry.height() >= from_height) {
             // Avoid inserting the same tx
@@ -238,7 +238,7 @@ std::vector<hash_digest> internal_database_basis<Clock>::get_history_txns(short_
             }
 
             auto data = db_value_to_data_chunk(value);
-            auto entry = domain::create<history_entry>(data);
+            auto entry = domain::create_old<history_entry>(data);
 
             if (from_height == 0 || entry.height() >= from_height) {
                 // Avoid inserting the same tx
@@ -333,7 +333,7 @@ result_code internal_database_basis<Clock>::remove_history_db(short_hash const&
     if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) {
 
         auto data = db_value_to_data_chunk(value);
-        auto entry = domain::create<history_entry>(data);
+        auto entry = domain::create_old<history_entry>(data);
 
         if (entry.height() == height) {
 
@@ -346,7 +346,7 @@ result_code internal_database_basis<Clock>::remove_history_db(short_hash const&
         while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) {
 
             auto data = db_value_to_data_chunk(value);
-            auto entry = domain::create<history_entry>(data);
+            auto entry = domain::create_old<history_entry>(data);
 
             if (entry.height() == height) {
                 if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) {

include/kth/database/databases/history_entry.hpp

@@ -40,26 +40,8 @@ class KD_API history_entry {
         factory_to_data(sink,id_, point_, point_kind_, height_, index_, value_or_checksum_ );
     }
 
-    bool from_data(const data_chunk& data);
-    bool from_data(std::istream& stream);
-
-    template <typename R, KTH_IS_READER(R)>
-    bool from_data(R& source) {
-        reset();
-
-        id_ = source.read_8_bytes_little_endian();
-        point_.from_data(source, false);
-        point_kind_ = static_cast<domain::chain::point_kind>(source.read_byte()),
-        height_ = source.read_4_bytes_little_endian();
-        index_ = source.read_4_bytes_little_endian();
-        value_or_checksum_ = source.read_8_bytes_little_endian();
-
-        if ( ! source) {
-            reset();
-        }
-
-        return source;
-    }
+    static
+    expect<history_entry> from_data(byte_reader& reader);
 
     static
     data_chunk factory_to_data(uint64_t id, domain::chain::point const& point, domain::chain::point_kind kind, uint32_t height, uint32_t index, uint64_t value_or_checksum);

include/kth/database/databases/internal_database.ipp

@@ -251,6 +251,7 @@ result_code internal_database_basis<Clock>::push_block(domain::chain::block cons
 #endif // ! defined(KTH_DB_READONLY)
 
 
+//TODO: change the function interface to return expect<utxo_entry>
 template <typename Clock>
 utxo_entry internal_database_basis<Clock>::get_utxo(domain::chain::output_point const& point, KTH_DB_txn* db_txn) const {
 
@@ -263,7 +264,7 @@ utxo_entry internal_database_basis<Clock>::get_utxo(domain::chain::output_point
         return utxo_entry{};
     }
 
-    return domain::create<utxo_entry>(db_value_to_data_chunk(value));
+    return domain::create_old<utxo_entry>(db_value_to_data_chunk(value));
 }
 
 template <typename Clock>
@@ -411,13 +412,13 @@ domain::chain::header::list internal_database_basis<Clock>::get_headers(uint32_t
     }
 
     auto data = db_value_to_data_chunk(value);
-    list.push_back(domain::create<domain::chain::header>(data));
+    list.push_back(domain::create_old<domain::chain::header>(data));
 
     while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) {
         auto height = *static_cast<uint32_t*>(kth_db_get_data(key));
         if (height > to) break;
         auto data = db_value_to_data_chunk(value);
-        list.push_back(domain::create<domain::chain::header>(data));
+        list.push_back(domain::create_old<domain::chain::header>(data));
     }
 
     kth_db_cursor_close(cursor);
@@ -523,10 +524,10 @@ result_code internal_database_basis<Clock>::insert_reorg_into_pool(utxo_pool_t&
     }
 
     auto entry_data = db_value_to_data_chunk(value);
-    auto entry = domain::create<utxo_entry>(entry_data);
+    auto entry = domain::create_old<utxo_entry>(entry_data);
 
     auto point_data = db_value_to_data_chunk(key_point);
-    auto point = domain::create<domain::chain::output_point>(point_data, KTH_INTERNAL_DB_WIRE);
+    auto point = domain::create_old<domain::chain::output_point>(point_data, KTH_INTERNAL_DB_WIRE);
     pool.insert({point, std::move(entry)});     //TODO(fernando): use emplace?
 
     return result_code::success;
@@ -674,7 +675,10 @@ bool internal_database_basis<Clock>::create_and_open_environment() {
     //     throw0(DB_ERROR(lmdb_error("Failed to set max number of readers: ", result).c_str()));
     // ----------------------------------------------------------------------------------------------------------------
 
-    auto res = kth_db_env_set_mapsize(env_, adjust_db_size(db_max_size_));
+    LOG_DEBUG(LOG_DATABASE, "DB max size (before adjust): ", db_max_size_);
+    auto const adjusted_db_size = adjust_db_size(db_max_size_);
+    LOG_DEBUG(LOG_DATABASE, "Adjusted DB size: ", adjusted_db_size);
+    auto res = kth_db_env_set_mapsize(env_, adjusted_db_size);
     if (res != KTH_DB_SUCCESS) {
         LOG_ERROR(LOG_DATABASE, "Error setting max memory map size. Verify do you have enough free space. [create_and_open_environment] ", static_cast<int32_t>(res));
         return false;
@@ -709,6 +713,8 @@ bool internal_database_basis<Clock>::create_and_open_environment() {
         mdb_flags |= KTH_DB_WRITEMAP | KTH_DB_MAPASYNC;
     }
 
+    LOG_DEBUG(LOG_DATABASE, "Opening LMDB Environment in directory: ", db_dir_.string());
+    LOG_DEBUG(LOG_DATABASE, "LMDB open flags: ", mdb_flags);
     res = kth_db_env_open(env_, db_dir_.string().c_str(), mdb_flags, env_open_mode_);
     return res == KTH_DB_SUCCESS;
 }

include/kth/database/databases/reorg_database.ipp

@@ -154,7 +154,7 @@ domain::chain::block internal_database_basis<Clock>::get_block_reorg(uint32_t he
     }
 
     auto data = db_value_to_data_chunk(value);
-    auto res = domain::create<domain::chain::block>(data);       //TODO(fernando): mover fuera de la DbTx
+    auto res = domain::create_old<domain::chain::block>(data);       //TODO(fernando): mover fuera de la DbTx
     return res;
 }
 

include/kth/database/databases/spend_database.ipp

@@ -39,7 +39,7 @@ domain::chain::input_point internal_database_basis<Clock>::get_spend(domain::cha
         return domain::chain::input_point{};
     }
 
-    auto res = domain::create<domain::chain::input_point>(data);
+    auto res = domain::create_old<domain::chain::input_point>(data);
     return res;
 }
 

include/kth/database/databases/transaction_database.ipp

@@ -76,7 +76,7 @@ transaction_entry internal_database_basis<Clock>::get_transaction(uint64_t id, K
     }
 
     auto data = db_value_to_data_chunk(value);
-    auto entry = domain::create<transaction_entry>(data);
+    auto entry = domain::create_old<transaction_entry>(data);
 
     return entry;
 }

include/kth/database/databases/transaction_entry.hpp

@@ -52,29 +52,8 @@ class KD_API transaction_entry {
         factory_to_data(sink, transaction_, height_, median_time_past_, position_ );
     }
 
-    bool from_data(const data_chunk& data);
-    bool from_data(std::istream& stream);
-
-
-    template <typename R, KTH_IS_READER(R)>
-    bool from_data(R& source) {
-        reset();
-
-#if defined(KTH_CACHED_RPC_DATA)
-        transaction_.from_data(source, false, true, false);
-#else
-        transaction_.from_data(source, false, true);
-#endif
-        height_ = source.read_4_bytes_little_endian();
-        median_time_past_ = source.read_4_bytes_little_endian();
-        position_ = read_position(source);
-
-        if ( ! source) {
-            reset();
-        }
-
-        return source;
-    }
+    static
+    expect<transaction_entry> from_data(byte_reader& reader);
 
     bool confirmed() const;
 
@@ -90,12 +69,7 @@ class KD_API transaction_entry {
     template <typename W, KTH_IS_WRITER(W)>
     static
     void factory_to_data(W& sink, domain::chain::transaction const& tx, uint32_t height, uint32_t median_time_past, uint32_t position) {
-#if defined(KTH_CACHED_RPC_DATA)
-        tx.to_data(sink, false, true, false);
-#else
-        tx.to_data(sink, false, true);
-#endif
-
+        tx.to_data(sink, false);
         sink.write_4_bytes_little_endian(height);
         sink.write_4_bytes_little_endian(median_time_past);
         write_position(sink, position);

include/kth/database/databases/transaction_unconfirmed_database.ipp

@@ -37,7 +37,7 @@ transaction_unconfirmed_entry internal_database_basis<Clock>::get_transaction_un
     }
 
     auto data = db_value_to_data_chunk(value);
-    auto res = domain::create<transaction_unconfirmed_entry>(data);
+    auto res = domain::create_old<transaction_unconfirmed_entry>(data);
 
     return res;
 }
@@ -68,12 +68,12 @@ std::vector<transaction_unconfirmed_entry> internal_database_basis<Clock>::get_a
     if ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == 0) {
 
         auto data = db_value_to_data_chunk(value);
-        auto res = domain::create<transaction_unconfirmed_entry>(data);
+        auto res = domain::create_old<transaction_unconfirmed_entry>(data);
         result.push_back(res);
 
         while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == 0) {
             auto data = db_value_to_data_chunk(value);
-            auto res = domain::create<transaction_unconfirmed_entry>(data);
+            auto res = domain::create_old<transaction_unconfirmed_entry>(data);
             result.push_back(res);
         }
     }

include/kth/database/databases/transaction_unconfirmed_entry.hpp

@@ -41,29 +41,8 @@ class KD_API transaction_unconfirmed_entry {
         factory_to_data(sink, transaction_, arrival_time_, height_);
     }
 
-    bool from_data(const data_chunk& data);
-    bool from_data(std::istream& stream);
-
-
-    template <typename R, KTH_IS_READER(R)>
-    bool from_data(R& source) {
-        reset();
-
-#if defined(KTH_CACHED_RPC_DATA)
-        transaction_.from_data(source, false, true, true);
-#else
-        transaction_.from_data(source, false, true);
-#endif
-        arrival_time_ = source.read_4_bytes_little_endian();
-
-        height_ = source.read_4_bytes_little_endian();
-
-        if ( ! source) {
-            reset();
-        }
-
-        return source;
-    }
+    static
+    expect<transaction_unconfirmed_entry> from_data(byte_reader& reader);
 
     static
     data_chunk factory_to_data(domain::chain::transaction const& tx, uint32_t arrival_time, uint32_t height);
@@ -74,16 +53,9 @@ class KD_API transaction_unconfirmed_entry {
     template <typename W, KTH_IS_WRITER(W)>
     static
     void factory_to_data(W& sink, domain::chain::transaction const& tx, uint32_t arrival_time, uint32_t height) {
-
-#if defined(KTH_CACHED_RPC_DATA)
-        tx.to_data(sink, false, true, true);
-#else
-        tx.to_data(sink, false, true);
-#endif
-
+        tx.to_data(sink, false);
         sink.write_4_bytes_little_endian(arrival_time);
         sink.write_4_bytes_little_endian(height);
-
     }
 
 private:

include/kth/database/databases/utxo_entry.hpp

@@ -36,24 +36,8 @@ class KD_API utxo_entry {
         to_data_fixed(sink, height_, median_time_past_, coinbase_);
     }
 
-    bool from_data(const data_chunk& data);
-    bool from_data(std::istream& stream);
-
-    template <typename R, KTH_IS_READER(R)>
-    bool from_data(R& source) {
-        reset();
-
-        output_.from_data(source, false);
-        height_ = source.read_4_bytes_little_endian();
-        median_time_past_ = source.read_4_bytes_little_endian();
-        coinbase_ = source.read_byte();
-
-        if ( ! source) {
-            reset();
-        }
-
-        return source;
-    }
+    static
+    expect<utxo_entry> from_data(byte_reader& reader);
 
     static
     data_chunk to_data_fixed(uint32_t height, uint32_t median_time_past, bool coinbase);