Automatically issue and import fast sync overlay certificates

Author: SpyCheese

adnl/adnl-node-id.hpp

@@ -37,7 +37,9 @@ class AdnlNodeIdShort {
   }
   explicit AdnlNodeIdShort(td::Bits256 value) : hash_(value) {
   }
-  explicit AdnlNodeIdShort(tl_object_ptr<ton_api::adnl_id_short> obj) : hash_(obj->id_) {
+  explicit AdnlNodeIdShort(tl_object_ptr<ton_api::adnl_id_short> &&obj) : hash_(obj->id_) {
+  }
+  explicit AdnlNodeIdShort(const tl_object_ptr<ton_api::adnl_id_short> &obj) : hash_(obj->id_) {
   }
 
   const auto &pubkey_hash() const {

dht-server/dht-server.cpp

@@ -171,8 +171,8 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
   }
   return ton::create_tl_object<ton::ton_api::engine_validator_config>(
       out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec),
-      ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
-      nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec));
+      ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), nullptr,
+      nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec));
 }
 
 td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
@@ -1195,7 +1195,8 @@ int main(int argc, char *argv[]) {
     SET_VERBOSITY_LEVEL(v);
   });
   p.add_option('V', "version", "shows dht-server build information", [&]() {
-    std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n";
+    std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1()
+              << ", Date: " << GitMetadata::CommitDate() << "]\n";
     std::exit(0);
   });
   p.add_option('h', "help", "prints_help", [&]() {

tl/generate/scheme/ton_api.tl

@@ -464,6 +464,8 @@ tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:
 tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof;
 
 tonNode.forgetPeer = tonNode.ForgetPeer;
+tonNode.newFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = tonNode.NewFastSyncMemberCertificate;
+
 
 ---functions---
 
@@ -501,6 +503,8 @@ tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success;
 
 tonNode.query = Object;
 
+tonNode.requestFastSyncOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int = overlay.MemberCertificate;
+
 ---types---
 
 // bit 0 - started
@@ -654,9 +658,11 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode
 engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave;
 engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig;
 engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate;
+engine.validator.fastSyncOverlayClient adnl_id:int256 slot:int = engine.validator.FastSyncOverlayClient;
 engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist;
 engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate)
-    collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig;
+    collator_node_whitelist:engine.validator.collatorNodeWhitelist
+    fast_sync_overlay_clients:(vector engine.validator.fastSyncOverlayClient) = engine.validator.ExtraConfig;
 engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl)
         dht:(vector engine.dht)
         validators:(vector engine.validator) collators:(vector engine.collator)
@@ -840,6 +846,8 @@ engine.validator.getCollationManagerStats = engine.validator.CollationManagerSta
 
 engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate;
 engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success;
+engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success;
+engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success;
 
 ---types---
 

tl/generate/scheme/ton_api.tlo

@@ -807,21 +807,21 @@ td::Status SignCertificateQuery::send() {
   auto sign = ton::create_serialize_tl_object<ton::ton_api::engine_validator_sign>(signer_.tl(), std::move(cid));
   auto pub = ton::create_serialize_tl_object<ton::ton_api::engine_validator_exportPublicKey>(signer_.tl());
   td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub),
-      td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
-        if (R.is_error()) {
-          td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error());
-        } else {
-          td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok());
-        }
-      }));
+                          td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+                            if (R.is_error()) {
+                              td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error());
+                            } else {
+                              td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok());
+                            }
+                          }));
   td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign),
-      td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
-        if (R.is_error()) {
-          td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error());
-        } else {
-          td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok());
-        }
-      }));
+                          td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
+                            if (R.is_error()) {
+                              td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error());
+                            } else {
+                              td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok());
+                            }
+                          }));
   return td::Status::OK();
 }
 
@@ -833,33 +833,32 @@ void SignCertificateQuery::receive_pubkey(td::BufferSlice R) {
   }
   pubkey_ = f.move_as_ok();
   has_pubkey_ = true;
-  if(has_signature_) {
+  if (has_signature_) {
     save_certificate();
   }
 }
 
-
 td::Status SignCertificateQuery::receive(td::BufferSlice data) {
   UNREACHABLE();
 }
 
 void SignCertificateQuery::receive_signature(td::BufferSlice R) {
   auto f = ton::fetch_tl_object<ton::ton_api::engine_validator_signature>(R.as_slice(), true);
-  if(f.is_error()){
+  if (f.is_error()) {
     handle_error(f.move_as_error_prefix("Failed to get signature: "));
     return;
   }
   signature_ = std::move(f.move_as_ok()->signature_);
-  if(has_pubkey_) {
+  if (has_pubkey_) {
     save_certificate();
   }
 }
 
 void SignCertificateQuery::save_certificate() {
-  auto c = ton::create_serialize_tl_object<ton::ton_api::overlay_certificate>(
-        std::move(pubkey_), expire_at_, max_size_, std::move(signature_));
+  auto c = ton::create_serialize_tl_object<ton::ton_api::overlay_certificate>(std::move(pubkey_), expire_at_, max_size_,
+                                                                              std::move(signature_));
   auto w = td::write_file(out_file_, c.as_slice());
-  if(w.is_error()) {
+  if (w.is_error()) {
     handle_error(w.move_as_error_prefix("Failed to write certificate to file: "));
     return;
   }
@@ -880,11 +879,8 @@ td::Status ImportCertificateQuery::send() {
   TRY_RESULT_PREFIX(cert, ton::fetch_tl_object<ton::ton_api::overlay_Certificate>(data.as_slice(), true),
                     "incorrect certificate");
   auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importCertificate>(
-                overlay_,
-                ton::create_tl_object<ton::ton_api::adnl_id_short>(id_),
-                ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(kh_.tl()),
-                std::move(cert)
-           );
+      overlay_, ton::create_tl_object<ton::ton_api::adnl_id_short>(id_),
+      ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(kh_.tl()), std::move(cert));
   td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
   return td::Status::OK();
 }
@@ -1031,18 +1027,16 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) {
   return td::Status::OK();
 }
 
-
 td::Status ImportCertificateQuery::receive(td::BufferSlice data) {
   TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
                     "received incorrect answer: ");
   td::TerminalIO::out() << "successfully sent certificate to overlay manager\n";
   return td::Status::OK();
 }
 
-
 td::Status SignShardOverlayCertificateQuery::run() {
   TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
-  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>() );
+  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>());
   TRY_RESULT_ASSIGN(key_, tokenizer_.get_token<ton::PublicKeyHash>());
   TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token<td::int32>());
   TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token<td::uint32>());
@@ -1052,8 +1046,8 @@ td::Status SignShardOverlayCertificateQuery::run() {
 }
 
 td::Status SignShardOverlayCertificateQuery::send() {
-  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_signShardOverlayCertificate>
-             (wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), expire_at_, max_size_);
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_signShardOverlayCertificate>(
+      wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), expire_at_, max_size_);
   td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
   return td::Status::OK();
 }
@@ -1062,7 +1056,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) {
   TRY_RESULT_PREFIX(c, ton::fetch_tl_object<ton::ton_api::overlay_certificate>(data.as_slice(), true),
                     "received incorrect cert: ");
   auto w = td::write_file(out_file_, data.as_slice());
-  if(w.is_error()) {
+  if (w.is_error()) {
     return w.move_as_error_prefix("Failed to write certificate to file: ");
   }
   td::TerminalIO::out() << "saved certificate\n";
@@ -1072,7 +1066,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) {
 
 td::Status ImportShardOverlayCertificateQuery::run() {
   TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token<td::int32>());
-  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>() );
+  TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token<td::int64>());
   TRY_RESULT_ASSIGN(key_, tokenizer_.get_token<ton::PublicKeyHash>());
   TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token<std::string>());
 
@@ -1083,8 +1077,8 @@ td::Status ImportShardOverlayCertificateQuery::send() {
   TRY_RESULT(data, td::read_file(in_file_));
   TRY_RESULT_PREFIX(cert, ton::fetch_tl_object<ton::ton_api::overlay_Certificate>(data.as_slice(), true),
                     "incorrect certificate");
-  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importShardOverlayCertificate>
-             (wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), std::move(cert));
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_importShardOverlayCertificate>(
+      wc_, shard_, ton::create_tl_object<ton::ton_api::engine_validator_keyHash>(key_.tl()), std::move(cert));
   td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
   return td::Status::OK();
 }
@@ -1908,3 +1902,42 @@ td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) {
   td::TerminalIO::out() << "success\n";
   return td::Status::OK();
 }
+
+td::Status AddFastSyncOverlayClientQuery::run() {
+  TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token<td::Bits256>());
+  TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token<td::int32>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status AddFastSyncOverlayClientQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_addFastSyncClient>(adnl_id_, slot_);
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status AddFastSyncOverlayClientQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}
+
+td::Status DelFastSyncOverlayClientQuery::run() {
+  TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token<td::Bits256>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status DelFastSyncOverlayClientQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_delFastSyncClient>(adnl_id_);
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}