Skip to content

[chore](cloud) Print new value json when set_value #57385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 29, 2025
Merged

[chore](cloud) Print new value json when set_value #57385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c555ede
[chore](cloud) Print new value json when set_value
gavinchou Oct 27, 2025
4377238
fix
gavinchou Oct 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 27 additions & 18 deletions cloud/src/meta-service/http_encode_key.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <google/protobuf/util/json_util.h>

#include <bit>
#include <chrono>
#include <fstream>
#include <iomanip>
#include <sstream>
Expand Down Expand Up @@ -240,15 +241,16 @@ static std::unordered_map<std::string_view,
};
// clang-format on

static MetaServiceResponseStatus encode_key(const brpc::URI& uri, std::string& key) {
static MetaServiceResponseStatus encode_key(const brpc::URI& uri, std::string* key,
std::string* key_type = nullptr) {
MetaServiceResponseStatus status;
status.set_code(MetaServiceCode::OK);
std::string_view key_type = http_query(uri, "key_type");
auto it = param_set.find(key_type);
std::string_view kt = http_query(uri, "key_type");
if (key_type != nullptr) *key_type = kt;
auto it = param_set.find(kt);
if (it == param_set.end()) {
status.set_code(MetaServiceCode::INVALID_ARGUMENT);
status.set_msg(fmt::format("key_type not supported: {}",
(key_type.empty() ? "(empty)" : key_type)));
status.set_msg(fmt::format("key_type not supported: {}", (kt.empty() ? "(empty)" : kt)));
return status;
}
auto& key_params = std::get<0>(it->second);
Expand All @@ -264,16 +266,18 @@ static MetaServiceResponseStatus encode_key(const brpc::URI& uri, std::string& k
params.emplace_back(p);
}
auto& key_encoding_function = std::get<1>(it->second);
key = key_encoding_function(params);
*key = key_encoding_function(params);
return status;
}

HttpResponse process_http_get_value(TxnKv* txn_kv, const brpc::URI& uri) {
std::string key;
std::string key_type;
if (auto hex_key = http_query(uri, "key"); !hex_key.empty()) {
key = unhex(hex_key);
key_type = http_query(uri, "key_type");
} else { // Encode key from params
auto st = encode_key(uri, key);
auto st = encode_key(uri, &key, &key_type);
if (st.code() != MetaServiceCode::OK) {
return http_json_reply(st);
}
Expand All @@ -285,7 +289,6 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const brpc::URI& uri) {
fmt::format("failed to create txn, err={}", err));
}

std::string_view key_type = http_query(uri, "key_type");
auto it = param_set.find(key_type);
if (it == param_set.end()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
Expand Down Expand Up @@ -330,17 +333,20 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const brpc::URI& uri) {
}

std::string handle_kv_output(std::string_view key, std::string_view value,
std::string_view original_value_json,
std::string_view original_value_json, std::string_view new_value_json,
std::string_view serialized_value_to_save) {
std::stringstream final_output;
final_output << "original_value_hex=" << hex(value) << "\n"
<< "key_hex=" << hex(key) << "\n"
final_output << "key_hex=" << hex(key) << "\n"
<< "original_value_json=" << original_value_json << "\n"
<< "changed_value_hex=" << hex(serialized_value_to_save) << "\n";
<< "new_value_json=" << new_value_json << "\n"
<< "original_value_hex=" << hex(value) << "\n"
<< "new_value_hex=" << hex(serialized_value_to_save) << "\n";
std::string final_json_str = final_output.str();
LOG(INFO) << final_json_str;
if (final_json_str.size() > 25000) {
std::string file_path = fmt::format("/tmp/{}.txt", hex(key));
using namespace std::chrono;
auto ts = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
std::string file_path = fmt::format("/tmp/{}_{}.txt", hex(key), ts);
LOG(INFO) << "write to file=" << file_path << ", key=" << hex(key)
<< " size=" << final_json_str.size();
try {
Expand All @@ -350,7 +356,7 @@ std::string handle_kv_output(std::string_view key, std::string_view value,
kv_file.close();
}
} catch (...) {
LOG(INFO) << "write tmp file failed.";
LOG(INFO) << "write tmp file failed: " << file_path;
}
}

Expand All @@ -363,10 +369,12 @@ HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) {
LOG(INFO) << "set value, body=" << body;

std::string key;
std::string key_type;
if (auto hex_key = http_query(uri, "key"); !hex_key.empty()) {
key = unhex(hex_key);
key_type = http_query(uri, "key_type");
} else { // Encode key from params
auto st = encode_key(uri, key);
auto st = encode_key(uri, &key, &key_type);
if (st.code() != MetaServiceCode::OK) {
return http_json_reply(st);
}
Expand All @@ -378,13 +386,13 @@ HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) {
fmt::format("failed to create txn, err={}", err));
}

std::string_view key_type = http_query(uri, "key_type");
auto it = param_set.find(key_type);
if (it == param_set.end()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
fmt::format("key_type not supported: {}",
(key_type.empty() ? "(empty)" : key_type)));
}

auto& json_parsing_function = std::get<3>(it->second);
std::shared_ptr<google::protobuf::Message> pb_to_save = json_parsing_function(body);
if (pb_to_save == nullptr) {
Expand Down Expand Up @@ -458,14 +466,15 @@ HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) {
LOG(WARNING) << "set_value saved, key=" << hex(key);

std::string final_json_str =
handle_kv_output(key, value.value(), original_value_json, serialized_value_to_save);
handle_kv_output(key, value.value(), original_value_json, proto_to_json(*pb_to_save),
serialized_value_to_save);

return http_text_reply(MetaServiceCode::OK, "", final_json_str);
}

HttpResponse process_http_encode_key(const brpc::URI& uri) {
std::string key;
auto st = encode_key(uri, key);
auto st = encode_key(uri, &key);
if (st.code() != MetaServiceCode::OK) {
return http_json_reply(st);
}
Expand Down
11 changes: 9 additions & 2 deletions cloud/test/http_encode_key_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ struct Input {
std::vector<std::string> key;
std::function<std::vector<std::string>()> gen_value;
std::string_view value;

std::string to_string() {
std::stringstream ss;
ss << "key_type=" << key_type << " param=" << param << " key=" << key[0]
<< " value=" << value;
return ss.str();
}
};

// clang-format off
Expand Down Expand Up @@ -618,15 +625,15 @@ TEST(HttpGetValueTest, process_http_get_value_test_cover_all_template) {
// std::cout << url << std::endl;
ASSERT_EQ(uri.SetHttpURL(url), 0); // clear and set query string
auto http_res = process_http_get_value(txn_kv.get(), uri);
EXPECT_EQ(http_res.status_code, 200);
EXPECT_EQ(http_res.status_code, 200) << url << " " << input.to_string();
// std::cout << http_res.body << std::endl;
EXPECT_EQ(http_res.body, input.value);
// Key mode
url = gen_url(input, false);
// std::cout << url << std::endl;
ASSERT_EQ(uri.SetHttpURL(url), 0); // clear and set query string
http_res = process_http_get_value(txn_kv.get(), uri);
EXPECT_EQ(http_res.status_code, 200);
EXPECT_EQ(http_res.status_code, 200) << url << " " << input.to_string();
// std::cout << http_res.body << std::endl;
EXPECT_EQ(http_res.body, input.value);
}
Expand Down
7 changes: 4 additions & 3 deletions cloud/test/meta_service_http_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2096,10 +2096,11 @@ TEST(HttpEncodeKeyTest, ProcessHttpSetValue) {
auto response = process_http_set_value(txn_kv.get(), &cntl);
EXPECT_EQ(response.status_code, 200) << response.msg;
std::stringstream final_json;
final_json << "original_value_hex=" << hex(initial_rowset_meta.SerializeAsString()) << "\n"
<< "key_hex=" << hex(initial_key) << "\n"
final_json << "key_hex=" << hex(initial_key) << "\n"
<< "original_value_json=" << proto_to_json(initial_rowset_meta) << "\n"
<< "changed_value_hex=" << hex(new_rowset_meta.SerializeAsString()) << "\n";
<< "new_value_json=" << proto_to_json(new_rowset_meta) << "\n"
<< "original_value_hex=" << hex(initial_rowset_meta.SerializeAsString()) << "\n"
<< "new_value_hex=" << hex(new_rowset_meta.SerializeAsString()) << "\n";
// std::cout << "xxx " << final_json.str() << std::endl;
EXPECT_EQ(response.body, final_json.str());

Expand Down
Loading