diff --git a/CHANGES b/CHANGES index 53dee85..b1eb4c9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,11 @@ +-- 2.1.2 (2024-02-17) +FEATURES +* show resource usage in Gazelle and Grafana report endpoints + +-- 2.1.1 (2024-02-15) +BUILD +* Detabbed source code in first step towards having the code lint cleanly, no code changes + -- 2.1.0 (2024-02-11) FEATURES * announce jitter is now a uniform distribution and can be modified on the fly diff --git a/Dockerfile b/Dockerfile index 5107f03..b6d0dd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN cmake -Wno-dev -S /srv -B /srv/build \ pkg-config \ && apt-get autoremove -y \ && apt-get clean -y \ + && mkdir -p /tmp/ocelot \ && mv /srv/build/ocelot /srv/ocelot \ && mv /srv/ocelot.conf.dist /srv/ocelot.conf diff --git a/ocelot.conf.dist b/ocelot.conf.dist index 813b77b..7517c05 100644 --- a/ocelot.conf.dist +++ b/ocelot.conf.dist @@ -34,4 +34,7 @@ schedule_interval = 3 log = false log_path = /tmp/ocelot +# the report path should be placed on a tmpfs partition in production +report_path = /tmp/ocelot + readonly = false diff --git a/src/config.cpp b/src/config.cpp index 3a29335..8135b32 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -100,6 +100,8 @@ void config::init() { add("log", false); add("log_path", "ocelot"); // path to where to write log + name of log (don't need to put file extension) + add("report_path", "/tmp"); // path for transfer to website (should be a tmpfs in production) + // Debugging add("readonly", false); } diff --git a/src/jemalloc_parse.cpp b/src/jemalloc_parse.cpp new file mode 100644 index 0000000..d83fae1 --- /dev/null +++ b/src/jemalloc_parse.cpp @@ -0,0 +1,203 @@ +#include +#include +#include +#include + +#include "jemalloc_parse.h" + +/* parse a size_t decimal text representation */ +const char *parse_sz(const char *in, const char *find, size_t *target) { + const char *p = strstr(in, find); + if (!p) { + // didn't find what we were looking for + return NULL; + } + p += strlen(find); + + // copy the digits to a null-terminated string in order to pass it to strtoul() + // the largest unsigned long decimal representation is 4294967295 (10 bytes) + char convert[11]; + char *c = convert; + while (isspace(*p)) { + p++; + } + while (isdigit(*p) && c < convert + sizeof(convert) - 1) { + *c++ = *p++; + } + *c = '\0'; + *target = strtoul(convert, &c, 10); + + return p + 1; // one past the last digit +} + +/* parse an unsigned long long decimal text representation */ +const char *parse_ull(const char *in, const char *find, unsigned long long *target) { + const char *p = strstr(in, find); + if (!p) { + // didn't find what we were looking for + return NULL; + } + p += strlen(find); + + // copy the digits to a null-terminated string in order to pass it to strtoull() + // the largest unsigned long long decimal representation is 18446744073709551615 (21 bytes) + char convert[22]; + char *c = convert; + while (isspace(*p)) { + p++; + } + while(isdigit(*p) && c < convert + sizeof(convert) - 1) { + *c++ = *p++; + } + *c = '\0'; + *target = strtoull(convert, &c, 10); + + return p + 1; // one past the last digit +} + +int jemalloc_parse(const char *in, struct ocelot_alloc_info *out) { + /* Parse the output of jemalloc_status() plain output + * This is an ugly hack; it would be faster to build directly what we want + * rather than walk down a formatted string that has generated lots of + * information that is discarded. + * If the input breaks, the error code indicates where the parsing failed. + */ + + // set everything to zero + memset (out, 0, sizeof(struct ocelot_alloc_info)); + + /* +___ Begin jemalloc statistics ___ +Version: "5.3.0-0-g54eaed1d8b56b1aa528be3bdd1877e59c56fa90c" +Build-time option settings + ... +Run-time option settings + ... +Profiling settings + ... +Arenas: 17 +Quantum size: 16 +Page size: 4096 +Maximum thread-cached size class: 32768 +Number of bin size classes: 36 +Number of thread-cache bin size classes: 41 +Number of large size classes: 196 + */ + if (!(in = strstr(in, "___ Begin jemalloc statistics ___"))) { + return 1; + } + + if (!(in = parse_sz(in, "Arenas: ", &out->nr_arena))) { + return 10; + } + if (!(in = parse_sz(in, "Number of bin size classes: ", &out->nr_bin_small))) { + return 11; + } + if (!(in = parse_sz(in, "Number of thread-cache bin size classes: ", &out->nr_bin_tc))) { + return 12; + } + if (!(in = parse_sz(in, "Number of large size classes: ", &out->nr_bin_large))) { + return 13; + } + + // Allocated: 895744, active: 1024000, metadata: 3047664 (n_thp 0), resident: 3985408, mapped: 7315456, retained: 1073152 + if (!(in = parse_ull(in, "Allocated: ", &out->mem_allocated))) { + return 20; + } + if (!(in = parse_ull(in, "active: ", &out->mem_active))) { + return 21; + } + if (!(in = parse_ull(in, "metadata: ", &out->mem_metadata))) { + return 22; + } + if (!(in = parse_ull(in, "resident: ", &out->mem_resident))) { + return 23; + } + if (!(in = parse_ull(in, "mapped: ", &out->mem_mapped))) { + return 24; + } + if (!(in = parse_ull(in, "retained: ", &out->mem_retained))) { + return 25; + } + + /* + allocated nmalloc (#/sec) ndalloc (#/sec) nrequests (#/sec) nfill (#/sec) nflush (#/sec) +small: 247840 131452 0 129036 0 24219 0 2402 0 8543 0 +large: 290816 815 0 808 0 971 0 815 0 2351 0 +total: 538656 132267 0 129844 0 + */ + if (!(in = strstr(in, "allocated nmalloc (#/sec) ndalloc (#/sec)"))) { + return 30; + } + if (!(in = strstr(in, "\nsmall:"))) { + return 31; + } + + if (!(in = parse_sz(in, " ", &out->small.allocated))) { + return 40; + } + if (!(in = parse_sz(in, " ", &out->small.nmalloc_total))) { + return 41; + } + if (!(in = parse_sz(in, " ", &out->small.ndalloc_rate))) { + return 42; + } + if (!(in = parse_sz(in, " ", &out->small.ndalloc_total))) { + return 43; + } + if (!(in = parse_sz(in, " ", &out->small.nrequests_rate))) { + return 44; + } + if (!(in = parse_sz(in, " ", &out->small.nrequests_total))) { + return 45; + } + if (!(in = parse_sz(in, " ", &out->small.nfill_rate))) { + return 46; + } + if (!(in = parse_sz(in, " ", &out->small.nfill_total))) { + return 47; + } + if (!(in = parse_sz(in, " ", &out->small.nflush_rate))) { + return 48; + } + if (!(in = parse_sz(in, " ", &out->small.nflush_total))) { + return 49; + } + + if (!(in = strstr(in, "\nlarge:"))) { + return 60; + } + + if (!(in = parse_sz(in, " ", &out->large.allocated))) { + return 70; + } + if (!(in = parse_sz(in, " ", &out->large.nmalloc_total))) { + return 71; + } + if (!(in = parse_sz(in, " ", &out->large.ndalloc_rate))) { + return 72; + } + if (!(in = parse_sz(in, " ", &out->large.ndalloc_total))) { + return 73; + } + if (!(in = parse_sz(in, " ", &out->large.nrequests_rate))) { + return 74; + } + if (!(in = parse_sz(in, " ", &out->large.nrequests_total))) { + return 75; + } + if (!(in = parse_sz(in, " ", &out->large.nfill_rate))) { + return 76; + } + if (!(in = parse_sz(in, " ", &out->large.nfill_total))) { + return 77; + } + if (!(in = parse_sz(in, " ", &out->large.nflush_rate))) { + return 78; + } + if (!(in = parse_sz(in, " ", &out->large.nflush_total))) { + return 79; + } + + return 0; +} diff --git a/src/jemalloc_parse.h b/src/jemalloc_parse.h new file mode 100644 index 0000000..6b15732 --- /dev/null +++ b/src/jemalloc_parse.h @@ -0,0 +1,37 @@ +#ifndef SRC_PARSE_JEMALLOC_H_ +#define SRC_PARSE_JEMALLOC_H_ + +struct ocelot_alloc_stat { + size_t allocated; + size_t nmalloc_total; + size_t nmalloc_rate; + size_t ndalloc_total; + size_t ndalloc_rate; + size_t nrequests_total; + size_t nrequests_rate; + size_t nfill_total; + size_t nfill_rate; + size_t nflush_total; + size_t nflush_rate; +}; + +struct ocelot_alloc_info { + size_t nr_arena; + size_t nr_bin_small; + size_t nr_bin_tc; + size_t nr_bin_large; + + unsigned long long mem_allocated; + unsigned long long mem_active; + unsigned long long mem_metadata; + unsigned long long mem_resident; + unsigned long long mem_mapped; + unsigned long long mem_retained; + + struct ocelot_alloc_stat small; + struct ocelot_alloc_stat large; +}; + +int jemalloc_parse(const char *in, struct ocelot_alloc_info *out); + +#endif // SRC_PARSE_JEMALLOC_H_ diff --git a/src/ocelot.cpp b/src/ocelot.cpp index 7586c2c..dcad0cc 100644 --- a/src/ocelot.cpp +++ b/src/ocelot.cpp @@ -27,7 +27,7 @@ static schedule *sched; struct stats_t stats; const char * version() { - return "2.1.1"; + return "2.1.3"; } static void create_daemon() { @@ -113,7 +113,7 @@ int main(int argc, char **argv) { conf_arg = true; conf_file_path = argv[++i]; } else if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) { - std::cout << "Ocelot version " << version() << std::endl; + std::cout << "Ocelot version " << version() << ", compiled " << __DATE__ << ' ' << __TIME__ << std::endl; return 0; } else { std::cout << "Usage: " << argv[0] << " [-v] [-c configfile] [--daemonize]" << std::endl; @@ -146,6 +146,10 @@ int main(int argc, char **argv) { auto combined_logger = std::make_shared("logger", begin(sinks), end(sinks)); // If we don't set flush on info, the file log takes a long while to actually flush combined_logger->flush_on(spdlog::level::info); + combined_logger->info( + std::string("Ocelot version ") + version() + + std::string(", compiled ") + __DATE__ + std::string(" ") + __TIME__ + ); spdlog::register_logger(combined_logger); db = new mysql(conf); diff --git a/src/report.cpp b/src/report.cpp index 0ad71c9..f18148a 100644 --- a/src/report.cpp +++ b/src/report.cpp @@ -1,16 +1,84 @@ // Copyright [2017-2024] Orpheus +#include +#include +#include +#include + +#include +#include // debug + +#include +#include #include +#include // for std::setfill() and std::setw() #include -#include +#include +#include #include "ocelot.h" #include "misc_functions.h" +#include "jemalloc_parse.h" #include "report.h" #include "response.h" #include "user.h" -std::string report(params_type ¶ms, user_list &users_list, client_opts_t &client_opts, unsigned int announce_interval, unsigned int announce_jitter) { +int jemalloc_fd; +std::mutex jemalloc_fd_mutex; + +/* Getting jemalloc stats back to Gazelle is a bit cumbersome + * because all jemalloc can do is write to a file descriptor. + * So we let it write to a file (which is why using a ramdisk + * is important) and then read it back to return as an HTTP + * response. + */ + +// callback function for malloc_stats_print() +void jemalloc_stats_export(void *opaque, const char *buf) { + if (jemalloc_fd > 0) { + write(jemalloc_fd, buf, strlen(buf)); + } +} + +int dump_jemalloc(const char *filename, const char *opts) { + // open the file descriptor + std::lock_guard guard(jemalloc_fd_mutex); + jemalloc_fd = open(filename, O_CREAT|O_WRONLY, 0600); + if (jemalloc_fd == -1) { + return jemalloc_fd; + } + + // dump stats + malloc_stats_print(jemalloc_stats_export, NULL, opts); + + // tidy up + int result = close(jemalloc_fd); + jemalloc_fd = 0; + return result; +} + +std::string report_jemalloc_plain(const char *opts, std::string path) { + std::string filename(path + "/jemalloc.json." + std::to_string(pthread_self())); + if (dump_jemalloc(filename.c_str(), opts) != 0) { + // return an empty string if something went bananas + unlink(filename.c_str()); + return std::string(""); + } + + // slurp the contents into a string + std::ifstream input(filename); + std::ostringstream output; + while(input >> output.rdbuf()) + ; + unlink(filename.c_str()); + output << "\n"; + + std::shared_ptr logger(spdlog::get("logger")); + logger->debug("jemalloc stats written to " + filename + ", size=" + std::to_string(output.str().size())); + return output.str(); +} + +std::string report(params_type ¶ms, user_list &users_list, unsigned int announce_interval, unsigned int announce_jitter) { std::stringstream output; std::string action = params["get"]; if (action == "stats") { @@ -27,6 +95,7 @@ std::string report(params_type ¶ms, user_list &users_list, client_opts_t &cl << (up_m < 10 ? "0" : "") << inttostr(up_m) << ':' << (up_s < 10 ? "0" : "") << inttostr(up_s) << "\n" << "version: " << version() << "\n" + << "jemalloc_version: " << JEMALLOC_VERSION_MAJOR << '.' << JEMALLOC_VERSION_MINOR << '.' << JEMALLOC_VERSION_BUGFIX << "\n" << stats.opened_connections << " connections opened\n" << stats.open_connections << " open connections\n" << stats.connection_rate << " connections/s\n" @@ -47,25 +116,20 @@ std::string report(params_type ¶ms, user_list &users_list, client_opts_t &cl << announce_interval << " announce interval\n" << announce_jitter << " announce jitter\n" ; - } else if (action == "prom_stats") { - time_t uptime = time(NULL) - stats.start_time; - output << "ocelot_uptime " << uptime << "\n" - << "ocelot_open_connections " << stats.open_connections << "\n" - << "ocelot_connection_rate " << stats.connection_rate << "\n" - << "ocelot_requests " << stats.requests << "\n" - << "ocelot_request_rate " << stats.request_rate << "\n" - << "ocelot_succ_announcements " << stats.succ_announcements << "\n" - << "ocelot_total_announcements " << stats.announcements << "\n" - << "ocelot_scrapes " << stats.scrapes << "\n" - << "ocelot_leechers " << stats.leechers << "\n" - << "ocelot_seeders " << stats.seeders << "\n" - << "ocelot_user_queue " << stats.user_queue_size << "\n" - << "ocelot_torrent_queue " << stats.torrent_queue_size << "\n" - << "ocelot_peer_queue " << stats.peer_queue_size << "\n" - << "ocelot_snatch_queue " << stats.snatch_queue_size << "\n" - << "ocelot_token_queue " << stats.token_queue_size << "\n" - << "ocelot_bytes_read " << stats.bytes_read << "\n" - << "ocelot_bytes_written " << stats.bytes_written << "\n#"; + + struct rusage r; + if (getrusage(RUSAGE_SELF, &r) == 0) { + output << r.ru_utime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_utime.tv_usec << " user time\n" + << r.ru_stime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_stime.tv_usec << " system time\n" + << r.ru_maxrss << " maximum resident set size\n" + << r.ru_minflt << " minor page faults\n" + << r.ru_majflt << " major page faults\n" + << r.ru_inblock << " blocks in\n" + << r.ru_oublock << " blocks out\n" + << r.ru_nvcsw << " voluntary context switches\n" + << r.ru_nivcsw << " involuntary context switches\n" + ; + } } else if (action == "user") { std::string key = params["key"]; if (key.empty()) { @@ -80,12 +144,89 @@ std::string report(params_type ¶ms, user_list &users_list, client_opts_t &cl << ",\"protected\":" << u->second->is_protected() << ",\"can_leech\":" << u->second->can_leech() << "}" << "\n"; - return response(output.str(), client_opts); + return output.str(); } } } else { output << "Invalid action\n"; } output << "success"; - return response(output.str(), client_opts); + return output.str(); +} + +std::string report_prom_stats(const char *jemalloc) { + std::stringstream output; + struct ocelot_alloc_info ji; + int result = jemalloc_parse(jemalloc, &ji); + + output << "ocelot_uptime " << time(NULL) - stats.start_time << "\n" + << "ocelot_version " << version() << "\n" + << "ocelot_open_connections " << stats.open_connections << "\n" + << "ocelot_connection_rate " << stats.connection_rate << "\n" + << "ocelot_requests " << stats.requests << "\n" + << "ocelot_request_rate " << stats.request_rate << "\n" + << "ocelot_succ_announcements " << stats.succ_announcements << "\n" + << "ocelot_total_announcements " << stats.announcements << "\n" + << "ocelot_scrapes " << stats.scrapes << "\n" + << "ocelot_leechers " << stats.leechers << "\n" + << "ocelot_seeders " << stats.seeders << "\n" + << "ocelot_user_queue " << stats.user_queue_size << "\n" + << "ocelot_torrent_queue " << stats.torrent_queue_size << "\n" + << "ocelot_peer_queue " << stats.peer_queue_size << "\n" + << "ocelot_snatch_queue " << stats.snatch_queue_size << "\n" + << "ocelot_token_queue " << stats.token_queue_size << "\n" + << "ocelot_bytes_read " << stats.bytes_read << "\n" + << "ocelot_bytes_written " << stats.bytes_written << "\n" + << "jemalloc_version " << JEMALLOC_VERSION_MAJOR << '.' << JEMALLOC_VERSION_MINOR << '.' << JEMALLOC_VERSION_BUGFIX << "\n" + << "jemalloc_parse_error " << result << "\n" + << "jemalloc_mem_allocated " << ji.mem_allocated << "\n" + << "jemalloc_arena_total " << ji.nr_arena << "\n" + << "jemalloc_bin_small_total " << ji.nr_bin_small << "\n" + << "jemalloc_bin_thread_cache_total " << ji.nr_bin_tc << "\n" + << "jemalloc_bin_large_total " << ji.nr_bin_large << "\n" + << "jemalloc_memory_allocated " << ji.mem_allocated << "\n" + << "jemalloc_memory_active " << ji.mem_active << "\n" + << "jemalloc_memory_metadata " << ji.mem_metadata << "\n" + << "jemalloc_memory_resident " << ji.mem_resident << "\n" + << "jemalloc_memory_mapped " << ji.mem_mapped << "\n" + << "jemalloc_memory_retained " << ji.mem_retained << "\n" + << "jemalloc_small_allocated " << ji.small.allocated << "\n" + << "jemalloc_small_nmalloc_total " << ji.small.nmalloc_total << "\n" + << "jemalloc_small_nmalloc_rate " << ji.small.nmalloc_rate << "\n" + << "jemalloc_small_ndalloc_total " << ji.small.ndalloc_total << "\n" + << "jemalloc_small_ndalloc_rate " << ji.small.ndalloc_rate << "\n" + << "jemalloc_small_nrequests_total " << ji.small.nrequests_total << "\n" + << "jemalloc_small_nrequests_rate " << ji.small.nrequests_rate << "\n" + << "jemalloc_small_nfill_total " << ji.small.nfill_total << "\n" + << "jemalloc_small_fill_rate " << ji.small.nfill_rate << "\n" + << "jemalloc_small_nflush_total " << ji.small.nflush_total << "\n" + << "jemalloc_small_nflush_rate " << ji.small.nflush_rate << "\n" + << "jemalloc_large_allocated " << ji.large.allocated << "\n" + << "jemalloc_large_nmalloc_total " << ji.large.nmalloc_total << "\n" + << "jemalloc_large_nmalloc_rate " << ji.large.nmalloc_rate << "\n" + << "jemalloc_large_ndalloc_total " << ji.large.ndalloc_total << "\n" + << "jemalloc_large_ndalloc_rate " << ji.large.ndalloc_rate << "\n" + << "jemalloc_large_nrequests_total " << ji.large.nrequests_total << "\n" + << "jemalloc_large_nrequests_rate " << ji.large.nrequests_rate << "\n" + << "jemalloc_large_nfill_total " << ji.large.nfill_total << "\n" + << "jemalloc_large_fill_rate " << ji.large.nfill_rate << "\n" + << "jemalloc_large_nflush_total " << ji.large.nflush_total << "\n" + << "jemalloc_large_nflush_rate " << ji.large.nflush_rate << "\n" + ; + + struct rusage r; + if (getrusage(RUSAGE_SELF, &r) == 0) { + output << "ocelot_time_user " << r.ru_utime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_utime.tv_usec << "\n" + "ocelot_time_system " << r.ru_stime.tv_sec << '.' << std::setfill('0') << std::setw(6) << r.ru_stime.tv_usec << "\n" + "ocelot_max_rss " << r.ru_maxrss << "\n" + "ocelot_minor_fault " << r.ru_minflt << "\n" + "ocelot_major_fault " << r.ru_majflt << "\n" + "ocelot_blk_in " << r.ru_inblock << "\n" + "ocelot_blk_out " << r.ru_oublock << "\n" + "ocelot_nvcsw " << r.ru_nvcsw << "\n" + "ocelot_nivcsw " << r.ru_nivcsw << "\n" + ; + } + output << '#'; + return output.str(); } diff --git a/src/report.h b/src/report.h index 1e3c53c..3bdaf53 100644 --- a/src/report.h +++ b/src/report.h @@ -8,9 +8,14 @@ std::string report( params_type ¶ms, user_list &users_list, - client_opts_t &client_opts, unsigned int announce_interval, unsigned int announce_jitter ); +// return a snapshot of jemalloc statistics +std::string report_jemalloc_plain(const char *opts, std::string path); + +// return output for a prometheus scrape +std::string report_prom_stats(const char *jemalloc_stats); + #endif // SRC_REPORT_H_ diff --git a/src/worker.cpp b/src/worker.cpp index 1e11759..40110b5 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -227,8 +227,25 @@ std::string worker::work(const std::string &input, std::string &ip, client_opts_ if (action == REPORT) { if (passkey == report_password) { - std::lock_guard ul_lock(db->user_list_mutex); - return report(params, users_list, client_opts, announce_interval, conf->get_uint("announce_jitter")); + if (params["get"] == "prom_stats") { + // exclude per-arena (a), destroyed merged (d), mutex (m) and extents (e) statistics + std::string jemalloc_stats(report_jemalloc_plain("adex", conf->get_str("report_path"))); + return response( + report_prom_stats(jemalloc_stats.c_str()), + client_opts + ); + } else if (params["jemalloc"] == "plain") { + return response( + report_jemalloc_plain("adex", conf->get_str("report_path")), + client_opts + ); + } else { + std::lock_guard ul_lock(db->user_list_mutex); + return response( + report(params, users_list, announce_interval, conf->get_uint("announce_jitter")), + client_opts + ); + } } else { return error("Authentication failure", client_opts); }