Prettify byte counts. Only build sqlshell if configured

Author: mattgodbolt

CMakeLists.txt

@@ -6,6 +6,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread")
 
 option(UseLTO "Use link-time optimization" OFF)
 option(Static "Statically link" OFF)
+option(BuildSqlShell "Build the sqlite shell" OFF)
 
 if(UseLTO)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto")
@@ -45,6 +46,8 @@ set(SOURCE_FILES
     src/ConsoleLog.cpp
     src/StringView.cpp
     src/StringView.h
+    src/PrettyBytes.h
+    src/PrettyBytes.cpp
     ext/sqlite/sqlite3.c)
 
 set(TEST_FILES
@@ -56,6 +59,7 @@ set(TEST_FILES
     tests/RegExpIndexerTest.cpp
     tests/TempDir.h
     tests/TempDir.cpp
+    tests/PrettyBytesTest.cpp
     tests/IndexTest.cpp
     tests/LogTest.cpp)
 
@@ -67,11 +71,13 @@ target_link_libraries(zindex libzindex ${ZLIB_LIBRARIES} dl)
 add_executable(zq src/zq.cpp)
 target_link_libraries(zq libzindex ${ZLIB_LIBRARIES} dl)
 
-add_executable(unit-tests ${TEST_FILES})
+add_executable(unit-tests ${TEST_FILES} )
 target_link_libraries(unit-tests libzindex ${ZLIB_LIBRARIES} dl)
 
-add_executable(sql-shell ext/sqlite/shell.c ext/sqlite/sqlite3.c)
-target_link_libraries(sql-shell dl)
+if(BuildSqlShell)
+    add_executable(sql-shell ext/sqlite/shell.c ext/sqlite/sqlite3.c)
+    target_link_libraries(sql-shell dl)
+endif(BuildSqlShell)
 
 enable_testing()
 add_test(NAME unit-tests

src/Index.cpp

@@ -18,6 +18,7 @@
 #include "IndexSink.h"
 #include "Log.h"
 #include "StringView.h"
+#include "PrettyBytes.h"
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -59,7 +60,7 @@ size_t makeWindow(uint8_t *out, size_t outSize, const uint8_t *in,
     return destLen;
 }
 
-void uncompress(const std::vector <uint8_t> &compressed, uint8_t *to,
+void uncompress(const std::vector<uint8_t> &compressed, uint8_t *to,
                 size_t len) {
     uLongf destLen = len;
     R(::uncompress(to, &len, &compressed[0], compressed.size()));
@@ -325,7 +326,7 @@ Index::Index() { }
 
 Index::~Index() { }
 
-Index::Index(std::unique_ptr < Impl > && imp) : impl_(std::move(imp)) { }
+Index::Index(std::unique_ptr<Impl> &&imp) : impl_(std::move(imp)) { }
 
 struct Index::Builder::Impl : LineSink {
     Log &log;
@@ -336,7 +337,7 @@ struct Index::Builder::Impl : LineSink {
     Sqlite::Statement addIndexSql;
     Sqlite::Statement addMetaSql;
     uint64_t indexEvery = DefaultIndexEvery;
-    std::unordered_map <std::string, std::unique_ptr<IndexHandler>> indexers;
+    std::unordered_map<std::string, std::unique_ptr<IndexHandler>> indexers;
 
     Impl(Log &log, File &&from, const std::string &fromPath,
          const std::string &indexFilename)
@@ -397,8 +398,8 @@ INSERT INTO Indexes VALUES(:name, :creationString, :isNumeric)
     }
 
     void build() {
-        log.info("Building index, generating a checkpoint every ", indexEvery,
-                 " bytes");
+        log.info("Building index, generating a checkpoint every ",
+                 PrettyBytes(indexEvery));
         struct stat compressedStat;
         if (fstat(fileno(from.get()), &compressedStat) != 0)
             throw ZlibError(Z_DATA_ERROR);
@@ -457,8 +458,9 @@ INSERT INTO LineOffsets VALUES(:line, :offset, :length))");
                 bool endOfBlock = zs.stream.data_type & 0x80;
                 bool lastBlockInStream = zs.stream.data_type & 0x40;
                 if (endOfBlock && !lastBlockInStream && needsIndex) {
-                    log.debug("Creating checkpoint at ", totalOut,
-                              " (compressed offset ", totalIn, ")");
+                    log.debug("Creating checkpoint at ", PrettyBytes(totalOut),
+                              " (compressed offset ", PrettyBytes(totalIn),
+                              ")");
                     if (totalOut != 0) {
                         // Flush previous information.
                         addIndex
@@ -482,8 +484,8 @@ INSERT INTO LineOffsets VALUES(:line, :offset, :length))");
                     char pc[16];
                     snprintf(pc, sizeof(pc) - 1, "%.2f",
                              (totalIn * 100.0) / compressedStat.st_size);
-                    log.info("Progress: ", totalIn, " of ",
-                             compressedStat.st_size,
+                    log.info("Progress: ", PrettyBytes(totalIn), " of ",
+                             PrettyBytes(compressedStat.st_size),
                              " (", pc, "%)");
                     nextProgress = now + LogProgressEverySecs;
                 }
@@ -597,9 +599,9 @@ Index Index::load(Log &log, File &&fromCompressed,
     Sqlite db(log);
     db.open(indexFilename.c_str(), true);
 
-    std::unique_ptr <Impl> impl(new Impl(log,
-                                         std::move(fromCompressed),
-                                         std::move(db)));
+    std::unique_ptr<Impl> impl(new Impl(log,
+                                        std::move(fromCompressed),
+                                        std::move(db)));
     impl->init(forceLoad);
     return Index(std::move(impl));
 }
@@ -608,7 +610,7 @@ void Index::getLine(uint64_t line, LineSink &sink) {
     impl_->getLine(line, sink);
 }
 
-void Index::getLines(const std::vector <uint64_t> &lines, LineSink &sink) {
+void Index::getLines(const std::vector<uint64_t> &lines, LineSink &sink) {
     for (auto line : lines) impl_->getLine(line, sink);
 }
 
@@ -618,7 +620,7 @@ void Index::queryIndex(const std::string &index, const std::string &query,
 }
 
 void Index::queryIndexMulti(const std::string &index,
-                            const std::vector <std::string> &queries,
+                            const std::vector<std::string> &queries,
                             LineSink &sink) {
     // TODO be a little smarter about this.
     for (auto query : queries) impl_->queryIndex(index, query, sink);

src/PrettyBytes.cpp

@@ -0,0 +1,17 @@
+#include "PrettyBytes.h"
+
+#include <iostream>
+#include <iomanip>
+
+using namespace std;
+
+ostream &operator<<(ostream &o, PrettyBytes pb) {
+    if (pb.bytes_ == 1) return o << "1 byte";
+    if (pb.bytes_ < 1024) return o << pb.bytes_ << " bytes";
+    o << setprecision(2) << fixed;
+    if (pb.bytes_ < 1024 * 1024)
+        return o << (pb.bytes_ / (1024.0)) << " KiB";
+    if (pb.bytes_ < 1024 * 1024 * 1024)
+        return o << (pb.bytes_ / (1024 * 1024.0)) << " MiB";
+    return o << (pb.bytes_ / (1024 * 1024 * 1024.0)) << " GiB";
+}

src/PrettyBytes.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <iosfwd>
+#include <cstdint>
+
+class PrettyBytes {
+    uint64_t bytes_;
+public:
+    explicit PrettyBytes(uint64_t bytes) : bytes_(bytes) { }
+
+    friend std::ostream &operator<<(std::ostream &, PrettyBytes pb);
+};

tests/PrettyBytesTest.cpp

@@ -0,0 +1,23 @@
+#include "PrettyBytes.h"
+
+#include "catch.hpp"
+#include <sstream>
+
+#define STR(XX) ([&]() ->std::string { \
+    std::stringstream STR__STREAM; \
+    STR__STREAM << XX; \
+    return STR__STREAM.str(); \
+})()
+
+TEST_CASE("pretty bytes", "[PrettyBytes]") {
+    CHECK(STR(PrettyBytes(0)) == "0 bytes");
+    CHECK(STR(PrettyBytes(1)) == "1 byte");
+    CHECK(STR(PrettyBytes(2)) == "2 bytes");
+    CHECK(STR(PrettyBytes(1023)) == "1023 bytes");
+    CHECK(STR(PrettyBytes(1024)) == "1.00 KiB");
+    CHECK(STR(PrettyBytes(1200)) == "1.17 KiB");
+    CHECK(STR(PrettyBytes(1200000)) == "1.14 MiB");
+    CHECK(STR(PrettyBytes(1200000000)) == "1.12 GiB");
+    CHECK(STR(PrettyBytes(std::numeric_limits<uint64_t>::max())) ==
+          "17179869184.00 GiB");
+}