Skip to content

Commit 796a274

Browse files
kleetuskleetus
authored
Merge pull request #27 from braydonf/0.12.1-bitcore-dboptions
Add options to configure block index database
2 parents 0c15767 + 69ea12c commit 796a274

File tree

8 files changed

+166
-24
lines changed

8 files changed

+166
-24
lines changed

‎src/dbwrapper.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@ void HandleError(const leveldb::Status& status) throw(dbwrapper_error)
2929
throw dbwrapper_error("Unknown database error");
3030
}
3131

32-
static leveldb::Options GetOptions(size_t nCacheSize)
32+
static leveldb::Options GetOptions(size_t nCacheSize, bool compression, int maxOpenFiles)
3333
{
3434
leveldb::Options options;
3535
options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
3636
options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
3737
options.filter_policy = leveldb::NewBloomFilterPolicy(10);
38-
options.compression = leveldb::kNoCompression;
39-
options.max_open_files = 64;
38+
options.compression = compression ? leveldb::kSnappyCompression : leveldb::kNoCompression;
39+
options.max_open_files = maxOpenFiles;
4040
if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
4141
// LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
4242
// on corruption in later versions.
@@ -45,14 +45,14 @@ static leveldb::Options GetOptions(size_t nCacheSize)
4545
return options;
4646
}
4747

48-
CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
48+
CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate, bool compression, int maxOpenFiles)
4949
{
5050
penv = NULL;
5151
readoptions.verify_checksums = true;
5252
iteroptions.verify_checksums = true;
5353
iteroptions.fill_cache = false;
5454
syncoptions.sync = true;
55-
options = GetOptions(nCacheSize);
55+
options = GetOptions(nCacheSize, compression, maxOpenFiles);
5656
options.create_if_missing = true;
5757
if (fMemory) {
5858
penv = leveldb::NewMemEnv(leveldb::Env::Default());

‎src/dbwrapper.h‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -169,14 +169,16 @@ class CDBWrapper
169169

170170
public:
171171
/**
172-
* @param[in] path Location in the filesystem where leveldb data will be stored.
173-
* @param[in] nCacheSize Configures various leveldb cache settings.
174-
* @param[in] fMemory If true, use leveldb's memory environment.
175-
* @param[in] fWipe If true, remove all existing data.
176-
* @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
177-
* with a zero'd byte array.
172+
* @param[in] path Location in the filesystem where leveldb data will be stored.
173+
* @param[in] nCacheSize Configures various leveldb cache settings.
174+
* @param[in] fMemory If true, use leveldb's memory environment.
175+
* @param[in] fWipe If true, remove all existing data.
176+
* @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
177+
* with a zero'd byte array.
178+
* @param[in] compression Enable snappy compression for the database
179+
* @param[in] maxOpenFiles The maximum number of open files for the database
178180
*/
179-
CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
181+
CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false, bool compression = false, int maxOpenFiles = 64);
180182
~CDBWrapper();
181183

182184
template <typename K, typename V>

‎src/init.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1294,18 +1294,33 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
12941294
}
12951295
}
12961296

1297+
// block tree db settings
1298+
int dbMaxOpenFiles = GetArg("-dbmaxopenfiles", DEFAULT_DB_MAX_OPEN_FILES);
1299+
bool dbCompression = GetBoolArg("-dbcompression", DEFAULT_DB_COMPRESSION);
1300+
1301+
LogPrintf("Block index database configuration:\n");
1302+
LogPrintf("* Using %d max open files\n", dbMaxOpenFiles);
1303+
LogPrintf("* Compression is %s\n", dbCompression ? "enabled" : "disabled");
1304+
12971305
// cache size calculations
12981306
int64_t nTotalCache = (GetArg("-dbcache", nDefaultDbCache) << 20);
12991307
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
13001308
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greated than nMaxDbcache
13011309
int64_t nBlockTreeDBCache = nTotalCache / 8;
1302-
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX))
1303-
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
1310+
if (GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX) || GetBoolArg("-spentindex", DEFAULT_SPENTINDEX)) {
1311+
// enable 3/4 of the cache if addressindex and/or spentindex is enabled
1312+
nBlockTreeDBCache = nTotalCache * 3 / 4;
1313+
} else {
1314+
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
1315+
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
1316+
}
1317+
}
13041318
nTotalCache -= nBlockTreeDBCache;
13051319
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
13061320
nTotalCache -= nCoinDBCache;
13071321
nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
13081322
LogPrintf("Cache configuration:\n");
1323+
LogPrintf("* Max cache setting possible %.1fMiB\n", nMaxDbCache);
13091324
LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
13101325
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
13111326
LogPrintf("* Using %.1fMiB for in-memory UTXO set\n", nCoinCacheUsage * (1.0 / 1024 / 1024));
@@ -1326,7 +1341,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
13261341
delete pcoinscatcher;
13271342
delete pblocktree;
13281343

1329-
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
1344+
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex, dbCompression, dbMaxOpenFiles);
13301345
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
13311346
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
13321347
pcoinsTip = new CCoinsViewCache(pcoinscatcher);

‎src/main.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ static const bool DEFAULT_TXINDEX = false;
115115
static const bool DEFAULT_ADDRESSINDEX = false;
116116
static const bool DEFAULT_TIMESTAMPINDEX = false;
117117
static const bool DEFAULT_SPENTINDEX = false;
118+
static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000;
119+
static const bool DEFAULT_DB_COMPRESSION = true;
118120
static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100;
119121

120122
static const bool DEFAULT_TESTSAFEMODE = false;

‎src/rpcrawtransaction.cpp‎

Lines changed: 86 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,10 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud
5959
out.push_back(Pair("addresses", a));
6060
}
6161

62-
void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
62+
void TxToJSONExpanded(const CTransaction& tx, const uint256 hashBlock, UniValue& entry,
63+
int nHeight = 0, int nConfirmations = 0, int nBlockTime = 0)
6364
{
65+
6466
uint256 txid = tx.GetHash();
6567
entry.push_back(Pair("txid", txid.GetHex()));
6668
entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)));
@@ -121,6 +123,63 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
121123
}
122124
entry.push_back(Pair("vout", vout));
123125

126+
if (!hashBlock.IsNull()) {
127+
entry.push_back(Pair("blockhash", hashBlock.GetHex()));
128+
129+
if (nConfirmations > 0) {
130+
entry.push_back(Pair("height", nHeight));
131+
entry.push_back(Pair("confirmations", nConfirmations));
132+
entry.push_back(Pair("time", nBlockTime));
133+
entry.push_back(Pair("blocktime", nBlockTime));
134+
} else {
135+
entry.push_back(Pair("height", -1));
136+
entry.push_back(Pair("confirmations", 0));
137+
}
138+
}
139+
140+
}
141+
142+
void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
143+
{
144+
145+
uint256 txid = tx.GetHash();
146+
entry.push_back(Pair("txid", txid.GetHex()));
147+
entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)));
148+
entry.push_back(Pair("version", tx.nVersion));
149+
entry.push_back(Pair("locktime", (int64_t)tx.nLockTime));
150+
151+
UniValue vin(UniValue::VARR);
152+
BOOST_FOREACH(const CTxIn& txin, tx.vin) {
153+
UniValue in(UniValue::VOBJ);
154+
if (tx.IsCoinBase())
155+
in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())));
156+
else {
157+
in.push_back(Pair("txid", txin.prevout.hash.GetHex()));
158+
in.push_back(Pair("vout", (int64_t)txin.prevout.n));
159+
UniValue o(UniValue::VOBJ);
160+
o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true)));
161+
o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())));
162+
in.push_back(Pair("scriptSig", o));
163+
}
164+
in.push_back(Pair("sequence", (int64_t)txin.nSequence));
165+
vin.push_back(in);
166+
}
167+
entry.push_back(Pair("vin", vin));
168+
169+
UniValue vout(UniValue::VARR);
170+
for (unsigned int i = 0; i < tx.vout.size(); i++) {
171+
const CTxOut& txout = tx.vout[i];
172+
UniValue out(UniValue::VOBJ);
173+
out.push_back(Pair("value", ValueFromAmount(txout.nValue)));
174+
out.push_back(Pair("valueSat", txout.nValue));
175+
out.push_back(Pair("n", (int64_t)i));
176+
UniValue o(UniValue::VOBJ);
177+
ScriptPubKeyToJSON(txout.scriptPubKey, o, true);
178+
out.push_back(Pair("scriptPubKey", o));
179+
vout.push_back(out);
180+
}
181+
entry.push_back(Pair("vout", vout));
182+
124183
if (!hashBlock.IsNull()) {
125184
entry.push_back(Pair("blockhash", hashBlock.GetHex()));
126185
BlockMap::iterator mi = mapBlockIndex.find(hashBlock);
@@ -206,18 +265,38 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp)
206265
+ HelpExampleRpc("getrawtransaction", "\"mytxid\", 1")
207266
);
208267

209-
LOCK(cs_main);
210-
211268
uint256 hash = ParseHashV(params[0], "parameter 1");
212269

213270
bool fVerbose = false;
214271
if (params.size() > 1)
215272
fVerbose = (params[1].get_int() != 0);
216273

217274
CTransaction tx;
275+
218276
uint256 hashBlock;
219-
if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true))
220-
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction");
277+
int nHeight = 0;
278+
int nConfirmations = 0;
279+
int nBlockTime = 0;
280+
281+
{
282+
LOCK(cs_main);
283+
if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true))
284+
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction");
285+
286+
BlockMap::iterator mi = mapBlockIndex.find(hashBlock);
287+
if (mi != mapBlockIndex.end() && (*mi).second) {
288+
CBlockIndex* pindex = (*mi).second;
289+
if (chainActive.Contains(pindex)) {
290+
nHeight = pindex->nHeight;
291+
nConfirmations = 1 + chainActive.Height() - pindex->nHeight;
292+
nBlockTime = pindex->GetBlockTime();
293+
} else {
294+
nHeight = -1;
295+
nConfirmations = 0;
296+
nBlockTime = pindex->GetBlockTime();
297+
}
298+
}
299+
}
221300

222301
string strHex = EncodeHexTx(tx);
223302

@@ -226,7 +305,8 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp)
226305

227306
UniValue result(UniValue::VOBJ);
228307
result.push_back(Pair("hex", strHex));
229-
TxToJSON(tx, hashBlock, result);
308+
TxToJSONExpanded(tx, hashBlock, result, nHeight, nConfirmations, nBlockTime);
309+
230310
return result;
231311
}
232312

‎src/test/dbwrapper_tests.cpp‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,49 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
4747
}
4848
}
4949

50+
BOOST_AUTO_TEST_CASE(dbwrapper_compression)
51+
{
52+
// Perform tests both with compression and without
53+
for (int i = 0; i < 2; i++) {
54+
bool compression = (bool)i;
55+
path ph = temp_directory_path() / unique_path();
56+
CDBWrapper dbw(ph, (1 << 20), true, false, false, compression);
57+
char key = 'k';
58+
uint256 in = GetRandHash();
59+
uint256 res;
60+
61+
BOOST_CHECK(dbw.Write(key, in));
62+
BOOST_CHECK(dbw.Read(key, res));
63+
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
64+
}
65+
}
66+
67+
BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_64)
68+
{
69+
path ph = temp_directory_path() / unique_path();
70+
CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 64);
71+
char key = 'k';
72+
uint256 in = GetRandHash();
73+
uint256 res;
74+
75+
BOOST_CHECK(dbw.Write(key, in));
76+
BOOST_CHECK(dbw.Read(key, res));
77+
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
78+
}
79+
80+
BOOST_AUTO_TEST_CASE(dbwrapper_maxopenfiles_1000)
81+
{
82+
path ph = temp_directory_path() / unique_path();
83+
CDBWrapper dbw(ph, (1 << 20), true, false, false, false, 1000);
84+
char key = 'k';
85+
uint256 in = GetRandHash();
86+
uint256 res;
87+
88+
BOOST_CHECK(dbw.Write(key, in));
89+
BOOST_CHECK(dbw.Read(key, res));
90+
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
91+
}
92+
5093
// Test batch operations
5194
BOOST_AUTO_TEST_CASE(dbwrapper_batch)
5295
{

‎src/txdb.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ static const char DB_REINDEX_FLAG = 'R';
3333
static const char DB_LAST_BLOCK = 'l';
3434

3535

36-
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true)
36+
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true, false, 64)
3737
{
3838
}
3939

@@ -75,7 +75,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
7575
return db.WriteBatch(batch);
7676
}
7777

78-
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
78+
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe, bool compression, int maxOpenFiles) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe, false, compression, maxOpenFiles) {
7979
}
8080

8181
bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {

‎src/txdb.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class CCoinsViewDB : public CCoinsView
5454
class CBlockTreeDB : public CDBWrapper
5555
{
5656
public:
57-
CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
57+
CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool compression = true, int maxOpenFiles = 1000);
5858
private:
5959
CBlockTreeDB(const CBlockTreeDB&);
6060
void operator=(const CBlockTreeDB&);

0 commit comments

Comments
 (0)