Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 6cc03e3

Browse files
marta-lokhovamarta-lokhova
committedNov 4, 2024·
Adapt the rest of the codebase to the new MISC database, which can be written to independently of ledger state database
1 parent 4e6f252 commit 6cc03e3

23 files changed

+287
-202
lines changed
 

‎src/database/test/DatabaseTests.cpp

+7-8
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ transactionTest(Application::pointer app)
3232
int a0 = a + 1;
3333
int a1 = a + 2;
3434

35-
auto& session = app->getDatabase().getSession();
35+
auto& session = app->getDatabase().getRawSession();
3636

3737
session << "DROP TABLE IF EXISTS test";
3838
session << "CREATE TABLE test (x INTEGER)";
@@ -104,7 +104,7 @@ checkMVCCIsolation(Application::pointer app)
104104

105105
int s2r1 = 0, s2r2 = 0, s2r3 = 0, s2r4 = 0;
106106

107-
auto& sess1 = app->getDatabase().getSession();
107+
auto& sess1 = app->getDatabase().getRawSession();
108108

109109
sess1 << "DROP TABLE IF EXISTS test";
110110
sess1 << "CREATE TABLE test (x INTEGER)";
@@ -217,7 +217,7 @@ TEST_CASE("postgres smoketest", "[db]")
217217
Application::pointer app = createTestApplication(clock, cfg);
218218
int a = 10, b = 0;
219219

220-
auto& session = app->getDatabase().getSession();
220+
auto& session = app->getDatabase().getRawSession();
221221

222222
SECTION("round trip")
223223
{
@@ -249,7 +249,7 @@ TEST_CASE("postgres smoketest", "[db]")
249249

250250
SECTION("postgres MVCC test")
251251
{
252-
app->getDatabase().getSession() << "drop table if exists test";
252+
app->getDatabase().getRawSession() << "drop table if exists test";
253253
checkMVCCIsolation(app);
254254
}
255255
}
@@ -279,7 +279,7 @@ TEST_CASE("postgres performance", "[db][pgperf][!hide]")
279279
try
280280
{
281281
Application::pointer app = createTestApplication(clock, cfg);
282-
auto& session = app->getDatabase().getSession();
282+
auto& session = app->getDatabase().getRawSession();
283283

284284
session << "drop table if exists txtest;";
285285
session << "create table txtest (a bigint, b bigint, c bigint, primary "
@@ -355,7 +355,6 @@ TEST_CASE("schema test", "[db]")
355355
Application::pointer app = createTestApplication(clock, cfg);
356356

357357
auto& db = app->getDatabase();
358-
auto dbv = db.getDBSchemaVersion();
359-
auto av = db.getAppSchemaVersion();
360-
REQUIRE(dbv == av);
358+
auto dbv = db.getMainDBSchemaVersion();
359+
REQUIRE(dbv == MAIN_SCHEMA_VERSION);
361360
}

‎src/herder/HerderImpl.cpp

+5-3
Original file line numberDiff line numberDiff line change
@@ -2057,16 +2057,18 @@ void
20572057
HerderImpl::persistUpgrades()
20582058
{
20592059
ZoneScoped;
2060+
releaseAssert(threadIsMain());
20602061
auto s = mUpgrades.getParameters().toJson();
2061-
mApp.getPersistentState().setState(PersistentState::kLedgerUpgrades, s);
2062+
mApp.getPersistentState().setState(PersistentState::kLedgerUpgrades, s,
2063+
mApp.getDatabase().getMiscSession());
20622064
}
20632065

20642066
void
20652067
HerderImpl::restoreUpgrades()
20662068
{
20672069
ZoneScoped;
2068-
std::string s =
2069-
mApp.getPersistentState().getState(PersistentState::kLedgerUpgrades);
2070+
std::string s = mApp.getPersistentState().getState(
2071+
PersistentState::kLedgerUpgrades, mApp.getDatabase().getMiscSession());
20702072
if (!s.empty())
20712073
{
20722074
Upgrades::UpgradeParameters p;

‎src/herder/HerderPersistence.h

-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,5 @@ class HerderPersistence
4949
static void dropAll(Database& db);
5050
static void deleteOldEntries(Database& db, uint32_t ledgerSeq,
5151
uint32_t count);
52-
static void createQuorumTrackingTable(soci::session& sess);
5352
};
5453
}

‎src/herder/HerderPersistenceImpl.cpp

+39-37
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,13 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
4747

4848
auto usedQSets = UnorderedMap<Hash, SCPQuorumSetPtr>{};
4949
auto& db = mApp.getDatabase();
50+
auto& sess = db.getMiscSession();
5051

51-
soci::transaction txscope(db.getSession());
52+
soci::transaction txscope(sess.session());
5253

5354
{
5455
auto prepClean = db.getPreparedStatement(
55-
"DELETE FROM scphistory WHERE ledgerseq =:l");
56+
"DELETE FROM scphistory WHERE ledgerseq =:l", sess);
5657

5758
auto& st = prepClean.statement();
5859
st.exchange(soci::use(seq));
@@ -92,7 +93,8 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
9293
auto prepEnv =
9394
db.getPreparedStatement("INSERT INTO scphistory "
9495
"(nodeid, ledgerseq, envelope) VALUES "
95-
"(:n, :l, :e)");
96+
"(:n, :l, :e)",
97+
sess);
9698
auto& st = prepEnv.statement();
9799
st.exchange(soci::use(nodeIDs, "n"));
98100
st.exchange(soci::use(seqs, "l"));
@@ -124,7 +126,7 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
124126
std::string qSetHHex(binToHex(qSetH));
125127

126128
auto prep = db.getPreparedStatement(
127-
"UPDATE quoruminfo SET qsethash = :h WHERE nodeid = :id");
129+
"UPDATE quoruminfo SET qsethash = :h WHERE nodeid = :id", sess);
128130
auto& st = prep.statement();
129131
st.exchange(soci::use(qSetHHex));
130132
st.exchange(soci::use(nodeIDStrKey));
@@ -136,7 +138,8 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
136138
if (st.get_affected_rows() != 1)
137139
{
138140
auto prepI = db.getPreparedStatement(
139-
"INSERT INTO quoruminfo (nodeid, qsethash) VALUES (:id, :h)");
141+
"INSERT INTO quoruminfo (nodeid, qsethash) VALUES (:id, :h)",
142+
sess);
140143
auto& stI = prepI.statement();
141144
stI.exchange(soci::use(nodeIDStrKey));
142145
stI.exchange(soci::use(qSetHHex));
@@ -158,7 +161,7 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
158161

159162
uint32_t lastSeenSeq;
160163
auto prepSelQSet = db.getPreparedStatement(
161-
"SELECT lastledgerseq FROM scpquorums WHERE qsethash = :h");
164+
"SELECT lastledgerseq FROM scpquorums WHERE qsethash = :h", sess);
162165
auto& stSel = prepSelQSet.statement();
163166
stSel.exchange(soci::into(lastSeenSeq));
164167
stSel.exchange(soci::use(qSetH));
@@ -177,7 +180,8 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
177180

178181
auto prepUpQSet = db.getPreparedStatement(
179182
"UPDATE scpquorums SET "
180-
"lastledgerseq = :l WHERE qsethash = :h");
183+
"lastledgerseq = :l WHERE qsethash = :h",
184+
sess);
181185

182186
auto& stUp = prepUpQSet.statement();
183187
stUp.exchange(soci::use(seq));
@@ -202,7 +206,8 @@ HerderPersistenceImpl::saveSCPHistory(uint32_t seq,
202206
auto prepInsQSet = db.getPreparedStatement(
203207
"INSERT INTO scpquorums "
204208
"(qsethash, lastledgerseq, qset) VALUES "
205-
"(:h, :l, :v);");
209+
"(:h, :l, :v);",
210+
sess);
206211

207212
auto& stIns = prepInsQSet.statement();
208213
stIns.exchange(soci::use(qSetH));
@@ -372,48 +377,45 @@ void
372377
HerderPersistence::dropAll(Database& db)
373378
{
374379
ZoneScoped;
375-
db.getSession() << "DROP TABLE IF EXISTS scphistory";
380+
db.getRawMiscSession() << "DROP TABLE IF EXISTS scphistory";
376381

377-
db.getSession() << "DROP TABLE IF EXISTS scpquorums";
382+
db.getRawMiscSession() << "DROP TABLE IF EXISTS scpquorums";
378383

379-
db.getSession() << "CREATE TABLE scphistory ("
380-
"nodeid CHARACTER(56) NOT NULL,"
381-
"ledgerseq INT NOT NULL CHECK (ledgerseq >= 0),"
382-
"envelope TEXT NOT NULL"
383-
")";
384+
db.getRawMiscSession() << "CREATE TABLE scphistory ("
385+
"nodeid CHARACTER(56) NOT NULL,"
386+
"ledgerseq INT NOT NULL CHECK (ledgerseq >= 0),"
387+
"envelope TEXT NOT NULL"
388+
")";
384389

385-
db.getSession() << "CREATE INDEX scpenvsbyseq ON scphistory(ledgerseq)";
390+
db.getRawMiscSession()
391+
<< "CREATE INDEX scpenvsbyseq ON scphistory(ledgerseq)";
386392

387-
db.getSession() << "CREATE TABLE scpquorums ("
388-
"qsethash CHARACTER(64) NOT NULL,"
389-
"lastledgerseq INT NOT NULL CHECK (lastledgerseq >= 0),"
390-
"qset TEXT NOT NULL,"
391-
"PRIMARY KEY (qsethash)"
392-
")";
393+
db.getRawMiscSession()
394+
<< "CREATE TABLE scpquorums ("
395+
"qsethash CHARACTER(64) NOT NULL,"
396+
"lastledgerseq INT NOT NULL CHECK (lastledgerseq >= 0),"
397+
"qset TEXT NOT NULL,"
398+
"PRIMARY KEY (qsethash)"
399+
")";
393400

394-
db.getSession()
401+
db.getRawMiscSession()
395402
<< "CREATE INDEX scpquorumsbyseq ON scpquorums(lastledgerseq)";
396403

397-
db.getSession() << "DROP TABLE IF EXISTS quoruminfo";
398-
}
399-
400-
void
401-
HerderPersistence::createQuorumTrackingTable(soci::session& sess)
402-
{
403-
sess << "CREATE TABLE quoruminfo ("
404-
"nodeid CHARACTER(56) NOT NULL,"
405-
"qsethash CHARACTER(64) NOT NULL,"
406-
"PRIMARY KEY (nodeid))";
404+
db.getRawMiscSession() << "DROP TABLE IF EXISTS quoruminfo";
405+
db.getRawMiscSession() << "CREATE TABLE quoruminfo ("
406+
"nodeid CHARACTER(56) NOT NULL,"
407+
"qsethash CHARACTER(64) NOT NULL,"
408+
"PRIMARY KEY (nodeid))";
407409
}
408410

409411
void
410412
HerderPersistence::deleteOldEntries(Database& db, uint32_t ledgerSeq,
411413
uint32_t count)
412414
{
413415
ZoneScoped;
414-
DatabaseUtils::deleteOldEntriesHelper(db.getSession(), ledgerSeq, count,
415-
"scphistory", "ledgerseq");
416-
DatabaseUtils::deleteOldEntriesHelper(db.getSession(), ledgerSeq, count,
417-
"scpquorums", "lastledgerseq");
416+
DatabaseUtils::deleteOldEntriesHelper(db.getRawMiscSession(), ledgerSeq,
417+
count, "scphistory", "ledgerseq");
418+
DatabaseUtils::deleteOldEntriesHelper(db.getRawMiscSession(), ledgerSeq,
419+
count, "scpquorums", "lastledgerseq");
418420
}
419421
}

‎src/herder/PendingEnvelopes.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,8 @@ PendingEnvelopes::getQSet(Hash const& hash)
745745
else
746746
{
747747
auto& db = mApp.getDatabase();
748-
qset = HerderPersistence::getQuorumSet(db, db.getSession(), hash);
748+
qset =
749+
HerderPersistence::getQuorumSet(db, db.getRawMiscSession(), hash);
749750
}
750751
if (qset)
751752
{
@@ -814,7 +815,7 @@ PendingEnvelopes::rebuildQuorumTrackerState()
814815
// see if we had some information for that node
815816
auto& db = mApp.getDatabase();
816817
auto h = HerderPersistence::getNodeQuorumSet(
817-
db, db.getSession(), id);
818+
db, db.getRawMiscSession(), id);
818819
if (h)
819820
{
820821
res = getQSet(*h);

‎src/herder/Upgrades.cpp

+10-10
Original file line numberDiff line numberDiff line change
@@ -690,22 +690,22 @@ Upgrades::timeForUpgrade(uint64_t time) const
690690
void
691691
Upgrades::dropAll(Database& db)
692692
{
693-
db.getSession() << "DROP TABLE IF EXISTS upgradehistory";
694-
db.getSession() << "CREATE TABLE upgradehistory ("
695-
"ledgerseq INT NOT NULL CHECK (ledgerseq >= 0), "
696-
"upgradeindex INT NOT NULL, "
697-
"upgrade TEXT NOT NULL, "
698-
"changes TEXT NOT NULL, "
699-
"PRIMARY KEY (ledgerseq, upgradeindex)"
700-
")";
701-
db.getSession()
693+
db.getRawSession() << "DROP TABLE IF EXISTS upgradehistory";
694+
db.getRawSession() << "CREATE TABLE upgradehistory ("
695+
"ledgerseq INT NOT NULL CHECK (ledgerseq >= 0), "
696+
"upgradeindex INT NOT NULL, "
697+
"upgrade TEXT NOT NULL, "
698+
"changes TEXT NOT NULL, "
699+
"PRIMARY KEY (ledgerseq, upgradeindex)"
700+
")";
701+
db.getRawSession()
702702
<< "CREATE INDEX upgradehistbyseq ON upgradehistory (ledgerseq);";
703703
}
704704

705705
void
706706
Upgrades::dropSupportUpgradeHistory(Database& db)
707707
{
708-
db.getSession() << "DROP TABLE IF EXISTS upgradehistory";
708+
db.getRawSession() << "DROP TABLE IF EXISTS upgradehistory";
709709
}
710710

711711
static void

‎src/herder/test/HerderTests.cpp

+6-5
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ TEST_CASE_VERSIONS("standalone", "[herder][acceptance]")
226226
app->getCommandHandler().manualCmd("setcursor?id=A1&cursor=1");
227227
app->getCommandHandler().manualCmd("maintenance?queue=true");
228228
auto& db = app->getDatabase();
229-
auto& sess = db.getSession();
229+
auto& sess = db.getRawSession();
230230

231231
app->getCommandHandler().manualCmd("setcursor?id=A2&cursor=3");
232232
app->getCommandHandler().manualCmd("maintenance?queue=true");
@@ -3395,15 +3395,15 @@ TEST_CASE("overlay parallel processing")
33953395
// soroban traffic
33963396
currLoadGenCount = loadGenDone.count();
33973397
auto secondLoadGenCount = secondLoadGenDone.count();
3398-
uint32_t const classicTxCount = 200;
3398+
uint32_t const txCount = 100;
33993399
// Generate Soroban txs from one node
34003400
loadGen.generateLoad(GeneratedLoadConfig::txLoad(
34013401
LoadGenMode::SOROBAN_UPLOAD, 50,
3402-
/* nTxs */ 500, desiredTxRate, /* offset */ 0));
3402+
/* nTxs */ txCount, desiredTxRate, /* offset */ 0));
34033403
// Generate classic txs from another node (with offset to prevent
34043404
// overlapping accounts)
34053405
secondLoadGen.generateLoad(GeneratedLoadConfig::txLoad(
3406-
LoadGenMode::PAY, 50, classicTxCount, desiredTxRate,
3406+
LoadGenMode::PAY, 50, txCount, desiredTxRate,
34073407
/* offset */ 50));
34083408

34093409
simulation->crankUntil(
@@ -5465,7 +5465,8 @@ TEST_CASE("SCP message capture from previous ledger", "[herder]")
54655465
// Prepare query
54665466
auto& db = node->getDatabase();
54675467
auto prep = db.getPreparedStatement(
5468-
"SELECT envelope FROM scphistory WHERE ledgerseq = :l");
5468+
"SELECT envelope FROM scphistory WHERE ledgerseq = :l",
5469+
db.getMiscSession());
54695470
auto& st = prep.statement();
54705471
st.exchange(soci::use(ledgerNum));
54715472
std::string envStr;

‎src/history/HistoryManagerImpl.cpp

+9-8
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ static std::string kSQLCreateStatement =
5959
void
6060
HistoryManager::dropAll(Database& db)
6161
{
62-
db.getSession() << "DROP TABLE IF EXISTS publishqueue;";
63-
soci::statement st = db.getSession().prepare << kSQLCreateStatement;
62+
db.getRawSession() << "DROP TABLE IF EXISTS publishqueue;";
63+
soci::statement st = db.getRawSession().prepare << kSQLCreateStatement;
6464
st.execute(true);
6565
}
6666

@@ -149,7 +149,8 @@ HistoryManagerImpl::dropSQLBasedPublish()
149149
// Migrate all the existing queued checkpoints to the new format
150150
{
151151
std::string state;
152-
auto prep = db.getPreparedStatement("SELECT state FROM publishqueue;");
152+
auto prep =
153+
db.getPreparedStatement("SELECT state FROM publishqueue;", sess);
153154
auto& st = prep.statement();
154155
st.exchange(soci::into(state));
155156
st.define_and_bind();
@@ -170,9 +171,9 @@ HistoryManagerImpl::dropSQLBasedPublish()
170171
for (auto const& checkpoint : checkpointLedgers)
171172
{
172173
auto begin = firstLedgerInCheckpointContaining(checkpoint);
173-
populateCheckpointFilesFromDB(mApp, sess, begin, freq,
174+
populateCheckpointFilesFromDB(mApp, sess.session(), begin, freq,
174175
mCheckpointBuilder);
175-
LedgerHeaderUtils::copyToStream(db, sess, begin, freq,
176+
LedgerHeaderUtils::copyToStream(db, sess.session(), begin, freq,
176177
mCheckpointBuilder);
177178
// Checkpoints in publish queue are complete, so we can finalize them
178179
mCheckpointBuilder.checkpointComplete(checkpoint);
@@ -184,17 +185,17 @@ HistoryManagerImpl::dropSQLBasedPublish()
184185
{
185186
// Then, reconstruct any partial checkpoints that haven't yet been
186187
// queued
187-
populateCheckpointFilesFromDB(mApp, sess,
188+
populateCheckpointFilesFromDB(mApp, sess.session(),
188189
firstLedgerInCheckpointContaining(lcl),
189190
freq, mCheckpointBuilder);
190-
LedgerHeaderUtils::copyToStream(db, sess,
191+
LedgerHeaderUtils::copyToStream(db, sess.session(),
191192
firstLedgerInCheckpointContaining(lcl),
192193
freq, mCheckpointBuilder);
193194
}
194195
db.clearPreparedStatementCache();
195196

196197
// Now it's safe to drop obsolete SQL tables
197-
sess << "DROP TABLE IF EXISTS publishqueue;";
198+
sess.session() << "DROP TABLE IF EXISTS publishqueue;";
198199
dropSupportTxHistory(db);
199200
dropSupportTxSetHistory(db);
200201
}

‎src/history/StateSnapshot.cpp

+5-4
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,11 @@ StateSnapshot::writeSCPMessages() const
5555
{
5656
ZoneScoped;
5757
std::unique_ptr<soci::session> snapSess(
58-
mApp.getDatabase().canUsePool()
59-
? std::make_unique<soci::session>(mApp.getDatabase().getPool())
60-
: nullptr);
61-
soci::session& sess(snapSess ? *snapSess : mApp.getDatabase().getSession());
58+
(mApp.getDatabase().canUsePool()
59+
? std::make_unique<soci::session>(mApp.getDatabase().getMiscPool())
60+
: nullptr));
61+
soci::session& sess(snapSess ? *snapSess
62+
: mApp.getDatabase().getRawMiscSession());
6263
soci::transaction tx(sess);
6364

6465
// The current "history block" is stored in _four_ files, one just ledger

0 commit comments

Comments
 (0)
Please sign in to comment.