Skip to content

Commit 9a6176c

Browse files
ThomasBradyThomasBrady
committed
Rebase
1 parent f2f33fc commit 9a6176c

9 files changed

+54
-39
lines changed

‎src/bucket/BucketBase.cpp

+5-4
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,8 @@ BucketBase::merge(BucketManager& bucketManager, uint32_t maxProtocolVersion,
306306
std::shared_ptr<BucketT> const& newBucket,
307307
std::vector<std::shared_ptr<BucketT>> const& shadows,
308308
bool keepTombstoneEntries, bool countMergeEvents,
309-
asio::io_context& ctx, bool doFsync)
309+
asio::io_context& ctx, bool doFsync,
310+
bool rewriteLiveToInitEntries)
310311
{
311312
BUCKET_TYPE_ASSERT(BucketT);
312313

@@ -353,7 +354,7 @@ BucketBase::merge(BucketManager& bucketManager, uint32_t maxProtocolVersion,
353354

354355
BucketOutputIterator<BucketT> out(bucketManager.getTmpDir(),
355356
keepTombstoneEntries, meta, mc, ctx,
356-
doFsync);
357+
doFsync, rewriteLiveToInitEntries);
357358

358359
BucketEntryIdCmp<BucketT> cmp;
359360
size_t iter = 0;
@@ -407,13 +408,13 @@ template std::shared_ptr<LiveBucket> BucketBase::merge<LiveBucket>(
407408
std::shared_ptr<LiveBucket> const& newBucket,
408409
std::vector<std::shared_ptr<LiveBucket>> const& shadows,
409410
bool keepTombstoneEntries, bool countMergeEvents, asio::io_context& ctx,
410-
bool doFsync);
411+
bool doFsync, bool rewriteLiveToInitEntries);
411412

412413
template std::shared_ptr<HotArchiveBucket> BucketBase::merge<HotArchiveBucket>(
413414
BucketManager& bucketManager, uint32_t maxProtocolVersion,
414415
std::shared_ptr<HotArchiveBucket> const& oldBucket,
415416
std::shared_ptr<HotArchiveBucket> const& newBucket,
416417
std::vector<std::shared_ptr<HotArchiveBucket>> const& shadows,
417418
bool keepTombstoneEntries, bool countMergeEvents, asio::io_context& ctx,
418-
bool doFsync);
419+
bool doFsync, bool rewriteLiveToInitEntries);
419420
}

‎src/bucket/BucketBase.h

+5-1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ class BucketBase : public NonMovableOrCopyable
6565
public:
6666
static constexpr ProtocolVersion
6767
FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION = ProtocolVersion::V_23;
68+
static constexpr ProtocolVersion
69+
FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY =
70+
ProtocolVersion::V_11;
6871

6972
// Create an empty bucket. The empty bucket has hash '000000...' and its
7073
// filename is the empty string.
@@ -114,7 +117,8 @@ class BucketBase : public NonMovableOrCopyable
114117
std::shared_ptr<BucketT> const& newBucket,
115118
std::vector<std::shared_ptr<BucketT>> const& shadows,
116119
bool keepTombstoneEntries, bool countMergeEvents,
117-
asio::io_context& ctx, bool doFsync);
120+
asio::io_context& ctx, bool doFsync,
121+
bool rewriteLiveToInitEntries = false);
118122

119123
static std::string randomBucketName(std::string const& tmpDir);
120124
static std::string randomBucketIndexName(std::string const& tmpDir);

‎src/bucket/BucketListBase.cpp

+7
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,13 @@ BucketListBase<BucketT>::keepTombstoneEntries(uint32_t level)
462462
return level < BucketListBase<BucketT>::kNumLevels - 1;
463463
}
464464

465+
template <typename BucketT>
466+
bool
467+
BucketListBase<BucketT>::rewriteLiveToInitEntries(uint32_t level)
468+
{
469+
return level < BucketListBase<BucketT>::kNumLevels - 1;
470+
}
471+
465472
template <typename BucketT>
466473
BucketLevel<BucketT> const&
467474
BucketListBase<BucketT>::getLevel(uint32_t i) const

‎src/bucket/BucketManager.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,7 @@ BucketManager::incrMergeCounters(MergeCounters const& delta)
359359

360360
bool
361361
BucketManager::renameBucketDirFile(std::filesystem::path const& src,
362-
std::filesystem::path const& dst)
362+
std::filesystem::path const& dst)
363363
{
364364
ZoneScoped;
365365
if (mConfig.DISABLE_XDR_FSYNC)

‎src/bucket/BucketOutputIterator.cpp

+26-26
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,10 @@ namespace stellar
2121
* hashes them while writing to either destination. Produces a Bucket when done.
2222
*/
2323
template <typename BucketT>
24-
BucketOutputIterator<BucketT>::BucketOutputIterator(std::string const& tmpDir,
25-
bool keepTombstoneEntries,
26-
BucketMetadata const& meta,
27-
MergeCounters& mc,
28-
asio::io_context& ctx,
29-
bool doFsync,
30-
bool rewriteLiveToInitEntries)
24+
BucketOutputIterator<BucketT>::BucketOutputIterator(
25+
std::string const& tmpDir, bool keepTombstoneEntries,
26+
BucketMetadata const& meta, MergeCounters& mc, asio::io_context& ctx,
27+
bool doFsync, bool rewriteLiveToInitEntries)
3128
: mFilename(BucketBase::randomBucketName(tmpDir))
3229
, mOut(ctx, doFsync)
3330
, mCtx(ctx)
@@ -159,27 +156,30 @@ BucketOutputIterator<BucketT>::put(typename BucketT::EntryT const& e)
159156
mBuf = std::make_unique<typename BucketT::EntryT>();
160157
}
161158

162-
if (mRewriteLiveToInitEntries && e.type() == LIVEENTRY &&
163-
protocolVersionStartsFrom(
164-
mMeta.ledgerVersion,
165-
Bucket::FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY))
166-
{
167-
// If mRewriteLiveToInitEntries, we also want to convert the LIVEENTRY
168-
// to an INITENTRY. This is because each level of the bucket list
169-
// contains only one entry per key, and per CAP-0020, INIT ENTRY implies
170-
// that either no entry with the same ledger key exists in an older
171-
// bucket. Therefore, all entries of type LIVEENTRY in the lowest level
172-
// are also of type INITENTRY.
173-
++mMergeCounters.mOutputIteratorLiveToInitConversions;
174-
// Make a copy of e and set the type of the new entry to INITENTRY.
175-
BucketEntry eCopy = e;
176-
eCopy.type(INITENTRY);
177-
*mBuf = eCopy;
178-
}
179-
else
159+
if constexpr (std::is_same_v<BucketT, LiveBucket>)
180160
{
181-
*mBuf = e;
161+
if (mRewriteLiveToInitEntries && e.type() == LIVEENTRY &&
162+
protocolVersionStartsFrom(
163+
mMeta.ledgerVersion,
164+
BucketT::FIRST_PROTOCOL_SUPPORTING_INITENTRY_AND_METAENTRY))
165+
{
166+
// If mRewriteLiveToInitEntries, we also want to convert the
167+
// LIVEENTRY to an INITENTRY. This is because each level of the
168+
// bucket list contains only one entry per key, and per CAP-0020,
169+
// INIT ENTRY implies that either no entry with the same ledger key
170+
// exists in an older bucket. Therefore, all entries of type
171+
// LIVEENTRY in the lowest level are also of type INITENTRY.
172+
++mMergeCounters.mOutputIteratorLiveToInitRewrites;
173+
++mMergeCounters.mOutputIteratorBufferUpdates;
174+
// Make a copy of e and set the type of the new entry to INITENTRY.
175+
typename BucketT::EntryT eCopy = e;
176+
eCopy.type(INITENTRY);
177+
*mBuf = eCopy;
178+
return;
179+
}
182180
}
181+
// In any case, replace *mBuf with e.
182+
*mBuf = e;
183183
++mMergeCounters.mOutputIteratorBufferUpdates;
184184
}
185185

‎src/bucket/BucketUtils.h

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ struct MergeCounters
6464
uint64_t mOutputIteratorTombstoneElisions{0};
6565
uint64_t mOutputIteratorBufferUpdates{0};
6666
uint64_t mOutputIteratorActualWrites{0};
67+
uint64_t mOutputIteratorLiveToInitRewrites{0};
6768
MergeCounters& operator+=(MergeCounters const& delta);
6869
bool operator==(MergeCounters const& other) const;
6970
};

‎src/bucket/FutureBucket.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,8 @@ FutureBucket<BucketT>::startMerge(Application& app, uint32_t maxProtocolVersion,
410410
auto res = BucketBase::merge(
411411
bm, maxProtocolVersion, curr, snap, shadows,
412412
BucketListBase<BucketT>::keepTombstoneEntries(level),
413-
countMergeEvents, ctx, doFsync, BucketList::rewriteLiveToInitEntries(level));
413+
countMergeEvents, ctx, doFsync,
414+
BucketListBase<BucketT>::rewriteLiveToInitEntries(level));
414415

415416
if (res)
416417
{

‎src/bucket/test/BucketListTests.cpp

+4-3
Original file line numberDiff line numberDiff line change
@@ -454,7 +454,8 @@ TEST_CASE_VERSIONS("hot archive bucket tombstones expire at bottom level",
454454
});
455455
}
456456

457-
TEST_CASE_VERSIONS("live bucket tombstones expire at bottom level",
457+
TEST_CASE_VERSIONS("live bucket tombstones expire at bottom level and live "
458+
"entries converted to init entries",
458459
"[bucket][bucketlist][tombstones]")
459460
{
460461
VirtualClock clock;
@@ -524,9 +525,9 @@ TEST_CASE_VERSIONS("live bucket tombstones expire at bottom level",
524525
// Assert that init entries are converted to live entries
525526
// at the lowest level.
526527
REQUIRE(e0.nLive == 0);
527-
REQUIRE(e0.nInit != 0);
528+
REQUIRE(e0.nInitOrArchived != 0);
528529
// But not the level above.
529-
REQUIRE(e1.nInit == 0);
530+
REQUIRE(e1.nInitOrArchived == 0);
530531
});
531532
}
532533

‎src/bucket/test/BucketManagerTests.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ class StopAndRestartBucketMergesTest
761761
CLOG_INFO(Bucket, "OutputIteratorTombstoneElisions: {}",
762762
mMergeCounters.mOutputIteratorTombstoneElisions);
763763
CLOG_INFO(Bucket, "OutputIteratorLiveToInitConversions: {}",
764-
mMergeCounters.mOutputIteratorLiveToInitConversions);
764+
mMergeCounters.mOutputIteratorLiveToInitRewrites);
765765
CLOG_INFO(Bucket, "OutputIteratorBufferUpdates: {}",
766766
mMergeCounters.mOutputIteratorBufferUpdates);
767767
CLOG_INFO(Bucket, "OutputIteratorActualWrites: {}",
@@ -917,8 +917,8 @@ class StopAndRestartBucketMergesTest
917917

918918
CHECK(mMergeCounters.mOutputIteratorTombstoneElisions ==
919919
other.mMergeCounters.mOutputIteratorTombstoneElisions);
920-
CHECK(mMergeCounters.mOutputIteratorLiveToInitConversions ==
921-
other.mMergeCounters.mOutputIteratorLiveToInitConversions);
920+
CHECK(mMergeCounters.mOutputIteratorLiveToInitRewrites ==
921+
other.mMergeCounters.mOutputIteratorLiveToInitRewrites);
922922
CHECK(mMergeCounters.mOutputIteratorBufferUpdates ==
923923
other.mMergeCounters.mOutputIteratorBufferUpdates);
924924
CHECK(mMergeCounters.mOutputIteratorActualWrites ==

0 commit comments

Comments
 (0)