diff --git a/azure-pipelines-full.yml b/azure-pipelines-full.yml index a44ab4cc2..ff963024a 100644 --- a/azure-pipelines-full.yml +++ b/azure-pipelines-full.yml @@ -8,7 +8,7 @@ jobs: pool: vmImage: windows-2022 displayName: 'C# (Windows)' - timeoutInMinutes: 150 + timeoutInMinutes: 150 strategy: maxParallel: 2 @@ -62,7 +62,7 @@ jobs: command: test projects: '**/*.test.csproj' arguments: '--configuration $(buildConfiguration) -l "console;verbosity=detailed"' - + - task: PublishTestResults@2 displayName: 'Publish Test Results' inputs: @@ -74,6 +74,7 @@ jobs: pool: vmImage: windows-2022 displayName: 'C++ (Windows)' + timeoutInMinutes: 90 strategy: maxParallel: 2 @@ -98,7 +99,7 @@ jobs: solution: 'cc/build/FASTER.sln' msbuildArguments: '/m /p:Configuration=$(buildConfiguration) /p:Platform=$(buildPlatform)' - - script: 'ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "in_memory"' + - script: 'ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration)' workingDirectory: 'cc/build' displayName: 'Run Ctest' @@ -106,6 +107,7 @@ jobs: pool: vmImage: ubuntu-22.04 displayName: 'C++ (Linux)' + timeoutInMinutes: 75 steps: - script: | @@ -117,10 +119,10 @@ jobs: mkdir -p build/Debug build/Release cd build/Debug cmake -DCMAKE_BUILD_TYPE=Debug ../.. - make -j + make -j2 cd ../../build/Release cmake -DCMAKE_BUILD_TYPE=Release ../.. - make -j + make -j2 displayName: 'Compile' - script: | ulimit -s 65536 @@ -130,7 +132,7 @@ jobs: - job: 'csharpLinux' pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-22.04 displayName: 'C# (Linux)' strategy: @@ -175,7 +177,7 @@ jobs: command: test projects: '**/*.test.csproj' arguments: '--configuration $(buildConfiguration) -l "console;verbosity=detailed" --filter "TestCategory=Smoke"' - + - task: PublishTestResults@2 displayName: 'Publish Test Results' inputs: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9e27d1e6c..83f0d9391 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,7 @@ jobs: pool: vmImage: windows-2022 displayName: 'C# (Windows)' - timeoutInMinutes: 75 + timeoutInMinutes: 75 strategy: maxParallel: 2 @@ -62,7 +62,7 @@ jobs: command: test projects: '**/*.test.csproj' arguments: '--configuration $(buildConfiguration) -l "console;verbosity=detailed" --filter:TestCategory=Smoke' - + - task: PublishTestResults@2 displayName: 'Publish Test Results' inputs: @@ -74,6 +74,7 @@ jobs: pool: vmImage: windows-2022 displayName: 'C++ (Windows)' + timeoutInMinutes: 90 strategy: maxParallel: 2 @@ -98,7 +99,7 @@ jobs: solution: 'cc/build/FASTER.sln' msbuildArguments: '/m /p:Configuration=$(buildConfiguration) /p:Platform=$(buildPlatform)' - - script: 'ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration) -R "in_memory"' + - script: 'ctest -j 1 --interactive-debug-mode 0 --output-on-failure -C $(buildConfiguration)' workingDirectory: 'cc/build' displayName: 'Run Ctest' @@ -106,6 +107,7 @@ jobs: pool: vmImage: ubuntu-22.04 displayName: 'C++ (Linux)' + timeoutInMinutes: 75 steps: - script: | @@ -117,10 +119,10 @@ jobs: mkdir -p build/Debug build/Release cd build/Debug cmake -DCMAKE_BUILD_TYPE=Debug ../.. - make -j + make -j2 cd ../../build/Release cmake -DCMAKE_BUILD_TYPE=Release ../.. - make -j + make -j2 displayName: 'Compile' - script: | ulimit -s 65536 @@ -130,7 +132,7 @@ jobs: - job: 'csharpLinux' pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-22.04 displayName: 'C# (Linux)' strategy: diff --git a/cc/src/core/checkpoint_state.h b/cc/src/core/checkpoint_state.h index 7a4efcef5..c4309b2f5 100644 --- a/cc/src/core/checkpoint_state.h +++ b/cc/src/core/checkpoint_state.h @@ -203,32 +203,40 @@ class CheckpointState { } public: - void IssueIndexPersistenceCallback() { - if (!index_persistence_callback) { - return; // no callback was provided - } - // TODO: status was always Status::Ok on original FASTER code -- normally should check if failed is true - if (callback_context) { - const auto& callback = reinterpret_cast(index_persistence_callback); - callback(callback_context, Status::Ok); - } else { - const auto& callback = reinterpret_cast(index_persistence_callback); - callback(Status::Ok); - } + auto GetIndexPersistenceCallback() { + return [cb = this->index_persistence_callback, ctxt = this->callback_context]() { + if (!cb) { + return; // no callback was provided + } + + // TODO: Status was always set to Status::Ok on the original FASTER code. + // Ideally, we should check if `failed` member is true + if (ctxt) { + const auto& callback = reinterpret_cast(cb); + callback(ctxt, Status::Ok); + } else { + const auto& callback = reinterpret_cast(cb); + callback(Status::Ok); + } + }; } - void IssueHybridLogPersistenceCallback(uint64_t serial_num) { - if (!hybrid_log_persistence_callback) { - return; // no callback was provided - } - // TODO: status was always Status::Ok on original FASTER code -- normally should check if failed is true - if (callback_context) { - const auto& callback = reinterpret_cast(hybrid_log_persistence_callback); - callback(callback_context, Status::Ok, serial_num); - } else { - const auto& callback = reinterpret_cast(hybrid_log_persistence_callback); - callback(Status::Ok, serial_num); - } + auto GetHybridLogPersistenceCallback() { + return [cb = this->hybrid_log_persistence_callback, ctxt = this->callback_context](uint64_t serial_num) { + if (!cb) { + return; // no callback was provided + } + + // TODO: Status was always set to Status::Ok on the original FASTER code. + // Ideally, we should check if `failed` member is true + if (ctxt) { + const auto& callback = reinterpret_cast(cb); + callback(ctxt, Status::Ok, serial_num); + } else { + const auto& callback = reinterpret_cast(cb); + callback(Status::Ok, serial_num); + } + }; } void CheckpointDone() { diff --git a/cc/src/core/faster.h b/cc/src/core/faster.h index 6564c7f3e..f09b38ba6 100644 --- a/cc/src/core/faster.h +++ b/cc/src/core/faster.h @@ -2932,7 +2932,7 @@ bool FasterKv::GlobalMoveToNextState(SystemState current_state) checkpoint_.failed = true; } // Notify the host that the index checkpoint has completed. - checkpoint_.IssueIndexPersistenceCallback(); + checkpoint_.GetIndexPersistenceCallback()(); break; case Phase::IN_PROGRESS: { assert(next_state.action != Action::CheckpointIndex); @@ -2991,13 +2991,14 @@ bool FasterKv::GlobalMoveToNextState(SystemState current_state) if(hash_index_.WriteCheckpointMetadata(checkpoint_) != Status::Ok) { checkpoint_.failed = true; } - //auto index_persistence_callback = checkpoint_.index_persistence_callback; - checkpoint_.IssueIndexPersistenceCallback(); + auto callback = checkpoint_.GetIndexPersistenceCallback(); // The checkpoint is done; we can reset the contexts now. (Have to reset contexts before // another checkpoint can be started.) checkpoint_.CheckpointDone(); - // Checkpoint is done--no more work for threads to do. + // Checkpoint is done -- no more work for threads to do. system_state_.store(SystemState{ Action::None, Phase::REST, next_state.version }); + // Notify the host that the index checkpoint has completed. + callback(); } break; default: @@ -3206,7 +3207,8 @@ void FasterKv::HandleSpecialPhases() { // Handle WAIT_FLUSH -> PERSISTENCE_CALLBACK and PERSISTENCE_CALLBACK -> PERSISTENCE_CALLBACK if(previous_state.phase == Phase::WAIT_FLUSH) { // Persistence callback - checkpoint_.IssueHybridLogPersistenceCallback(prev_thread_ctx().serial_num); + auto callback = checkpoint_.GetHybridLogPersistenceCallback(); + callback(prev_thread_ctx().serial_num); // Thread has finished checkpointing. thread_ctx().phase = Phase::REST; // Thread ack that it has finished checkpointing. diff --git a/cc/src/core/persistent_memory_malloc.h b/cc/src/core/persistent_memory_malloc.h index 25bcef976..7ef224ce1 100644 --- a/cc/src/core/persistent_memory_malloc.h +++ b/cc/src/core/persistent_memory_malloc.h @@ -396,8 +396,10 @@ class PersistentMemoryMalloc { // Shift head address to tail address Address desired_head_address{ tail_address.page(), 0 }; - if(flushed_until_address.load() < desired_head_address) { - throw std::runtime_error{ "flushed_until_address < tail_address" }; + while (flushed_until_address.load() < desired_head_address) { + epoch_->ProtectAndDrain(); + disk->TryComplete(); + std::this_thread::yield(); } Address old_head_address; diff --git a/cc/test/cold_index_test.cc b/cc/test/cold_index_test.cc index 20c41aabc..62361a60b 100644 --- a/cc/test/cold_index_test.cc +++ b/cc/test/cold_index_test.cc @@ -35,18 +35,22 @@ INSTANTIATE_TEST_CASE_P( ColdIndexTests, ColdIndexTestParams, ::testing::Values( + // ================================================================== + // NOTE: Some tests are disabled for CI -- enable locally if needed + // ================================================================== + // w/o compaction - std::make_tuple(2048, 192_MiB, 0.4, false), + //std::make_tuple(2048, 192_MiB, 0.4, false), std::make_tuple(2048, 192_MiB, 0.0, false), - std::make_tuple((1 << 20), 192_MiB, 0.4, false), + //std::make_tuple((1 << 20), 192_MiB, 0.4, false), std::make_tuple((1 << 20), 192_MiB, 0.0, false), // \w atomic hash chunk updates - std::make_tuple(2048, 1_GiB, 0.9, false), - std::make_tuple((1 << 20), 1_GiB, 0.9, false), + //std::make_tuple(2048, 1_GiB, 0.9, false), + //std::make_tuple((1 << 20), 1_GiB, 0.9, false), // \w compaction - std::make_tuple(2048, 192_MiB, 0.4, true), + //std::make_tuple(2048, 192_MiB, 0.4, true), std::make_tuple(2048, 192_MiB, 0.0, true), - std::make_tuple((1 << 20), 192_MiB, 0.4, true), + //std::make_tuple((1 << 20), 192_MiB, 0.4, true), std::make_tuple((1 << 20), 192_MiB, 0.0, true), // \w compaction & atomic hash chunk updates std::make_tuple(2048, 1_GiB, 0.9, true), @@ -62,9 +66,9 @@ INSTANTIATE_TEST_CASE_P( ColdIndexRecoveryTests, ColdIndexRecoveryTestParams, ::testing::Values( - std::make_tuple(2048, 192_MiB, 0.4), + //std::make_tuple(2048, 192_MiB, 0.4), std::make_tuple(2048, 192_MiB, 0.0), - std::make_tuple((1 << 20), 192_MiB, 0.4), + //std::make_tuple((1 << 20), 192_MiB, 0.4), std::make_tuple((1 << 20), 192_MiB, 0.0) ) ); @@ -299,7 +303,7 @@ TEST_P(ColdIndexTestParams, UpsertRead_Serial) { rc_config.enabled = false; HlogCompactionConfig compaction_config{ - 250ms, 0.8, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.8, 0.1, 128_MiB, 768_MiB, 2, auto_compaction }; faster_t::IndexConfig index_config{ table_size, 256_MiB, 0.6 }; faster_t store{ index_config, log_mem_size, ROOT_PATH, log_mutable_frac, @@ -428,7 +432,7 @@ TEST_P(ColdIndexTestParams, ConcurrentUpsertAndRead) { faster_t store{ index_config, log_mem_size, ROOT_PATH, log_mutable_frac }; static constexpr size_t kNumRecords = 250'000; - static constexpr size_t kNumThreads = 8; + static constexpr size_t kNumThreads = 2; static std::atomic records_read{ 0 }; static std::atomic records_updated{ 0 }; diff --git a/cc/test/compact_lookup_test.cc b/cc/test/compact_lookup_test.cc index c24c4683e..45f07557a 100644 --- a/cc/test/compact_lookup_test.cc +++ b/cc/test/compact_lookup_test.cc @@ -42,6 +42,10 @@ static std::string ROOT_PATH{ "test_compact_lookup_store/" }; #endif // Parameterized test definition for in-memory tests +// ================================================================== +// NOTE: Some tests are disabled for CI -- enable locally if needed +// ================================================================== + // class CompactLookupParameterizedInMemTestFixture : public ::testing::TestWithParam> { }; @@ -50,9 +54,9 @@ INSTANTIATE_TEST_CASE_P( CompactLookupParameterizedInMemTestFixture, ::testing::Values( // Truncate after compaction -- single thread - std::pair(true, 1), - // Truncate after compaction -- 8 threads - std::pair(true, 8)) + //std::pair(true, 1), + // Truncate after compaction -- 2 threads + std::pair(true, 2)) ); // Parameterized test definition for on-disk tests @@ -64,11 +68,11 @@ INSTANTIATE_TEST_CASE_P( CompactLookupParameterizedOnDiskTestFixture, ::testing::Values( // Truncate after compaction, single thread - std::make_tuple(true, false, 1), - // Truncate after compaction, 8 threads - std::make_tuple(true, false, 8), - // Truncate after compaction, 8 threads w/ checkpoint - std::make_tuple(true, true, 8)) + //std::make_tuple(true, false, 1), + // Truncate after compaction, 2 threads + std::make_tuple(true, false, 2), + // Truncate after compaction, 2 threads w/ checkpoint + std::make_tuple(true, true, 2)) ); @@ -1355,7 +1359,7 @@ TEST_P(CompactLookupParameterizedOnDiskTestFixture, OnDiskRmw) { CreateNewLogDir(ROOT_PATH, log_fp); // NOTE: deliberately keeping the hash index small to test hash-chain chasing correctness - faster_t store{ 2048, (1 << 20) * 192, log_fp, 0.4 }; + faster_t store{ 2048, (1 << 20) * 256, log_fp, 0.4 }; uint32_t num_records = 20000; // ~160 MB of data bool shift_begin_address = std::get<0>(GetParam()); @@ -1984,7 +1988,7 @@ TEST_P(CompactLookupParameterizedOnDiskTestFixture, OnDiskVariableLengthKey) { std::string log_fp; CreateNewLogDir(ROOT_PATH, log_fp); - faster_t store{ (1 << 20), (1 << 20) * 192, log_fp, 0.4 }; + faster_t store{ 2048, (1 << 20) * 192, log_fp, 0.4 }; uint32_t numRecords = 12500; // will occupy ~512 MB space in store bool shift_begin_address = std::get<0>(GetParam()); diff --git a/cc/test/f2_recovery_test.cc b/cc/test/f2_recovery_test.cc index f2f2ffeaa..0215a2c63 100644 --- a/cc/test/f2_recovery_test.cc +++ b/cc/test/f2_recovery_test.cc @@ -45,29 +45,32 @@ INSTANTIATE_TEST_CASE_P( HotColdRecoveryTests, HotColdRecoveryTestParam, ::testing::Values( + // ================================================================== + // NOTE: Some tests are disabled for CI -- enable locally if needed + // ================================================================== + ///== w/o read-cache // Small hash hot & cold indices - std::make_tuple(2048, false, false, false), - std::make_tuple(2048, false, true, false), - std::make_tuple(2048, true, false, false), - std::make_tuple(2048, true, true, false), + //std::make_tuple(8192, false, false, false), + //std::make_tuple(8192, false, true, false), + //std::make_tuple(8192, true, false, false), + //std::make_tuple(8192, true, true, false), // Large hot & cold indices - std::make_tuple((1 << 22), false, false, false), - std::make_tuple((1 << 22), false, true, false), - std::make_tuple((1 << 22), true, false, false), - std::make_tuple((1 << 22), true, true, false), + //std::make_tuple((1 << 22), false, false, false), + //std::make_tuple((1 << 22), false, true, false), + //std::make_tuple((1 << 22), true, false, false), + //std::make_tuple((1 << 22), true, true, false), ///== w/ read-cache // Small hash hot & cold indices - std::make_tuple(2048, false, false, true), - std::make_tuple(2048, false, true, true), - std::make_tuple(2048, true, false, true), - std::make_tuple(2048, true, true, true), + std::make_tuple(8192, false, false, true), + std::make_tuple(8192, false, true, true), + std::make_tuple(8192, true, false, true), + std::make_tuple(8192, true, true, true), // Large hot & cold indices - std::make_tuple((1 << 22), false, false, true), - std::make_tuple((1 << 22), false, true, true), - std::make_tuple((1 << 22), true, false, true), - std::make_tuple((1 << 22), true, true, true) - + std::make_tuple((1 << 20), false, false, true), + std::make_tuple((1 << 20), false, true, true), + std::make_tuple((1 << 20), true, false, true), + std::make_tuple((1 << 20), true, true, true) ) ); @@ -335,16 +338,16 @@ TEST_P(HotColdRecoveryTestParam, CheckpointAndRecoverySerial) { ++num_threads_persistent; }; - // checkpoint (transition from REST to INDEX_CHKPT) - log_debug("Requesting checkpoint..."); - ASSERT_TRUE(store.Checkpoint(hybrid_log_persistence_callback, token, lazy_checkpoint)); - log_debug("Checkpoint requested!"); - if (!auto_compaction) { // perform cold-cold compaction store.CompactColdLog(store.cold_store.hlog.safe_read_only_address.control(), true); } + // checkpoint (transition from REST to INDEX_CHKPT) + log_debug("Requesting checkpoint..."); + ASSERT_TRUE(store.Checkpoint(hybrid_log_persistence_callback, token, lazy_checkpoint)); + log_debug("Checkpoint requested!"); + while(num_threads_persistent < 1) { store.CompletePending(false); } diff --git a/cc/test/f2_test.cc b/cc/test/f2_test.cc index 97201120f..c7578171d 100644 --- a/cc/test/f2_test.cc +++ b/cc/test/f2_test.cc @@ -48,16 +48,20 @@ INSTANTIATE_TEST_CASE_P( HotColdTests, HotColdParameterizedTestParam, ::testing::Values( + // ================================================================== + // NOTE: Some tests are disabled for CI -- enable locally if needed + // ================================================================== + // Small hash hot & cold indices - std::make_tuple(2048, false, false), - std::make_tuple(2048, false, true), - std::make_tuple(2048, true, false), - std::make_tuple(2048, true, true), + //std::make_tuple(8192, false, false), + //std::make_tuple(8192, true, false), + //std::make_tuple(8192, false, true), + std::make_tuple(8192, true, true), // Large hot & cold indices - std::make_tuple((1 << 22), false, false), - std::make_tuple((1 << 22), false, true), - std::make_tuple((1 << 22), true, false), - std::make_tuple((1 << 22), true, true) + //std::make_tuple((1 << 20), false, false), + //std::make_tuple((1 << 20), true, false), + //std::make_tuple((1 << 20), false, true), + std::make_tuple((1 << 20), true, true) ) ); @@ -68,6 +72,7 @@ static std::string root_path{ "test_f2_store/" }; #endif static constexpr uint64_t kCompletePendingInterval = 128; +static constexpr uint8_t kNumCompactionThreads = 2; /// Upsert context required to insert data for unit testing. template @@ -267,9 +272,9 @@ TEST_P(HotColdParameterizedTestParam, UpsertRead) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 768_MiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ table_size, 256_MiB, 0.6 }; f2_t store{ table_size, 192_MiB, hot_fp, @@ -408,9 +413,9 @@ TEST_P(HotColdParameterizedTestParam, HotColdCompaction) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 2_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 2_GiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ 8192, 256_MiB, 0.6 }; f2_t store{ table_size, 192_MiB, hot_fp, @@ -505,9 +510,9 @@ TEST_P(HotColdParameterizedTestParam, UpsertDelete) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 768_MiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ table_size, 256_MiB, 0.6 }; f2_t store{ table_size, 192_MiB, hot_fp, @@ -628,12 +633,12 @@ TEST_P(HotColdParameterizedTestParam, Rmw) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 768_MiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ table_size, 256_MiB, 0.6 }; - f2_t store{ table_size, 192_MiB, hot_fp, + f2_t store{ table_size, 256_MiB, hot_fp, cold_index_config, 192_MiB, cold_fp, 0.4, 0, rc_config, f2_compaction_config }; @@ -812,9 +817,9 @@ TEST_P(HotColdParameterizedTestParam, ConcurrentOps) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 768_MiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ table_size, 256_MiB, 0.6 }; f2_t store{ table_size, 192_MiB, hot_fp, @@ -1152,9 +1157,9 @@ TEST_P(HotColdParameterizedTestParam, VariableLengthKey) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 768_MiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ table_size, 256_MiB, 0.6 }; f2_t store{ table_size, 192_MiB, hot_fp, @@ -1527,9 +1532,9 @@ TEST_P(HotColdParameterizedTestParam, VariableLengthValue) { F2CompactionConfig f2_compaction_config; f2_compaction_config.hot_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 256_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 256_MiB, kNumCompactionThreads, auto_compaction }; f2_compaction_config.cold_store = HlogCompactionConfig{ - 250ms, 0.9, 0.1, 128_MiB, 768_MiB, 4, auto_compaction }; + 250ms, 0.9, 0.1, 128_MiB, 768_MiB, kNumCompactionThreads, auto_compaction }; f2_t::ColdIndexConfig cold_index_config{ table_size, 256_MiB, 0.6 }; f2_t store{ table_size, 192_MiB, hot_fp, diff --git a/cc/test/paging_test.h b/cc/test/paging_test.h index 0cac92409..c684dde5f 100644 --- a/cc/test/paging_test.h +++ b/cc/test/paging_test.h @@ -29,13 +29,23 @@ INSTANTIATE_TEST_CASE_P( PagingTests, PagingTestParam, ::testing::Values( - std::make_tuple(2048, false, false), - std::make_tuple((1 << 20), false, false), - std::make_tuple(2048, true, true), + // ================================================================== + // NOTE: Some tests are disabled for CI -- enable locally if needed + // ================================================================== + + // === w/o auto-compaction + //std::make_tuple(8192, false, false), + //std::make_tuple((1 << 20), false, false), + // === w/ auto-compaction + std::make_tuple(8192, true, true), std::make_tuple((1 << 20), true, true) ) ); +static constexpr uint64_t kRefreshInterval = 256; +static constexpr uint64_t kCompletePendingInterval = 256; +static constexpr uint8_t kNumCompactionThreads = 2; + TEST_P(PagingTestParam, UpsertRead_Serial) { class Key { public: @@ -226,7 +236,7 @@ TEST_P(PagingTestParam, UpsertRead_Serial) { }; HlogCompactionConfig compaction_config{ - 250ms, 0.9, 0.2, 256_MiB, 1_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.2, 256_MiB, 1_GiB, kNumCompactionThreads, auto_compaction }; store_t store{ table_size, 256_MiB, ROOT_PATH, 0.5, rc_config, compaction_config }; @@ -244,7 +254,7 @@ TEST_P(PagingTestParam, UpsertRead_Serial) { ASSERT_TRUE(false); }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -261,7 +271,7 @@ TEST_P(PagingTestParam, UpsertRead_Serial) { ++records_read; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -287,7 +297,7 @@ TEST_P(PagingTestParam, UpsertRead_Serial) { ASSERT_TRUE(false); }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -313,7 +323,7 @@ TEST_P(PagingTestParam, UpsertRead_Serial) { ++records_read; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -526,7 +536,7 @@ TEST_P(PagingTestParam, UpsertRead_Concurrent) { }; HlogCompactionConfig compaction_config{ - 250ms, 0.9, 0.2, 256_MiB, 1_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.2, 256_MiB, 1_GiB, kNumCompactionThreads, auto_compaction }; store_t store{ table_size, 256_MiB, ROOT_PATH, 0.5, rc_config, compaction_config }; @@ -547,7 +557,7 @@ TEST_P(PagingTestParam, UpsertRead_Concurrent) { ASSERT_TRUE(false); }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store_->Refresh(); } @@ -584,7 +594,7 @@ TEST_P(PagingTestParam, UpsertRead_Concurrent) { ++records_read; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -620,8 +630,8 @@ TEST_P(PagingTestParam, UpsertRead_Concurrent) { // Restart session store.StartSession(); - // Delete some old copies of records (160 MB) that we no longer need. - static constexpr uint64_t kNewBeginAddress{ 167772160L }; + // Delete some old copies of records (96 MiB) that we no longer need. + static constexpr uint64_t kNewBeginAddress{ 100663296 }; static std::atomic truncated{ false }; static std::atomic complete{ false }; @@ -650,7 +660,7 @@ TEST_P(PagingTestParam, UpsertRead_Concurrent) { ++records_read; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -793,7 +803,7 @@ TEST_P(PagingTestParam, Rmw) { }; HlogCompactionConfig compaction_config{ - 250ms, 0.9, 0.2, 256_MiB, 1_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.2, 256_MiB, 1_GiB, kNumCompactionThreads, auto_compaction }; store_t store{ table_size, 256_MiB, ROOT_PATH, 0.5, rc_config, compaction_config }; @@ -814,7 +824,7 @@ TEST_P(PagingTestParam, Rmw) { ++records_touched; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -842,7 +852,7 @@ TEST_P(PagingTestParam, Rmw) { ++records_touched; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -987,7 +997,7 @@ TEST_P(PagingTestParam, Rmw_Large) { }; HlogCompactionConfig compaction_config{ - 250ms, 0.9, 0.2, 256_MiB, 1_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.2, 256_MiB, 1_GiB, kNumCompactionThreads, auto_compaction }; store_t store{ table_size, 256_MiB, ROOT_PATH, 0.5, rc_config, compaction_config }; @@ -1007,7 +1017,7 @@ TEST_P(PagingTestParam, Rmw_Large) { ++records_touched; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -1035,7 +1045,7 @@ TEST_P(PagingTestParam, Rmw_Large) { ++records_touched; }; - if(idx % 256 == 0) { + if(idx % kRefreshInterval == 0) { store.Refresh(); } @@ -1203,7 +1213,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent) { ASSERT_EQ(Status::Ok, result); }; - if(idx % 256 == 0) { + if(idx % kCompletePendingInterval == 0) { store_->CompletePending(false); } @@ -1227,7 +1237,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent) { ASSERT_EQ(7 * kNumThreads, context->counter); }; - if(idx % 256 == 0) { + if(idx % kCompletePendingInterval == 0) { store_->CompletePending(false); } @@ -1253,7 +1263,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent) { ASSERT_EQ(13 * kNumThreads, context->counter); }; - if(idx % 256 == 0) { + if(idx % kCompletePendingInterval == 0) { store_->CompletePending(false); } @@ -1287,7 +1297,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent) { }; HlogCompactionConfig compaction_config{ - 250ms, 0.9, 0.2, 256_MiB, 1_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.2, 256_MiB, 1_GiB, kNumCompactionThreads, auto_compaction }; store_t store{ table_size, 256_MiB, ROOT_PATH, 0.5, rc_config, compaction_config }; @@ -1480,7 +1490,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent_Large) { ASSERT_EQ(Status::Ok, result); }; - if(idx % 256 == 0) { + if(idx % kCompletePendingInterval == 0) { store_->CompletePending(false); } @@ -1504,7 +1514,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent_Large) { ASSERT_EQ(7 * kNumThreads, context->counter); }; - if(idx % 256 == 0) { + if(idx % kCompletePendingInterval == 0) { store_->CompletePending(false); } @@ -1530,7 +1540,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent_Large) { ASSERT_EQ(13 * kNumThreads, context->counter); }; - if(idx % 256 == 0) { + if(idx % kCompletePendingInterval == 0) { store_->CompletePending(false); } @@ -1564,7 +1574,7 @@ TEST_P(PagingTestParam, Rmw_Concurrent_Large) { }; HlogCompactionConfig compaction_config{ - 250ms, 0.9, 0.2, 256_MiB, 1_GiB, 4, auto_compaction }; + 250ms, 0.9, 0.2, 256_MiB, 1_GiB, kNumCompactionThreads, auto_compaction }; store_t store{ table_size, 256_MiB, ROOT_PATH, 0.5, rc_config, compaction_config };