Skip to content

Commit f23e98b

Browse files
committed
it: adjust and enable HeartbeatFailed
Since we implemented metrics, we can enable HeartbeatFailed test with some adjustments to log filtering. This test seems to fail under valgrind. This is why I enable it to run next to other test that cannot be run under valgrind. Note: The original test seems to be flaky for Cassandra. The following scenario occurred: 1. node2 is paused 2. keepaliver notifies the pool refiller about that 3. refiller removes the connection to node2 (metrics::total_connections -= 1) 4. in the test, we read get_metrics().total_connections < initial_connections - we go out of the loop 5. refiller tries to open a connection again (metrics::total_connections += 1) 6. we read get_metrics().total_connections, and expect total_connections to be less than initial_connections - but it is not. This is why, to combat this, I adjusted the test so the same metrics snapshot is used to leave the loop and make an assertion. In this case, the aforementioned "unlucky" scenario will not happen.
1 parent 5304ee2 commit f23e98b

File tree

2 files changed

+18
-5
lines changed

2 files changed

+18
-5
lines changed

Makefile

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ SCYLLA_TEST_FILTER := $(subst ${SPACE},${EMPTY},ClusterTests.*\
3636
:UseKeyspaceCaseSensitiveTests.Integration_Cassandra_ConnectWithKeyspace)
3737
endif
3838

39+
ifndef SCYLLA_NO_VALGRIND_TEST_FILTER
40+
SCYLLA_NO_VALGRIND_TEST_FILTER := $(subst ${SPACE},${EMPTY},AsyncTests.Integration_Cassandra_Simple\
41+
:HeartbeatTests.Integration_Cassandra_HeartbeatFailed)
42+
endif
43+
3944
ifndef CASSANDRA_TEST_FILTER
4045
CASSANDRA_TEST_FILTER := $(subst ${SPACE},${EMPTY},ClusterTests.*\
4146
:BasicsTests.*\
@@ -72,6 +77,11 @@ CASSANDRA_TEST_FILTER := $(subst ${SPACE},${EMPTY},ClusterTests.*\
7277
:UseKeyspaceCaseSensitiveTests.Integration_Cassandra_ConnectWithKeyspace)
7378
endif
7479

80+
ifndef CASSANDRA_NO_VALGRIND_TEST_FILTER
81+
CASSANDRA_NO_VALGRIND_TEST_FILTER := $(subst ${SPACE},${EMPTY},AsyncTests.Integration_Cassandra_Simple\
82+
:HeartbeatTests.Integration_Cassandra_HeartbeatFailed)
83+
endif
84+
7585
ifndef CCM_COMMIT_ID
7686
# TODO: change it back to master/next when https://github.com/scylladb/scylla-ccm/issues/646 is fixed.
7787
export CCM_COMMIT_ID := 5392dd68
@@ -226,7 +236,7 @@ endif
226236
@echo "Running integration tests on scylla ${SCYLLA_VERSION}"
227237
valgrind --error-exitcode=123 --leak-check=full --errors-for-leak-kinds=definite build/cassandra-integration-tests --scylla --version=${SCYLLA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${SCYLLA_TEST_FILTER}"
228238
@echo "Running timeout sensitive tests on scylla ${SCYLLA_VERSION}"
229-
build/cassandra-integration-tests --scylla --version=${SCYLLA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="AsyncTests.Integration_Cassandra_Simple"
239+
build/cassandra-integration-tests --scylla --version=${SCYLLA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${SCYLLA_NO_VALGRIND_TEST_FILTER}"
230240

231241
run-test-integration-cassandra: prepare-integration-test download-ccm-cassandra-image install-java8-if-missing
232242
ifdef DONT_REBUILD_INTEGRATION_BIN
@@ -237,7 +247,7 @@ endif
237247
@echo "Running integration tests on cassandra ${CASSANDRA_VERSION}"
238248
valgrind --error-exitcode=123 --leak-check=full --errors-for-leak-kinds=definite build/cassandra-integration-tests --version=${CASSANDRA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${CASSANDRA_TEST_FILTER}"
239249
@echo "Running timeout sensitive tests on cassandra ${CASSANDRA_VERSION}"
240-
build/cassandra-integration-tests --version=${CASSANDRA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="AsyncTests.Integration_Cassandra_Simple"
250+
build/cassandra-integration-tests --version=${CASSANDRA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${CASSANDRA_NO_VALGRIND_TEST_FILTER}"
241251

242252
run-test-unit: install-cargo-if-missing _update-rust-tooling
243253
@cd ${CURRENT_DIR}/scylla-rust-wrapper; cargo test

tests/src/integration/tests/test_heartbeat.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,19 +83,22 @@ CASSANDRA_INTEGRATION_TEST_F(HeartbeatTests, HeartbeatDisabled) {
8383
CASSANDRA_INTEGRATION_TEST_F(HeartbeatTests, HeartbeatFailed) {
8484
CHECK_FAILURE;
8585

86-
logger_.add_critera("Failed to send a heartbeat within connection idle interval.");
86+
logger_.add_critera("Timed out while waiting for response to keepalive request");
8787
Cluster cluster =
8888
default_cluster().with_connection_heartbeat_interval(1).with_connection_idle_timeout(5);
8989
connect(cluster);
9090

9191
cass_uint64_t initial_connections = session_.metrics().stats.total_connections;
9292
pause_node(2);
9393
start_timer();
94-
while (session_.metrics().stats.total_connections >= initial_connections &&
94+
95+
CassMetrics metrics = session_.metrics();
96+
while (metrics.stats.total_connections >= initial_connections &&
9597
elapsed_time() < 60000) {
9698
session_.execute_async(SELECT_ALL_SYSTEM_LOCAL_CQL); // Simply execute statements ignore any
9799
// error that can occur from paused node
100+
metrics = session_.metrics();
98101
}
99-
EXPECT_LT(session_.metrics().stats.total_connections, initial_connections);
102+
EXPECT_LT(metrics.stats.total_connections, initial_connections);
100103
EXPECT_GE(logger_.count(), 1u);
101104
}

0 commit comments

Comments
 (0)