it: adjust and enable HeartbeatFailed

Since we implemented metrics, we can enable HeartbeatFailed test with some
adjustments to log filtering.

This test seems to fail under valgrind. This is why I enable it to run
next to other test that cannot be run under valgrind.

Note: The original test seems to be flaky for Cassandra. The following scenario
occurred:
1. node2 is paused
2. keepaliver notifies the pool refiller about that
3. refiller removes the connection to node2 (metrics::total_connections -= 1)
4. in the test, we read get_metrics().total_connections < initial_connections - we go out of the loop
5. refiller tries to open a connection again (metrics::total_connections += 1)
6. we read get_metrics().total_connections, and expect total_connections to be
less than initial_connections - but it is not.

This is why, to combat this, I adjusted the test so the same metrics snapshot
is used to leave the loop and make an assertion. In this case, the aforementioned
"unlucky" scenario will not happen.

Author: muzarski

Makefile

@@ -36,6 +36,11 @@ SCYLLA_TEST_FILTER := $(subst ${SPACE},${EMPTY},ClusterTests.*\
 :UseKeyspaceCaseSensitiveTests.Integration_Cassandra_ConnectWithKeyspace)
 endif
 
+ifndef SCYLLA_NO_VALGRIND_TEST_FILTER
+SCYLLA_NO_VALGRIND_TEST_FILTER := $(subst ${SPACE},${EMPTY},AsyncTests.Integration_Cassandra_Simple\
+:HeartbeatTests.Integration_Cassandra_HeartbeatFailed)
+endif
+
 ifndef CASSANDRA_TEST_FILTER
 CASSANDRA_TEST_FILTER := $(subst ${SPACE},${EMPTY},ClusterTests.*\
 :BasicsTests.*\
@@ -72,6 +77,11 @@ CASSANDRA_TEST_FILTER := $(subst ${SPACE},${EMPTY},ClusterTests.*\
 :UseKeyspaceCaseSensitiveTests.Integration_Cassandra_ConnectWithKeyspace)
 endif
 
+ifndef CASSANDRA_NO_VALGRIND_TEST_FILTER
+CASSANDRA_NO_VALGRIND_TEST_FILTER := $(subst ${SPACE},${EMPTY},AsyncTests.Integration_Cassandra_Simple\
+:HeartbeatTests.Integration_Cassandra_HeartbeatFailed)
+endif
+
 ifndef CCM_COMMIT_ID
 	# TODO: change it back to master/next when https://github.com/scylladb/scylla-ccm/issues/646 is fixed.
 	export CCM_COMMIT_ID := 5392dd68
@@ -226,7 +236,7 @@ endif
 	@echo "Running integration tests on scylla ${SCYLLA_VERSION}"
 	valgrind --error-exitcode=123 --leak-check=full --errors-for-leak-kinds=definite build/cassandra-integration-tests --scylla --version=${SCYLLA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${SCYLLA_TEST_FILTER}"
 	@echo "Running timeout sensitive tests on scylla ${SCYLLA_VERSION}"
-	build/cassandra-integration-tests --scylla --version=${SCYLLA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="AsyncTests.Integration_Cassandra_Simple"
+	build/cassandra-integration-tests --scylla --version=${SCYLLA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${SCYLLA_NO_VALGRIND_TEST_FILTER}"
 
 run-test-integration-cassandra: prepare-integration-test download-ccm-cassandra-image install-java8-if-missing
 ifdef DONT_REBUILD_INTEGRATION_BIN
@@ -237,7 +247,7 @@ endif
 	@echo "Running integration tests on cassandra ${CASSANDRA_VERSION}"
 	valgrind --error-exitcode=123 --leak-check=full --errors-for-leak-kinds=definite build/cassandra-integration-tests --version=${CASSANDRA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${CASSANDRA_TEST_FILTER}"
 	@echo "Running timeout sensitive tests on cassandra ${CASSANDRA_VERSION}"
-	build/cassandra-integration-tests --version=${CASSANDRA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="AsyncTests.Integration_Cassandra_Simple"
+	build/cassandra-integration-tests --version=${CASSANDRA_VERSION} --category=CASSANDRA --verbose=ccm --gtest_filter="${CASSANDRA_NO_VALGRIND_TEST_FILTER}"
 
 run-test-unit: install-cargo-if-missing _update-rust-tooling
 	@cd ${CURRENT_DIR}/scylla-rust-wrapper; cargo test

tests/src/integration/tests/test_heartbeat.cpp

@@ -83,19 +83,22 @@ CASSANDRA_INTEGRATION_TEST_F(HeartbeatTests, HeartbeatDisabled) {
 CASSANDRA_INTEGRATION_TEST_F(HeartbeatTests, HeartbeatFailed) {
   CHECK_FAILURE;
 
-  logger_.add_critera("Failed to send a heartbeat within connection idle interval.");
+  logger_.add_critera("Timed out while waiting for response to keepalive request");
   Cluster cluster =
       default_cluster().with_connection_heartbeat_interval(1).with_connection_idle_timeout(5);
   connect(cluster);
 
   cass_uint64_t initial_connections = session_.metrics().stats.total_connections;
   pause_node(2);
   start_timer();
-  while (session_.metrics().stats.total_connections >= initial_connections &&
+
+  CassMetrics metrics = session_.metrics();
+  while (metrics.stats.total_connections >= initial_connections &&
          elapsed_time() < 60000) {
     session_.execute_async(SELECT_ALL_SYSTEM_LOCAL_CQL); // Simply execute statements ignore any
                                                          // error that can occur from paused node
+    metrics = session_.metrics();
   }
-  EXPECT_LT(session_.metrics().stats.total_connections, initial_connections);
+  EXPECT_LT(metrics.stats.total_connections, initial_connections);
   EXPECT_GE(logger_.count(), 1u);
 }