ZTS: update the relevant mmp test cases

- mmp_concurrent_import: added test case to verify that concurrent
  import correctness.  The pool may only be imported once.

- mmp_exported_import: an activity check is now required for pools
  which were cleanly exported if the system and pool hostids don't
  match.

- mmp_inactive_import: an activity check is now required for any
  pool which wasn't cleanly exported, even if the system and pool
  hostids match.

- mmp_on_uberblocks: updated expected uberblocks to take in to account
  the value MMP_INTERVAL_DEFAULT is set too.

- mmp_reset_interval: reduce the number of iterations from 10 to 3.
  This is sufficient to verify functionality and significantly speeds
  up the test.

- mmp_on_uberblocks: adjust the thresholds and increase the runtime
  to avoid false positives observed in CI.

- Update tests to use 'zhack action idle' instead of ztest to improve
  the reliability of the tests.

- Add additional log_note messages to test cases which have multiple
  verification steps to make it clear which portion of a test failed
  when reviewing the logs.

- Replace default_setup/cleanup_noexit calls with 'zpool create' and
  'zpool destroy' calls to avoid additional unnecessary dataset
  creation work.

- Update activity/noactivity check helper functions to use the
  ZFS_LOAD_INFO_DEBUG information now available from 'zpool import'
  to determine if this activity check ran and why.  This is more
  reliable in the CI than measuring the runtime.

- Removed all mmp tests from the zts-report.py exceptions list.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

Author: behlendorf

module/zfs/spa.c

@@ -4099,8 +4099,8 @@ spa_activity_check_tryimport(spa_t *spa, uberblock_t *spa_ub,
 		hrtime_t extra_delay = MMP_IMPORT_VERIFY_ITERS *
 		    MSEC2NSEC(MMP_INTERVAL_VALID(spa_ub) ?
 		    MMP_INTERVAL(spa_ub) : MMP_MIN_INTERVAL);
-		cmn_err(CE_NOTE, "pool '%s' multihost activity check "
-		    "required, %llu seconds remaining", spa_load_name(spa),
+		cmn_err(CE_NOTE, "pool '%s' activity check required, "
+		    "%llu seconds remaining", spa_load_name(spa),
 		    (u_longlong_t)MAX(NSEC2SEC(import_delay + extra_delay), 1));
 		spa_import_progress_set_notes(spa, "Checking MMP activity, "
 		    "waiting %llu ms", (u_longlong_t)NSEC2MSEC(import_delay));

tests/runfiles/linux.run

@@ -159,8 +159,10 @@ tags = ['functional', 'mmap']
 tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
     'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history',
-    'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk']
+    'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk',
+    'mmp_concurrent_import']
 tags = ['functional', 'mmp']
+timeout = 1200
 
 [tests/functional/mount:Linux]
 tests = ['umount_unlinked_drain', 'mount_loopback']

tests/test-runner/bin/zts-report.py.in

@@ -247,7 +247,6 @@ maybe = {
     'l2arc/persist_l2arc_005_pos': ['FAIL', known_reason],
     'largest_pool/largest_pool_001_pos': ['FAIL', known_reason],
     'mmap/mmap_sync_001_pos': ['FAIL', known_reason],
-    'mmp/mmp_on_uberblocks': ['FAIL', known_reason],
     'pam/setup': ['SKIP', "pamtester might be not available"],
     'pool_checkpoint/checkpoint_discard_busy': ['FAIL', 11946],
     'projectquota/setup': ['SKIP', exec_reason],
@@ -366,9 +365,6 @@ elif sys.platform.startswith('linux'):
         'io/io_uring': ['SKIP', 'io_uring support required'],
         'limits/filesystem_limit': ['SKIP', known_reason],
         'limits/snapshot_limit': ['SKIP', known_reason],
-        'mmp/mmp_active_import': ['FAIL', known_reason],
-        'mmp/mmp_exported_import': ['FAIL', known_reason],
-        'mmp/mmp_inactive_import': ['FAIL', known_reason],
         'stat/statx_dioalign': ['SKIP', 'statx_reason'],
     })
 

tests/zfs-tests/tests/Makefile.am

@@ -1719,6 +1719,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/mmap/setup.ksh \
 	functional/mmp/cleanup.ksh \
 	functional/mmp/mmp_active_import.ksh \
+	functional/mmp/mmp_concurrent_import.ksh \
 	functional/mmp/mmp_exported_import.ksh \
 	functional/mmp/mmp_hostid.ksh \
 	functional/mmp/mmp_inactive_import.ksh \

tests/zfs-tests/tests/functional/mmp/cleanup.ksh

@@ -24,6 +24,10 @@
 
 verify_runnable "global"
 
+if [ -e $HOSTID_FILE.save ]; then
+	mv -f ${HOSTID_FILE}.save $HOSTID_FILE
+fi
+
 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF
 
 log_pass "mmp cleanup passed"

tests/zfs-tests/tests/functional/mmp/mmp.cfg

@@ -25,14 +25,16 @@ export DISK=${DISKS%% *}
 export HOSTID_FILE="/etc/hostid"
 export HOSTID1=01234567
 export HOSTID2=89abcdef
+export HOSTID3=aaaabbbb
+export HOSTID4=ccccdddd
 
 export TXG_TIMEOUT_LONG=5000
 export TXG_TIMEOUT_DEFAULT=5
 
 export MMP_POOL=mmppool
 export MMP_DIR=$TEST_BASE_DIR/mmp
 export MMP_CACHE=$MMP_DIR/zpool.cache
-export MMP_ZTEST_LOG=$MMP_DIR/ztest.log
+export MMP_ZHACK_LOG=$MMP_DIR/zhack.log
 export MMP_HISTORY=100
 export MMP_HISTORY_OFF=0
 
@@ -43,5 +45,3 @@ export MMP_INTERVAL_MIN=100
 export MMP_IMPORT_INTERVALS=20
 export MMP_FAIL_INTERVALS_DEFAULT=10
 export MMP_FAIL_INTERVALS_MIN=2
-
-export MMP_TEST_DURATION_DEFAULT=$((MMP_IMPORT_INTERVALS*MMP_INTERVAL_DEFAULT/1000))

tests/zfs-tests/tests/functional/mmp/mmp.kshlib

@@ -99,11 +99,11 @@ function mmp_pool_create_simple # pool dir
 	log_must zpool set multihost=on $pool
 }
 
-function mmp_pool_create # pool dir
+function mmp_pool_create_zhack # pool dir
 {
 	typeset pool=${1:-$MMP_POOL}
 	typeset dir=${2:-$MMP_DIR}
-	typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool"
+	typeset opts="-d $dir action idle -t120 $pool"
 
 	mmp_pool_create_simple $pool $dir
 
@@ -112,11 +112,11 @@ function mmp_pool_create # pool dir
 	log_must mmp_clear_hostid
 	log_must mmp_set_hostid $HOSTID2
 
-	log_note "Starting ztest in the background as hostid $HOSTID1"
-	log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &"
+	log_note "Starting zhack in the background as hostid $HOSTID1"
+	log_must eval "ZFS_HOSTID=$HOSTID1 zhack $opts >$MMP_ZHACK_LOG 2>&1 &"
 
 	while ! is_pool_imported "$pool" "-d $dir"; do
-		log_must pgrep ztest
+		log_must pgrep zhack
 		log_must sleep 5
 	done
 }
@@ -126,10 +126,10 @@ function mmp_pool_destroy # pool dir
 	typeset pool=${1:-$MMP_POOL}
 	typeset dir=${2:-$MMP_DIR}
 
-	ZTESTPID=$(pgrep ztest)
-	if [ -n "$ZTESTPID" ]; then
-		log_must kill $ZTESTPID
-		wait $ZTESTPID
+	ZHACKPID=$(pgrep zhack)
+	if [ -n "$ZHACKPID" ]; then
+		log_must kill $ZHACKPID
+		wait $ZHACKPID
 	fi
 
 	if poolexists $pool; then
@@ -158,33 +158,34 @@ function import_no_activity_check # pool opts
 	typeset pool=$1
 	typeset opts=$2
 
-	typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1))
-
-	SECONDS=0
-	zpool import $opts $pool
+	RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool)
 	typeset rc=$?
 
-	if [[ $SECONDS -gt $max_duration ]]; then
-		log_fail "ERROR: import_no_activity_check unexpected activity \
-check (${SECONDS}s gt $max_duration)"
+	# mmp_result: 3 (ESRCH) no activity check was run not required
+	# mmp_result: 6 (ENXIO) no activity check was run hostid not set
+	if ! echo "$RESULT" | grep -q "mmp_result: 3" &&
+	   ! echo "$RESULT" | grep -q "mmp_result: 6"; then
+		log_note "ERROR: $RESULT"
+		log_fail "ERROR: import_no_activity_check unexpected activity check"
 	fi
 
 	return $rc
 }
 
-function import_activity_check # pool opts act_test_duration
+function import_activity_check # pool opts
 {
 	typeset pool=$1
 	typeset opts=$2
-	typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT}
 
-	SECONDS=0
-	zpool import $opts $pool
+	RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool)
 	typeset rc=$?
 
-	if [[ $SECONDS -le $min_duration ]]; then
-		log_fail "ERROR: import_activity_check expected activity check \
-(${SECONDS}s le min_duration $min_duration)"
+	# mmp_result: 0 (Success) check was run no activity detected
+	# mmp_result: 121 (EREMOTEIO) check was run activity detected
+	# mmp_result: 4 (EINTR) check was run but interrupted by user
+	if ! echo "$RESULT" | grep -q "mmp_result: 0"; then
+		log_note "ERROR: $RESULT"
+		log_fail "ERROR: import_activity_check expected activity check"
 	fi
 
 	return $rc

tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh

@@ -24,10 +24,10 @@
 #	with one hostid be importable by a host with a different hostid.
 #
 # STRATEGY:
-#	 1. Simulate an active pool on another host with ztest.
+#	 1. Simulate an active pool on another host with zhack.
 #	 2. Verify 'zpool import' reports an active pool.
 #	 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool.
-#	 4. Kill ztest to make pool eligible for import.
+#	 4. Kill zhack to make pool eligible for import.
 #	 5. Verify 'zpool import' fails with the expected error message.
 #	 6. Verify 'zpool import $MMP_POOL' fails with the expected message.
 #	 7. Verify 'zpool import -f $MMP_POOL' can now import the pool.
@@ -44,32 +44,35 @@ function cleanup
 {
 	mmp_pool_destroy $MMP_POOL $MMP_DIR
 	log_must mmp_clear_hostid
-	ZTESTPID=$(pgrep ztest)
-	if [ -n "$ZTESTPID" ]; then
-		for pid in $ZTESTPID; do
+	ZHACKPID=$(pgrep zhack)
+	if [ -n "$ZHACKPID" ]; then
+		for pid in $ZHACKPID; do
 			log_must kill -9 $pid
 		done
 	else
-		# if ztest not running and log present, ztest crashed
-		if [ -f $MMP_ZTEST_LOG ]; then
-			log_note "ztest appears to have crashed.  Tail of log:"
-			tail -n 50 $MMP_ZTEST_LOG
+		# if zhack is not running and log present, zhack crashed
+		if [ -f $MMP_ZHACK_LOG ]; then
+			log_note "zhack appears to have crashed.  Tail of log:"
+			tail -n 50 $MMP_ZHACK_LOG
 		fi
 	fi
 }
 
 log_assert "multihost=on|off active pool activity checks"
 log_onexit cleanup
 
-# 1. Simulate an active pool on another host with ztest.
+# 1. Simulate an active pool on another host with zhack.
+log_note "Simulate an active pool on another host with zhack"
 mmp_pool_destroy $MMP_POOL $MMP_DIR
-mmp_pool_create $MMP_POOL $MMP_DIR
+mmp_pool_create_zhack $MMP_POOL $MMP_DIR
 
 # 2. Verify 'zpool import' reports an active pool.
+log_note "Verify 'zpool import' reports an active pool"
 log_must mmp_set_hostid $HOSTID2
 log_must is_pool_imported $MMP_POOL "-d $MMP_DIR"
 
 # 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool.
+log_note "Verify 'zpool import [-f] $MMP_POOL' cannot import the pool"
 MMP_IMPORTED_MSG="Cannot import '$MMP_POOL': pool is imported"
 
 log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG"
@@ -84,21 +87,23 @@ for i in {1..10}; do
 	    "$MMP_IMPORTED_MSG"
 done
 
-# 4. Kill ztest to make pool eligible for import.  Poll with 'zpool status'.
-ZTESTPID=$(pgrep ztest)
-if [ -n "$ZTESTPID" ]; then
-	log_must kill -9 $ZTESTPID
+# 4. Kill zhack to make pool eligible for import.  Poll with 'zpool status'.
+log_note "Kill zhack to make pool eligible for import.  Poll with 'zpool status'"
+ZHACKPID=$(pgrep zhack)
+if [ -n "$ZHACKPID" ]; then
+	log_must kill -9 $ZHACKPID
 fi
 log_must wait_pool_imported $MMP_POOL "-d $MMP_DIR"
-if [ -f $MMP_ZTEST_LOG ]; then
-	log_must rm $MMP_ZTEST_LOG
+if [ -f $MMP_ZHACK_LOG ]; then
+	log_must rm $MMP_ZHACK_LOG
 fi
 
 # 5. Verify 'zpool import' fails with the expected error message, when
 #    - hostid=0:        - configuration error
 #    - hostid=matches   - safe to import the pool
 #    - hostid=different - previously imported on a different system
 #
+log_note "Verify 'zpool import' fails with the expected error message"
 log_must mmp_clear_hostid
 MMP_IMPORTED_MSG="Set a unique system hostid"
 log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" "$MMP_IMPORTED_MSG"
@@ -113,13 +118,16 @@ MMP_IMPORTED_MSG="The pool was last accessed by another system."
 log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "status" "$MMP_IMPORTED_MSG"
 
 # 6. Verify 'zpool import $MMP_POOL' fails with the expected message.
+log_note "Verify 'zpool import $MMP_POOL' fails with the expected message"
 MMP_IMPORTED_MSG="pool was previously in use from another system."
 log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG"
 
 # 7. Verify 'zpool import -f $MMP_POOL' can now import the pool.
+log_note "Verify 'zpool import -f $MMP_POOL' can now import the pool"
 log_must import_activity_check $MMP_POOL "-f -d $MMP_DIR"
 
 # 8 Verify pool may be exported/imported without -f argument.
+log_note "Verify pool may be exported/imported without -f argument"
 log_must zpool export $MMP_POOL
 log_must import_no_activity_check $MMP_POOL "-d $MMP_DIR"