ZTS: update the relevant mmp test cases

- mmp_exported_import: an activity check is now required for pools
  which were cleanly exported if the system and pool hostids don't
  match.

- mmp_inactive_import: an activity check is now required for any
  pool which wasn't cleanly exported, even if the system and pool
  hostids match.

- mmp_on_uberblocks: updated expected uberblocks to take in to account
  the value MMP_INTERVAL_DEFAULT is set too.

- mmp_reset_interval: reduce the number of iterations from 10 to 3.
  This is sufficient to verify functionality and significantly speeds
  up the test.

- mmp_on_uberblocks: reduce threshold from 7 to 6 to avoid false
  positives observed in CI on ubuntu 22.04.

- Update tests to use 'zhack action idle' instead of ztest to improve
  the reliability of the tests.

- Add additional log_note messages to test cases which have multiple
  verification steps to make it clear which portion of a test failed
  when reviewing the logs.

- Replace default_setup/cleanup_noexit calls with 'zpool create' and
  'zpool destroy' calls to avoid additional unnecessary dataset
  creation work.

- Update activity/noactivity check helper functions to use the
  ZFS_LOAD_INFO_DEBUG information now available from 'zpool import'
  to determine if this activity check ran and why.  This is more
  reliable in the CI than measuring the runtime.

- Set MMP_INTERVAL_DEFAULT=500 by default to reduce the total runtime.

- Removed all mmp tests from the zts-report.py exceptions list.

- Enabled MMP tests to run on FreeBSD.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

Author: behlendorf

module/zfs/spa.c

@@ -4100,8 +4100,8 @@ spa_activity_check_tryimport(spa_t *spa, uberblock_t *spa_ub,
 		hrtime_t extra_delay = MMP_IMPORT_VERIFY_ITERS *
 		    MSEC2NSEC(MMP_INTERVAL_VALID(spa_ub) ?
 		    MMP_INTERVAL(spa_ub) : MMP_MIN_INTERVAL);
-		cmn_err(CE_NOTE, "pool '%s' multihost activity check "
-		    "required, %llu seconds remaining", spa_load_name(spa),
+		cmn_err(CE_NOTE, "pool '%s' activity check required, "
+		    "%llu seconds remaining", spa_load_name(spa),
 		    (u_longlong_t)MAX(NSEC2SEC(import_delay + extra_delay), 1));
 		spa_import_progress_set_notes(spa, "Checking MMP activity, "
 		    "waiting %llu ms", (u_longlong_t)NSEC2MSEC(import_delay));

tests/runfiles/common.run

@@ -824,6 +824,14 @@ tests = ['mmap_mixed', 'mmap_read_001_pos', 'mmap_seek_001_pos',
     'mmap_sync_001_pos', 'mmap_write_001_pos', 'mmap_ftruncate']
 tags = ['functional', 'mmap']
 
+[tests/functional/mmp]
+tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
+    'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
+    'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history',
+    'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk']
+tags = ['functional', 'mmp']
+timeout = 1200
+
 [tests/functional/mount]
 tests = ['umount_001', 'umountall_001']
 tags = ['functional', 'mount']

tests/runfiles/linux.run

@@ -155,13 +155,6 @@ tags = ['functional', 'luks']
 tests = ['mmap_libaio_001_pos', 'mmap_sync_001_pos']
 tags = ['functional', 'mmap']
 
-[tests/functional/mmp:Linux]
-tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
-    'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
-    'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history',
-    'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk']
-tags = ['functional', 'mmp']
-
 [tests/functional/mount:Linux]
 tests = ['umount_unlinked_drain', 'mount_loopback']
 tags = ['functional', 'mount']

tests/test-runner/bin/zts-report.py.in

@@ -247,7 +247,6 @@ maybe = {
     'l2arc/persist_l2arc_005_pos': ['FAIL', known_reason],
     'largest_pool/largest_pool_001_pos': ['FAIL', known_reason],
     'mmap/mmap_sync_001_pos': ['FAIL', known_reason],
-    'mmp/mmp_on_uberblocks': ['FAIL', known_reason],
     'pam/setup': ['SKIP', "pamtester might be not available"],
     'pool_checkpoint/checkpoint_discard_busy': ['FAIL', 11946],
     'projectquota/setup': ['SKIP', exec_reason],
@@ -366,9 +365,6 @@ elif sys.platform.startswith('linux'):
         'io/io_uring': ['SKIP', 'io_uring support required'],
         'limits/filesystem_limit': ['SKIP', known_reason],
         'limits/snapshot_limit': ['SKIP', known_reason],
-        'mmp/mmp_active_import': ['FAIL', known_reason],
-        'mmp/mmp_exported_import': ['FAIL', known_reason],
-        'mmp/mmp_inactive_import': ['FAIL', known_reason],
         'stat/statx_dioalign': ['SKIP', 'statx_reason'],
     })
 

tests/zfs-tests/tests/functional/mmp/cleanup.ksh

@@ -24,6 +24,10 @@
 
 verify_runnable "global"
 
+if [ -e $HOSTID_FILE.save ]; then
+	mv -f ${HOSTID_FILE}.save $HOSTID_FILE
+fi
+
 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF
 
 log_pass "mmp cleanup passed"

tests/zfs-tests/tests/functional/mmp/mmp.cfg

@@ -32,16 +32,14 @@ export TXG_TIMEOUT_DEFAULT=5
 export MMP_POOL=mmppool
 export MMP_DIR=$TEST_BASE_DIR/mmp
 export MMP_CACHE=$MMP_DIR/zpool.cache
-export MMP_ZTEST_LOG=$MMP_DIR/ztest.log
+export MMP_ZTEST_LOG=$MMP_DIR/zhack.log
 export MMP_HISTORY=100
 export MMP_HISTORY_OFF=0
 
 export MMP_INTERVAL_HOUR=$((60*60*1000))
-export MMP_INTERVAL_DEFAULT=1000
+export MMP_INTERVAL_DEFAULT=500
 export MMP_INTERVAL_MIN=100
 
 export MMP_IMPORT_INTERVALS=20
 export MMP_FAIL_INTERVALS_DEFAULT=10
 export MMP_FAIL_INTERVALS_MIN=2
-
-export MMP_TEST_DURATION_DEFAULT=$((MMP_IMPORT_INTERVALS*MMP_INTERVAL_DEFAULT/1000))

tests/zfs-tests/tests/functional/mmp/mmp.kshlib

@@ -99,11 +99,11 @@ function mmp_pool_create_simple # pool dir
 	log_must zpool set multihost=on $pool
 }
 
-function mmp_pool_create # pool dir
+function mmp_pool_create_zhack # pool dir
 {
 	typeset pool=${1:-$MMP_POOL}
 	typeset dir=${2:-$MMP_DIR}
-	typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool"
+	typeset opts="-d $dir action idle -t120 $pool"
 
 	mmp_pool_create_simple $pool $dir
 
@@ -112,11 +112,11 @@ function mmp_pool_create # pool dir
 	log_must mmp_clear_hostid
 	log_must mmp_set_hostid $HOSTID2
 
-	log_note "Starting ztest in the background as hostid $HOSTID1"
-	log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &"
+	log_note "Starting zhack in the background as hostid $HOSTID1"
+	log_must eval "ZFS_HOSTID=$HOSTID1 zhack $opts >$MMP_ZTEST_LOG 2>&1 &"
 
 	while ! is_pool_imported "$pool" "-d $dir"; do
-		log_must pgrep ztest
+		log_must pgrep zhack
 		log_must sleep 5
 	done
 }
@@ -126,7 +126,7 @@ function mmp_pool_destroy # pool dir
 	typeset pool=${1:-$MMP_POOL}
 	typeset dir=${2:-$MMP_DIR}
 
-	ZTESTPID=$(pgrep ztest)
+	ZTESTPID=$(pgrep zhack)
 	if [ -n "$ZTESTPID" ]; then
 		log_must kill $ZTESTPID
 		wait $ZTESTPID
@@ -158,33 +158,34 @@ function import_no_activity_check # pool opts
 	typeset pool=$1
 	typeset opts=$2
 
-	typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1))
-
-	SECONDS=0
-	zpool import $opts $pool
+	RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool)
 	typeset rc=$?
 
-	if [[ $SECONDS -gt $max_duration ]]; then
-		log_fail "ERROR: import_no_activity_check unexpected activity \
-check (${SECONDS}s gt $max_duration)"
+	# mmp_result: 3 (ESRCH) no activity check was run not required
+	# mmp_result: 6 (ENXIO) no activity check was run hostid not set
+	if ! echo "$RESULT" | grep -q "mmp_result: 3" &&
+	   ! echo "$RESULT" | grep -q "mmp_result: 6"; then
+		log_note "ERROR: $RESULT"
+		log_fail "ERROR: import_no_activity_check unexpected activity check"
 	fi
 
 	return $rc
 }
 
-function import_activity_check # pool opts act_test_duration
+function import_activity_check # pool opts
 {
 	typeset pool=$1
 	typeset opts=$2
-	typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT}
 
-	SECONDS=0
-	zpool import $opts $pool
+	RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool)
 	typeset rc=$?
 
-	if [[ $SECONDS -le $min_duration ]]; then
-		log_fail "ERROR: import_activity_check expected activity check \
-(${SECONDS}s le min_duration $min_duration)"
+	# mmp_result: 0 (Success) check was run no activity detected
+	# mmp_result: 121 (EREMOTEIO) check was run activity detected
+	# mmp_result: 4 (EINTR) check was run but interrupted by user
+	if ! echo "$RESULT" | grep -q "mmp_result: 0"; then
+		log_note "ERROR: $RESULT"
+		log_fail "ERROR: import_activity_check expected activity check"
 	fi
 
 	return $rc

tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh

@@ -24,10 +24,10 @@
 #	with one hostid be importable by a host with a different hostid.
 #
 # STRATEGY:
-#	 1. Simulate an active pool on another host with ztest.
+#	 1. Simulate an active pool on another host with zhack.
 #	 2. Verify 'zpool import' reports an active pool.
 #	 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool.
-#	 4. Kill ztest to make pool eligible for import.
+#	 4. Kill zhack to make pool eligible for import.
 #	 5. Verify 'zpool import' fails with the expected error message.
 #	 6. Verify 'zpool import $MMP_POOL' fails with the expected message.
 #	 7. Verify 'zpool import -f $MMP_POOL' can now import the pool.
@@ -44,15 +44,15 @@ function cleanup
 {
 	mmp_pool_destroy $MMP_POOL $MMP_DIR
 	log_must mmp_clear_hostid
-	ZTESTPID=$(pgrep ztest)
+	ZTESTPID=$(pgrep zhack)
 	if [ -n "$ZTESTPID" ]; then
 		for pid in $ZTESTPID; do
 			log_must kill -9 $pid
 		done
 	else
-		# if ztest not running and log present, ztest crashed
+		# if zhack is not running and log present, zhack crashed
 		if [ -f $MMP_ZTEST_LOG ]; then
-			log_note "ztest appears to have crashed.  Tail of log:"
+			log_note "zhack appears to have crashed.  Tail of log:"
 			tail -n 50 $MMP_ZTEST_LOG
 		fi
 	fi
@@ -61,15 +61,18 @@ function cleanup
 log_assert "multihost=on|off active pool activity checks"
 log_onexit cleanup
 
-# 1. Simulate an active pool on another host with ztest.
+# 1. Simulate an active pool on another host with zhack.
+log_note "Simulate an active pool on another host with zhack"
 mmp_pool_destroy $MMP_POOL $MMP_DIR
-mmp_pool_create $MMP_POOL $MMP_DIR
+mmp_pool_create_zhack $MMP_POOL $MMP_DIR
 
 # 2. Verify 'zpool import' reports an active pool.
+log_note "Verify 'zpool import' reports an active pool"
 log_must mmp_set_hostid $HOSTID2
 log_must is_pool_imported $MMP_POOL "-d $MMP_DIR"
 
 # 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool.
+log_note "Verify 'zpool import [-f] $MMP_POOL' cannot import the pool"
 MMP_IMPORTED_MSG="Cannot import '$MMP_POOL': pool is imported"
 
 log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG"
@@ -84,8 +87,9 @@ for i in {1..10}; do
 	    "$MMP_IMPORTED_MSG"
 done
 
-# 4. Kill ztest to make pool eligible for import.  Poll with 'zpool status'.
-ZTESTPID=$(pgrep ztest)
+# 4. Kill zhack to make pool eligible for import.  Poll with 'zpool status'.
+log_note "Kill zhack to make pool eligible for import.  Poll with 'zpool status'"
+ZTESTPID=$(pgrep zhack)
 if [ -n "$ZTESTPID" ]; then
 	log_must kill -9 $ZTESTPID
 fi
@@ -99,6 +103,7 @@ fi
 #    - hostid=matches   - safe to import the pool
 #    - hostid=different - previously imported on a different system
 #
+log_note "Verify 'zpool import' fails with the expected error message"
 log_must mmp_clear_hostid
 MMP_IMPORTED_MSG="Set a unique system hostid"
 log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" "$MMP_IMPORTED_MSG"
@@ -113,13 +118,16 @@ MMP_IMPORTED_MSG="The pool was last accessed by another system."
 log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "status" "$MMP_IMPORTED_MSG"
 
 # 6. Verify 'zpool import $MMP_POOL' fails with the expected message.
+log_note "Verify 'zpool import $MMP_POOL' fails with the expected message"
 MMP_IMPORTED_MSG="pool was previously in use from another system."
 log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG"
 
 # 7. Verify 'zpool import -f $MMP_POOL' can now import the pool.
+log_note "Verify 'zpool import -f $MMP_POOL' can now import the pool"
 log_must import_activity_check $MMP_POOL "-f -d $MMP_DIR"
 
 # 8 Verify pool may be exported/imported without -f argument.
+log_note "Verify pool may be exported/imported without -f argument"
 log_must zpool export $MMP_POOL
 log_must import_no_activity_check $MMP_POOL "-d $MMP_DIR"
 

tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh

@@ -28,7 +28,7 @@
 #	3. Verify multihost=off and hostids differ (no activity check)
 #	4. Verify multihost=off and hostid zero allowed (no activity check)
 #	5. Verify multihost=on and hostids match (no activity check)
-#	6. Verify multihost=on and hostids differ (no activity check)
+#	6. Verify multihost=on and hostids differ (activity check)
 #	7. Verify multihost=on and hostid zero fails (no activity check)
 #
 
@@ -40,7 +40,7 @@ verify_runnable "both"
 
 function cleanup
 {
-	default_cleanup_noexit
+	datasetexists $TESTPOOL && destroy_pool $TESTPOOL
 	log_must mmp_clear_hostid
 }
 
@@ -49,9 +49,10 @@ log_onexit cleanup
 
 # 1. Create a zpool
 log_must mmp_set_hostid $HOSTID1
-default_setup_noexit $DISK
+log_must zpool create -f $TESTPOOL $DISK
 
 # 2. Verify multihost=off and hostids match (no activity check)
+log_note "Verify multihost=off and hostids match (no activity check)"
 log_must zpool set multihost=off $TESTPOOL
 
 for opt in "" "-f"; do
@@ -60,6 +61,7 @@ for opt in "" "-f"; do
 done
 
 # 3. Verify multihost=off and hostids differ (no activity check)
+log_note "Verify multihost=off and hostids differ (no activity check)"
 for opt in "" "-f"; do
 	log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1
 	log_must zpool export $TESTPOOL
@@ -69,6 +71,7 @@ for opt in "" "-f"; do
 done
 
 # 4. Verify multihost=off and hostid zero allowed (no activity check)
+log_note "Verify multihost=off and hostid zero allowed (no activity check)"
 log_must mmp_clear_hostid
 
 for opt in "" "-f"; do
@@ -77,6 +80,7 @@ for opt in "" "-f"; do
 done
 
 # 5. Verify multihost=on and hostids match (no activity check)
+log_note "Verify multihost=on and hostids match (no activity check)"
 log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1
 log_must zpool set multihost=on $TESTPOOL
 
@@ -85,16 +89,18 @@ for opt in "" "-f"; do
 	log_must import_no_activity_check $TESTPOOL $opt
 done
 
-# 6. Verify multihost=on and hostids differ (no activity check)
+# 6. Verify multihost=on and hostids differ (activity check)
+log_note "Verify multihost=on and hostids differ (activity check)"
 for opt in "" "-f"; do
 	log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1
 	log_must zpool export $TESTPOOL
 	log_must mmp_clear_hostid
 	log_must mmp_set_hostid $HOSTID2
-	log_must import_no_activity_check $TESTPOOL $opt
+	log_must import_activity_check $TESTPOOL $opt
 done
 
 # 7. Verify multihost=on and hostid zero fails (no activity check)
+log_note "Verify multihost=on and hostids differ (no activity check)"
 log_must zpool export $TESTPOOL
 log_must mmp_clear_hostid
 

tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh

@@ -40,7 +40,7 @@ verify_runnable "both"
 
 function cleanup
 {
-	default_cleanup_noexit
+	datasetexists $MMP_POOL && destroy_pool $MMP_POOL
 	log_must rm $MMP_DIR/file.{0,1,2,3,4,5}
 	log_must rmdir $MMP_DIR
 	log_must mmp_clear_hostid
@@ -56,7 +56,7 @@ log_must mkdir -p $MMP_DIR
 log_must truncate -s $MINVDEVSIZE $MMP_DIR/file.{0,1,2,3,4,5}
 
 # 1. Create a non-redundant pool
-log_must zpool create $MMP_POOL $MMP_DIR/file.0
+log_must zpool create -f $MMP_POOL $MMP_DIR/file.0
 
 # 2. Create an 'etc' dataset containing a valid hostid file; caching is
 #    disabled on the dataset to force the hostid to be read from disk.
@@ -71,23 +71,27 @@ mntpnt_fs=$(get_prop mountpoint $MMP_POOL/fs)
 log_must mkfile 1M $mntpnt_fs/file
 
 # 4. Verify multihost cannot be enabled until the /etc/hostid is linked
+log_note "Verify multihost cannot be enabled until the /etc/hostid is linked"
 log_mustnot zpool set multihost=on $MMP_POOL
 log_mustnot ls -l $HOSTID_FILE
 log_must ln -s $mntpnt_etc/hostid $HOSTID_FILE
 log_must zpool set multihost=on $MMP_POOL
 
 # 5. Verify vdevs may be attached and detached
+log_note "Verify vdevs may be attached and detached"
 log_must zpool attach $MMP_POOL $MMP_DIR/file.0 $MMP_DIR/file.1
 log_must zpool detach $MMP_POOL $MMP_DIR/file.1
 
 # 6. Verify normal, cache, log and special vdevs can be added
+log_note "Verify normal, cache, log and special vdevs can be added"
 log_must zpool add $MMP_POOL $MMP_DIR/file.1
 log_must zpool add $MMP_POOL $MMP_DIR/file.2
 log_must zpool add $MMP_POOL cache $MMP_DIR/file.3
 log_must zpool add $MMP_POOL log $MMP_DIR/file.4
 log_must zpool add $MMP_POOL special $MMP_DIR/file.5
 
 # 7. Verify normal, cache, and log vdevs can be removed
+log_note "Verify normal, cache, and log vdevs can be removed"
 log_must zpool remove $MMP_POOL $MMP_DIR/file.2
 log_must zpool remove $MMP_POOL $MMP_DIR/file.3
 log_must zpool remove $MMP_POOL $MMP_DIR/file.4