btrfs-progs: scrub: improve stability during device disconnections

This commit enhances the scrub command to handle temporary device disconnections
more gracefully, improving stability and preventing data loss:

- Preserve maximum last_physical position during resume after interruptions
- Handle device disconnection errors (ENODEV, ENOTCONN, EIO) as interrupted
  rather than canceled, allowing scrub to resume from the last position
- Add retry logic for temporarily unavailable devices with automatic reconnection
- Force more frequent progress saving when device issues are detected
- Preserve progress data during temporary disconnections (e.g., USB hub resets)

These changes ensure that scrub operations can survive transient hardware issues
and resume properly without losing progress, making the filesystem maintenance
more robust in environments with less reliable storage devices.

Author: DocMAX

cmds/scrub.c

@@ -816,7 +816,18 @@ static struct scrub_progress *scrub_resumed_stats(struct scrub_progress *data,
 	_SCRUB_SUM(dest, data, malloc_errors);
 	_SCRUB_SUM(dest, data, uncorrectable_errors);
 	_SCRUB_SUM(dest, data, corrected_errors);
-	_SCRUB_COPY(dest, data, last_physical);
+	
+	/*
+	 * Preserve the maximum last_physical position from resumed or current data.
+	 * This handles the case where last_physical was reset to 0 due to device
+	 * disconnection but we still want to resume from the highest position
+	 * we actually reached.
+	 */
+	if (data->resumed->p.last_physical > data->scrub_args.progress.last_physical)
+		dest->scrub_args.progress.last_physical = data->resumed->p.last_physical;
+	else
+		dest->scrub_args.progress.last_physical = data->scrub_args.progress.last_physical;
+		
 	dest->stats.canceled = data->stats.canceled;
 	dest->stats.finished = data->stats.finished;
 	dest->stats.t_resumed = data->stats.t_start;
@@ -968,10 +979,23 @@ static void *scrub_one_dev(void *ctx)
 	sp->stats.duration = tv.tv_sec - sp->stats.t_start;
 	sp->stats.canceled = !!ret;
 	sp->ioctl_errno = errno;
+	
+	/*
+	 * For device disconnection errors, preserve the progress by marking
+	 * as interrupted rather than canceled, to allow resume to continue
+	 * from the last position
+	 */
+	if (ret && (errno == ENODEV || errno == ENOTCONN || errno == EIO)) {
+		sp->stats.canceled = 0;
+		sp->stats.finished = 0;  /* Mark as interrupted for resume */
+	} else {
+		sp->stats.canceled = !!ret;
+		sp->stats.finished = 1;
+	}
+	
 	ret = pthread_mutex_lock(&sp->progress_mutex);
 	if (ret)
 		return ERR_PTR(-ret);
-	sp->stats.finished = 1;
 	ret = pthread_mutex_unlock(&sp->progress_mutex);
 	if (ret)
 		return ERR_PTR(-ret);
@@ -1051,12 +1075,26 @@ static void *scrub_progress_cycle(void *ctx)
 		gettimeofday(&tv, NULL);
 		this = (this + 1)%2;
 		last = (last + 1)%2;
+		
 		for (i = 0; i < ndev; ++i) {
 			sp = &spc->progress[this * ndev + i];
 			sp_last = &spc->progress[last * ndev + i];
 			sp_shared = &spc->shared_progress[i];
+			
 			if (sp->stats.finished)
 				continue;
+			
+			/*
+			 * For devices with recent connection issues, try to
+			 * reconnect by retrying the progress ioctl a few times
+			 * in case the device comes back online
+			 */
+			int retry_count = 0;
+			if (sp_last->ioctl_errno == ENODEV || sp_last->ioctl_errno == ENOTCONN) {
+				retry_count = 3;
+			}
+			
+retry_progress:
 			progress_one_dev(sp);
 			sp->stats.duration = tv.tv_sec - sp->stats.t_start;
 			if (!sp->ret)
@@ -1066,11 +1104,27 @@ static void *scrub_progress_cycle(void *ctx)
 				ret = -sp->ioctl_errno;
 				goto out;
 			}
+			
+			/*
+			 * If device is temporarily unavailable and we have retries left,
+			 * wait a moment and try again
+			 */
+			if (retry_count > 0 && (sp->ioctl_errno == ENODEV || sp->ioctl_errno == ENOTCONN)) {
+				struct timespec sleep_time = {0, 500000000}; /* 0.5 seconds */
+				nanosleep(&sleep_time, NULL);
+				retry_count--;
+				goto retry_progress;
+			}
+			
 			/*
 			 * scrub finished or device removed, check the
 			 * finished flag. if unset, just use the last
 			 * result we got for the current write and go
 			 * on. flag should be set on next cycle, then.
+			 *
+			 * For device removal (ENODEV), preserve the last_physical
+			 * position in case this was caused by a temporary
+			 * disconnection like USB hub reset.
 			 */
 			perr = pthread_setcancelstate(
 					PTHREAD_CANCEL_DISABLE, &old);
@@ -1080,6 +1134,13 @@ static void *scrub_progress_cycle(void *ctx)
 			if (perr)
 				goto out;
 			if (!sp_shared->stats.finished) {
+				/*
+				 * Preserve the last_physical position to avoid
+				 * losing progress on temporary disconnections
+				 */
+				if (sp->ioctl_errno == ENODEV && sp_last->scrub_args.progress.last_physical > 0) {
+					sp_shared->scrub_args.progress.last_physical = sp_last->scrub_args.progress.last_physical;
+				}
 				perr = pthread_mutex_unlock(
 						&sp_shared->progress_mutex);
 				if (perr)
@@ -1120,8 +1181,24 @@ static void *scrub_progress_cycle(void *ctx)
 		}
 		if (!spc->do_record)
 			continue;
-		ret = scrub_write_progress(spc->write_mutex, fsid,
-					   &spc->progress[this * ndev], ndev);
+		
+		/*
+		 * Force progress saving more frequently if we have device issues
+		 * to prevent data loss during temporary disconnections
+		 */
+		int force_write = 0;
+		for (i = 0; i < ndev; ++i) {
+			struct scrub_progress *sp_check = &spc->progress[this * ndev + i];
+			if (sp_check->ioctl_errno == ENODEV || sp_check->ioctl_errno == ENOTCONN) {
+				force_write = 1;
+				break;
+			}
+		}
+		
+		if (force_write || (tv.tv_sec % 30) == 0) {
+			ret = scrub_write_progress(spc->write_mutex, fsid,
+						   &spc->progress[this * ndev], ndev);
+		}
 		if (ret)
 			goto out;
 	}