From f03f5271251a230fedc51e7da144b63b05729dd1 Mon Sep 17 00:00:00 2001 From: rhassaine Date: Fri, 13 Mar 2026 09:52:08 +0000 Subject: [PATCH] Default stageOutMode to 'copy' for Google Batch executor On Google Batch, task outputs are unstaged from local scratch to a gcsfuse-mounted work directory, which is always a cross-device operation. The default 'move' mode uses 'mv' which fails in two scenarios: - When output declarations include both a directory and files inside it, the directory is moved first (with all contents), causing subsequent file moves to fail with 'No such file or directory' - When staged input files are symlinks pointing back to the work directory, 'mv' detects source and target as the same file Using 'copy' mode avoids both issues at no additional I/O cost since cross-device 'mv' already performs a copy internally. Co-Authored-By: Claude Opus 4.6 Signed-off-by: rhassaine --- .../cloud/google/batch/GoogleBatchScriptLauncher.groovy | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index 9db0bf9fb2..79badde792 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -26,6 +26,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cloud.google.GoogleOpts import nextflow.executor.BashWrapperBuilder +import nextflow.executor.SimpleFileCopyStrategy import nextflow.extension.FilesEx import nextflow.processor.TaskBean import nextflow.processor.TaskRun @@ -56,6 +57,13 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc GoogleBatchScriptLauncher(TaskBean bean, Path remoteBinDir) { super(bean) + // Default to 'copy' stage-out mode on Google Batch. Task outputs are + // unstaged from local scratch to a gcsfuse-mounted work directory, which + // is always a cross-device operation. Using 'mv' for this fails when + // output declarations overlap (directory moved before its children) or + // when staged input symlinks resolve to the target path. + if( !bean.stageOutMode ) + ((SimpleFileCopyStrategy) copyStrategy).stageoutMode = 'copy' // keep track the google storage work dir this.remoteWorkDir = (CloudStoragePath) bean.workDir this.remoteBinDir = toContainerMount(remoteBinDir)