feat(nf-google): make the retry behaviour more consistent, and centralized CLI transport detection

Signed-off-by: Tomiles <116039+tomiles@users.noreply.github.com>

Author: tomiles

plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchFileCopyStrategy.groovy

@@ -48,7 +48,7 @@ class GoogleBatchFileCopyStrategy extends SimpleFileCopyStrategy {
      * CLI stage-in when a CLI transport is selected and {@code stageInMode=copy}.
      */
     private boolean useCliStageInCopy() {
-        return batchConfig.stageInCopyTransport in [BatchConfig.COPY_TRANSPORT_GCLOUD, BatchConfig.COPY_TRANSPORT_GSUTIL] && 'copy' == stageinMode
+        return BatchConfig.isCliCopyTransport(batchConfig.stageInCopyTransport) && 'copy' == stageinMode
     }
 
     private String effectiveStageOutMode() {
@@ -58,7 +58,7 @@ class GoogleBatchFileCopyStrategy extends SimpleFileCopyStrategy {
     private boolean shouldUseCliStageOutCopy() {
         if( effectiveStageOutMode() != 'copy' )
             return false
-        return batchConfig.stageOutCopyTransport in [BatchConfig.COPY_TRANSPORT_GCLOUD, BatchConfig.COPY_TRANSPORT_GSUTIL]
+        return BatchConfig.isCliCopyTransport(batchConfig.stageOutCopyTransport)
     }
 
     private boolean needsGoogleBatchBashLib() {
@@ -90,7 +90,9 @@ class GoogleBatchFileCopyStrategy extends SimpleFileCopyStrategy {
             return super.stageInputFile(path, targetName)
         final gsUri = gsUriForCliStageIn(path)
         if( gsUri != null ) {
-            return "downloads+=(\"nxf_cp_retry nxf_gs_download '${gsUri}' ${Escape.path(targetName)}\")"
+            return batchConfig.maxTransferAttempts > 1
+                    ? "downloads+=(\"nxf_cp_retry nxf_gs_download '${gsUri}' ${Escape.path(targetName)}\")"
+                    : "downloads+=(\"nxf_gs_download '${gsUri}' ${Escape.path(targetName)}\")"
         }
         return super.stageInputFile(path, targetName)
     }

plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy

@@ -62,6 +62,7 @@ class BatchConfig implements ConfigScope {
     static final String COPY_TRANSPORT_GSUTIL = 'gsutil'
 
     static final private List<String> VALID_COPY_TRANSPORTS = List.of(COPY_TRANSPORT_POSIX, COPY_TRANSPORT_GCLOUD, COPY_TRANSPORT_GSUTIL)
+    static final private List<String> CLI_COPY_TRANSPORTS = List.of(COPY_TRANSPORT_GCLOUD, COPY_TRANSPORT_GSUTIL)
 
     static final private int DEFAULT_MAX_SPOT_ATTEMPTS = 0
 
@@ -253,7 +254,11 @@ class BatchConfig implements ConfigScope {
      * Load {@link nextflow.cloud.google.batch.GoogleBatchFileCopyStrategy} when any transport requests CLI object-storage copy.
      */
     boolean usesGoogleBatchStaging() {
-        return [stageInCopyTransport, stageOutCopyTransport].any { it in [COPY_TRANSPORT_GCLOUD, COPY_TRANSPORT_GSUTIL] }
+        return [stageInCopyTransport, stageOutCopyTransport].any { isCliCopyTransport(it) }
+    }
+
+    static boolean isCliCopyTransport(String transport) {
+        return transport in CLI_COPY_TRANSPORTS
     }
 
     BatchRetryConfig getRetryConfig() { retry }

plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchFileCopyStrategyTest.groovy

@@ -43,6 +43,24 @@ class GoogleBatchFileCopyStrategyTest extends Specification {
         def path = Paths.get('/mnt/disks/b/data/in.txt')
         def copy = new GoogleBatchFileCopyStrategy(bean, batch)
 
+        expect:
+        copy.stageInputFile(path, 'in.txt') == 'downloads+=("nxf_gs_download \'gs://b/data/in.txt\' in.txt")'
+    }
+
+    def 'should build gs download staging line with retry when maxTransferAttempts > 1' () {
+        given:
+        def batch = new BatchConfig([
+            stageInCopyTransport: 'gcloud',
+            maxTransferAttempts: 3
+        ])
+        def bean = Mock(TaskBean) {
+            getWorkDir() >> Paths.get('/mnt/disks/w/x')
+            getTargetDir() >> Paths.get('/mnt/disks/w/x')
+            getStageInMode() >> 'copy'
+        }
+        def path = Paths.get('/mnt/disks/b/data/in.txt')
+        def copy = new GoogleBatchFileCopyStrategy(bean, batch)
+
         expect:
         copy.stageInputFile(path, 'in.txt') == 'downloads+=("nxf_cp_retry nxf_gs_download \'gs://b/data/in.txt\' in.txt")'
     }