Skip to content

Commit 3c78a54

Browse files
Aksookimsin98
authored andcommitted
fix DataSink to S3 buckets
Due to how pathlib.Path handles path strings, S3 bucket paths are missing a /.
1 parent 7dd3b37 commit 3c78a54

File tree

1 file changed

+8
-14
lines changed

1 file changed

+8
-14
lines changed

nipype/interfaces/io.py

+8-14
Original file line numberDiff line numberDiff line change
@@ -440,27 +440,20 @@ def _check_s3_base_dir(self):
440440
is not a valid S3 path, defaults to '<N/A>'
441441
"""
442442

443-
# Init variables
444-
s3_str = "s3://"
443+
# NOTE: due to pathlib.Path in traits.Directory, S3 bucket paths
444+
# follow the format 's3:/bucket_name/...'
445+
s3_str = "s3:/"
445446
bucket_name = "<N/A>"
446447
base_directory = self.inputs.base_directory
447448

448449
if not isdefined(base_directory):
449450
s3_flag = False
450451
return s3_flag, bucket_name
451452

452-
# Explicitly lower-case the "s3"
453+
# Check for 's3:/' in base dir
453454
if base_directory.lower().startswith(s3_str):
454-
base_dir_sp = base_directory.split("/")
455-
base_dir_sp[0] = base_dir_sp[0].lower()
456-
base_directory = "/".join(base_dir_sp)
457-
458-
# Check if 's3://' in base dir
459-
if base_directory.startswith(s3_str):
460-
# Expects bucket name to be 's3://bucket_name/base_dir/..'
461-
bucket_name = base_directory.split(s3_str)[1].split("/")[0]
455+
bucket_name = base_directory[len(s3_str):].split("/")[0]
462456
s3_flag = True
463-
# Otherwise it's just a normal datasink
464457
else:
465458
s3_flag = False
466459

@@ -618,8 +611,9 @@ def _upload_to_s3(self, bucket, src, dst):
618611

619612
from botocore.exceptions import ClientError
620613

621-
# Init variables
622-
s3_str = "s3://"
614+
# NOTE: due to pathlib.Path in traits.Directory, S3 bucket paths
615+
# follow the format 's3:/bucket_name/...'
616+
s3_str = "s3:/"
623617
s3_prefix = s3_str + bucket.name
624618

625619
# Explicitly lower-case the "s3"

0 commit comments

Comments
 (0)