@@ -440,27 +440,20 @@ def _check_s3_base_dir(self):
440
440
is not a valid S3 path, defaults to '<N/A>'
441
441
"""
442
442
443
- # Init variables
444
- s3_str = "s3://"
443
+ # NOTE: due to pathlib.Path in traits.Directory, S3 bucket paths
444
+ # follow the format 's3:/bucket_name/...'
445
+ s3_str = "s3:/"
445
446
bucket_name = "<N/A>"
446
447
base_directory = self .inputs .base_directory
447
448
448
449
if not isdefined (base_directory ):
449
450
s3_flag = False
450
451
return s3_flag , bucket_name
451
452
452
- # Explicitly lower-case the "s3"
453
+ # Check for 's3:/' in base dir
453
454
if base_directory .lower ().startswith (s3_str ):
454
- base_dir_sp = base_directory .split ("/" )
455
- base_dir_sp [0 ] = base_dir_sp [0 ].lower ()
456
- base_directory = "/" .join (base_dir_sp )
457
-
458
- # Check if 's3://' in base dir
459
- if base_directory .startswith (s3_str ):
460
- # Expects bucket name to be 's3://bucket_name/base_dir/..'
461
- bucket_name = base_directory .split (s3_str )[1 ].split ("/" )[0 ]
455
+ bucket_name = base_directory [len (s3_str ):].split ("/" )[0 ]
462
456
s3_flag = True
463
- # Otherwise it's just a normal datasink
464
457
else :
465
458
s3_flag = False
466
459
@@ -618,8 +611,9 @@ def _upload_to_s3(self, bucket, src, dst):
618
611
619
612
from botocore .exceptions import ClientError
620
613
621
- # Init variables
622
- s3_str = "s3://"
614
+ # NOTE: due to pathlib.Path in traits.Directory, S3 bucket paths
615
+ # follow the format 's3:/bucket_name/...'
616
+ s3_str = "s3:/"
623
617
s3_prefix = s3_str + bucket .name
624
618
625
619
# Explicitly lower-case the "s3"
0 commit comments