@@ -203,7 +203,8 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True):
203
203
# boto expects it to be.
204
204
# If the delim is not None, then both text and binary modes
205
205
# work.
206
- _download_to_local (boto_conn , s3_path , fp , num_result_dir , delim = delim )
206
+ _download_to_local (boto_conn , s3_path , fp , num_result_dir , delim = delim ,
207
+ skip_data_avail_check = isinstance (self , PrestoCommand ))
207
208
else :
208
209
fp .write ("," .join (r ['result_location' ]))
209
210
@@ -1176,7 +1177,7 @@ def _read_iteratively(key_instance, fp, delim):
1176
1177
return
1177
1178
1178
1179
1179
- def _download_to_local (boto_conn , s3_path , fp , num_result_dir , delim = None ):
1180
+ def _download_to_local (boto_conn , s3_path , fp , num_result_dir , delim = None , skip_data_avail_check = False ):
1180
1181
'''
1181
1182
Downloads the contents of all objects in s3_path into fp
1182
1183
@@ -1253,14 +1254,15 @@ def _is_complete_data_available(bucket_paths, num_result_dir):
1253
1254
#It is a folder
1254
1255
key_prefix = m .group (2 )
1255
1256
bucket_paths = bucket .list (key_prefix )
1256
- complete_data_available = _is_complete_data_available (bucket_paths , num_result_dir )
1257
- while complete_data_available is False and retries > 0 :
1258
- retries = retries - 1
1259
- log .info ("Results dir is not available on s3. Retry: " + str (6 - retries ))
1260
- time .sleep (10 )
1257
+ if not skip_data_avail_check :
1261
1258
complete_data_available = _is_complete_data_available (bucket_paths , num_result_dir )
1262
- if complete_data_available is False :
1263
- raise Exception ("Results file not available on s3 yet. This can be because of s3 eventual consistency issues." )
1259
+ while complete_data_available is False and retries > 0 :
1260
+ retries = retries - 1
1261
+ log .info ("Results dir is not available on s3. Retry: " + str (6 - retries ))
1262
+ time .sleep (10 )
1263
+ complete_data_available = _is_complete_data_available (bucket_paths , num_result_dir )
1264
+ if complete_data_available is False :
1265
+ raise Exception ("Results file not available on s3 yet. This can be because of s3 eventual consistency issues." )
1264
1266
1265
1267
for one_path in bucket_paths :
1266
1268
name = one_path .name
0 commit comments