Skip to content

Commit 3451b35

Browse files
committed
fix: usr: SDK-35: Skip complete data check for presto. Pull Request #148.
Before fetching the query result, hive needs to perform a data check to make sure result data is complete. However, the same is not required for presto and it should skip the checking while fetching presto result.
1 parent 94b7a4d commit 3451b35

File tree

1 file changed

+11
-9
lines changed

1 file changed

+11
-9
lines changed

qds_sdk/commands.py

+11-9
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,8 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True):
203203
# boto expects it to be.
204204
# If the delim is not None, then both text and binary modes
205205
# work.
206-
_download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=delim)
206+
_download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=delim,
207+
skip_data_avail_check=isinstance(self, PrestoCommand))
207208
else:
208209
fp.write(",".join(r['result_location']))
209210

@@ -1176,7 +1177,7 @@ def _read_iteratively(key_instance, fp, delim):
11761177
return
11771178

11781179

1179-
def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None):
1180+
def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None, skip_data_avail_check=False):
11801181
'''
11811182
Downloads the contents of all objects in s3_path into fp
11821183
@@ -1253,14 +1254,15 @@ def _is_complete_data_available(bucket_paths, num_result_dir):
12531254
#It is a folder
12541255
key_prefix = m.group(2)
12551256
bucket_paths = bucket.list(key_prefix)
1256-
complete_data_available = _is_complete_data_available(bucket_paths, num_result_dir)
1257-
while complete_data_available is False and retries > 0:
1258-
retries = retries - 1
1259-
log.info("Results dir is not available on s3. Retry: " + str(6-retries))
1260-
time.sleep(10)
1257+
if not skip_data_avail_check:
12611258
complete_data_available = _is_complete_data_available(bucket_paths, num_result_dir)
1262-
if complete_data_available is False:
1263-
raise Exception("Results file not available on s3 yet. This can be because of s3 eventual consistency issues.")
1259+
while complete_data_available is False and retries > 0:
1260+
retries = retries - 1
1261+
log.info("Results dir is not available on s3. Retry: " + str(6-retries))
1262+
time.sleep(10)
1263+
complete_data_available = _is_complete_data_available(bucket_paths, num_result_dir)
1264+
if complete_data_available is False:
1265+
raise Exception("Results file not available on s3 yet. This can be because of s3 eventual consistency issues.")
12641266

12651267
for one_path in bucket_paths:
12661268
name = one_path.name

0 commit comments

Comments
 (0)