buildupload: bump retry period to 5 minutes

It's incredibly expensive when we flake on something at the very end of
the pipeline when uploading S3 artifacts; all the created artifacts are
lost and we have to rerun a whole new build.

We currently only retry for 10 seconds, which makes sense for truly
transient flakes but for uploads, given the stakes, let's be more
resilient to flakes that could take a bit longer to resolve as well,
like DNS resolution issues.

Retry for 5 minutes, with an exponential backoff of up to 20 seconds.

Author: jlebon

src/cmd-buildupload

@@ -22,7 +22,13 @@ CACHE_MAX_AGE_ARTIFACT = 60 * 60 * 24 * 365
 # set metadata caching to 5m
 CACHE_MAX_AGE_METADATA = 60 * 5
 from cosalib.builds import Builds, BUILDFILES
-from cosalib.cmdlib import load_json, retry_stop, retry_boto_exception, retry_callback  # noqa: E402
+from cosalib.cmdlib import (
+    load_json,
+    retry_stop_long,
+    retry_wait_long,
+    retry_boto_exception,
+    retry_callback
+)
 
 
 def main():
@@ -188,7 +194,8 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix):
             dry_run=args.dry_run)
 
 
-@retry(stop=retry_stop, retry=retry_boto_exception, before_sleep=retry_callback)
+@retry(stop=retry_stop_long, wait=retry_wait_long,
+       retry=retry_boto_exception, before_sleep=retry_callback)
 def s3_check_exists(s3_client, bucket, key, dry_run=False):
     print(f"Checking if bucket '{bucket}' has key '{key}'")
     try:
@@ -205,7 +212,8 @@ def s3_check_exists(s3_client, bucket, key, dry_run=False):
     return True
 
 
-@retry(stop=retry_stop, retry=retry_boto_exception, retry_error_callback=retry_callback)
+@retry(stop=retry_stop_long, wait=retry_wait_long,
+       retry=retry_boto_exception, retry_error_callback=retry_callback)
 def s3_copy(s3_client, src, bucket, key, max_age, acl, extra_args={}, dry_run=False):
     extra_args = dict(extra_args)
     if 'ContentType' not in extra_args:

src/cosalib/cmdlib.py

@@ -24,7 +24,7 @@
 from flufl.lock import Lock
 
 from tenacity import (
-    stop_after_delay, stop_after_attempt, retry_if_exception_type)
+    stop_after_delay, stop_after_attempt, retry_if_exception_type, wait_exponential)
 
 gi.require_version("RpmOstree", "1.0")
 from gi.repository import RpmOstree
@@ -39,6 +39,9 @@
 LOCK_DEFAULT_LIFETIME = datetime.timedelta(weeks=52)
 
 retry_stop = (stop_after_delay(10) | stop_after_attempt(5))
+# for operations that want to be more persistent
+retry_stop_long = (stop_after_delay(60 * 5))  # 5 minutes
+retry_wait_long = (wait_exponential(max=10))
 retry_boto_exception = (retry_if_exception_type(ConnectionClosedError) |
                       retry_if_exception_type(ConnectTimeoutError) |
                       retry_if_exception_type(IncompleteReadError) |