Improve Azure deployment stability (#26)

* Handle eventual consistency errors during Datahub list cluster definitions in CDP
* Handle eventual consistency errors during Azure cross-account credential creation in CDP
 
Signed-off-by: Daniel Chaffelson <[email protected]>

Author: Chaffelson

src/cdpy/common.py

@@ -198,6 +198,7 @@ def _warning_format(message, category, filename, lineno, line=None):
             'DELETE_FAILED',
             'Error',  # DW
             'installation:failed',  # ML
+            'provision:failed',  # ML
             'deprovision:failed',  # ML
             'BAD_HEALTH'  # DF
         ]

src/cdpy/datahub.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from cdpy.common import CdpSdkBase, Squelch
+from cdpy.common import CdpSdkBase, Squelch, CdpError, CdpWarning
 
 
 class CdpyDatahub(CdpSdkBase):
@@ -28,8 +28,25 @@ def describe_all_clusters(self, environment_name=None):
             return [self.describe_cluster(cluster['clusterName']) for cluster in clusters_listing]
         return clusters_listing
 
-    def list_cluster_templates(self):
-        return self.sdk.call(svc='datahub', func='list_cluster_templates', ret_field='clusterTemplates')
+    def list_cluster_templates(self, retries=3, delay=5):
+        # Intermittent timeout issue in CDP 7.2.10, should be reverted to bare listing in 7.2.12
+        resp = self.sdk.call(
+            svc='datahub', func='list_cluster_templates', ret_field='clusterTemplates',
+            ret_error=True
+        )
+        if isinstance(resp, CdpError):
+            if retries > 0:
+                if str(resp.status_code) == '500' and resp.error_code == 'UNKNOWN':
+                    retries = retries - 1
+                    self.sdk.throw_warning(
+                        CdpWarning('Got likely CDP Control Plane eventual consistency error, %d retries left...'
+                                   % (retries))
+                    )
+                    self.sdk.sleep(delay)
+                    return self.list_cluster_templates(retries, delay)
+            else:
+                self.sdk.throw_error(resp)
+        return resp
 
     def describe_cluster_template(self, name):
         return self.sdk.call(svc='datahub', func='describe_cluster_template', ret_field='clusterTemplate', squelch=[

src/cdpy/environments.py

@@ -276,16 +276,32 @@ def create_aws_credential(self, name, role, description, retries=3, delay=2):
                 self.sdk.throw_error(resp)
         return resp
 
-    def create_azure_credential(self, name, subscription, tenant, application, secret):
-        return self.sdk.call(
-            svc='environments', func='create_azure_credential', squelch=[
+    def create_azure_credential(self, name, subscription, tenant, application, secret, retries=3, delay=5):
+        resp = self.sdk.call(
+            svc='environments', func='create_azure_credential', ret_error=True, squelch=[
                 Squelch(field='violations', value='Credential already exists with name',
                         warning='Credential with this name already exists', default=None)],
             credentialName=name,
             subscriptionId=subscription,
             tenantId=tenant,
             appBased={'applicationId': application, 'secretKey': secret}
         )
+        if isinstance(resp, CdpError):
+            if retries > 0:
+                consistency_violations = [
+                    'You may have sent your authentication request to the wrong tenant'
+                ]
+                if any(x in str(resp.violations) for x in consistency_violations):
+                    retries = retries - 1
+                    self.sdk.throw_warning(
+                        CdpWarning('Got likely Azure eventual consistency error [%s], %d retries left...'
+                                   % (str(resp.violations), retries))
+                    )
+                    self.sdk.sleep(delay)
+                    return self.create_azure_credential(name, subscription, tenant, application, secret, retries, delay)
+            else:
+                self.sdk.throw_error(resp)
+        return resp
 
     def create_gcp_credential(self, name, key_file):
         return self.sdk.call(