From d10d70e2427514481288ebc1d5c9099b6a022f33 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Tue, 10 Sep 2024 20:47:26 +1000 Subject: [PATCH] add exception for kcpt creation failure --- magnum_cluster_api/hacks.py | 72 ++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/magnum_cluster_api/hacks.py b/magnum_cluster_api/hacks.py index 75867d5f..face1161 100644 --- a/magnum_cluster_api/hacks.py +++ b/magnum_cluster_api/hacks.py @@ -23,7 +23,9 @@ from tenacity import Retrying, retry_if_result, stop_after_delay, wait_fixed from magnum_cluster_api import objects, utils +from oslo_log import log as logging +LOG = logging.getLogger(__name__) CERTIFICATE_EXPIRY_DAYS_FIX_APPLIED = False @@ -42,28 +44,56 @@ def set_certificate_expiry_days( if "certificatesExpiryDays" in rollout_before: continue - # NOTE(mnaser): Since the KubeadmControlPlaneTemplate is immutable, we need to - # delete the object and re-create it. - kcpt.delete() + # Backup the original object in case we need to restore it + original_kcpt = kcpt.obj.copy() - del kcpt.obj["metadata"]["uid"] - kcpt.obj["spec"]["template"]["spec"].setdefault("rolloutBefore", {}) - kcpt.obj["spec"]["template"]["spec"]["rolloutBefore"][ - "certificatesExpiryDays" - ] = 21 + try: + # NOTE(mnaser): Since the KubeadmControlPlaneTemplate is immutable, we need to + # delete the object and re-create it. + kcpt.delete() - # Use tenacity to wait for kcpt to be created - for attempt in Retrying( - retry=retry_if_result(lambda result: result is None), - stop=stop_after_delay(10), - wait=wait_fixed(1), - ): - with attempt: - utils.kube_apply_patch(kcpt) - new_kcpt = objects.KubeadmControlPlaneTemplate.objects( - api, namespace="magnum-system" - ).get(name=kcpt.obj["metadata"]["name"]) - if not attempt.retry_state.outcome.failed: - attempt.retry_state.set_result(new_kcpt) + del kcpt.obj["metadata"]["uid"] + kcpt.obj["spec"]["template"]["spec"].setdefault("rolloutBefore", {}) + kcpt.obj["spec"]["template"]["spec"]["rolloutBefore"][ + "certificatesExpiryDays" + ] = 21 + + # Use tenacity to wait for kcpt to be created + for attempt in Retrying( + retry=retry_if_result(lambda result: result is None), + stop=stop_after_delay(10), + wait=wait_fixed(1), + ): + with attempt: + utils.kube_apply_patch(kcpt) + new_kcpt = objects.KubeadmControlPlaneTemplate.objects( + api, namespace="magnum-system" + ).get(name=kcpt.obj["metadata"]["name"]) + if not attempt.retry_state.outcome.failed: + attempt.retry_state.set_result(new_kcpt) + except Exception as e: + LOG.exception( + "Failed to set certificate expiry days for kcpt %s: %s", + kcpt.obj["metadata"]["name"], + str(e), + ) + del original_kcpt["metadata"]["uid"] + # Use tenacity to wait for kcpt to be created + for attempt in Retrying( + retry=retry_if_result(lambda result: result is None), + stop=stop_after_delay(10), + wait=wait_fixed(1), + ): + with attempt: + utils.kube_apply_patch(original_kcpt) + new_kcpt = objects.KubeadmControlPlaneTemplate.objects( + api, namespace="magnum-system" + ).get(name=original_kcpt.obj["metadata"]["name"]) + if not attempt.retry_state.outcome.failed: + attempt.retry_state.set_result(new_kcpt) + LOG.info( + "Recreated kcpt %s with original values", + kcpt.obj["metadata"]["name"], + ) CERTIFICATE_EXPIRY_DAYS_FIX_APPLIED = True