add retry logic

cmd/manager/main.go

@@ -47,6 +47,12 @@ var (
 	deleteCNRRequeue                 = app.Flag("delete-cnr-requeue", "How often to check if a CNR can be deleted").Default("24h").Duration()
 	defaultCNScyclingExpiry          = app.Flag("default-cns-cycling-expiry", "Fail the CNS if it has been cycling for this long").Default("3h").Duration()
 	unhealthyPodTerminationThreshold = app.Flag("unhealthy-pod-termination-after", "How long to tolerate an un-evictable yet unhealthy pod before forcefully removing it").Default("5m").Duration()
+
+	// Retry configuration flags
+	awsRetryEnabled   = app.Flag("aws-retry-enabled", "Enable retry logic for transient AWS API errors").Default("true").Bool()
+	awsMaxRetries     = app.Flag("aws-max-retries", "Maximum number of retry attempts for transient AWS errors").Default("5").Int()
+	awsInitialDelayMs = app.Flag("aws-initial-delay-ms", "Initial delay in milliseconds before the first retry (exponential backoff)").Default("5000").Int()
+	awsMaxDelayMs     = app.Flag("aws-max-delay-ms", "Maximum delay in milliseconds between retry attempts").Default("60000").Int()
 )
 
 var log = logf.Log.WithName("cmd")
@@ -95,8 +101,8 @@ func main() {
 	// Register the custom metrics
 	metrics.Register(mgr.GetClient(), log, *namespace)
 
-	// Setup the cloud provider
-	cloudProvider, err := builder.BuildCloudProvider(*cloudProviderName, logger)
+	// Setup the cloud provider with retry configuration
+	cloudProvider, err := builder.BuildCloudProviderWithRetryConfig(*cloudProviderName, logger, *awsRetryEnabled, *awsMaxRetries, *awsInitialDelayMs, *awsMaxDelayMs)
 	if err != nil {
 		log.Error(err, "Unable to build cloud provider")
 		os.Exit(1)

pkg/cloudprovider/aws/aws.go

@@ -59,25 +59,27 @@ func verifyIfErrorOccurred(apiErr error, expectedMessage ...string) (bool, error
 }
 
 func verifyIfErrorOccurredWithDefaults(apiErr error, expectedMessage string) (bool, error) {
-	skip_errs := []string{
+	skipErrs := []string{
 		// default errors we wanted to skip
 		"is not in correct state",
 		expectedMessage,
 	}
-	return verifyIfErrorOccurred(apiErr, skip_errs...)
+	return verifyIfErrorOccurred(apiErr, skipErrs...)
 }
 
 type provider struct {
 	autoScalingService autoscalingiface.AutoScalingAPI
 	ec2Service         ec2iface.EC2API
 	logger             logr.Logger
+	retryConfig        RetryConfig
 }
 
 type autoscalingGroups struct {
 	autoScalingService autoscalingiface.AutoScalingAPI
 	ec2Service         ec2iface.EC2API
 	groups             []*autoscaling.Group
 	logger             logr.Logger
+	retryConfig        RetryConfig
 }
 
 type instance struct {
@@ -93,9 +95,17 @@ func (p *provider) Name() string {
 
 // GetNodeGroups gets a Autoscaling groups
 func (p *provider) GetNodeGroups(names []string) (cloudprovider.NodeGroups, error) {
-	result, err := p.autoScalingService.DescribeAutoScalingGroups(&autoscaling.DescribeAutoScalingGroupsInput{
-		AutoScalingGroupNames: aws.StringSlice(names),
-	})
+	var result *autoscaling.DescribeAutoScalingGroupsOutput
+	var err error
+
+	// Retry with exponential backoff for transient errors
+	err = retryOnTransientErrorWithConfig(func() error {
+		result, err = p.autoScalingService.DescribeAutoScalingGroups(&autoscaling.DescribeAutoScalingGroupsInput{
+			AutoScalingGroupNames: aws.StringSlice(names),
+		})
+		return err
+	}, p.logger, p.retryConfig)
+
 	if err != nil {
 		return nil, err
 	}
@@ -110,6 +120,7 @@ func (p *provider) GetNodeGroups(names []string) (cloudprovider.NodeGroups, erro
 		autoScalingService: p.autoScalingService,
 		ec2Service:         p.ec2Service,
 		logger:             p.logger,
+		retryConfig:        p.retryConfig,
 	}, nil
 }
 
@@ -129,11 +140,18 @@ func (p *provider) InstancesExist(providerIDs []string) (map[string]interface{},
 		instanceIDs = append(instanceIDs, instanceID)
 	}
 
-	output, err := p.ec2Service.DescribeInstances(
-		&ec2.DescribeInstancesInput{
-			InstanceIds: aws.StringSlice(instanceIDs),
-		},
-	)
+	var output *ec2.DescribeInstancesOutput
+	var err error
+
+	// Retry with exponential backoff for transient errors
+	err = retryOnTransientErrorWithConfig(func() error {
+		output, err = p.ec2Service.DescribeInstances(
+			&ec2.DescribeInstancesInput{
+				InstanceIds: aws.StringSlice(instanceIDs),
+			},
+		)
+		return err
+	}, p.logger, p.retryConfig)
 
 	if err != nil {
 		return nil, err
@@ -161,9 +179,14 @@ func (p *provider) TerminateInstance(providerID string) error {
 		return err
 	}
 
-	_, err = p.ec2Service.TerminateInstances(&ec2.TerminateInstancesInput{
-		InstanceIds: aws.StringSlice([]string{instanceID}),
-	})
+	// Retry with exponential backoff for transient errors
+	err = retryOnTransientErrorWithConfig(func() error {
+		_, err := p.ec2Service.TerminateInstances(&ec2.TerminateInstancesInput{
+			InstanceIds: aws.StringSlice([]string{instanceID}),
+		})
+		return err
+	}, p.logger, p.retryConfig)
+
 	return err
 }