feat: reorder rpcs if they fail all retries (#132)

## feat: reorder rpcs if they fail all retries

Currently if the default RPC (or any other RPC) starts failing after the
client initialisation the multi-client will still keep the same order
for RPCs possibly using the same bad RPC on the next call. This change
reorders the RPCs and changes the default RPC when a request failed in a
way that all RPCs that failed will go down on the RPC's backup list and
the successful RPC will become the primary. If the primary RPC was
successful or there are no backups, no re-ordering will be performed.

### Test
Durable Pipeline:
https://github.com/smartcontractkit/chainlink-deployments/actions/runs/15415638318

___
Issue:[ CLD-261](https://smartcontract-it.atlassian.net/browse/CLD-261)

Author: DimitriosNaikopoulos

.changeset/ready-items-dance.md

@@ -0,0 +1,5 @@
+---
+"chainlink-deployments-framework": minor
+---
+
+feat: reorder bad RPCs if they fail all retries

deployment/multiclient.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"sync"
 	"time"
 
 	"github.com/avast/retry-go/v4"
@@ -66,6 +67,7 @@ type MultiClient struct {
 	RetryConfig RetryConfig
 	lggr        logger.Logger
 	chainName   string
+	mu          sync.RWMutex
 }
 
 // rpcHealthCheck performs a basic health check on the RPC client by calling eth_blockNumber
@@ -334,7 +336,7 @@ func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*t
 		}
 	}
 
-	for _, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) {
+	for _, client := range mc.clients() {
 		go waitMined(client, tx)
 	}
 	var receipt *types.Receipt
@@ -355,29 +357,33 @@ func (mc *MultiClient) WaitMined(ctx context.Context, tx *types.Transaction) (*t
 func (mc *MultiClient) retryWithBackups(ctx context.Context, opName string, op func(context.Context, *ethclient.Client) error) error {
 	var err error
 	traceID := uuid.New()
-	for i, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) {
+
+	for rpcIndex, client := range mc.clients() {
 		retryCount := 0
 		err2 := retry.Do(func() error {
 			timeoutCtx, cancel := ensureTimeout(ctx, mc.RetryConfig.Timeout)
 			defer cancel()
 
 			err = op(timeoutCtx, client)
 			if err != nil {
-				mc.lggr.Warnf("traceID %q: chain %q: op: %q: client index %d: failed execution - retryable error '%s'", traceID.String(), mc.chainName, opName, i, MaybeDataErr(err))
+				mc.lggr.Warnf("traceID %q: chain %q: op: %q: client index %d: failed execution - retryable error '%s'", traceID.String(), mc.chainName, opName, rpcIndex, MaybeDataErr(err))
 				return err
 			}
 
+			// If the operation was successful, check if we need to reorder the RPCs
+			mc.reorderRPCs(rpcIndex)
+
 			return nil
 		}, retry.Attempts(mc.RetryConfig.Attempts), retry.Delay(mc.RetryConfig.Delay),
 			retry.OnRetry(func(n uint, err error) { retryCount++ }))
 		if err2 == nil {
 			if retryCount > 0 {
-				mc.lggr.Infof("traceID %q: chain %q: op: %q: client index %d: successfully executed after %d retry", traceID.String(), mc.chainName, opName, i, retryCount)
+				mc.lggr.Infof("traceID %q: chain %q: op: %q: client index %d: successfully executed after %d retry", traceID.String(), mc.chainName, opName, rpcIndex, retryCount)
 			}
 
 			return nil
 		}
-		mc.lggr.Infof("traceID %q: chain %q: op: %q: client index %d: failed, trying next client", traceID.String(), mc.chainName, opName, i)
+		mc.lggr.Infof("traceID %q: chain %q: op: %q: client index %d: failed, trying next client", traceID.String(), mc.chainName, opName, rpcIndex)
 	}
 
 	return errors.Join(err, fmt.Errorf("all backup clients failed for chain %q", mc.chainName))
@@ -431,3 +437,36 @@ func ensureTimeout(parent context.Context, timeout time.Duration) (context.Conte
 	// create a new context with the specified timeout
 	return context.WithTimeout(parent, timeout)
 }
+
+// reorderRPCs reorders the RPCs based on the latest call.
+// If the default RPC failed all attempts, it will be moved to the end of the backup list.
+// If backup RPCs also failed, they will be moved to the end of the backup list.
+// If the primary RPC worked, it will remain the first in the list.
+func (mc *MultiClient) reorderRPCs(rpcIndex int) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	if rpcIndex < 1 || len(mc.Backups) == 0 {
+		return // No need to reorder if the first RPC is still the default or we don't have backups
+	}
+
+	// Find the index of the backupRPC
+	newDefaultRPCIndex := rpcIndex - 1
+	newDefaultRPC := mc.Backups[newDefaultRPCIndex]
+
+	// Reorder the failed backups to the end of the list
+	reordered := make([]*ethclient.Client, 0, len(mc.Backups))
+	reordered = append(reordered, mc.Backups[newDefaultRPCIndex+1:]...)
+	reordered = append(reordered, mc.Backups[:newDefaultRPCIndex]...)
+	reordered = append(reordered, mc.Client)
+
+	mc.Backups = reordered
+	mc.Client = newDefaultRPC
+}
+
+func (mc *MultiClient) clients() []*ethclient.Client {
+	mc.mu.RLock()
+	defer mc.mu.RUnlock()
+
+	return append([]*ethclient.Client{mc.Client}, mc.Backups...)
+}

deployment/multiclient_test.go

@@ -295,3 +295,103 @@ func TestEnsureTimeout(t *testing.T) {
 		})
 	}
 }
+func TestMultiClient_reorderRPCs(t *testing.T) {
+	t.Parallel()
+
+	// Create some test clients with different memory addresses for identification
+	client0 := ethclient.NewClient(nil) // primary
+	client1 := ethclient.NewClient(nil) // backup 0
+	client2 := ethclient.NewClient(nil) // backup 1
+	client3 := ethclient.NewClient(nil) // backup 2
+
+	rpcClients := []*ethclient.Client{
+		client1, // backup 0
+		client2, // backup 1
+		client3, // backup 2
+	}
+
+	tests := []struct {
+		name                string
+		backups             []*ethclient.Client
+		newDefaultClientIdx int
+		expectedClient      *ethclient.Client
+		expectedBackups     []*ethclient.Client
+	}{
+		{
+			name:                "Move first backup to primary",
+			backups:             rpcClients,
+			newDefaultClientIdx: 1,
+			expectedClient:      client1,
+			expectedBackups: []*ethclient.Client{
+				client2,
+				client3,
+				client0,
+			},
+		},
+		{
+			name:                "Move middle backup to primary",
+			backups:             rpcClients,
+			newDefaultClientIdx: 2,
+			expectedClient:      client2,
+			expectedBackups: []*ethclient.Client{
+				client3,
+				client1,
+				client0,
+			},
+		},
+		{
+			name:                "Move last backup to primary",
+			backups:             rpcClients,
+			newDefaultClientIdx: 3,
+			expectedClient:      client3,
+			expectedBackups: []*ethclient.Client{
+				client1,
+				client2,
+				client0,
+			},
+		},
+		{
+			name:                "Keep primary unchanged",
+			backups:             rpcClients,
+			newDefaultClientIdx: 0,
+			expectedClient:      client0,
+			expectedBackups: []*ethclient.Client{
+				client1,
+				client2,
+				client3,
+			},
+		},
+		{
+			name:                "Keep primary unchanged when no backups",
+			backups:             []*ethclient.Client{},
+			newDefaultClientIdx: 1,
+			expectedClient:      client0,
+			expectedBackups:     []*ethclient.Client{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			mc := &MultiClient{
+				Client:  client0,
+				Backups: tt.backups,
+				lggr:    logger.Test(t),
+			}
+
+			// Call the method being tested
+			mc.reorderRPCs(tt.newDefaultClientIdx)
+
+			// Verify the results
+			assert.Same(t, tt.expectedClient, mc.Client, "Primary client should be the selected backup")
+			require.Len(t, mc.Backups, len(tt.expectedBackups), "Backup count should remain the same")
+
+			// Check that backups are in the expected order
+			for i, expected := range tt.expectedBackups {
+				assert.Same(t, expected, mc.Backups[i],
+					"Backup at position %d should be as expected", i)
+			}
+		})
+	}
+}