Various updates: move measurements fallback logic, add a delay between round-robin failover, update README & runbooks, make settings generic

README.md

@@ -237,7 +237,7 @@ make clean-e2e-nitro
 
 ## Verifying Enclave Images
 
-The enclave build process is publicly verifiable so users can trust what runs inside the enclave. Cutting a release branch produces a GitHub Actions-generated Docker image ([example](https://github.com/smartcontractkit/confidential-compute/releases/tag/v1.3.0)). Because the image is built by GitHub from transparent source, it can be used to create reproducible [Enclave Image Files](https://docs.aws.amazon.com/enclaves/latest/user/building-eif.html).
+The enclave build process is publicly verifiable so users can trust what runs inside the enclave. Cutting a release branch produces a GitHub Actions-generated Docker image ([example](https://github.com/smartcontractkit/chainlink-confidential-compute/releases/tag/v1.3.0)). Because the image is built by GitHub from transparent source, it can be used to create reproducible [Enclave Image Files](https://docs.aws.amazon.com/enclaves/latest/user/building-eif.html).
 
 ```bash
 # Create measurements — produces [ENCLAVE_NAME].eif.measurements.json

capabilities/framework/config_update_test.go

@@ -30,15 +30,15 @@ func (s *stubEnclaveClient) GetPublicKeys(ctx context.Context, requestID [32]byt
 func (s *stubEnclaveClient) ExecuteBatch(context.Context, []types.SignedComputeRequest, [][32]byte) ([]types.ExecuteResponse, error) {
 	return nil, nil
 }
-func (s *stubEnclaveClient) UpdateNodes([]types.Enclave) {}
+func (s *stubEnclaveClient) UpdateNodes(context.Context, []types.Enclave) error { return nil }
 func (s *stubEnclaveClient) UpdateConfig(context.Context, types.UpdateConfigRequest) error {
 	return nil
 }
 func (s *stubEnclaveClient) GetConfigs(context.Context) ([]types.EnclaveConfig, error) {
 	return nil, nil
 }
 func (s *stubEnclaveClient) GetCacheStats() map[string]interface{} { return nil }
-func (s *stubEnclaveClient) Close() error                        { return nil }
+func (s *stubEnclaveClient) Close() error                          { return nil }
 
 func TestBroadcastConfigUpdate_DropsWhenProposalInFlight(t *testing.T) {
 	stub := &stubEnclaveClient{}

capabilities/framework/executor.go

@@ -653,7 +653,7 @@ func (e *RealExecutor) Execute(ctx context.Context, protoBytes []byte, secrets [
 
 		runLggr.Debugw("fetching enclave params")
 		enclaveParamsStart := time.Now()
-		enclaveParams, err := e.getEnclaveParams(ctx, reqID, metrics)
+		enclaveParams, err := e.getEnclaveParams(ctx, reqID)
 		enclaveParamsDuration = time.Since(enclaveParamsStart)
 
 		if err != nil {
@@ -761,15 +761,6 @@ func (e *RealExecutor) Execute(ctx context.Context, protoBytes []byte, secrets [
 		innerLggr.Debugw("enclave execution succeeded",
 			"duration_ms", enclaveExecuteDuration.Milliseconds())
 
-		if executeResponses[0].AttestationFallbackUsed {
-			innerLggr.Warnw("attestation validation used fallback measurements",
-				"endpoint", "execute")
-			metrics.Emit("attestation_validation_fallback_used", map[string]any{
-				"endpoint":   "execute",
-				"enclave.id": enclaveIDStr,
-			})
-		}
-
 		if err := e.validateEnclaveSigners(executeResponses[0].Config); err != nil {
 			return fmt.Errorf("execute response config validation failed for request %x: %w", reqID, err)
 		}
@@ -1050,8 +1041,16 @@ func (e *RealExecutor) EnsureFreshEnclaves(ctx context.Context) error {
 	}
 
 	sortEnclaveNodes(nodes)
-	e.enclaveClient.UpdateNodes(nodes)
-	e.enclaves = nodes
+	if err := e.enclaveClient.UpdateNodes(ctx, nodes); err != nil {
+		e.lggr.Warnw("node update failed, continuing to use old nodes & measurements",
+			"endpoint", "publicKeys",
+			"error", err)
+		e.metrics.Emit("attestation_validation_fallback_used", map[string]any{
+			"endpoint": "publicKeys",
+		})
+	} else {
+		e.enclaves = nodes
+	}
 
 	vaultDONPossibleFaultyNodes, err := getVaultDONPossibleFaultyNodes(ctx, e.vaultDON.Capability, int(localNode.WorkflowDON.F))
 	if err != nil {
@@ -1342,7 +1341,7 @@ func checkRequirements(req *sdk.Requirements) types.RequirementsChecker {
 	}
 }
 
-func (e *RealExecutor) getEnclaveParams(ctx context.Context, reqID [32]byte, metrics types.Emitter) (*EnclaveParams, error) {
+func (e *RealExecutor) getEnclaveParams(ctx context.Context, reqID [32]byte) (*EnclaveParams, error) {
 	reqIDHex := hex.EncodeToString(reqID[:])
 	ephemeralPubKeyResponse, err := e.enclaveClient.GetPublicKeys(ctx, reqID, checkRequirements(nil))
 	if err != nil {
@@ -1371,15 +1370,6 @@ func (e *RealExecutor) getEnclaveParams(ctx context.Context, reqID [32]byte, met
 	}
 	selectedEphemeralPublicKey := selectedEnclaveResponse.PublicKeys[mostRecentPubKeyIndex]
 	selectedEnclaveID := selectedEnclaveResponse.EnclaveID
-	if selectedEnclaveResponse.AttestationFallbackUsed {
-		e.lggr.Warnw("attestation validation used fallback measurements",
-			"endpoint", "publicKeys",
-			"enclave.id", hex.EncodeToString(selectedEnclaveID[:]))
-		metrics.Emit("attestation_validation_fallback_used", map[string]any{
-			"endpoint":   "publicKeys",
-			"enclave.id": hex.EncodeToString(selectedEnclaveID[:]),
-		})
-	}
 	e.lggr.Infow("selected enclave public key",
 		"reqID", reqIDHex,
 		"enclaveID", hex.EncodeToString(selectedEnclaveID[:]),

capabilities/framework/executor_test.go

@@ -60,7 +60,7 @@ type MockEnclaveClient struct {
 
 	GetPublicKeysFunc func(ctx context.Context, requestID [32]byte, checkRequirements enclavetypes.RequirementsChecker) ([]enclavetypes.EnclavePublicKeyData, error)
 	ExecuteBatchFunc  func(ctx context.Context, reqs []enclavetypes.SignedComputeRequest, enclaveIDs [][32]byte) ([]enclavetypes.ExecuteResponse, error)
-	UpdateNodesFunc   func(nodes []enclavetypes.Enclave)
+	UpdateNodesFunc   func(ctx context.Context, nodes []enclavetypes.Enclave) error
 	UpdateConfigFunc  func(ctx context.Context, update enclavetypes.UpdateConfigRequest) error
 	GetConfigsFunc    func(ctx context.Context) ([]enclavetypes.EnclaveConfig, error)
 }
@@ -122,10 +122,11 @@ func (m *MockEnclaveClient) commonExecuteBatchReturn(t *testing.T) ([]enclavetyp
 		}}, nil
 }
 
-func (m *MockEnclaveClient) UpdateNodes(nodes []enclavetypes.Enclave) {
+func (m *MockEnclaveClient) UpdateNodes(ctx context.Context, nodes []enclavetypes.Enclave) error {
 	if m.UpdateNodesFunc != nil {
-		m.UpdateNodesFunc(nodes)
+		return m.UpdateNodesFunc(ctx, nodes)
 	}
+	return nil
 }
 
 func (m *MockEnclaveClient) UpdateConfig(ctx context.Context, update enclavetypes.UpdateConfigRequest) error {
@@ -614,7 +615,7 @@ func TestExecutor_Execute(t *testing.T) {
 		assert.NotEmpty(t, completedCalls[0]["workflow.owner"], "enclave metrics should include workflow.owner")
 		assert.NotEmpty(t, completedCalls[0]["enclave.id"], "enclave metrics should include enclave.id")
 		assert.Equal(t, WORKFLOW_NAME, completedCalls[0]["workflow.name"], "enclave metrics should include workflow.name")
-			assert.Equal(t, "enclave", completedCalls[0]["component"], "enclave metrics should include component tag")
+		assert.Equal(t, "enclave", completedCalls[0]["component"], "enclave metrics should include component tag")
 		assert.Equal(t, WORKFLOW_ORG_ID, completedCalls[0]["org.id"], "enclave metrics should include org.id")
 
 		var testOutput framework.TestOutput
@@ -1121,6 +1122,11 @@ func TestExecutor_Execute_EmitsAttestationFallbackMetric(t *testing.T) {
 	}
 
 	mockEnclaveClient := &MockEnclaveClient{}
+	// UpdateNodes failing means a new node was unreachable or misconfigured;
+	// EnsureFreshEnclaves should keep the old nodes and emit the fallback metric.
+	mockEnclaveClient.UpdateNodesFunc = func(ctx context.Context, nodes []enclavetypes.Enclave) error {
+		return errors.New("public key fetch failed")
+	}
 	mockEnclaveClient.GetPublicKeysFunc = func(ctx context.Context, requestID [32]byte, checkRequirements enclavetypes.RequirementsChecker) ([]enclavetypes.EnclavePublicKeyData, error) {
 		return []enclavetypes.EnclavePublicKeyData{{
 			PublicKeyResponse: enclavetypes.PublicKeyResponse{
@@ -1132,17 +1138,11 @@ func TestExecutor_Execute_EmitsAttestationFallbackMetric(t *testing.T) {
 				},
 				Attestation: []byte("mock_attestation_data"),
 			},
-			EnclaveID:               mockEnclaveID,
-			AttestationFallbackUsed: true,
+			EnclaveID: mockEnclaveID,
 		}}, nil
 	}
 	mockEnclaveClient.ExecuteBatchFunc = func(ctx context.Context, reqs []enclavetypes.SignedComputeRequest, enclaveIDs [][32]byte) ([]enclavetypes.ExecuteResponse, error) {
-		responses, err := mockEnclaveClient.commonExecuteBatchReturn(t)
-		if err != nil {
-			return nil, err
-		}
-		responses[0].AttestationFallbackUsed = true
-		return responses, nil
+		return mockEnclaveClient.commonExecuteBatchReturn(t)
 	}
 
 	mockVaultDON := framework.VaultDON{
@@ -1156,17 +1156,8 @@ func TestExecutor_Execute_EmitsAttestationFallbackMetric(t *testing.T) {
 
 	require.Contains(t, mockMetrics.EmitRecords, "attestation_validation_fallback_used")
 	records := mockMetrics.EmitRecords["attestation_validation_fallback_used"]
-	require.Len(t, records, 2)
-
-	endpoints := map[string]bool{}
-	for _, record := range records {
-		endpoint, ok := record["endpoint"].(string)
-		require.True(t, ok)
-		endpoints[endpoint] = true
-		assert.NotEmpty(t, record["enclave.id"])
-	}
-	assert.True(t, endpoints["publicKeys"])
-	assert.True(t, endpoints["execute"])
+	require.NotEmpty(t, records)
+	assert.Equal(t, "publicKeys", records[0]["endpoint"])
 }
 
 func TestExecutor_ExecuteFailAfterRetry(t *testing.T) {
@@ -1645,9 +1636,10 @@ func TestEnsureFreshEnclaves_ConfigurationChange(t *testing.T) {
 		mockEnclaveClient.ExecuteBatchFunc = func(ctx context.Context, reqs []enclavetypes.SignedComputeRequest, enclaveIDs [][32]byte) ([]enclavetypes.ExecuteResponse, error) {
 			return mockEnclaveClient.commonExecuteBatchReturn(t)
 		}
-		mockEnclaveClient.UpdateNodesFunc = func(nodes []enclavetypes.Enclave) {
+		mockEnclaveClient.UpdateNodesFunc = func(ctx context.Context, nodes []enclavetypes.Enclave) error {
 			updateNodesCallCount++
 			lastUpdatedNodes = nodes
+			return nil
 		}
 
 		mockVaultDONCapability := &MockVaultDONCapability{}
@@ -1750,8 +1742,9 @@ func TestEnsureFreshEnclaves_ConfigurationChange(t *testing.T) {
 		mockEnclaveClient.ExecuteBatchFunc = func(ctx context.Context, reqs []enclavetypes.SignedComputeRequest, enclaveIDs [][32]byte) ([]enclavetypes.ExecuteResponse, error) {
 			return mockEnclaveClient.commonExecuteBatchReturn(t)
 		}
-		mockEnclaveClient.UpdateNodesFunc = func(nodes []enclavetypes.Enclave) {
+		mockEnclaveClient.UpdateNodesFunc = func(ctx context.Context, nodes []enclavetypes.Enclave) error {
 			lastUpdatedNodes = nodes
+			return nil
 		}
 
 		mockVaultDONCapability := &MockVaultDONCapability{}

enclave-client/client.go

@@ -13,7 +13,7 @@ import (
 type EnclaveClient interface {
 	GetPublicKeys(ctx context.Context, requestID [32]byte, checkRequirements types.RequirementsChecker) ([]types.EnclavePublicKeyData, error)
 	ExecuteBatch(ctx context.Context, reqs []types.SignedComputeRequest, enclaveIDs [][32]byte) ([]types.ExecuteResponse, error)
-	UpdateNodes(nodes []types.Enclave)
+	UpdateNodes(ctx context.Context, nodes []types.Enclave) error
 	UpdateConfig(ctx context.Context, update types.UpdateConfigRequest) error
 	GetConfigs(ctx context.Context) ([]types.EnclaveConfig, error)
 	GetCacheStats() map[string]interface{}