Add retries test

Author: AlCutter

internal/mirror/gateway/mirror_gateway_test.go

@@ -44,68 +44,85 @@ func (f fakeWitnessGroup) WitnessEndpoints() map[string][]note.Verifier {
 }
 
 func TestGateway(t *testing.T) {
-	opts := tessera.NewAppendOptions().
-		WithCheckpointInterval(100 * time.Millisecond).
-		WithCheckpointRepublishInterval(100 * time.Millisecond)
-
-	testLog, shutdown := testonly.NewTestLog(t, opts)
-	defer func() {
-		_ = shutdown(t.Context())
-	}()
-
-	// Create a log with some entries.
-	const size = 5
-	var f tessera.IndexFuture
-	for i := range size {
-		entry := tessera.NewEntry(fmt.Appendf(nil, "entry-%d", i))
-		f = testLog.Appender.Add(t.Context(), entry)
-	}
-	a := tessera.NewPublicationAwaiter(t.Context(), testLog.LogReader.ReadCheckpoint, 100*time.Millisecond)
-	if _, _, err := a.Await(t.Context(), f); err != nil {
-		t.Fatalf("failed to add entry: %v", err)
-	}
-	goalCP, err := testLog.LogReader.ReadCheckpoint(t.Context())
-	if err != nil {
-		t.Fatalf("failed to read checkpoint: %v", err)
-	}
+	for _, tc := range []struct {
+		name       string
+		numMirrors int
+		failCount  int
+	}{
+		{
+			name:       "multiple mirrors",
+			numMirrors: 3,
+		},
+		{
+			name:       "retry on transient error",
+			numMirrors: 1,
+			failCount:  2,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			opts := tessera.NewAppendOptions().
+				WithCheckpointInterval(100 * time.Millisecond).
+				WithCheckpointRepublishInterval(100 * time.Millisecond)
+
+			testLog, shutdown := testonly.NewTestLog(t, opts)
+			defer func() {
+				_ = shutdown(t.Context())
+			}()
+
+			// Create a log with some entries.
+			const size = 5
+			var f tessera.IndexFuture
+			for i := range size {
+				entry := tessera.NewEntry(fmt.Appendf(nil, "entry-%d", i))
+				f = testLog.Appender.Add(t.Context(), entry)
+			}
+			a := tessera.NewPublicationAwaiter(t.Context(), testLog.LogReader.ReadCheckpoint, 100*time.Millisecond)
+			if _, _, err := a.Await(t.Context(), f); err != nil {
+				t.Fatalf("failed to add entry: %v", err)
+			}
+			goalCP, err := testLog.LogReader.ReadCheckpoint(t.Context())
+			if err != nil {
+				t.Fatalf("failed to read checkpoint: %v", err)
+			}
 
-	const numMirrors = 3
-	var verifiers []note.Verifier
-	endpoints := make(map[string][]note.Verifier)
+			var verifiers []note.Verifier
+			endpoints := make(map[string][]note.Verifier)
 
-	for i := range numMirrors {
-		signer, verifier := mustNewKeypair(t, fmt.Sprintf("Mirror-%d", i))
-		server := startMockMirror(t, signer, testLog.SigVerifier)
-		defer server.Close()
+			for i := range tc.numMirrors {
+				signer, verifier := mustNewKeypair(t, fmt.Sprintf("Mirror-%d", i))
+				server := startMockMirror(t, signer, testLog.SigVerifier, tc.failCount)
+				defer server.Close()
 
-		endpoints[server.URL] = []note.Verifier{verifier}
-		verifiers = append(verifiers, verifier)
-	}
+				endpoints[server.URL] = []note.Verifier{verifier}
+				verifiers = append(verifiers, verifier)
+			}
 
-	policy := fakeWitnessGroup{
-		endpoints: endpoints,
-	}
+			policy := fakeWitnessGroup{
+				endpoints: endpoints,
+			}
 
-	g := gateway.NewGateway(t.Context(), http.DefaultClient, policy, testLog.LogReader, "test")
+			g := gateway.NewGateway(t.Context(), http.DefaultClient, policy, testLog.LogReader, "test")
 
-	// Call CosignCheckpoint and gather signatures.
-	sigCh := g.CosignCheckpoint(t.Context(), goalCP, size)
-	var cosigs []byte
-	for sig := range sigCh {
-		cosigs = append(cosigs, sig...)
-	}
+			// Call CosignCheckpoint and gather signatures.
+			sigCh := g.CosignCheckpoint(t.Context(), goalCP, size)
+			var cosigs []byte
+			for sig := range sigCh {
+				cosigs = append(cosigs, sig...)
+			}
 
-	// Verify cosignatures.
-	fullCP := append(slices.Clone(goalCP), cosigs...)
-	cp, _, n, err := log.ParseCheckpoint(fullCP, testLog.SigVerifier.Name(), testLog.SigVerifier, verifiers...)
-	if err != nil {
-		t.Fatalf("failed to verify cosigned checkpoint: %v", err)
-	}
-	if got, want := len(n.Sigs), 1+numMirrors; got != want {
-		t.Errorf("note signatures: got %d, want %d", got, want)
-	}
-	if got, want := uint64(cp.Size), uint64(size); got != want {
-		t.Errorf("checkpoint size: got %d, want %d", got, want)
+			// Verify cosignatures.
+			fullCP := append(slices.Clone(goalCP), cosigs...)
+			cp, _, n, err := log.ParseCheckpoint(fullCP, testLog.SigVerifier.Name(), testLog.SigVerifier, verifiers...)
+			if err != nil {
+				t.Fatalf("failed to verify cosigned checkpoint: %v", err)
+			}
+			if got, want := len(n.Sigs), 1+tc.numMirrors; got != want {
+				t.Errorf("note signatures: got %d, want %d", got, want)
+			}
+			if got, want := uint64(cp.Size), uint64(size); got != want {
+				t.Errorf("checkpoint size: got %d, want %d", got, want)
+			}
+		})
 	}
 }
 
@@ -126,28 +143,37 @@ func mustNewKeypair(t *testing.T, name string) (f_note.Signer, note.Verifier) {
 	return s, v
 }
 
-func startMockMirror(t *testing.T, signer note.Signer, logVerifier note.Verifier) *httptest.Server {
+func startMockMirror(t *testing.T, signer note.Signer, logVerifier note.Verifier, failCount int) *httptest.Server {
 	t.Helper()
 	var mu sync.Mutex
 	var pendingCP []byte
+	attempts := 0
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+
+		if strings.HasSuffix(r.URL.Path, "/add-checkpoint") || strings.HasSuffix(r.URL.Path, "/add-entries") {
+			if attempts < failCount {
+				attempts++
+				w.WriteHeader(http.StatusInternalServerError)
+				_, _ = w.Write([]byte("mock error"))
+				return
+			}
+		}
+
 		if strings.HasSuffix(r.URL.Path, "/add-checkpoint") {
 			body, _ := io.ReadAll(r.Body)
 			parts := bytes.SplitN(body, []byte("\n\n"), 2)
 			if len(parts) == 2 {
-				mu.Lock()
 				pendingCP = parts[1]
-				mu.Unlock()
 			}
 			w.WriteHeader(http.StatusOK)
 			return
 		}
 		if strings.HasSuffix(r.URL.Path, "/add-entries") {
 			_, _ = io.Copy(io.Discard, r.Body)
-			mu.Lock()
 			cp := pendingCP
-			mu.Unlock()
 
 			if len(cp) == 0 {
 				w.WriteHeader(http.StatusBadRequest)