rafttest: test lazy replication

Epic: none
Release note: none

Author: pav-kv

pkg/raft/rafttest/BUILD.bazel

@@ -20,6 +20,7 @@ go_library(
         "interaction_env_handler_raft_log.go",
         "interaction_env_handler_raftstate.go",
         "interaction_env_handler_report_unreachable.go",
+        "interaction_env_handler_send_msgapp.go",
         "interaction_env_handler_send_snapshot.go",
         "interaction_env_handler_set_randomized_election_timeout.go",
         "interaction_env_handler_stabilize.go",

pkg/raft/rafttest/interaction_env_handler.go

@@ -198,6 +198,12 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string {
 		// propose-conf-change 2 v1=true
 		// v5
 		err = env.handleProposeConfChange(t, d)
+
+	case "send-msg-app":
+		// Send a MsgApp from the leader to a peer.
+		// Example: send-msg-app 1 to=2 lo=10 hi=20
+		err = env.handleSendMsgApp(t, d)
+
 	case "report-unreachable":
 		// Calls <1st>.ReportUnreachable(<2nd>).
 		//

pkg/raft/rafttest/interaction_env_handler_add_nodes.go

@@ -57,6 +57,8 @@ func (env *InteractionEnv) handleAddNodes(t *testing.T, d datadriven.TestData) e
 				arg.Scan(t, i, &snap.Data)
 			case "async-storage-writes":
 				arg.Scan(t, i, &cfg.AsyncStorageWrites)
+			case "lazy-replication":
+				arg.Scan(t, i, &cfg.LazyReplication)
 			case "prevote":
 				arg.Scan(t, i, &cfg.PreVote)
 			case "checkquorum":

pkg/raft/rafttest/interaction_env_handler_send_msgapp.go

@@ -0,0 +1,49 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package rafttest
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/raft"
+	pb "github.com/cockroachdb/cockroach/pkg/raft/raftpb"
+	"github.com/cockroachdb/datadriven"
+	"github.com/stretchr/testify/require"
+)
+
+func (env *InteractionEnv) handleSendMsgApp(t *testing.T, d datadriven.TestData) error {
+	require.Len(t, d.CmdArgs, 4)
+	from := firstAsNodeIdx(t, d)
+	var to int
+	var lo, hi uint64
+	d.ScanArgs(t, "to", &to)
+	d.ScanArgs(t, "lo", &lo)
+	d.ScanArgs(t, "hi", &hi)
+	return env.SendMsgApp(from, pb.PeerID(to), lo, hi)
+}
+
+func (env *InteractionEnv) SendMsgApp(from int, to pb.PeerID, lo, hi uint64) error {
+	rn := env.Nodes[from].RawNode
+	snap := rn.LogSnapshot()
+	ls, err := snap.LogSlice(lo, hi, math.MaxUint64)
+	if err != nil {
+		return err
+	}
+	if msg, ok := rn.SendMsgApp(to, ls); ok {
+		env.Output.WriteString(raft.DescribeMessage(msg, defaultEntryFormatter))
+		env.Messages = append(env.Messages, msg)
+	} else {
+		env.Output.WriteString(fmt.Sprintf("could not send MsgApp (%d,%d] to %d", lo, hi, to))
+	}
+	return nil
+}

pkg/raft/testdata/lazy_replication.txt

@@ -1,13 +1,14 @@
-# This is a prep test for the next commit, to demonstrate the difference between
-# the eager and lazy replication.
+# This test demonstrates the "lazy replication" feature. The leader sends MsgApp
+# messages to StateReplication peers only when requested explicitly by the
+# application.
 
 # Skip logging the boilerplate. Set up a raft group of 3 nodes, and elect node 1
 # as the leader. Nodes 2 and 3 are the followers.
 log-level none
 ----
 ok
 
-add-nodes 3 voters=(1,2,3) index=10
+add-nodes 3 voters=(1,2,3) index=10 lazy-replication=true
 ----
 ok
 
@@ -32,87 +33,118 @@ propose 1 data-2
 ----
 ok
 
-# Both entries are sent to the followers eagerly, in two separate MsgApp
-# messages. The entries are committed, and the followers' logs converge to the
-# leader's.
+# NB: no entries are sent to the followers yet.
 stabilize
 ----
 > 1 handling Ready
   Ready MustSync=true:
   Entries:
   1/12 EntryNormal "data-1"
   1/13 EntryNormal "data-2"
-  Messages:
-  1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data-1"]
-  1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data-1"]
-  1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data-2"]
-  1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data-2"]
+
+# Attempt to send a misaligned MsgApp. No-op.
+send-msg-app 1 to=2 lo=10 hi=13
+----
+could not send MsgApp (10,13] to 2
+
+# Send a MsgApp to node 2, containing both entries.
+send-msg-app 1 to=2 lo=11 hi=13
+----
+1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[
+  1/12 EntryNormal "data-1"
+  1/13 EntryNormal "data-2"
+]
+
+# Send a MsgApp to node 3, containing only one entry.
+send-msg-app 1 to=3 lo=11 hi=12
+----
+1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data-1"]
+
+# The followers receive the entries and reply to the leader. The leader commits
+# both entries, but the replication flow to node 3 still has one pending entry.
+stabilize
+----
 > 2 receiving messages
-  1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data-1"]
-  1->2 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data-2"]
+  1->2 MsgApp Term:1 Log:1/11 Commit:11 Entries:[
+    1/12 EntryNormal "data-1"
+    1/13 EntryNormal "data-2"
+  ]
 > 3 receiving messages
   1->3 MsgApp Term:1 Log:1/11 Commit:11 Entries:[1/12 EntryNormal "data-1"]
-  1->3 MsgApp Term:1 Log:1/12 Commit:11 Entries:[1/13 EntryNormal "data-2"]
 > 2 handling Ready
   Ready MustSync=true:
   Entries:
   1/12 EntryNormal "data-1"
   1/13 EntryNormal "data-2"
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/12 Commit:11
   2->1 MsgAppResp Term:1 Log:0/13 Commit:11
 > 3 handling Ready
   Ready MustSync=true:
   Entries:
   1/12 EntryNormal "data-1"
-  1/13 EntryNormal "data-2"
   Messages:
   3->1 MsgAppResp Term:1 Log:0/12 Commit:11
-  3->1 MsgAppResp Term:1 Log:0/13 Commit:11
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/12 Commit:11
   2->1 MsgAppResp Term:1 Log:0/13 Commit:11
   3->1 MsgAppResp Term:1 Log:0/12 Commit:11
-  3->1 MsgAppResp Term:1 Log:0/13 Commit:11
 > 1 handling Ready
   Ready MustSync=true:
   HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
   CommittedEntries:
   1/12 EntryNormal "data-1"
   1/13 EntryNormal "data-2"
   Messages:
-  1->2 MsgApp Term:1 Log:1/13 Commit:12
-  1->3 MsgApp Term:1 Log:1/13 Commit:12
   1->2 MsgApp Term:1 Log:1/13 Commit:13
-  1->3 MsgApp Term:1 Log:1/13 Commit:13
+  1->3 MsgApp Term:1 Log:1/12 Commit:13
 > 2 receiving messages
-  1->2 MsgApp Term:1 Log:1/13 Commit:12
   1->2 MsgApp Term:1 Log:1/13 Commit:13
 > 3 receiving messages
-  1->3 MsgApp Term:1 Log:1/13 Commit:12
-  1->3 MsgApp Term:1 Log:1/13 Commit:13
+  1->3 MsgApp Term:1 Log:1/12 Commit:13
 > 2 handling Ready
   Ready MustSync=true:
   HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
   CommittedEntries:
   1/12 EntryNormal "data-1"
   1/13 EntryNormal "data-2"
   Messages:
-  2->1 MsgAppResp Term:1 Log:0/13 Commit:12
   2->1 MsgAppResp Term:1 Log:0/13 Commit:13
 > 3 handling Ready
   Ready MustSync=true:
-  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
+  HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1
   CommittedEntries:
   1/12 EntryNormal "data-1"
+  Messages:
+  3->1 MsgAppResp Term:1 Log:0/12 Commit:12
+> 1 receiving messages
+  2->1 MsgAppResp Term:1 Log:0/13 Commit:13
+  3->1 MsgAppResp Term:1 Log:0/12 Commit:12
+
+# One entry still to be replicated to node 3.
+status 1
+----
+1: StateReplicate match=13 next=14 sentCommit=11 matchCommit=11
+2: StateReplicate match=13 next=14 sentCommit=13 matchCommit=13
+3: StateReplicate match=12 next=13 sentCommit=13 matchCommit=12
+
+# Replicate it.
+send-msg-app 1 to=3 lo=12 hi=13
+----
+1->3 MsgApp Term:1 Log:1/12 Commit:13 Entries:[1/13 EntryNormal "data-2"]
+
+stabilize
+----
+> 3 receiving messages
+  1->3 MsgApp Term:1 Log:1/12 Commit:13 Entries:[1/13 EntryNormal "data-2"]
+> 3 handling Ready
+  Ready MustSync=true:
+  HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1
+  Entries:
+  1/13 EntryNormal "data-2"
+  CommittedEntries:
   1/13 EntryNormal "data-2"
   Messages:
-  3->1 MsgAppResp Term:1 Log:0/13 Commit:12
   3->1 MsgAppResp Term:1 Log:0/13 Commit:13
 > 1 receiving messages
-  2->1 MsgAppResp Term:1 Log:0/13 Commit:12
-  2->1 MsgAppResp Term:1 Log:0/13 Commit:13
-  3->1 MsgAppResp Term:1 Log:0/13 Commit:12
   3->1 MsgAppResp Term:1 Log:0/13 Commit:13
 
 # The leader has converged to a fully replicated state.