Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions changelog/25.0/25.0.0/summary.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
- **[Minor Changes](#minor-changes)**
- **[VReplication](#minor-changes-vreplication)**
- [Default data protection for `_reverse` workflow cancel/complete](#vreplication-reverse-workflow-data-protection)
- [`vdiff show --only-summary` omits the per-table row-sample report](#vreplication-vdiff-only-summary)
- **[VTGate](#minor-changes-vtgate)**
- [Ingress bytes in query LogStats](#vtgate-logstats-ingress-bytes)
- [New controls for cross-keyspace reads](#vtgate-cross-keyspace-reads)
Expand Down Expand Up @@ -170,6 +171,16 @@ The `--keep-data` flag help text has been updated to note this default explicitl

See [#19906](https://github.com/vitessio/vitess/pull/19906) for details.

#### <a id="vreplication-vdiff-only-summary"/>`vdiff show --only-summary` omits the per-table row-sample report</a>

`vtctldclient vdiff ... show` now accepts an `--only-summary` flag. When set, the per-table diff report has its sampled-row arrays (`MismatchedRowsSample`, `ExtraRowsSourceSample`, `ExtraRowsTargetSample`) stripped on the tablet before the response is built, while the scalar counters (processed, matching, mismatched, and extra rows) and all other summary fields are preserved.

The sampled-row arrays carry actual row data, including large `BLOB`/`JSON` columns, and `vdiff show` aggregates them across every target shard. For diffs over tables with large rows this could push the aggregated response past gRPC message limits and make `vdiff show` fail outright, leaving no way to read the summary or mismatch state. `--only-summary` lets callers that only need progress and the mismatch state avoid transferring the samples while keeping the reported counts accurate.

The option is exposed as `only_summary` on the `VDiffShowRequest` (vtctld) and `VDiffReportOptions` (tablet) protobuf messages. It is opt-in and backward compatible: without the flag, the full report is returned as before.

See [#20870](https://github.com/vitessio/vitess/pull/20870) for details.

### <a id="minor-changes-vtgate"/>VTGate</a>

#### <a id="vtgate-logstats-ingress-bytes"/>Ingress bytes in query LogStats</a>
Expand Down
7 changes: 5 additions & 2 deletions go/cmd/vtctldclient/command/vreplication/vdiff/vdiff.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,9 @@ var (
}{}

showOptions = struct {
Arg string
Verbose bool
Arg string
Verbose bool
OnlySummary bool
}{}

stopOptions = struct {
Expand Down Expand Up @@ -645,6 +646,7 @@ func commandShow(cmd *cobra.Command, args []string) error {
Workflow: common.BaseOptions.Workflow,
TargetKeyspace: common.BaseOptions.TargetKeyspace,
Arg: showOptions.Arg,
OnlySummary: showOptions.OnlySummary,
})
if err != nil {
return err
Expand Down Expand Up @@ -709,6 +711,7 @@ func registerCommands(root *cobra.Command) {
base.AddCommand(resume)

show.Flags().BoolVar(&showOptions.Verbose, "verbose", false, "Show verbose output in summaries")
show.Flags().BoolVar(&showOptions.OnlySummary, "only-summary", false, "Omit the per-table diff report body and return only the vdiff and per-table summary state. Useful for large diffs where the report can exceed gRPC message limits.")
Comment thread
pedroalb marked this conversation as resolved.
Outdated
Comment thread
pedroalb marked this conversation as resolved.
Outdated
base.AddCommand(show)

stop.Flags().StringSliceVar(&stopOptions.TargetShards, "target-shards", nil, "The target shards to stop the vdiff on; default is all shards.")
Expand Down
23 changes: 19 additions & 4 deletions go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 34 additions & 0 deletions go/vt/proto/tabletmanagerdata/tabletmanagerdata_vtproto.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 16 additions & 3 deletions go/vt/proto/vtctldata/vtctldata.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 34 additions & 0 deletions go/vt/proto/vtctldata/vtctldata_vtproto.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions go/vt/vtctl/workflow/vdiff.go
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,11 @@ func (s *Server) VDiffShow(ctx context.Context, req *vtctldatapb.VDiffShowReques
Workflow: req.Workflow,
Action: string(vdiff.ShowAction),
ActionArg: req.Arg,
Options: &tabletmanagerdatapb.VDiffOptions{
ReportOptions: &tabletmanagerdatapb.VDiffReportOptions{
OnlySummary: req.GetOnlySummary(),
},
},
}

ts, err := s.buildTrafficSwitcher(ctx, req.TargetKeyspace, req.Workflow)
Expand Down
22 changes: 19 additions & 3 deletions go/vt/vttablet/tabletmanager/vdiff/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,27 @@ func (vde *Engine) PerformVDiffAction(ctx context.Context, req *tabletmanagerdat
return resp, nil
}

func (vde *Engine) getVDiffSummary(vdiffID int64, dbClient binlogplayer.DBClient) (*query.QueryResult, error) {
// vdiffSummaryQuery returns the summary query to run. When onlySummary is true
// it returns the variant that strips the row-sample arrays from the per-table
// report, keeping the scalar counters. The samples are the part that can grow
// very large (they carry sampled row data, including large blob/JSON columns)
// and, fanned out across many target shards, can push the aggregated response
// past gRPC message limits. Stripping them lets callers that only need the
// vdiff/table state, counts and has_mismatch avoid transferring that data while
// keeping the summary counters accurate. All other summary columns are
// unaffected.
func vdiffSummaryQuery(onlySummary bool) string {
if onlySummary {
return sqlVDiffSummaryOnly
}
return sqlVDiffSummary
}

@mattlord mattlord Aug 25, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit, but summary feels very overloaded here to the point that it loses all meaning. Maybe we could at least call sqlVDiffSummaryOnly something like sqlVDiffSummaryMinimal or sqlVDiffSummaryWithoutSamples? Or maybe generally we use Full and Minimal like we have in some code here.


func (vde *Engine) getVDiffSummary(vdiffID int64, dbClient binlogplayer.DBClient, reportOpts *tabletmanagerdatapb.VDiffReportOptions) (*query.QueryResult, error) {
var qr *sqltypes.Result
var err error

query, err := sqlparser.ParseAndBind(sqlVDiffSummary, sqltypes.Int64BindVariable(vdiffID), sqltypes.StringBindVariable(vde.dbName))
query, err := sqlparser.ParseAndBind(vdiffSummaryQuery(reportOpts.GetOnlySummary()), sqltypes.Int64BindVariable(vdiffID), sqltypes.StringBindVariable(vde.dbName))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -336,7 +352,7 @@ func (vde *Engine) handleShowAction(ctx context.Context, dbClient binlogplayer.D
case 1:
row := qr.Named().Row()
vdiffID, _ := row["id"].ToInt64()
summary, err := vde.getVDiffSummary(vdiffID, dbClient)
summary, err := vde.getVDiffSummary(vdiffID, dbClient, req.GetOptions().GetReportOptions())
resp.Output = summary
if err != nil {
return err
Expand Down
42 changes: 42 additions & 0 deletions go/vt/vttablet/tabletmanager/vdiff/action_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package vdiff
import (
"context"
"fmt"
"strings"
"testing"
"time"

Expand All @@ -34,6 +35,47 @@ import (
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
)

// TestVDiffSummaryQuery asserts that the two summary-query variants differ only
// in how they project the report column: the full variant returns the stored
// report as-is, while the summary-only variant strips just the row-sample
// arrays and keeps the scalar counters. The assertions are structural (split on
// the report select-expression and compare the surrounding column list and
// FROM/WHERE) so harmless SQL formatting changes don't break the test.
func TestVDiffSummaryQuery(t *testing.T) {
const (
fullReportExpr = `vdt.report as report`
summaryReportExpr = `JSON_REMOVE(vdt.report, '$.MismatchedRowsSample', '$.ExtraRowsSourceSample', '$.ExtraRowsTargetSample') as report`
)

full := vdiffSummaryQuery(false)
summary := vdiffSummaryQuery(true)

// Both must be bindable (report/columns aside, the FROM/WHERE has the %a
// placeholders that ParseAndBind fills).
require.Contains(t, full, "%a", "bind placeholders must be preserved for ParseAndBind")
require.Contains(t, summary, "%a", "bind placeholders must be preserved for ParseAndBind")

// Each variant uses its own report select-expression and not the other's.
require.Contains(t, full, fullReportExpr, "full query must select the stored report as-is")
require.NotContains(t, full, "JSON_REMOVE", "full query must not strip the report")
require.Contains(t, summary, summaryReportExpr, "summary-only query must strip the row-sample arrays from the report")

// The summary-only variant must strip the sample arrays but must not touch
// the scalar counters, so the summary counts stay accurate.
for _, sample := range []string{"MismatchedRowsSample", "ExtraRowsSourceSample", "ExtraRowsTargetSample"} {
require.Contains(t, summaryReportExpr, sample, "expected sample array %q to be stripped", sample)
}

// The two variants must be identical everywhere except the report
// select-expression: swapping in the full expression must reproduce the
// full query exactly, so no other column or clause can silently differ.
require.Equal(t,
full,
strings.Replace(summary, summaryReportExpr, fullReportExpr, 1),
"summary-only must differ from the full query only in the report select-expression",
)
Comment thread
pedroalb marked this conversation as resolved.
}

func TestPerformVDiffAction(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
Expand Down
Loading
Loading