simulate: --repeats runs every scenario k times for pass@k / pass^k

One attempt per scenario cannot tell a flaky scenario from a broken one.
--repeats K asks the server for K attempts of every scenario in one run;
the CLI groups attempts under their scenario in the job list and report,
labels each attempt, and adds a pass@k (passed at least once) / pass^k
(passed every time) line to the counts header and summaries, computed
over the scenarios whose attempts have all finished.

The exit code is unchanged: any failed attempt fails the run.

Author: u9g

autocomplete/fish_autocomplete

@@ -222,6 +222,7 @@ complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subco
 complete -x -c lk -n '__fish_seen_subcommand_from agent a; and not __fish_seen_subcommand_from init create dockerfile config deploy promote status update restart rollback logs tail delete destroy versions list secrets update-secrets private-link start dev console daemon simulate help h' -a 'simulate' -d 'Run agent simulations against LiveKit Cloud'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l num-simulations -s n -r -d 'Number of scenarios to generate'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l concurrency -r -d 'Max simulations running in parallel (default: server-side limit)'
+complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l repeats -s k -r -d 'Run every scenario `K` times and report pass@k (passed at least once) and pass^k (passed every time). Requires --scenarios'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l scenarios -r -d 'Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent\'s source'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l yes -s y -d 'Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)'
 complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l agent-name -r -d 'Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or "" to target the project\'s default agent (the one that auto-joins every room). Requires --scenarios.'

cmd/lk/simulate.go

@@ -88,6 +88,11 @@ var simulateCommand = &cli.Command{
 			Name:  "concurrency",
 			Usage: "Max simulations running in parallel (default: server-side limit)",
 		},
+		&cli.IntFlag{
+			Name:    "repeats",
+			Aliases: []string{"k"},
+			Usage:   "Run every scenario `K` times and report pass@k (passed at least once) and pass^k (passed every time). Requires --scenarios",
+		},
 		&cli.StringFlag{
 			Name:  "scenarios",
 			Usage: "Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent's source",
@@ -266,6 +271,7 @@ type simulateConfig struct {
 	pc             *config.ProjectConfig
 	numSimulations int32
 	concurrency    int32
+	repeats        int32
 	mode           simulateMode
 	simulationMode livekit.SimulationMode
 	agentName      string
@@ -372,11 +378,15 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 
 	numSimulations := int32(cmd.Int("num-simulations"))
 	concurrency := int32(cmd.Int("concurrency"))
+	repeats := int32(cmd.Int("repeats"))
 	liveAgentName := cmd.String("agent-name")
 
 	// never auto-discovered: an explicit --scenarios file is the source of
 	// truth, otherwise scenarios are generated from the agent's source
 	scenariosPath := cmd.String("scenarios")
+	if repeats > 1 && scenariosPath == "" {
+		return fmt.Errorf("--repeats requires --scenarios (generated scenarios are not repeated)")
+	}
 
 	var (
 		agentName   string
@@ -449,6 +459,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 		pc:             pc,
 		numSimulations: numSimulations,
 		concurrency:    concurrency,
+		repeats:        repeats,
 		mode:           mode,
 		simulationMode: simulationMode,
 		agentName:      agentName,
@@ -645,6 +656,7 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek
 	}
 	if c.mode == modeScenarios {
 		req.ScenarioGroup = c.scenarioGroup
+		req.Repeats = c.repeats
 	}
 
 	resp, err := c.client.CreateSimulationRun(ctx, req)
@@ -798,15 +810,92 @@ func cancelSimulationRun(client *lksdk.AgentSimulationClient, runID string) {
 	}
 }
 
-// sortedJobs orders a run's jobs by job ID: the backend's ordering shuffles
-// rows as statuses change.
+// sortedJobs orders a run's jobs so every attempt of a scenario sits together,
+// scenarios in scenarios.yaml order, then by job ID: the backend's ordering
+// shuffles rows as statuses change.
 func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job {
+	order := make(map[string]int, len(run.GetScenarioGroup().GetScenarios()))
+	for i, sc := range run.GetScenarioGroup().GetScenarios() {
+		order[sc.GetId()] = i
+	}
 	jobs := make([]*livekit.SimulationRun_Job, len(run.Jobs))
 	copy(jobs, run.Jobs)
-	sort.Slice(jobs, func(i, j int) bool { return jobs[i].GetId() < jobs[j].GetId() })
+	sort.Slice(jobs, func(i, j int) bool {
+		a, b := jobs[i], jobs[j]
+		if a.GetScenarioId() != b.GetScenarioId() {
+			return order[a.GetScenarioId()] < order[b.GetScenarioId()]
+		}
+		if a.GetAttempt() != b.GetAttempt() {
+			return a.GetAttempt() < b.GetAttempt()
+		}
+		return a.GetId() < b.GetId()
+	})
 	return jobs
 }
 
+// scenarioPassCounts folds a repeated run's jobs into scenarios: how many
+// have every attempt finished, and of those how many passed at least once
+// (pass@k) and every time (pass^k). Zero scenarios when the run did not
+// repeat, so callers can skip the line.
+func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll int) {
+	k := int(run.GetRepeats())
+	if k < 2 {
+		return
+	}
+	type tally struct{ terminal, passed int }
+	tallies := map[string]*tally{}
+	for _, j := range run.Jobs {
+		if j.GetScenarioId() == "" {
+			continue
+		}
+		t := tallies[j.GetScenarioId()]
+		if t == nil {
+			t = &tally{}
+			tallies[j.GetScenarioId()] = t
+		}
+		if isTerminalJobStatus(j.Status) {
+			t.terminal++
+		}
+		if j.Status == livekit.SimulationRun_Job_STATUS_COMPLETED {
+			t.passed++
+		}
+	}
+	for _, t := range tallies {
+		if t.terminal < k {
+			continue
+		}
+		scenarios++
+		if t.passed > 0 {
+			passAny++
+		}
+		if t.passed == k {
+			passAll++
+		}
+	}
+	return
+}
+
+// passRateLine is the pass@k / pass^k summary, or "" when the run did not
+// repeat or no scenario has finished every attempt.
+func passRateLine(run *livekit.SimulationRun) string {
+	scenarios, passAny, passAll := scenarioPassCounts(run)
+	if scenarios == 0 {
+		return ""
+	}
+	k := run.GetRepeats()
+	return fmt.Sprintf("pass@%d %d/%d (%.2f), pass^%d %d/%d (%.2f)",
+		k, passAny, scenarios, float64(passAny)/float64(scenarios),
+		k, passAll, scenarios, float64(passAll)/float64(scenarios))
+}
+
+// attemptSuffix marks a job's attempt when the run repeated scenarios.
+func attemptSuffix(run *livekit.SimulationRun, job *livekit.SimulationRun_Job) string {
+	if run.GetRepeats() < 2 || job.GetAttempt() == 0 {
+		return ""
+	}
+	return fmt.Sprintf(" (attempt %d/%d)", job.GetAttempt(), run.GetRepeats())
+}
+
 func simulationJobCounts(run *livekit.SimulationRun) (total, done, passed, failed int) {
 	if run == nil {
 		return

cmd/lk/simulate_ci.go

@@ -207,6 +207,9 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
 		}
 		total, _, passed, failedN := simulationJobCounts(run)
 		fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN)
+		if line := passRateLine(run); line != "" {
+			fmt.Fprintln(out.ResultWriter(), line)
+		}
 	}
 
 	if brokenAgent && agent != nil {

cmd/lk/simulate_repeats_test.go

@@ -0,0 +1,68 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/stretchr/testify/require"
+)
+
+func attemptJob(id, scenario string, attempt int32, status livekit.SimulationRun_Job_Status) *livekit.SimulationRun_Job {
+	return &livekit.SimulationRun_Job{Id: id, ScenarioId: scenario, Attempt: attempt, Status: status}
+}
+
+func TestScenarioPassCounts(t *testing.T) {
+	const (
+		done    = livekit.SimulationRun_Job_STATUS_COMPLETED
+		failed  = livekit.SimulationRun_Job_STATUS_FAILED
+		running = livekit.SimulationRun_Job_STATUS_RUNNING
+	)
+	run := &livekit.SimulationRun{Repeats: 2, Jobs: []*livekit.SimulationRun_Job{
+		attemptJob("SRJ_1", "SCN_a", 1, done), attemptJob("SRJ_2", "SCN_a", 2, done), // pass^k
+		attemptJob("SRJ_3", "SCN_b", 1, done), attemptJob("SRJ_4", "SCN_b", 2, failed), // pass@k only
+		attemptJob("SRJ_5", "SCN_c", 1, failed), attemptJob("SRJ_6", "SCN_c", 2, failed), // neither
+		attemptJob("SRJ_7", "SCN_d", 1, done), attemptJob("SRJ_8", "SCN_d", 2, running), // not finished
+	}}
+
+	scenarios, passAny, passAll := scenarioPassCounts(run)
+	require.Equal(t, 3, scenarios)
+	require.Equal(t, 2, passAny)
+	require.Equal(t, 1, passAll)
+	require.Equal(t, "pass@2 2/3 (0.67), pass^2 1/3 (0.33)", passRateLine(run))
+
+	run.Repeats = 1
+	require.Equal(t, "", passRateLine(run))
+}
+
+func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) {
+	run := &livekit.SimulationRun{
+		Repeats:       2,
+		ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_b"}, {Id: "SCN_a"}}},
+		Jobs: []*livekit.SimulationRun_Job{
+			attemptJob("SRJ_1", "SCN_a", 2, 0),
+			attemptJob("SRJ_2", "SCN_b", 2, 0),
+			attemptJob("SRJ_3", "SCN_a", 1, 0),
+			attemptJob("SRJ_4", "SCN_b", 1, 0),
+		},
+	}
+	var ids []string
+	for _, j := range sortedJobs(run) {
+		ids = append(ids, j.Id)
+	}
+	require.Equal(t, []string{"SRJ_4", "SRJ_2", "SRJ_3", "SRJ_1"}, ids)
+	require.Equal(t, " (attempt 1/2)", attemptSuffix(run, run.Jobs[3]))
+}

cmd/lk/simulate_report.go

@@ -173,7 +173,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess)
 		return
 	}
 
-	for i, job := range run.Jobs {
+	for i, job := range sortedJobs(run) {
 		icon := "⏺"
 		switch job.Status {
 		case livekit.SimulationRun_Job_STATUS_COMPLETED:
@@ -186,6 +186,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess)
 		if label == "" {
 			label = fmt.Sprintf("Job %d", i+1)
 		}
+		label += attemptSuffix(run, job)
 
 		fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id)
 
@@ -251,6 +252,9 @@ func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.S
 	fmt.Fprintln(w)
 	fmt.Fprintln(w, "::group::Summary")
 	fmt.Fprintf(w, "%d total, %d passed, %d failed\n", total, passed, failed)
+	if line := passRateLine(run); line != "" {
+		fmt.Fprintln(w, line)
+	}
 
 	if summary.GoingWell != "" {
 		fmt.Fprintln(w)

cmd/lk/simulate_tui.go

@@ -1409,6 +1409,9 @@ func (m *simulateModel) renderCounts() string {
 	if running > 0 {
 		parts = append(parts, yellowStyle().Render(fmt.Sprintf("%d running", running)))
 	}
+	if line := passRateLine(m.run); line != "" {
+		parts = append(parts, boldStyle.Render(line))
+	}
 
 	elapsed := ""
 	if !m.startTime.IsZero() {
@@ -1474,7 +1477,7 @@ func (m *simulateModel) renderJobList() string {
 	maxWidth := 0
 	for i := winStart; i < winEnd; i++ {
 		ij := jobs[i]
-		row := rowData{ij: ij, label: jobLabel(ij.job)}
+		row := rowData{ij: ij, label: jobLabel(ij.job) + attemptSuffix(m.run, ij.job)}
 		rows = append(rows, row)
 		w := lipgloss.Width(fmt.Sprintf("  ⏺ %3d. %s %s", ij.origIdx, ij.job.Id, row.label))
 		if w > maxWidth {
@@ -1556,7 +1559,7 @@ func (m *simulateModel) buildMatrixRows() []matrixRow {
 	}
 	for i := winStart; i < winEnd; i++ {
 		ij := jobs[i]
-		label := jobLabel(ij.job)
+		label := jobLabel(ij.job) + attemptSuffix(m.run, ij.job)
 		iconCh, iconStyle := jobStatusIcon(ij.job)
 		line := fmt.Sprintf("  %c %3d. %s %s", iconCh, ij.origIdx, ij.job.Id, label)
 		rows = append(rows, matrixRow{
@@ -1598,7 +1601,7 @@ func (m *simulateModel) renderDetail() string {
 	b.WriteString("\n")
 	fmt.Fprintf(&b, "  %s %s %s\n",
 		jobIcon(job),
-		boldStyle.Render(fmt.Sprintf("Job %d", origIdx)),
+		boldStyle.Render(fmt.Sprintf("Job %d", origIdx)+attemptSuffix(m.run, job)),
 		dimStyle.Render(job.Id),
 	)
 	if url := simulationJobDashboardURL(m.projectID(), m.runID, job.Id); url != "" {