Skip to content

Commit dbd3886

Browse files
authored
simulate: list, view and export subcommands (#968)
* simulate: view and export are subcommands, not flags --view ID and --export ID each replaced the whole run action from inside a flag, and audio inherited both. A flag that takes an ID and swaps the action is a subcommand, so they become `lk agent simulate view <run-id>` and `lk agent simulate export <run-id>`, matching the noun/verb shape of the rest of lk. The exit hints print the new forms. * simulate: list a project's recent runs `lk agent simulate list` prints the API's first page (newest first) with ID, created time, status, mode and agent name, or the raw response with --json, so a run ID can be found without the dashboard. * simulate: --limit on list pages through runs `lk agent simulate list` fetched only the API's first page (20 runs). --limit walks next_page_token until that many runs are collected, the same loop as `lk egress list`; unset keeps the single-page default. --json now prints the collected runs wrapped in the List response so the field formatting is unchanged. * simulate: text subcommand names the default mode `lk agent simulate text` runs the same text simulation as bare `lk agent simulate`, so the mode can be spelled out the same way as `simulate audio`. Bare `simulate` is unchanged. * simulate: bare command prints help instead of running `lk agent simulate` with no subcommand shows the subcommand list and does nothing else; a run needs `text` or `audio`. The project is not resolved for the help path.
1 parent 1b8616d commit dbd3886

3 files changed

Lines changed: 188 additions & 43 deletions

File tree

autocomplete/fish_autocomplete

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -224,15 +224,23 @@ complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcomma
224224
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l concurrency -r -d 'Max simulations running in parallel (default: server-side limit)'
225225
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l scenarios -r -d 'Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent\'s source'
226226
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l yes -s y -d 'Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)'
227-
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l view -r -d 'Open a pre-existing simulation'
228-
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l export -r -d 'Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished'
229227
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l agent-name -r -d 'Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or "" to target the project\'s default agent (the one that auto-joins every room). Requires --scenarios.'
230228
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l help -s h -d 'show help'
231-
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from audio' -a 'audio' -d 'Simulate speech-to-speech interactions using the agent\'s full audio pipeline'
229+
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from text audio list view export' -a 'text' -d 'Simulate text-only interactions'
230+
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from text' -f -l help -s h -d 'show help'
231+
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from text audio list view export' -a 'audio' -d 'Simulate speech-to-speech interactions using the agent\'s full audio pipeline'
232232
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l background-noise -d 'Mix ambient noise into the simulated user\'s audio'
233233
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l low-quality-microphone -d 'Publish the simulated user\'s audio as a low-quality microphone would capture it'
234234
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l packet-loss -d 'Drop packets from the simulated user\'s audio track'
235235
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from audio' -f -l help -s h -d 'show help'
236+
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from text audio list view export' -a 'list' -d 'List the project\'s most recent simulation runs'
237+
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from list' -f -l limit -r -d 'maximum number of runs to return. If unset, defaults to API page size'
238+
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from list' -f -l json -s j -d 'Output as JSON'
239+
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from list' -f -l help -s h -d 'show help'
240+
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from text audio list view export' -a 'view' -d 'Open a pre-existing simulation run'
241+
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from view' -f -l help -s h -d 'show help'
242+
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from text audio list view export' -a 'export' -d 'Print a finished run and its exact per-job chat contexts as JSON. Nothing is run or polled'
243+
complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and __fish_seen_subcommand_from export' -f -l help -s h -d 'show help'
236244
complete -x -c lk -n '__fish_seen_subcommand_from agent a; and not __fish_seen_subcommand_from init create dockerfile config deploy promote status update restart rollback logs tail delete destroy versions list secrets update-secrets private-link start dev console daemon simulate help h' -a 'help' -d 'Shows a list of commands or help for one command'
237245
complete -c lk -n '__fish_seen_subcommand_from analytics' -f -l experimental -d 'Enable experimental features'
238246
complete -c lk -n '__fish_seen_subcommand_from analytics' -f -l help -s h -d 'show help'

cmd/lk/simulate.go

Lines changed: 80 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,16 @@ const (
5656
)
5757

5858
var simulateCommand = &cli.Command{
59-
Name: "simulate",
60-
Usage: "Run agent simulations against LiveKit Cloud",
61-
ArgsUsage: "[entrypoint]",
59+
Name: "simulate",
60+
Usage: "Run agent simulations against LiveKit Cloud",
6261
// Hide the implicit `help` subcommand so shell completion falls back to
6362
// native filename completion for the entrypoint arg (see startCommand).
6463
HideHelpCommand: true,
6564
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
65+
// Bare `simulate` only prints help; no project is needed for that.
66+
if cmd.Args().Len() == 0 {
67+
return nil, nil
68+
}
6669
pc, err := loadProjectDetails(cmd)
6770
if err != nil {
6871
return nil, err
@@ -71,9 +74,9 @@ var simulateCommand = &cli.Command{
7174
return nil, nil
7275
},
7376
Action: func(ctx context.Context, cmd *cli.Command) error {
74-
return runSimulate(ctx, cmd, livekit.SimulationMode_SIMULATION_MODE_TEXT)
77+
return cli.ShowSubcommandHelp(cmd)
7578
},
76-
Commands: []*cli.Command{simulateAudioCommand},
79+
Commands: []*cli.Command{simulateTextCommand, simulateAudioCommand, simulateListCommand, simulateViewCommand, simulateExportCommand},
7780
Flags: []cli.Flag{
7881
&cli.IntFlag{
7982
Name: "num-simulations",
@@ -93,21 +96,24 @@ var simulateCommand = &cli.Command{
9396
Aliases: []string{"y"},
9497
Usage: "Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)",
9598
},
96-
&cli.StringFlag{
97-
Name: "view",
98-
Usage: "Open a pre-existing simulation",
99-
},
100-
&cli.StringFlag{
101-
Name: "export",
102-
Usage: "Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished",
103-
},
10499
&cli.StringFlag{
105100
Name: "agent-name",
106101
Usage: "Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or \"\" to target the project's default agent (the one that auto-joins every room). Requires --scenarios.",
107102
},
108103
},
109104
}
110105

106+
var simulateTextCommand = &cli.Command{
107+
Name: "text",
108+
Usage: "Simulate text-only interactions",
109+
Description: "Options on lk agent simulate apply here too, e.g. --scenarios and --agent-name.",
110+
ArgsUsage: "[entrypoint]",
111+
HideHelpCommand: true,
112+
Action: func(ctx context.Context, cmd *cli.Command) error {
113+
return runSimulate(ctx, cmd, livekit.SimulationMode_SIMULATION_MODE_TEXT)
114+
},
115+
}
116+
111117
// simulateAudioCommand inherits every flag on `simulate`: flags there are
112118
// persistent (cli.FlagBase.Local defaults to false), so they parse on either
113119
// side of the subcommand name.
@@ -136,6 +142,60 @@ var simulateAudioCommand = &cli.Command{
136142
},
137143
}
138144

145+
// Run flags on `simulate` are persistent, so these subcommands parse them too;
146+
// they take a run ID instead and ignore the rest.
147+
var simulateViewCommand = &cli.Command{
148+
Name: "view",
149+
Usage: "Open a pre-existing simulation run",
150+
ArgsUsage: "<run-id>",
151+
HideHelpCommand: true,
152+
Action: runSimulateView,
153+
}
154+
155+
var simulateExportCommand = &cli.Command{
156+
Name: "export",
157+
Usage: "Print a finished run and its exact per-job chat contexts as JSON. Nothing is run or polled",
158+
ArgsUsage: "<run-id>",
159+
HideHelpCommand: true,
160+
Action: func(ctx context.Context, cmd *cli.Command) error {
161+
runID, err := simulateRunIDArg(cmd)
162+
if err != nil {
163+
return err
164+
}
165+
return exportSimulationRunJSON(ctx, simulateProjectConfig, runID)
166+
},
167+
}
168+
169+
func simulateRunIDArg(cmd *cli.Command) (string, error) {
170+
runID := cmd.Args().First()
171+
if runID == "" {
172+
return "", fmt.Errorf("%s requires a run ID", cmd.Name)
173+
}
174+
return runID, nil
175+
}
176+
177+
// runSimulateView opens a pre-existing run: nothing is spawned, so no agent
178+
// project, entrypoint or scenarios are needed.
179+
func runSimulateView(ctx context.Context, cmd *cli.Command) error {
180+
runID, err := simulateRunIDArg(cmd)
181+
if err != nil {
182+
return err
183+
}
184+
pc := simulateProjectConfig
185+
simCfg := &simulateConfig{
186+
ctx: ctx,
187+
client: lksdk.NewAgentSimulationClient(serverURL, pc.APIKey, pc.APISecret),
188+
pc: pc,
189+
mode: modeView,
190+
simulationMode: livekit.SimulationMode_SIMULATION_MODE_TEXT,
191+
viewModeRunID: runID,
192+
}
193+
if !isInteractive() {
194+
return runSimulateCI(ctx, simCfg)
195+
}
196+
return runSimulateTUI(simCfg)
197+
}
198+
139199
// writeGeneratedScenariosTemp writes a generated run's scenarios to a temp
140200
// scenarios.yaml; "" when the run carries none.
141201
func writeGeneratedScenariosTemp(run *livekit.SimulationRun) (string, error) {
@@ -213,7 +273,7 @@ type simulateConfig struct {
213273
entrypoint string
214274
scenarioGroup *livekit.ScenarioGroup
215275
scenariosPath string // path to the --scenarios file (empty when generating from source)
216-
viewModeRunID string // non-empty when --view opens a pre-existing run
276+
viewModeRunID string // non-empty when `view` opens a pre-existing run
217277
liveAgent bool // --agent-name: run against an already-running agent, don't spawn one
218278
warnings []string // config-level warnings surfaced at setup (e.g. ignored flags)
219279

@@ -309,19 +369,8 @@ func buildTaskExists(projectDir string) (bool, error) {
309369
func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.SimulationMode) error {
310370
pc := simulateProjectConfig
311371

312-
// --export is a one-shot read of a finished run, so it short-circuits
313-
// every other flag: no agent, no run creation, no polling.
314-
if cmd.IsSet("export") {
315-
exportRunID := cmd.String("export")
316-
if exportRunID == "" {
317-
return fmt.Errorf("--export requires a run ID")
318-
}
319-
return exportSimulationRunJSON(ctx, pc, exportRunID)
320-
}
321-
322372
numSimulations := int32(cmd.Int("num-simulations"))
323373
concurrency := int32(cmd.Int("concurrency"))
324-
runID := cmd.String("view")
325374
liveAgentName := cmd.String("agent-name")
326375

327376
// never auto-discovered: an explicit --scenarios file is the source of
@@ -346,9 +395,6 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
346395
}
347396
liveAgent = true
348397
agentName = liveAgentName
349-
} else if runID != "" {
350-
// --view opens a pre-existing run: nothing is spawned, so no agent
351-
// project or entrypoint is needed.
352398
} else {
353399
agentName = generateAgentName()
354400
projectDir, projectType, err = agentfs.DetectProjectRoot(".")
@@ -383,14 +429,9 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
383429
}
384430
}
385431

386-
var mode simulateMode
387-
switch {
388-
case runID != "":
389-
mode = modeView
390-
case scenarioGroup != nil && len(scenarioGroup.Scenarios) > 0:
432+
mode := modeGenerateFromSource
433+
if scenarioGroup != nil && len(scenarioGroup.Scenarios) > 0 {
391434
mode = modeScenarios
392-
default:
393-
mode = modeGenerateFromSource
394435
}
395436

396437
if mode == modeGenerateFromSource {
@@ -415,7 +456,6 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
415456
entrypoint: entrypoint,
416457
scenarioGroup: scenarioGroup,
417458
scenariosPath: scenariosPath,
418-
viewModeRunID: runID,
419459
liveAgent: liveAgent,
420460
warnings: simulateConfigWarnings(mode, numSimulations),
421461
}
@@ -688,12 +728,12 @@ func dashboardBaseURL() string {
688728
// --project would resolve those.
689729
// The binary name comes from argv[0] so a renamed or path-qualified lk is
690730
// reproduced verbatim.
691-
func simulateCommandHint(flag, runID string) string {
731+
func simulateCommandHint(subcommand, runID string) string {
692732
binary := "lk"
693733
if len(os.Args) > 0 && os.Args[0] != "" {
694734
binary = os.Args[0]
695735
}
696-
hint := binary + " agent simulate " + flag + " " + runID
736+
hint := binary + " agent simulate " + subcommand + " " + runID
697737
if simulateProjectConfig != nil && simulateProjectConfig.Name != "" {
698738
hint += " --project " + simulateProjectConfig.Name
699739
}
@@ -704,11 +744,11 @@ func simulateCommandHint(flag, runID string) string {
704744
}
705745

706746
func viewCommandHint(runID string) string {
707-
return simulateCommandHint("--view", runID)
747+
return simulateCommandHint("view", runID)
708748
}
709749

710750
func exportCommandHint(runID string) string {
711-
return simulateCommandHint("--export", runID) + " > " + runID + ".json"
751+
return simulateCommandHint("export", runID) + " > " + runID + ".json"
712752
}
713753

714754
// In view mode the re-open hint would echo the command the user just ran, so

cmd/lk/simulate_list.go

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright 2026 LiveKit, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package main
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"strings"
21+
22+
"github.com/urfave/cli/v3"
23+
24+
"github.com/livekit/livekit-cli/v2/pkg/util"
25+
"github.com/livekit/protocol/livekit"
26+
lksdk "github.com/livekit/server-sdk-go/v2"
27+
)
28+
29+
var simulateListCommand = &cli.Command{
30+
Name: "list",
31+
Usage: "List the project's most recent simulation runs",
32+
HideHelpCommand: true,
33+
Action: listSimulationRuns,
34+
Flags: []cli.Flag{
35+
&cli.IntFlag{
36+
Name: "limit",
37+
Usage: "maximum number of runs to return. If unset, defaults to API page size",
38+
},
39+
jsonFlag,
40+
},
41+
}
42+
43+
// listSimulationRuns prints runs newest first, without jobs. Pass/fail detail
44+
// lives in `view`.
45+
func listSimulationRuns(ctx context.Context, cmd *cli.Command) error {
46+
pc := simulateProjectConfig
47+
client := lksdk.NewAgentSimulationClient(serverURL, pc.APIKey, pc.APISecret)
48+
49+
limit := cmd.Int("limit")
50+
var runs []*livekit.SimulationRun
51+
var resp *livekit.SimulationRun_List_Response
52+
for resp == nil || (len(runs) < limit && resp.NextPageToken.GetToken() != "") {
53+
req := &livekit.SimulationRun_List_Request{ProjectId: pc.ProjectId}
54+
if resp != nil {
55+
req.PageToken = &livekit.TokenPagination{Token: resp.NextPageToken.GetToken()}
56+
}
57+
pageCtx, cancel := context.WithTimeout(ctx, simulationAPITimeout)
58+
var err error
59+
resp, err = client.ListSimulationRuns(pageCtx, req)
60+
cancel()
61+
if err != nil {
62+
return fmt.Errorf("unable to list simulation runs: %w", err)
63+
}
64+
runs = append(runs, resp.Runs...)
65+
}
66+
if limit > 0 && len(runs) > limit {
67+
runs = runs[:limit]
68+
}
69+
70+
if cmd.Bool("json") {
71+
util.PrintJSON(&livekit.SimulationRun_List_Response{Runs: runs})
72+
return nil
73+
}
74+
75+
if len(runs) == 0 {
76+
out.Status("No simulation runs found")
77+
return nil
78+
}
79+
80+
var rows [][]string
81+
for _, run := range runs {
82+
rows = append(rows, []string{
83+
run.GetId(),
84+
formatDeployedAt(run.GetCreatedAt().AsTime()),
85+
strings.TrimPrefix(run.GetStatus().String(), "STATUS_"),
86+
strings.TrimPrefix(run.GetMode().String(), "SIMULATION_MODE_"),
87+
run.GetAgentName(),
88+
})
89+
}
90+
91+
t := util.CreateTable().
92+
Headers("ID", "Created At", "Status", "Mode", "Agent").
93+
Rows(rows...)
94+
out.Result(t)
95+
fmt.Fprintf(out.StatusWriter(), "To open a run: %s\n", viewCommandHint("<run-id>"))
96+
return nil
97+
}

0 commit comments

Comments
 (0)