Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .agents/skills/cli-e2e-testing/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ Tests rely on a deterministic shell prompt for "command finished" detection:

## Step 4 — Diagnose the "Y/n input race"

This is the most-observed flake class so far. It is what broke `ChannelUpdateWorkflowTests.UpdateProjectChannelToStable_TypeScript_PreviewsStablePackagesAndPreservesChannel` on PR #17522 (run `26489967289`, job `78006625708`).
This is the most-observed flake class so far. It is what broke `ChannelUpdateWorkflowTests.UpdateToStable_TypeScript_PreviewsStablePkgsAndKeepsChannel` on PR #17522 (run `26489967289`, job `78006625708`).

### Symptom

Expand Down
43 changes: 17 additions & 26 deletions .github/workflows/cli-e2e-recording-comment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -407,20 +407,15 @@ jobs:
# Look up test outcome from TRX data.
# .cast files are named after the test method name (via [CallerMemberName] in CreateTestTerminal),
# so the filename matches the method name key in the outcomes JSON.
# Per-link label carries the outcome too, so a recording URL copied out of
# the table still tells reviewers whether it represents a pass or a failure.
TEST_OUTCOME=$(jq -r --arg name "$filename" '.[$name] // "Unknown"' test_outcomes.json)
if [ "$TEST_OUTCOME" = "Passed" ]; then
STATUS_EMOJI="✅"
LINK_LABEL="Recording"
TEST_PASS_COUNT=$((TEST_PASS_COUNT + 1))
elif [ "$TEST_OUTCOME" = "Failed" ]; then
STATUS_EMOJI="❌"
LINK_LABEL="Recording"
TEST_FAIL_COUNT=$((TEST_FAIL_COUNT + 1))
else
STATUS_EMOJI="❔"
LINK_LABEL="Recording"
TEST_UNKNOWN_COUNT=$((TEST_UNKNOWN_COUNT + 1))
fi

Expand Down Expand Up @@ -456,30 +451,26 @@ jobs:
fi
fi

# Build the job and artifacts link cells for the table
JOB_CELL=""
if [ -n "$JOB_URL" ]; then
JOB_CELL="[#${JOB_ID}](${JOB_URL})"
else
JOB_CELL="—"
fi
ARTIFACTS_CELL=""
if [ -n "$ARTIFACTS_URL" ]; then
ARTIFACTS_CELL="[Logs](${ARTIFACTS_URL})"
else
ARTIFACTS_CELL="—"
fi

# Build the table row once; append to both tables as needed.
# Build the detail cell with links separated by ·
DETAIL_PARTS=""
if [ -n "$ASCIINEMA_URL" ]; then
ROW="| ${STATUS_EMOJI} | ${safe_filename} | [${LINK_LABEL}](${ASCIINEMA_URL}) | ${JOB_CELL} | ${ARTIFACTS_CELL} |"
DETAIL_PARTS="[Recording](${ASCIINEMA_URL})"
echo "Uploaded: $ASCIINEMA_URL"
UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
else
ROW="| ${STATUS_EMOJI} | ${safe_filename} | ⚠️ Upload failed | ${JOB_CELL} | ${ARTIFACTS_CELL} |"
DETAIL_PARTS="⚠️ Upload failed"
echo "Failed to upload $castfile after $MAX_UPLOAD_RETRIES attempts"
FAIL_COUNT=$((FAIL_COUNT + 1))
fi
if [ -n "$JOB_URL" ]; then
DETAIL_PARTS="${DETAIL_PARTS} · [Job](${JOB_URL})"
fi
if [ -n "$ARTIFACTS_URL" ]; then
DETAIL_PARTS="${DETAIL_PARTS} · [CLI logs](${ARTIFACTS_URL})"
fi

# Build the table row once; append to both tables as needed.
ROW="| ${STATUS_EMOJI} | ${safe_filename} | ${DETAIL_PARTS} |"

TABLE_BODY="${TABLE_BODY}
${ROW}"
Expand Down Expand Up @@ -537,8 +528,8 @@ jobs:
FAILED_SECTION="
### ❌ Failed Tests

| Status | Test | Recording | Job | Artifacts |
|--------|------|-----------|-----|-----------|${FAILED_TESTS_BODY}
| - | Test | Detail |
|--------|------|-----------|${FAILED_TESTS_BODY}
"
fi

Expand All @@ -548,8 +539,8 @@ jobs:
<details>
<summary>View all recordings</summary>

| Status | Test | Recording | Job | Artifacts |
|--------|------|-----------|-----|-----------|${TABLE_BODY}
| - | Test | Detail |
|--------|------|-----------|${TABLE_BODY}

---
<sub>📹 Recordings uploaded automatically from [CI run #${RUN_ID}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID})</sub>
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Cli.EndToEnd.Tests/AgentCommandTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ await auto.WaitUntilAsync(
/// aren't part of the regression and are covered by the broader integration test.
/// </summary>
[Fact]
public async Task AgentInitCommand_NonInteractive_BundleOnlySkillsBeyondCliCatalog_AreInstallable()
public async Task AgentInit_NonInteractive_BundleOnlySkillsNotInCatalog()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
4 changes: 2 additions & 2 deletions tests/Aspire.Cli.EndToEnd.Tests/CSharpProjectModeInitTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public sealed class CSharpProjectModeInitTests(ITestOutputHelper output)
[Theory]
[InlineData("Test.sln")]
[InlineData("Test.slnx")]
public async Task AspireInitWithSolutionFileGeneratesAppHostThatBuildsAgainstChannelHive(string solutionFileName)
public async Task AspireInit_SolutionFile_BuildsAgainstChannelHive(string solutionFileName)
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -127,7 +127,7 @@ await auto.RunCommandAsync(
/// </remarks>
[CaptureWorkspaceOnFailure]
[Fact]
public async Task AspireInitWithExistingAppHostDirRecreatesMissingNuGetConfigAndPreservesFiles()
public async Task AspireInit_ExistingAppHostDir_RecreatesNuGetConfigKeepsFiles()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
8 changes: 4 additions & 4 deletions tests/Aspire.Cli.EndToEnd.Tests/ChannelUpdateWorkflowTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public sealed class ChannelUpdateWorkflowTests(ITestOutputHelper output)
{
[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_TypeScript_PreviewsStablePackagesAndPreservesChannel()
public async Task UpdateToStable_TypeScript_PreviewsStablePkgsAndKeepsChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -227,7 +227,7 @@ await File.WriteAllTextAsync(appHostPath,

[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_CSharpSingleFileInit_PreservesAspireConfigChannel()
public async Task UpdateToStable_CSharpSingleFileInit_KeepsConfigChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -270,7 +270,7 @@ public async Task UpdateProjectChannelToStable_CSharpSingleFileInit_PreservesAsp

[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_CSharpEmptyAppHost_PreservesAspireConfigChannel()
public async Task UpdateToStable_CSharpEmptyAppHost_KeepsConfigChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -310,7 +310,7 @@ public async Task UpdateProjectChannelToStable_CSharpEmptyAppHost_PreservesAspir

[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_TypeScriptSingleFileInit_PreservesAspireConfigChannel()
public async Task UpdateToStable_TypeScriptSingleFileInit_KeepsConfigChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public async Task AgentInit_InstallsPlaywrightCli_AndGeneratesSkillFiles()
/// to be dropped in the CLI process's current working directory.
/// </summary>
[Fact]
public async Task AgentInit_WhenCwdDiffersFromWorkspaceRoot_PlacesSkillFilesInWorkspaceRoot()
public async Task AgentInit_CwdDiffersFromRoot_PlacesSkillsInWorkspaceRoot()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Cli.EndToEnd.Tests/ResourceCommandTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ public async Task ResourceCommand_FailsWhenInteractionServiceIsRequired()

[Fact]
[CaptureWorkspaceOnFailure]
public async Task ResourceCommand_FailedExecution_DisplaysAppHostLogPathAndLogContainsEntries()
public async Task ResourceCommand_FailedExec_ShowsLogPathAndLogHasEntries()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Cli.EndToEnd.Tests/TypeScriptPolyglotTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ await auto.WaitUntilAsync(s =>

[Fact]
[CaptureWorkspaceOnFailure]
public async Task CreateTypeScriptAppHostWithViteApp_AllowsGuestAppPackageManagerToDiffer()
public async Task TypeScriptAppHostWithVite_AllowsDifferentGuestPkgManager()
{
const string appHostToolchain = "pnpm";
const string guestToolchain = "npm";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public sealed class UpdateChannelNuGetConfigOrderingTests(ITestOutputHelper outp
{
[Fact]
[CaptureWorkspaceOnFailure]
public async Task AspireUpdateAppliesAllPackageEditsBeforeRestoringWhenNuGetConfigGainsSourceMapping()
public async Task AspireUpdate_AppliesPkgEditsBeforeRestore_OnSourceMapping()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
Loading