Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .agents/skills/cli-e2e-testing/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ Tests rely on a deterministic shell prompt for "command finished" detection:

## Step 4 — Diagnose the "Y/n input race"

This is the most-observed flake class so far. It is what broke `ChannelUpdateWorkflowTests.UpdateProjectChannelToStable_TypeScript_PreviewsStablePackagesAndPreservesChannel` on PR #17522 (run `26489967289`, job `78006625708`).
This is the most-observed flake class so far. It is what broke `ChannelUpdateWorkflowTests.UpdateToStable_TypeScript_PreviewsStablePkgsAndKeepsChannel` on PR #17522 (run `26489967289`, job `78006625708`).

### Symptom

Expand Down
43 changes: 17 additions & 26 deletions .github/workflows/cli-e2e-recording-comment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -407,20 +407,15 @@ jobs:
# Look up test outcome from TRX data.
# .cast files are named after the test method name (via [CallerMemberName] in CreateTestTerminal),
# so the filename matches the method name key in the outcomes JSON.
# Per-link label carries the outcome too, so a recording URL copied out of
# the table still tells reviewers whether it represents a pass or a failure.
TEST_OUTCOME=$(jq -r --arg name "$filename" '.[$name] // "Unknown"' test_outcomes.json)
if [ "$TEST_OUTCOME" = "Passed" ]; then
STATUS_EMOJI="✅"
LINK_LABEL="Recording"
TEST_PASS_COUNT=$((TEST_PASS_COUNT + 1))
elif [ "$TEST_OUTCOME" = "Failed" ]; then
STATUS_EMOJI="❌"
LINK_LABEL="Recording"
TEST_FAIL_COUNT=$((TEST_FAIL_COUNT + 1))
else
STATUS_EMOJI="❔"
LINK_LABEL="Recording"
TEST_UNKNOWN_COUNT=$((TEST_UNKNOWN_COUNT + 1))
fi

Expand Down Expand Up @@ -456,30 +451,26 @@ jobs:
fi
fi

# Build the job and artifacts link cells for the table
JOB_CELL=""
if [ -n "$JOB_URL" ]; then
JOB_CELL="[#${JOB_ID}](${JOB_URL})"
else
JOB_CELL="—"
fi
ARTIFACTS_CELL=""
if [ -n "$ARTIFACTS_URL" ]; then
ARTIFACTS_CELL="[Logs](${ARTIFACTS_URL})"
else
ARTIFACTS_CELL="—"
fi

# Build the table row once; append to both tables as needed.
# Build the detail cell with links separated by <br />
DETAIL_PARTS=""
if [ -n "$ASCIINEMA_URL" ]; then
ROW="| ${STATUS_EMOJI} | ${safe_filename} | [${LINK_LABEL}](${ASCIINEMA_URL}) | ${JOB_CELL} | ${ARTIFACTS_CELL} |"
DETAIL_PARTS="[Recording](${ASCIINEMA_URL})"
echo "Uploaded: $ASCIINEMA_URL"
UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
else
ROW="| ${STATUS_EMOJI} | ${safe_filename} | ⚠️ Upload failed | ${JOB_CELL} | ${ARTIFACTS_CELL} |"
DETAIL_PARTS="⚠️ Upload failed"
echo "Failed to upload $castfile after $MAX_UPLOAD_RETRIES attempts"
FAIL_COUNT=$((FAIL_COUNT + 1))
fi
if [ -n "$JOB_URL" ]; then
DETAIL_PARTS="${DETAIL_PARTS} <br /> [Job](${JOB_URL})"
fi
if [ -n "$ARTIFACTS_URL" ]; then
DETAIL_PARTS="${DETAIL_PARTS} <br /> [CLI logs](${ARTIFACTS_URL})"
fi

# Build the table row once; append to both tables as needed.
ROW="| ${STATUS_EMOJI} | ${safe_filename} | ${DETAIL_PARTS} |"

TABLE_BODY="${TABLE_BODY}
${ROW}"
Expand Down Expand Up @@ -537,8 +528,8 @@ jobs:
FAILED_SECTION="
### ❌ Failed Tests

| Status | Test | Recording | Job | Artifacts |
|--------|------|-----------|-----|-----------|${FAILED_TESTS_BODY}
| - | Test | Detail |
|--------|------|-----------|${FAILED_TESTS_BODY}
"
fi

Expand All @@ -548,8 +539,8 @@ jobs:
<details>
<summary>View all recordings</summary>

| Status | Test | Recording | Job | Artifacts |
|--------|------|-----------|-----|-----------|${TABLE_BODY}
| - | Test | Detail |
|--------|------|-----------|${TABLE_BODY}

---
<sub>📹 Recordings uploaded automatically from [CI run #${RUN_ID}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID})</sub>
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Cli.EndToEnd.Tests/AgentCommandTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ await auto.WaitUntilAsync(
/// aren't part of the regression and are covered by the broader integration test.
/// </summary>
[Fact]
public async Task AgentInitCommand_NonInteractive_BundleOnlySkillsBeyondCliCatalog_AreInstallable()
public async Task AgentInit_NonInteractive_BundleOnlySkillsNotInCatalog()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
4 changes: 2 additions & 2 deletions tests/Aspire.Cli.EndToEnd.Tests/CSharpProjectModeInitTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public sealed class CSharpProjectModeInitTests(ITestOutputHelper output)
[Theory]
[InlineData("Test.sln")]
[InlineData("Test.slnx")]
public async Task AspireInitWithSolutionFileGeneratesAppHostThatBuildsAgainstChannelHive(string solutionFileName)
public async Task AspireInit_SolutionFile_BuildsAgainstChannelHive(string solutionFileName)
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -127,7 +127,7 @@ await auto.RunCommandAsync(
/// </remarks>
[CaptureWorkspaceOnFailure]
[Fact]
public async Task AspireInitWithExistingAppHostDirRecreatesMissingNuGetConfigAndPreservesFiles()
public async Task AspireInit_ExistingAppHostDir_RecreatesNuGetConfigKeepsFiles()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
8 changes: 4 additions & 4 deletions tests/Aspire.Cli.EndToEnd.Tests/ChannelUpdateWorkflowTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public sealed class ChannelUpdateWorkflowTests(ITestOutputHelper output)
{
[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_TypeScript_PreviewsStablePackagesAndPreservesChannel()
public async Task UpdateToStable_TypeScript_PreviewsStablePkgsAndKeepsChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -227,7 +227,7 @@ await File.WriteAllTextAsync(appHostPath,

[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_CSharpSingleFileInit_PreservesAspireConfigChannel()
public async Task UpdateToStable_CSharpSingleFileInit_KeepsConfigChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -270,7 +270,7 @@ public async Task UpdateProjectChannelToStable_CSharpSingleFileInit_PreservesAsp

[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_CSharpEmptyAppHost_PreservesAspireConfigChannel()
public async Task UpdateToStable_CSharpEmptyAppHost_KeepsConfigChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down Expand Up @@ -310,7 +310,7 @@ public async Task UpdateProjectChannelToStable_CSharpEmptyAppHost_PreservesAspir

[Fact]
[CaptureWorkspaceOnFailure]
public async Task UpdateProjectChannelToStable_TypeScriptSingleFileInit_PreservesAspireConfigChannel()
public async Task UpdateToStable_TypeScriptSingleFileInit_KeepsConfigChannel()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public async Task AgentInit_InstallsPlaywrightCli_AndGeneratesSkillFiles()
/// to be dropped in the CLI process's current working directory.
/// </summary>
[Fact]
public async Task AgentInit_WhenCwdDiffersFromWorkspaceRoot_PlacesSkillFilesInWorkspaceRoot()
public async Task AgentInit_CwdDiffersFromRoot_PlacesSkillsInWorkspaceRoot()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Cli.EndToEnd.Tests/ResourceCommandTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ public async Task ResourceCommand_FailsWhenInteractionServiceIsRequired()

[Fact]
[CaptureWorkspaceOnFailure]
public async Task ResourceCommand_FailedExecution_DisplaysAppHostLogPathAndLogContainsEntries()
public async Task ResourceCommand_FailedExec_ShowsLogPathAndLogHasEntries()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Cli.EndToEnd.Tests/TypeScriptPolyglotTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ await auto.WaitUntilAsync(s =>

[Fact]
[CaptureWorkspaceOnFailure]
public async Task CreateTypeScriptAppHostWithViteApp_AllowsGuestAppPackageManagerToDiffer()
public async Task TypeScriptAppHostWithVite_AllowsDifferentGuestPkgManager()
{
const string appHostToolchain = "pnpm";
const string guestToolchain = "npm";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public sealed class UpdateChannelNuGetConfigOrderingTests(ITestOutputHelper outp
{
[Fact]
[CaptureWorkspaceOnFailure]
public async Task AspireUpdateAppliesAllPackageEditsBeforeRestoringWhenNuGetConfigGainsSourceMapping()
public async Task AspireUpdate_AppliesPkgEditsBeforeRestore_OnSourceMapping()
{
var repoRoot = CliE2ETestHelpers.GetRepoRoot();
var strategy = CliInstallStrategy.Detect(output.WriteLine);
Expand Down
Loading