Skip to content
Open
11 changes: 10 additions & 1 deletion src/Aspire.Cli/Commands/RunCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,14 @@ internal sealed class RunCommand : BaseCommand
private bool _isDetachMode;
private const int MaxDisplayedAppHostStartupOutputLines = 80;

private static readonly TimeSpan s_appHostStartupCancellationTimeout = TimeSpan.FromSeconds(5);
private static readonly TimeSpan s_runProcessTerminationTimeout = TimeSpan.FromSeconds(3);
private static readonly TimeSpan s_runProcessShutdownTimeout =
TimeSpan.FromTicks(s_runProcessTerminationTimeout.Ticks * 2) + TimeSpan.FromSeconds(1);

// Startup cancellation waits for the AppHost run task to complete after it stops the
// directly-owned guest and server processes. Each stop gets its own run shutdown budget.
private static readonly TimeSpan s_appHostStartupCancellationTimeout =
TimeSpan.FromTicks(s_runProcessShutdownTimeout.Ticks * 2);

// Guest AppHosts can bring up the temporary server/backchannel and then fail immediately
// afterward when the guest startup process hits a syntax, pre-execute, or model validation
Expand Down Expand Up @@ -278,6 +285,8 @@ protected override async Task<CommandResult> ExecuteAsync(ParseResult parseResul
WorkingDirectory = ExecutionContext.WorkingDirectory,
BuildCompletionSource = buildCompletionSource,
BackchannelCompletionSource = backchannelCompletionSource,
ProcessTerminationTimeout = s_runProcessTerminationTimeout,
ProcessShutdownTimeout = s_runProcessShutdownTimeout,
};
ProfilingTelemetry.AddCurrentContextToEnvironment(context.EnvironmentVariables);
if (captureProfile)
Expand Down
109 changes: 94 additions & 15 deletions src/Aspire.Cli/Processes/ProcessShutdownService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,83 @@ internal sealed class ProcessShutdownService(
private static readonly TimeSpan s_processTerminationTimeout = TimeSpan.FromSeconds(10);
private static readonly TimeSpan s_processTerminationPollInterval = TimeSpan.FromMilliseconds(250);

internal static TimeSpan DefaultProcessTerminationTimeout => s_processTerminationTimeout;

public Task<bool> StopProcessTreeAsync(
int pid,
DateTimeOffset? startTime,
bool includeStartTimeForDcp,
CancellationToken cancellationToken)
{
return StopProcessTreeAsync(
pid,
startTime,
includeStartTimeForDcp,
forceKillEntireProcessTree: !OperatingSystem.IsWindows(),
processTerminationTimeout: s_processTerminationTimeout,
cancellationToken);
}

public Task<bool> StopProcessTreeAsync(
int pid,
DateTimeOffset? startTime,
bool includeStartTimeForDcp,
bool forceKillEntireProcessTree,
CancellationToken cancellationToken)
{
return StopProcessTreeAsync(
pid,
startTime,
includeStartTimeForDcp,
forceKillEntireProcessTree,
processTerminationTimeout: s_processTerminationTimeout,
cancellationToken);
}

public Task<bool> StopProcessTreeAsync(
int pid,
DateTimeOffset? startTime,
bool includeStartTimeForDcp,
bool forceKillEntireProcessTree,
TimeSpan processTerminationTimeout,
CancellationToken cancellationToken)
{
return StopProcessesAsync(
[new ProcessTarget(pid, startTime)],
[new ProcessTarget(pid, startTime)],
token => RequestProcessTreeGracefulShutdownAsync(pid, startTime, includeStartTimeForDcp, token),
forceKillEntireProcessTree,
processTerminationTimeout,
cancellationToken);
}

public Task<bool> StopProcessGroupAsync(
int pid,
DateTimeOffset? startTime,
bool forceKillEntireProcessTree,
CancellationToken cancellationToken)
{
return StopProcessGroupAsync(
pid,
startTime,
forceKillEntireProcessTree,
processTerminationTimeout: s_processTerminationTimeout,
cancellationToken);
}

public Task<bool> StopProcessGroupAsync(
int pid,
DateTimeOffset? startTime,
bool forceKillEntireProcessTree,
TimeSpan processTerminationTimeout,
CancellationToken cancellationToken)
{
return StopProcessesAsync(
[new ProcessTarget(pid, startTime)],
[new ProcessTarget(pid, startTime)],
token => RequestProcessGroupGracefulShutdownAsync(pid, startTime, token),
forceKillEntireProcessTree,
processTerminationTimeout,
cancellationToken);
}

Expand All @@ -59,6 +127,8 @@ public async Task<bool> StopAppHostAsync(
processesToMonitor: [appHostProcess],
processesToForceKill,
token => RequestAppHostGracefulShutdownAsync(appHostInfo, requestRpcStopAsync, token),
forceKillEntireProcessTree: !OperatingSystem.IsWindows(),
processTerminationTimeout: s_processTerminationTimeout,
cancellationToken).ConfigureAwait(false);
}

Expand All @@ -71,37 +141,33 @@ internal async Task<bool> StopProcessesAsync(
processesToMonitorAndKill,
processesToMonitorAndKill,
requestGracefulShutdownAsync,
forceKillEntireProcessTree: !OperatingSystem.IsWindows(),
processTerminationTimeout: s_processTerminationTimeout,
cancellationToken).ConfigureAwait(false);
}

private async Task<bool> StopProcessesAsync(
IReadOnlyCollection<ProcessTarget> processesToMonitor,
IReadOnlyCollection<ProcessTarget> processesToForceKill,
Func<CancellationToken, Task<bool>> requestGracefulShutdownAsync,
bool forceKillEntireProcessTree,
TimeSpan processTerminationTimeout,
CancellationToken cancellationToken)
{
var gracefulShutdownRequested = await TryRequestGracefulShutdownAsync(requestGracefulShutdownAsync, cancellationToken).ConfigureAwait(false);
if (gracefulShutdownRequested && await MonitorProcessesForTerminationAsync(processesToMonitor, cancellationToken).ConfigureAwait(false))
if (gracefulShutdownRequested && await MonitorProcessesForTerminationAsync(processesToMonitor, processTerminationTimeout, cancellationToken).ConfigureAwait(false))
{
ForceKillRemainingProcesses(processesToForceKill.Except(processesToMonitor), afterTimeout: false);
ForceKillRemainingProcesses(processesToForceKill.Except(processesToMonitor), afterTimeout: false, killEntireProcessTree: forceKillEntireProcessTree);
return true;
}

ForceKillRemainingProcesses(processesToForceKill, afterTimeout: true);
ForceKillRemainingProcesses(processesToForceKill, afterTimeout: true, killEntireProcessTree: forceKillEntireProcessTree);

return await MonitorProcessesForTerminationAsync(processesToMonitor, cancellationToken).ConfigureAwait(false);
return await MonitorProcessesForTerminationAsync(processesToMonitor, processTerminationTimeout, cancellationToken).ConfigureAwait(false);
}

private void ForceKillRemainingProcesses(IEnumerable<ProcessTarget> processes, bool afterTimeout)
private void ForceKillRemainingProcesses(IEnumerable<ProcessTarget> processes, bool afterTimeout, bool killEntireProcessTree)
{
// On Unix the AppHost's process tree does not include DCP (it is launched in its own
// session/process group), so a tree kill of the AppHost is safe: DCP will detect the
// AppHost exiting and gracefully tear down its own children. The same applies to the
// launcher CLI handle - any leftover `dotnet run` / AppHost descendants get cleaned up.
// On Windows DCP is an in-tree descendant of the AppHost, so we must single-process-kill
// here and rely on the graceful DCP `stop-process-tree` path for orderly resource cleanup.
var killEntireProcessTree = !OperatingSystem.IsWindows();

foreach (var process in processes.Distinct())
{
if (afterTimeout)
Expand Down Expand Up @@ -211,6 +277,16 @@ private async Task<bool> RequestProcessTreeGracefulShutdownAsync(
return true;
}

private Task<bool> RequestProcessGroupGracefulShutdownAsync(
int pid,
DateTimeOffset? startTime,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
ProcessSignaler.RequestGracefulShutdownForProcessGroup(pid, startTime, logger);
return Task.FromResult(true);
}

internal async Task<bool> TryStopProcessTreeWithDcpAsync(int pid, DateTimeOffset? startTime, bool includeStartTime, CancellationToken cancellationToken)
{
using var process = ProcessSignaler.TryGetRunningProcess(pid, startTime, logger);
Expand Down Expand Up @@ -275,10 +351,13 @@ internal async Task<bool> TryStopProcessTreeWithDcpAsync(int pid, DateTimeOffset
return true;
}

private async Task<bool> MonitorProcessesForTerminationAsync(IReadOnlyCollection<ProcessTarget> processes, CancellationToken cancellationToken)
private async Task<bool> MonitorProcessesForTerminationAsync(
IReadOnlyCollection<ProcessTarget> processes,
TimeSpan processTerminationTimeout,
CancellationToken cancellationToken)
{
var startTime = timeProvider.GetUtcNow();
while (timeProvider.GetUtcNow() - startTime < s_processTerminationTimeout)
while (timeProvider.GetUtcNow() - startTime < processTerminationTimeout)
{
if (processes.All(IsProcessStopped))
{
Expand Down
10 changes: 10 additions & 0 deletions src/Aspire.Cli/Projects/AppHostProjectContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ internal sealed class AppHostProjectContext
/// </summary>
public required DirectoryInfo WorkingDirectory { get; init; }

/// <summary>
/// Gets the per-process termination monitoring timeout for AppHost-owned child processes.
/// </summary>
public TimeSpan? ProcessTerminationTimeout { get; init; }

/// <summary>
/// Gets the total shutdown coordination timeout for AppHost-owned child processes.
/// </summary>
public TimeSpan? ProcessShutdownTimeout { get; init; }

/// <summary>
/// Gets or sets the output collector for capturing stdout/stderr.
/// Project implementations populate this during execution.
Expand Down
88 changes: 80 additions & 8 deletions src/Aspire.Cli/Projects/AppHostServerSession.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Diagnostics;
using Aspire.Cli.Configuration;
using Aspire.Cli.Processes;
using Aspire.Cli.Telemetry;
using Aspire.Cli.Utils;
using Aspire.Hosting;
Expand All @@ -23,6 +24,9 @@ internal sealed class AppHostServerSession : IAppHostServerSession
private readonly ProfilingTelemetry.ActivityScope _activity;
private readonly ProfilingTelemetry? _profilingTelemetry;
private readonly IDisposable? _projectLifetime;
private readonly ProcessShutdownService? _processShutdownService;
private readonly TimeSpan? _processTerminationTimeout;
private readonly TimeSpan? _processShutdownTimeout;
private IAppHostRpcClient? _rpcClient;
private bool _disposed;

Expand All @@ -34,7 +38,10 @@ internal AppHostServerSession(
ILogger logger,
ProfilingTelemetry.ActivityScope activity = default,
ProfilingTelemetry? profilingTelemetry = null,
IDisposable? projectLifetime = null)
IDisposable? projectLifetime = null,
ProcessShutdownService? processShutdownService = null,
TimeSpan? processTerminationTimeout = null,
TimeSpan? processShutdownTimeout = null)
{
_serverProcess = serverProcess;
_output = output;
Expand All @@ -44,6 +51,9 @@ internal AppHostServerSession(
_activity = activity;
_profilingTelemetry = profilingTelemetry;
_projectLifetime = projectLifetime;
_processShutdownService = processShutdownService;
_processTerminationTimeout = processTerminationTimeout;
_processShutdownTimeout = processShutdownTimeout;
}

/// <inheritdoc />
Expand All @@ -68,13 +78,19 @@ internal AppHostServerSession(
/// <param name="debug">Whether to enable debug logging for the server.</param>
/// <param name="logger">The logger to use for lifecycle diagnostics.</param>
/// <param name="profilingTelemetry">Optional profiling telemetry for the server process lifetime.</param>
/// <param name="processShutdownService">Optional shared process shutdown coordinator.</param>
/// <param name="processTerminationTimeout">Optional per-process termination monitoring timeout.</param>
/// <param name="processShutdownTimeout">Optional total shutdown coordination timeout.</param>
/// <returns>The started AppHost server session.</returns>
internal static AppHostServerSession Start(
IAppHostServerProject appHostServerProject,
Dictionary<string, string>? environmentVariables,
bool debug,
ILogger logger,
ProfilingTelemetry? profilingTelemetry = null)
ProfilingTelemetry? profilingTelemetry = null,
ProcessShutdownService? processShutdownService = null,
TimeSpan? processTerminationTimeout = null,
TimeSpan? processShutdownTimeout = null)
{
var currentPid = Environment.ProcessId;
var serverEnvironmentVariables = environmentVariables is null
Expand Down Expand Up @@ -127,7 +143,10 @@ internal static AppHostServerSession Start(
logger,
activity,
profilingTelemetry,
appHostServerProject as IDisposable);
appHostServerProject as IDisposable,
processShutdownService,
processTerminationTimeout,
processShutdownTimeout);
}

/// <inheritdoc />
Expand Down Expand Up @@ -156,15 +175,50 @@ public async ValueTask DisposeAsync()

if (!_serverProcess.HasExited)
{
try
var stopped = false;
if (_processShutdownService is not null)
{
_serverProcess.Kill(entireProcessTree: true);
_activity.SetError("AppHost server process was terminated during session disposal.");
if (TryGetServerProcessStartTime(out var serverProcessStartTime))
{
var processTerminationTimeout = _processTerminationTimeout ?? ProcessShutdownService.DefaultProcessTerminationTimeout;
var processShutdownTimeout = _processShutdownTimeout ?? GetProcessShutdownTimeout(processTerminationTimeout);
using var shutdownCts = new CancellationTokenSource(processShutdownTimeout);
try
{
_logger.LogInformation("Requesting shutdown of AppHost server process {ProcessId}", _serverProcess.Id);
stopped = await _processShutdownService.StopProcessGroupAsync(
_serverProcess.Id,
serverProcessStartTime,
forceKillEntireProcessTree: !OperatingSystem.IsWindows(),
processTerminationTimeout,
shutdownCts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (shutdownCts.IsCancellationRequested)
{
_logger.LogWarning("Timed out waiting for AppHost server process {ProcessId} shutdown.", _serverProcess.Id);
}
}
else
{
stopped = true;
}
}
catch (Exception ex)

if (!stopped && !_serverProcess.HasExited)
{
_logger.LogDebug(ex, "Error killing AppHost server process");
try
{
var killEntireProcessTree = !OperatingSystem.IsWindows();
_logger.LogInformation("Killing AppHost server process {ProcessId} (entireProcessTree={EntireProcessTree})", _serverProcess.Id, killEntireProcessTree);
_serverProcess.Kill(entireProcessTree: killEntireProcessTree);
_activity.SetError("AppHost server process was terminated during session disposal.");
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Error killing AppHost server process");
}
}

}

if (_serverProcess.HasExited)
Expand All @@ -176,6 +230,24 @@ public async ValueTask DisposeAsync()
_projectLifetime?.Dispose();
_activity.Dispose();
}

private bool TryGetServerProcessStartTime(out DateTimeOffset? startTime)
{
try
{
startTime = new DateTimeOffset(_serverProcess.StartTime);
return true;
}
catch (InvalidOperationException)
{
// The process exited between the HasExited check and shutdown coordination.
startTime = null;
return false;
}
}

private static TimeSpan GetProcessShutdownTimeout(TimeSpan processTerminationTimeout)
=> TimeSpan.FromTicks(processTerminationTimeout.Ticks * 2) + TimeSpan.FromSeconds(1);
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,8 @@ public async Task<AppHostServerPrepareResult> PrepareAsync(
WindowStyle = ProcessWindowStyle.Minimized,
UseShellExecute = false,
CreateNoWindow = true
};
}.CreateNewProcessGroupOnWindows();

startInfo.ArgumentList.Add("exec");
startInfo.ArgumentList.Add(assemblyPath);

Expand Down
2 changes: 2 additions & 0 deletions src/Aspire.Cli/Projects/ExtensionGuestLauncher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public ExtensionGuestLauncher(
DirectoryInfo workingDirectory,
IDictionary<string, string> environmentVariables,
CancellationToken cancellationToken,
TimeSpan? processTerminationTimeout = null,
TimeSpan? processShutdownTimeout = null,
Func<Task>? afterLaunchAsync = null)
{
// Prepend the runtime command (e.g., "npx") as the first argument so the
Expand Down
Loading
Loading