Merge development into PR #2311

Resolve the FilamentCoverageSpoolResolver constructor conflict by retaining both the status-cache dependency from development and the reviewed spool deadline settings dependency.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3f293177-2aeb-4fc3-9faf-49bb53032dc3

Author: Frost

src/infra/Services/Spoolman/FilamentCoverageSpoolResolver.cs

@@ -6,6 +6,7 @@
 using Farm.Infrastructure.Services.Interfaces;
 using Farm.Infrastructure.Services.Mutations;
 using Farm.Infrastructure.Services.Printers;
+using Farm.Infrastructure.Settings;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
@@ -20,9 +21,23 @@ public sealed class FilamentCoverageSpoolResolver(
     ILogger<FilamentCoverageSpoolResolver> logger,
     AppDbContext? db = null,
     IMutationWatermarkReader? watermarkReader = null,
-    IPrinterStatusCacheReader? statusCache = null) : IFilamentCoverageSpoolResolver
+    IPrinterStatusCacheReader? statusCache = null,
+    ISettingsService? settingsService = null) : IFilamentCoverageSpoolResolver
 {
-    private const int MaxConcurrentSourceRequests = 4;
+    /// <summary>
+    /// Upper bound on spool sources resolved at once. Distinct sources are
+    /// independent hosts, so this exists only to stop a large farm creating
+    /// unbounded fan-out against the network.
+    ///
+    /// <para>
+    /// This is NOT what keeps the endpoint inside the mobile client's readiness budget.
+    /// Dark sources each hold a slot for the full per-source timeout, so N of them
+    /// serialise into <c>ceil(N / MaxConcurrentSourceRequests)</c> waves and total latency
+    /// still grows with fleet size. <see cref="SpoolCoverageSettings.FleetResolveTimeoutMs"/>
+    /// is what bounds the endpoint, at any fleet size.
+    /// </para>
+    /// </summary>
+    private const int MaxConcurrentSourceRequests = 8;
 
     internal const string ReasonSpoolmanUnconfigured = "spoolman-unconfigured";
     internal const string ReasonSourceUnavailable = "spool-source-unavailable";
@@ -34,6 +49,67 @@ public sealed class FilamentCoverageSpoolResolver(
     private readonly AppDbContext? _db = db;
     private readonly IMutationWatermarkReader? _watermarkReader = watermarkReader;
     private readonly IPrinterStatusCacheReader? _statusCache = statusCache;
+    private readonly ISettingsService? _settingsService = settingsService;
+
+    /// <summary>
+    /// The read deadlines for one resolve operation. Captured once per call so every
+    /// source in a fan-out uses the same values: <c>SettingsService.Get</c> enumerates a
+    /// shared dictionary, so reading it from each of the concurrent source tasks can race
+    /// a concurrent settings save and fall back to defaults for some sources but not
+    /// others.
+    /// </summary>
+    private readonly record struct SpoolReadBudget(TimeSpan PerSource, TimeSpan Fleet);
+
+    /// <summary>
+    /// Resolves the configured read deadlines, falling back to the
+    /// <see cref="SpoolCoverageSettings"/> defaults when settings are unavailable.
+    /// Coverage must never inherit the backend's print-control timeout for this
+    /// read-only projection.
+    /// </summary>
+    private SpoolReadBudget ReadBudget()
+    {
+        SpoolCoverageSettings settings = new();
+        try
+        {
+            if (_settingsService?.Get<SpoolCoverageSettings>() is SpoolCoverageSettings configured)
+            {
+                settings = configured;
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogDebug(ex, "[FilamentCoverage] Falling back to default spool read budget");
+        }
+
+        // Clamp both ends. Validate() enforces these ranges on the write path, but a row
+        // persisted by another path (a migration, a direct edit) must not be able to
+        // reintroduce the very stall this budget exists to prevent.
+        return new SpoolReadBudget(
+            TimeSpan.FromMilliseconds(Math.Clamp(settings.SpoolSourceTimeoutMs, 250, 30_000)),
+            TimeSpan.FromMilliseconds(Math.Clamp(settings.FleetResolveTimeoutMs, 1_000, 60_000)));
+    }
+
+    /// <summary>
+    /// Renders a spool source's URL for logging with any embedded userinfo removed.
+    /// <see cref="LogSanitizer"/> only defeats log forging; it does not strip credentials,
+    /// and a server URL is free text an operator may have typed as
+    /// <c>http://user:secret@host</c>.
+    /// </summary>
+    private static string? DescribeSource(string? serverUrl)
+    {
+        if (string.IsNullOrWhiteSpace(serverUrl))
+        {
+            return LogSanitizer.Sanitize(serverUrl);
+        }
+
+        if (Uri.TryCreate(serverUrl, UriKind.Absolute, out Uri? uri) && !string.IsNullOrEmpty(uri.UserInfo))
+        {
+            UriBuilder redacted = new(uri) { UserName = string.Empty, Password = string.Empty };
+            return LogSanitizer.Sanitize(redacted.Uri.ToString());
+        }
+
+        return LogSanitizer.Sanitize(serverUrl);
+    }
 
     public async Task<FilamentCoverageSpoolSnapshot> ResolveSpoolAsync(
         CanonicalSpoolIdentity identity,
@@ -43,6 +119,10 @@ public async Task<FilamentCoverageSpoolSnapshot> ResolveSpoolAsync(
             .CaptureAsync(_watermarkReader, _logger, "source-qualified filament spool", ct)
             .ConfigureAwait(false);
         HashSet<int> spoolIds = [identity.SpoolId];
+
+        // Single source, so the fleet budget adds nothing beyond the per-source deadline;
+        // pass the caller token through as the budget token.
+        SpoolReadBudget budget = ReadBudget();
         Dictionary<int, FilamentCoverageSpoolSnapshot> resolved;
 
         if (identity.SourceKind == SpoolSourceKind.Central)
@@ -80,7 +160,7 @@ public async Task<FilamentCoverageSpoolSnapshot> ResolveSpoolAsync(
                     ReasonSourceUnavailable);
             }
 
-            resolved = await ResolveCentralAsync(spoolIds, originWatermark, ct).ConfigureAwait(false);
+            resolved = await ResolveCentralAsync(spoolIds, originWatermark, budget, ct, ct).ConfigureAwait(false);
         }
         else
         {
@@ -113,6 +193,8 @@ public async Task<FilamentCoverageSpoolSnapshot> ResolveSpoolAsync(
                         SpoolIds = { identity.SpoolId },
                     },
                     originWatermark,
+                    budget,
+                    ct,
                     ct).ConfigureAwait(false);
             }
             catch (OperationCanceledException)
@@ -201,9 +283,13 @@ public async Task<FilamentCoverageSpoolSnapshot> ResolveSpoolAsync(
 
         var request = new SourceRequest(selection.NativeClient, selection.ServerUrl);
         _ = request.SpoolIds.Add(spoolId);
+
+        // Single source, so the fleet budget adds nothing beyond the per-source deadline;
+        // pass the caller token through as the budget token.
+        SpoolReadBudget budget = ReadBudget();
         Dictionary<int, FilamentCoverageSpoolSnapshot> resolved = selection.Key.Native
-            ? await ResolveNativeAsync(request, originWatermark, ct).ConfigureAwait(false)
-            : await ResolveCentralAsync(request.SpoolIds, originWatermark, ct).ConfigureAwait(false);
+            ? await ResolveNativeAsync(request, originWatermark, budget, ct, ct).ConfigureAwait(false)
+            : await ResolveCentralAsync(request.SpoolIds, originWatermark, budget, ct, ct).ConfigureAwait(false);
 
         return resolved.TryGetValue(spoolId, out FilamentCoverageSpoolSnapshot? snapshot)
             ? snapshot
@@ -280,16 +366,53 @@ public async Task<IReadOnlyDictionary<Guid, IReadOnlyDictionary<int, FilamentCov
 
         // Distinct spool sources are independent HTTP adapters. Resolve them in
         // parallel without allowing a large farm to create unbounded fan-out.
+        //
+        // The gate alone cannot bound this endpoint: dark sources each hold a slot for
+        // the full per-source timeout, so they serialise into successive timeout waves
+        // and total latency grows with fleet size. The fleet deadline below is what makes
+        // the bound hold at any size — when it expires, sources still in flight degrade
+        // to "unavailable" and the projection returns the coverage it already has.
+        SpoolReadBudget budget = ReadBudget();
+        using CancellationTokenSource fleetCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
+        fleetCts.CancelAfter(budget.Fleet);
+        CancellationToken budgetToken = fleetCts.Token;
+
         using SemaphoreSlim sourceRequestGate = new(MaxConcurrentSourceRequests);
         Task<KeyValuePair<SourceKey, Dictionary<int, FilamentCoverageSpoolSnapshot>>>[] pendingSources =
             requests.Select(async pair =>
             {
-                await sourceRequestGate.WaitAsync(ct).ConfigureAwait(false);
+                try
+                {
+                    await sourceRequestGate.WaitAsync(budgetToken).ConfigureAwait(false);
+                }
+                catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+                {
+                    // The fleet deadline expired while this source was still queued behind
+                    // the gate. Report it as unavailable rather than failing the projection.
+                    return KeyValuePair.Create(pair.Key, Failure(pair.Value.SpoolIds, pair.Key.Native, ReasonSourceUnavailable));
+                }
+                catch (OperationCanceledException)
+                {
+                    // The CALLER cancelled while this source was still queued. WaitAsync throws
+                    // carrying the linked budget token, so normalise to the caller's token and
+                    // keep all three cancellation exits uniform.
+                    //
+                    // This is hardening, not a repair of observable behaviour: ResolveAsync
+                    // surfaces cancellation through Task.WhenAll, which reports the LOWEST-INDEX
+                    // cancelled task's token, and indices 0..MaxConcurrentSourceRequests-1 always
+                    // win the gate synchronously and so are always in flight. A queued source can
+                    // therefore never be the task whose token escapes. Normalising anyway means
+                    // the guarantee does not silently depend on that Task.WhenAll ordering detail
+                    // if the fan-out ever consumes sources in completion order instead.
+                    ct.ThrowIfCancellationRequested();
+                    throw;
+                }
+
                 try
                 {
                     Dictionary<int, FilamentCoverageSpoolSnapshot> resolved = pair.Key.Native
-                        ? await ResolveNativeAsync(pair.Value, originWatermark, ct).ConfigureAwait(false)
-                        : await ResolveCentralAsync(pair.Value.SpoolIds, originWatermark, ct).ConfigureAwait(false);
+                        ? await ResolveNativeAsync(pair.Value, originWatermark, budget, budgetToken, ct).ConfigureAwait(false)
+                        : await ResolveCentralAsync(pair.Value.SpoolIds, originWatermark, budget, budgetToken, ct).ConfigureAwait(false);
                     return KeyValuePair.Create(pair.Key, resolved);
                 }
                 finally
@@ -334,13 +457,31 @@ public async Task<IReadOnlyDictionary<Guid, IReadOnlyDictionary<int, FilamentCov
     private async Task<Dictionary<int, FilamentCoverageSpoolSnapshot>> ResolveNativeAsync(
         SourceRequest request,
         long? originWatermark,
+        SpoolReadBudget budget,
+        CancellationToken budgetToken,
         CancellationToken ct)
     {
+        TimeSpan timeout = budget.PerSource;
         try
         {
+            // A powered-down printer that still holds its address black-holes packets
+            // instead of refusing them, so this read must carry its own deadline. Without
+            // it the call inherits the backend's print-control timeout (60s) and one dark
+            // printer stalls the entire fleet projection. Linking off the fleet budget (not
+            // the caller token) also cuts this read short when the overall deadline expires.
+            using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(budgetToken);
+            linked.CancelAfter(timeout);
+
             string? json = await request.NativeClient!
-                .GetSpoolmanSpoolsAsync(request.ServerUrl!, ct)
+                .GetSpoolmanSpoolsAsync(request.ServerUrl!, linked.Token)
                 .ConfigureAwait(false);
+
+            // The Moonraker client swallows every exception from its Spoolman proxy and
+            // reports a cancelled or timed-out call as a null body, so cancellation has to
+            // be re-surfaced explicitly. Checking the LINKED token covers both cases; the
+            // catch filters below then separate our timeout from a caller cancellation.
+            linked.Token.ThrowIfCancellationRequested();
+
             if (json is null)
             {
                 return Failure(request.SpoolIds, true, ReasonSourceUnavailable);
@@ -351,20 +492,36 @@ private async Task<Dictionary<int, FilamentCoverageSpoolSnapshot>> ResolveNative
                 .ToDictionary(g => g.Key, g => g.First());
             return BuildSnapshots(request.SpoolIds, spools, true, originWatermark);
         }
+        catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+        {
+            // Either this source's own deadline or the fleet deadline. Both degrade; only a
+            // genuine caller cancellation propagates, via the rethrow below.
+            _logger.LogDebug(
+                "[FilamentCoverage] Native Spoolman source timed out after {TimeoutMs}ms at {ServerUrl}",
+                timeout.TotalMilliseconds,
+                DescribeSource(request.ServerUrl));
+            return Failure(request.SpoolIds, true, ReasonSourceUnavailable);
+        }
         catch (OperationCanceledException)
         {
+            // Reached only when the CALLER cancelled. Rethrow carrying the caller's token
+            // rather than the internal linked one, so callers can identify their own
+            // cancellation from ex.CancellationToken.
+            ct.ThrowIfCancellationRequested();
             throw;
         }
         catch (Exception ex)
         {
-            _logger.LogDebug(ex, "[FilamentCoverage] Native Spoolman source unavailable at {ServerUrl}", LogSanitizer.Sanitize(request.ServerUrl));
+            _logger.LogDebug(ex, "[FilamentCoverage] Native Spoolman source unavailable at {ServerUrl}", DescribeSource(request.ServerUrl));
             return Failure(request.SpoolIds, true, ReasonSourceUnavailable);
         }
     }
 
     private async Task<Dictionary<int, FilamentCoverageSpoolSnapshot>> ResolveCentralAsync(
         HashSet<int> spoolIds,
         long? originWatermark,
+        SpoolReadBudget budget,
+        CancellationToken budgetToken,
         CancellationToken ct)
     {
         SpoolmanConfigDto? config = _spoolmanService.GetConfig();
@@ -373,8 +530,14 @@ private async Task<Dictionary<int, FilamentCoverageSpoolSnapshot>> ResolveCentra
             return Failure(spoolIds, false, ReasonSpoolmanUnconfigured);
         }
 
+        TimeSpan timeout = budget.PerSource;
         try
         {
+            // Bound the whole paged read, not each page: an unreachable central Spoolman
+            // must not stall coverage any longer than a dark printer does.
+            using CancellationTokenSource linked = CancellationTokenSource.CreateLinkedTokenSource(budgetToken);
+            linked.CancelAfter(timeout);
+
             const int pageSize = 500;
             Dictionary<int, SpoolmanSpoolDto> found = [];
             int offset = 0;
@@ -388,7 +551,15 @@ private async Task<Dictionary<int, FilamentCoverageSpoolSnapshot>> ResolveCentra
                         Offset = offset,
                         AllowArchived = true,
                     },
-                    ct).ConfigureAwait(false);
+                    linked.Token).ConfigureAwait(false);
+
+                // SpoolmanService.ListSpoolsAsync catches every exception - including
+                // cancellation - and returns an EMPTY page. Without this check a timed-out
+                // read would fall through to BuildSnapshots and be reported as
+                // `spool-not-found`, i.e. an affirmative "that spool does not exist" claim
+                // about a source we never actually reached. Re-surface cancellation here so
+                // the catch filters below degrade to `spool-source-unavailable` instead.
+                linked.Token.ThrowIfCancellationRequested();
 
                 foreach (SpoolmanSpoolDto spool in page.Items.Where(spool => spoolIds.Contains(spool.Id)))
                 {
@@ -402,8 +573,19 @@ private async Task<Dictionary<int, FilamentCoverageSpoolSnapshot>> ResolveCentra
 
             return BuildSnapshots(spoolIds, found, false, originWatermark);
         }
+        catch (OperationCanceledException) when (!ct.IsCancellationRequested)
+        {
+            _logger.LogDebug(
+                "[FilamentCoverage] Central Spoolman source timed out after {TimeoutMs}ms",
+                timeout.TotalMilliseconds);
+            return Failure(spoolIds, false, ReasonSourceUnavailable);
+        }
         catch (OperationCanceledException)
         {
+            // Reached only when the CALLER cancelled. Rethrow carrying the caller's token
+            // rather than the internal linked one, so callers can identify their own
+            // cancellation from ex.CancellationToken.
+            ct.ThrowIfCancellationRequested();
             throw;
         }
         catch (Exception ex)