M21.3: watcher survives mempool transport failures and recreates

app/Services/Blockchain/MempoolClient.php

@@ -3,6 +3,7 @@
 namespace App\Services\Blockchain;
 
 use Illuminate\Http\Client\Pool;
+use Illuminate\Http\Client\Response;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
 
@@ -51,12 +52,16 @@ public function transactionsForAddresses(string $network, array $addresses): arr
         foreach ($addresses as $address) {
             $response = $responses[$address] ?? null;
 
-            if (! $response || ! $response->ok()) {
+            // A transport-level failure leaves the ConnectionException itself
+            // in the pool slot rather than a Response (#187); treat it like a
+            // non-2xx reply for this address and keep going.
+            if (! $response instanceof Response || ! $response->ok()) {
                 Log::warning('Mempool transactions fetch failed', [
                     'network' => $network,
                     'address' => $address,
-                    'status' => $response?->status(),
-                    'body' => $response?->body(),
+                    'status' => $response instanceof Response ? $response->status() : null,
+                    'body' => $response instanceof Response ? $response->body() : null,
+                    'error' => $response instanceof \Throwable ? $response->getMessage() : null,
                 ]);
 
                 $transactions[$address] = [];

bootstrap/app.php

@@ -19,9 +19,10 @@
 return Application::configure(basePath: dirname(__DIR__))
     ->withEvents(discover: false)
     ->withSchedule(function (Schedule $schedule): void {
+        // 10-minute mutex expiry: a scheduler recreate mid-run strands the lock (default 24 h); a real run takes seconds (#188).
         $schedule->command('wallet:watch-payments')
             ->everyMinute()
-            ->withoutOverlapping()
+            ->withoutOverlapping(10)
             ->runInBackground();
         $schedule->command('invoices:send-past-due-alerts')->dailyAt('02:00');
 

docs/ops/RUNNING_THE_SERVER.md

@@ -20,10 +20,11 @@ Recreating is also what makes an `.env` change take at all: the container rebuil
 
 `docker compose ps` reports health per service: the app answers a PHP-FPM ping, while the queue and scheduler answer on a heartbeat their worker loops refresh — so a worker still running but wedged reads `unhealthy` rather than fine.
 
-Two things that look like faults and are not:
+Three things that read wrong at a glance:
 
 - **Healthy containers are not proof that scheduled work is running.** The watcher's own run stamp is the liveness signal, and the support dashboard's stale tile reads from it. Watch the stamp advance.
 - **A high restart count on the queue container is expected.** The worker runs with `--max-time=3600` and exits hourly by design, so roughly one restart per hour of uptime is normal. Compare the count against uptime before treating it as a symptom.
+- **Recreating the scheduler mid-run strands its `withoutOverlapping` mutex until it expires.** Every later tick skips as "has mutex" while everything reads healthy. The deploy script clears it after bringing services up; after any recreate, confirm the watcher stamp advances within a couple of minutes.
 
 ---
 

scripts/deploy.sh

@@ -13,7 +13,7 @@ TAG="${1:?usage: deploy.sh <image-tag>}"
 IMAGE="ghcr.io/n8bar/cryptozing-app"
 
 FILES=(-f compose.production.yaml)
-# Our alpha deployment layers the site container in; self-hosters won't have it.
+# Our deployment layers the site container in; self-hosters won't have it.
 [ -f compose.alpha.yaml ] && FILES+=(-f compose.alpha.yaml)
 
 # Persist the tag so later compose invocations keep serving it.
@@ -30,6 +30,10 @@ docker compose "${FILES[@]}" run --rm app php artisan migrate --force
 
 docker compose "${FILES[@]}" up -d --remove-orphans
 
+# Recreating the scheduler mid-run strands its withoutOverlapping mutex (#188).
+docker compose "${FILES[@]}" exec -T app php artisan schedule:clear-cache \
+    || echo "WARN: schedule:clear-cache failed; clear framework/schedule-* rows from cache_locks by hand (#188)" >&2
+
 # §4.2.4: nothing may still be running another tag of the app image.
 stale=$(docker ps --format '{{.Names}} {{.Image}}' \
     | awk -v img="$IMAGE" -v want="$IMAGE:$TAG" '$2 ~ "^"img && $2 != want')

tests/Feature/Console/ScheduleTest.php

@@ -22,6 +22,8 @@ public function test_wallet_watch_payments_command_is_scheduled(): void
         $this->assertNotNull($event, 'wallet:watch-payments should be scheduled.');
         $this->assertSame('* * * * *', $event->expression);
         $this->assertTrue($event->withoutOverlapping);
+        // #188: a stranded mutex must clear within one interval plus a margin, not the 24 h default.
+        $this->assertSame(10, $event->expiresAt);
         $this->assertTrue($event->runInBackground);
     }
 }

tests/Feature/Wallet/MempoolClientTest.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace Tests\Feature\Wallet;
+
+use App\Services\Blockchain\MempoolClient;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+use Tests\TestCase;
+
+class MempoolClientTest extends TestCase
+{
+    private const BASE = 'https://mempool.example/testnet4/api';
+
+    private function client(): MempoolClient
+    {
+        return new MempoolClient([
+            'mempool' => ['testnet4_base' => self::BASE, 'timeout' => 1.0],
+        ]);
+    }
+
+    /**
+     * #187: when a pooled request fails at the transport level, Http::pool()
+     * hands back the ConnectionException itself in that slot. The client must
+     * treat it as a failed fetch for that address and keep the rest of the batch.
+     */
+    public function test_transport_failure_for_one_address_does_not_abort_the_batch(): void
+    {
+        Http::fake([
+            self::BASE.'/address/tb1qdown/txs' => Http::failedConnection('cURL error 28: Connection timed out'),
+            self::BASE.'/address/tb1qup/txs' => Http::response([
+                ['txid' => 'tx-up', 'status' => ['confirmed' => true], 'vout' => []],
+            ], 200),
+        ]);
+
+        Log::spy();
+
+        $transactions = $this->client()->transactionsForAddresses('testnet4', ['tb1qdown', 'tb1qup']);
+
+        $this->assertSame([], $transactions['tb1qdown']);
+        $this->assertSame('tx-up', $transactions['tb1qup'][0]['txid']);
+
+        Log::shouldHaveReceived('warning')
+            ->once()
+            ->withArgs(function (string $message, array $context): bool {
+                return $message === 'Mempool transactions fetch failed'
+                    && $context['address'] === 'tb1qdown'
+                    && $context['status'] === null
+                    && str_contains((string) $context['error'], 'cURL error 28');
+            });
+    }
+}