fix(installer): detect stale background-task pids

ods/installers/lib/background-tasks.sh

@@ -42,12 +42,45 @@ pid = int(sys.argv[3])
 description = sys.argv[4]
 log_file = sys.argv[5]
 
+
+def _proc_start(pid):
+    """Start-time token for a pid, or None when it cannot be read.
+
+    Used to detect pid reuse: os.kill(pid, 0) only proves some process owns
+    the number, so a reaped+recycled pid would otherwise report the task as
+    still running. starttime (field 22 of /proc/<pid>/stat) is stable per
+    process; on non-Linux fall back to `ps -o lstart`.
+    """
+    stat_path = "/proc/%d/stat" % pid
+    try:
+        with open(stat_path) as f:
+            rest = f.read().rsplit(")", 1)[1].split()
+        return rest[19]
+    except (OSError, ValueError, IndexError):
+        pass
+    try:
+        import subprocess
+
+        out = subprocess.run(
+            ["ps", "-p", str(pid), "-o", "lstart="],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if out.returncode == 0:
+            return out.stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return None
+
+
 tasks = json.loads(registry_path.read_text())
 tasks.append({
     "id": task_id,
     "pid": pid,
     "description": description,
     "log_file": log_file,
+    "start_time": _proc_start(pid),
     "status": "running"
 })
 fd, tmp_path = tempfile.mkstemp(dir=str(registry_path.parent), suffix=".tmp")
@@ -96,6 +129,38 @@ if not task:
 
 pid = task["pid"]
 
+
+def _proc_start(pid):
+    stat_path = "/proc/%d/stat" % pid
+    try:
+        with open(stat_path) as f:
+            rest = f.read().rsplit(")", 1)[1].split()
+        return rest[19]
+    except (OSError, ValueError, IndexError):
+        pass
+    try:
+        import subprocess
+
+        out = subprocess.run(
+            ["ps", "-p", str(pid), "-o", "lstart="],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if out.returncode == 0:
+            return out.stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return None
+
+
+# Check if the recorded process is still running. Guard against pid reuse
+# (os.kill(pid, 0) alone would match an unrelated recycled pid): only treat
+# it as running when the pid's start time still matches the recorded one.
+recorded_start = task.get("start_time")
+if recorded_start is not None and _proc_start(pid) != recorded_start:
+    sys.exit(1)  # Completed (or never matched the recorded process)
+
 # Check if process is still running
 try:
     os.kill(pid, 0)
@@ -173,22 +238,51 @@ for task in tasks:
     task_id = task["id"]
     pid = task["pid"]
     desc = task["description"]
-
-    # Check if still running
-    try:
-        os.kill(pid, 0)
-        status = "running"
-    except OSError:
-        log_file = task.get("log_file", "")
-        if log_file and Path(log_file).exists():
-            log_content = Path(log_file).read_text()
-            if "ERROR" in log_content or "failed" in log_content:
-                status = "failed"
+
+    # Check if still running. Guard against pid reuse (see bg_task_status):
+    # a live pid whose start time differs from the recorded one is treated as
+    # completed rather than "running".
+    def _proc_start(pid):
+        stat_path = "/proc/%d/stat" % pid
+        try:
+            with open(stat_path) as f:
+                rest = f.read().rsplit(")", 1)[1].split()
+            return rest[19]
+        except (OSError, ValueError, IndexError):
+            pass
+        try:
+            import subprocess
+
+            out = subprocess.run(
+                ["ps", "-p", str(pid), "-o", "lstart="],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if out.returncode == 0:
+                return out.stdout.strip()
+        except (OSError, subprocess.SubprocessError):
+            pass
+        return None
+
+    recorded_start = task.get("start_time")
+    if recorded_start is not None and _proc_start(pid) != recorded_start:
+        status = "completed"
+    else:
+        try:
+            os.kill(pid, 0)
+            status = "running"
+        except OSError:
+            log_file = task.get("log_file", "")
+            if log_file and Path(log_file).exists():
+                log_content = Path(log_file).read_text()
+                if "ERROR" in log_content or "failed" in log_content:
+                    status = "failed"
+                else:
+                    status = "completed"
             else:
                 status = "completed"
-        else:
-            status = "completed"
-
+
     print(f"  [{task_id}] {desc}: {status}")
 PY
 }

ods/tests/test-background-task-stale-pid.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Regression: bg_task_status must not report a recycled pid as "running".
+#
+# os.kill(pid, 0) only proves SOME process owns the number. If a task's pid is
+# reaped and then recycled by an unrelated process, the stale-probe still
+# returns running, so bg_task_wait spins to its timeout (~20 min) instead of
+# completing. The fix records the process start time at registration and only
+# treats a live pid as the task when its start time still matches.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIB="$SCRIPT_DIR/../installers/lib/background-tasks.sh"
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+pass() { echo -e "${GREEN}\u2713${NC} $1"; }
+fail() { echo -e "${RED}\u2717${NC} $1"; exit 1; }
+info() { echo -e "${BLUE}\u2139${NC} $1"; }
+
+[[ -f "$LIB" ]] || fail "background-tasks.sh not found"
+
+# This check depends on /proc/<pid>/stat (Linux/procfs). On hosts without it
+# the start_time is null and the pid-reuse guard is skipped, so skip there.
+if [[ ! -e /proc/self/stat ]]; then
+    info "procfs not available; skipping stale-pid regression (CI is Linux)"
+    exit 0
+fi
+
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+
+export BG_TASK_REGISTRY="$TMP/registry.json"
+
+# Minimal stubs required by the lib at source time.
+ai() { :; }
+ai_ok() { :; }
+ai_warn() { :; }
+ai_bad() { :; }
+
+# shellcheck source=background-tasks.sh
+. "$LIB"
+
+# A long-lived task: its pid must be reported as running.
+sleep 60 &
+LIVE_PID=$!
+bg_task_start "task-live" "$LIVE_PID" "live" "$TMP/live.log" >/dev/null
+
+set +e
+bg_task_status "task-live"; live_rc=$?
+set -e
+[[ $live_rc -eq 0 ]] || fail "live task not reported running (rc=$live_rc)"
+
+# Simulate the recorded task being gone and its pid reused by another process:
+# steal the live task's pid and record a start_time that no longer matches the
+# current one, then a live pid must NOT be reported as the task.
+python3 - "$BG_TASK_REGISTRY" <<'PY'
+import json
+import sys
+
+path = sys.argv[1]
+tasks = json.load(open(path))
+for t in tasks:
+    if t["id"] == "task-live":
+        t["start_time"] = "00000000000000000000000000000000000000000"
+json.dump(tasks, open(path, "w"))
+PY
+
+set +e
+bg_task_status "task-live"; stale_rc=$?
+set -e
+if [[ $stale_rc -eq 0 ]]; then
+    fail "recycled pid still reported running (rc=0)"
+fi
+pass "recycled pid is detected as not running (rc=$stale_rc)"