Release 0.1.8

CHANGELOG.md

@@ -6,6 +6,15 @@ Notable changes are documented here. The format follows
 
 ## [Unreleased]
 
+## [0.1.8] - 2026-07-26
+
+### Fixed
+
+- OSD and client tracers now stop and remove their remote runner files when the
+  controlling SSH connection closes, including forced TUI termination.
+- Starting a tracer removes matching runner processes left by an interrupted
+  earlier session.
+
 ## [0.1.7] - 2026-07-26
 
 ### Changed
@@ -136,7 +145,8 @@ Initial release.
 - Cross-platform controller (Linux, macOS, Windows) with cargo-dist release
   archives that bundle the cephtrace tracers.
 
-[Unreleased]: https://github.com/xtrusia/cephlens/compare/v0.1.7...HEAD
+[Unreleased]: https://github.com/xtrusia/cephlens/compare/v0.1.8...HEAD
+[0.1.8]: https://github.com/xtrusia/cephlens/compare/v0.1.7...v0.1.8
 [0.1.7]: https://github.com/xtrusia/cephlens/compare/v0.1.6...v0.1.7
 [0.1.6]: https://github.com/xtrusia/cephlens/compare/v0.1.5...v0.1.6
 [0.1.5]: https://github.com/xtrusia/cephlens/compare/v0.1.4...v0.1.5

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cephlens"
-version = "0.1.7"
+version = "0.1.8"
 edition = "2024"
 rust-version = "1.88.0"
 authors = ["cephlens contributors"]

README.md

@@ -256,9 +256,12 @@ trace path placeholders:
 ```
 
 The osdtrace runner script is written under
-`~/.cache/cephlens/runner/cephlens-runner-*.sh` on each remote host and is
-removed on stop, quit, or TTL expiry. The optional downloaded `osdtrace` binary
-is stored under `~/.cephlens/bin/osdtrace`.
+`~/.cache/cephlens/runner/cephlens-runner-*.sh` on each remote host. Interactive
+trace runners watch the SSH connection and remove their tracer process tree and
+runner files when that connection closes. Starting a tracer also removes a
+matching runner left by an older interrupted session. TTL expiry remains the
+fallback when SSH has not detected a network failure. The optional downloaded
+`osdtrace` binary is stored under `~/.cephlens/bin/osdtrace`.
 
 Automatic `osdtrace` download is disabled unless `osdtrace_url` is configured.
 When a download is required, cephlens requires `osdtrace_sha256` and verifies the

src/app.rs

@@ -2,7 +2,7 @@ use std::{
     collections::{HashMap, VecDeque},
     io::{BufRead, BufReader, Read, Write},
     path::PathBuf,
-    process::{Command as ProcessCommand, Stdio},
+    process::{ChildStdin, Command as ProcessCommand, Stdio},
     sync::{
         Arc,
         atomic::{AtomicBool, Ordering},
@@ -25,7 +25,7 @@ use crate::{
     radostrace::{RadosEvent, parse_rados_event},
     runner::{
         CleanupResult, cleanup_trace_runners_async, cleanup_trace_runners_wait, install_trace_host,
-        probe_trace_host, trace_runner_install_command, trace_runner_script, trace_threshold_label,
+        probe_trace_host, trace_runner_install_command, trace_threshold_label,
     },
     session::{
         TRACE_KFS_LOG, TRACE_OSD_LOG, TRACE_RADOS_LOG, append_snapshot, append_trace_line,
@@ -1021,7 +1021,7 @@ fn spawn_client_trace_runner(
             .arg("--")
             .arg(&host)
             .arg(remote)
-            .stdin(Stdio::null())
+            .stdin(Stdio::piped())
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
             .spawn();
@@ -1033,6 +1033,7 @@ fn spawn_client_trace_runner(
                 return;
             }
         };
+        let connection = spawn_connection_heartbeat(child.stdin.take(), Arc::clone(&stop));
 
         let mut readers = Vec::new();
         if let Some(stderr) = child.stderr.take() {
@@ -1072,6 +1073,7 @@ fn spawn_client_trace_runner(
             }
         }
 
+        join_connection_heartbeat(connection);
         let _ = child.wait();
         for reader in readers {
             let _ = reader.join();
@@ -1158,7 +1160,7 @@ fn spawn_trace_runner(
     stop: Arc<AtomicBool>,
 ) {
     thread::spawn(move || {
-        let command = trace_runner_install_command(&session, latency_ms, ttl_secs);
+        let command = trace_runner_install_command(&session, latency_ms, ttl_secs, true);
         let remote = format!("sh -c {}", shell_quote(&command));
         let child_result = ProcessCommand::new("ssh")
             .args([
@@ -1190,16 +1192,7 @@ fn spawn_trace_runner(
             }
         };
 
-        if let Some(mut stdin) = child.stdin.take()
-            && let Err(err) = stdin.write_all(trace_runner_script().as_bytes())
-        {
-            let _ = child.kill();
-            let _ = tx.send(WorkerMsg::TraceDone {
-                host,
-                message: format!("failed to upload runner: {err}"),
-            });
-            return;
-        }
+        let connection = spawn_connection_heartbeat(child.stdin.take(), Arc::clone(&stop));
 
         if let Some(stderr) = child.stderr.take() {
             let err_tx = tx.clone();
@@ -1260,6 +1253,7 @@ fn spawn_trace_runner(
                 Ok(None) => thread::sleep(Duration::from_millis(100)),
             }
         };
+        join_connection_heartbeat(connection);
         while let Ok(line) = line_rx.try_recv() {
             let trimmed = line.trim();
             if !trimmed.is_empty() {
@@ -1273,6 +1267,28 @@ fn spawn_trace_runner(
     });
 }
 
+fn spawn_connection_heartbeat(
+    stdin: Option<ChildStdin>,
+    stop: Arc<AtomicBool>,
+) -> Option<thread::JoinHandle<()>> {
+    stdin.map(|mut stdin| {
+        thread::spawn(move || {
+            while !stop.load(Ordering::SeqCst) {
+                if stdin.write_all(b"\n").is_err() {
+                    break;
+                }
+                thread::sleep(Duration::from_secs(1));
+            }
+        })
+    })
+}
+
+fn join_connection_heartbeat(handle: Option<thread::JoinHandle<()>>) {
+    if let Some(handle) = handle {
+        let _ = handle.join();
+    }
+}
+
 fn record_trace_event(app: &mut App, event: &TraceEvent) {
     let now_bucket = Utc::now().timestamp() / TRACE_BUCKET_SECS;
     let retention_secs = app.trace_ttl_secs.max(app.trace_window_secs);

src/lab.rs

@@ -11,7 +11,7 @@ use crate::{
     collect::{collect_snapshot, run_bench},
     config::ResolvedConfig,
     report::build_report,
-    runner::{trace_runner_install_command, trace_runner_script},
+    runner::trace_runner_install_command,
     session::{
         TRACE_KFS_LOG, TRACE_OSD_LOG, TRACE_RADOS_LOG, append_snapshot, append_trace_line,
         create_session_dir, session_snapshot_path,
@@ -55,8 +55,8 @@ pub(crate) fn run_lab(
                 session_dir.clone(),
                 host.clone(),
                 TRACE_OSD_LOG,
-                trace_runner_install_command(&session, cfg.trace_latency_ms, trace_ttl_secs),
-                Some(trace_runner_script().to_owned()),
+                trace_runner_install_command(&session, cfg.trace_latency_ms, trace_ttl_secs, false),
+                None,
                 "__CEPHLENS_TRACE_ERROR__",
             ));
         }

src/runner.rs

@@ -3,7 +3,7 @@ use std::{process::Command as ProcessCommand, thread};
 use crate::{
     ssh::ssh_capture,
     trace::{TraceInstallConfig, TraceTarget, parse_trace_target, trace_install_command},
-    util::{shell_quote, short},
+    util::{remote_pidfile_cleanup_functions, shell_quote, short},
 };
 
 #[derive(Clone, Debug)]
@@ -109,33 +109,22 @@ pub(crate) fn report_cleanup_results(results: &[CleanupResult]) {
 
 fn trace_runner_cleanup_command(session: &str) -> String {
     let safe_session = safe_session_id(session);
+    trace_runner_cleanup_for_glob(&format!(
+        "\"$HOME/.cache/cephlens/runner/cephlens-runner-{safe_session}.pid\""
+    ))
+}
+
+fn trace_runner_cleanup_for_glob(pidfile_glob: &str) -> String {
     format!(
         r#"
-runner="$HOME/.cache/cephlens/runner/cephlens-runner-{safe_session}.sh"
-pidfile="$HOME/.cache/cephlens/runner/cephlens-runner-{safe_session}.pid"
-pids=""
-if [ -f "$pidfile" ]; then
-  pids=$(cat "$pidfile" 2>/dev/null || true)
-fi
-if [ -n "$pids" ]; then
-  for pid in $pids; do
-    children=$(pgrep -P "$pid" 2>/dev/null || true)
-    if [ -n "$children" ]; then
-      kill -TERM $children 2>/dev/null || true
-    fi
-    kill -TERM "$pid" 2>/dev/null || true
-  done
-  sleep 1
-  for pid in $pids; do
-    children=$(pgrep -P "$pid" 2>/dev/null || true)
-    if [ -n "$children" ]; then
-      kill -KILL $children 2>/dev/null || true
-    fi
-    kill -KILL "$pid" 2>/dev/null || true
-  done
-fi
-rm -f "$runner" "$pidfile" 2>/dev/null || true
-"#
+{cleanup_functions}
+for pidfile in {pidfile_glob}; do
+  [ -f "$pidfile" ] || continue
+  runner="${{pidfile%.pid}}.sh"
+  cephlens_cleanup_pidfile "$pidfile" "$runner" "$runner"
+done
+"#,
+        cleanup_functions = remote_pidfile_cleanup_functions(),
     )
 }
 
@@ -199,19 +188,25 @@ pub(crate) fn trace_runner_install_command(
     session: &str,
     latency_ms: u64,
     ttl_secs: u64,
+    watch_connection: bool,
 ) -> String {
     let safe_session = safe_session_id(session);
+    let cleanup =
+        trace_runner_cleanup_for_glob("\"$HOME\"/.cache/cephlens/runner/cephlens-runner-*.pid");
+    let script = shell_quote(trace_runner_script());
+    let watch_connection = u8::from(watch_connection);
     format!(
         r#"
 set -eu
+{cleanup}
 dir="$HOME/.cache/cephlens/runner"
 mkdir -p "$dir"
 runner="$dir/cephlens-runner-{safe_session}.sh"
 pidfile="$dir/cephlens-runner-{safe_session}.pid"
-cat > "$runner"
+printf '%s' {script} > "$runner"
 chmod 700 "$runner"
 echo "__CEPHLENS_RUNNER__ installed $runner"
-exec "$runner" {latency_ms} {ttl_secs} "$pidfile"
+exec "$runner" {latency_ms} {ttl_secs} "$pidfile" {watch_connection}
 "#
     )
 }
@@ -221,9 +216,11 @@ pub(crate) fn trace_runner_script() -> &'static str {
 latency_ms="${1:-1}"
 ttl_secs="${2:-1800}"
 pidfile="${3:-}"
+watch_connection="${4:-0}"
 runner_path="$0"
 trace_pid=""
 ttl_pid=""
+connection_pid=""
 
 if [ -n "$pidfile" ]; then
   printf '%s\n' "$$" > "$pidfile"
@@ -235,8 +232,14 @@ cleanup() {
   if [ -n "$ttl_pid" ]; then
     kill "$ttl_pid" 2>/dev/null || true
   fi
+  if [ -n "$connection_pid" ]; then
+    kill "$connection_pid" 2>/dev/null || true
+  fi
   if [ -n "$trace_pid" ]; then
-    kill "$trace_pid" 2>/dev/null || sudo -n kill "$trace_pid" 2>/dev/null || true
+    trace_pids=$(cephlens_process_tree "$trace_pid")
+    for target_pid in $trace_pids; do
+      sudo -n kill -TERM "$target_pid" 2>/dev/null || kill -TERM "$target_pid" 2>/dev/null || true
+    done
     wait "$trace_pid" 2>/dev/null || true
   fi
   rm -f "$runner_path" 2>/dev/null || true
@@ -252,6 +255,25 @@ cleanup() {
 trap cleanup INT TERM HUP EXIT
 echo "__CEPHLENS_RUNNER__ starting ttl=${ttl_secs}s latency_ms=${latency_ms}" >&2
 
+cephlens_process_tree() {
+  root_pid=$1
+  printf '%s\n' "$root_pid"
+  for child_pid in $(pgrep -P "$root_pid" 2>/dev/null || true); do
+    cephlens_process_tree "$child_pid"
+  done
+}
+
+if [ "$watch_connection" = "1" ]; then
+  parent_pid=$$
+  exec 3<&0
+  (
+    while IFS= read -r _ <&3; do :; done
+    kill -TERM "$parent_pid" 2>/dev/null || true
+  ) &
+  connection_pid=$!
+  exec 3<&-
+fi
+
 if ! sudo -n true 2>/dev/null; then
   echo "__CEPHLENS_TRACE_ERROR__ sudo -n unavailable"
   exit 126
@@ -316,4 +338,14 @@ mod tests {
         assert!(script.contains("echo \"__CEPHLENS_RUNNER__ ttl expired\" >&2"));
         assert!(script.contains("echo \"__CEPHLENS_RUNNER__ osdtrace_pid=$trace_pid\" >&2"));
     }
+
+    #[test]
+    fn trace_runner_replaces_stale_runner_and_watches_connection() {
+        let command = trace_runner_install_command("new-session", 1, 30, true);
+        let script = trace_runner_script();
+
+        assert!(command.contains("cephlens-runner-*.pid"));
+        assert!(script.contains("while IFS= read -r"));
+        assert!(script.contains("cephlens_process_tree \"$trace_pid\""));
+    }
 }