From 6f7d0c69281be248031ac6e5dcb5763e61be5cd8 Mon Sep 17 00:00:00 2001
From: sakridge <stephen@solana.com>
Date: Mon, 23 Dec 2019 14:31:57 -0800
Subject: [PATCH] Move cleanup to a script so it doesn't kill itself (#7603)

---
 net/net.sh            | 37 ++++++++++++-------------------------
 net/remote/cleanup.sh | 25 +++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 25 deletions(-)
 create mode 100755 net/remote/cleanup.sh

diff --git a/net/net.sh b/net/net.sh
index 98121c5ac2ef55..ac33e770b79997 100755
--- a/net/net.sh
+++ b/net/net.sh
@@ -456,10 +456,16 @@ startCommon() {
     "
   fi
   [[ -z "$externalNodeSshKey" ]] || ssh-copy-id -f -i "$externalNodeSshKey" "${sshOptions[@]}" "solana@$ipAddress"
+  syncScripts "$ipAddress"
+}
+
+syncScripts() {
+  echo "rsyncing scripts... to $ipAddress"
+  declare ipAddress=$1
   rsync -vPrc -e "ssh ${sshOptions[*]}" \
     --exclude 'net/log*' \
     "$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
-    "$ipAddress":~/solana/
+    "$ipAddress":~/solana/ > /dev/null
 }
 
 startBootstrapLeader() {
@@ -870,33 +876,14 @@ stopNode() {
 
   echo "--- Stopping node: $ipAddress"
   echo "stop log: $logFile"
+  syncScripts "$ipAddress"
   (
+    # Since cleanup.sh does a pkill, we cannot pass the command directly,
+    # otherwise the process which is doing the killing will be killed because
+    # the script itself will match the pkill pattern
     set -x
     # shellcheck disable=SC2029 # It's desired that PS4 be expanded on the client side
-    ssh "${sshOptions[@]}" "$ipAddress" "
-      PS4=\"$PS4\"
-      set -x
-      ! tmux list-sessions || tmux kill-session
-      declare sudo=
-      if sudo true; then
-        sudo=\"sudo -n\"
-      fi
-
-      for pid in solana/*.pid; do
-        pgid=\$(ps opgid= \$(cat \$pid) | tr -d '[:space:]')
-        if [[ -n \$pgid ]]; then
-          \$sudo kill -- -\$pgid
-        fi
-      done
-      if [[ -f solana/netem.cfg ]]; then
-        solana/scripts/netem.sh delete < solana/netem.cfg
-        rm -f solana/netem.cfg
-      fi
-      solana/scripts/net-shaper.sh force_cleanup
-      for pattern in solana- remote- iftop validator client node; do
-        pkill -9 -f \$pattern
-      done
-    "
+    ssh "${sshOptions[@]}" "$ipAddress" "PS4=\"$PS4\" ./solana/net/remote/cleanup.sh"
   ) >> "$logFile" 2>&1 &
 
   declare pid=$!
diff --git a/net/remote/cleanup.sh b/net/remote/cleanup.sh
new file mode 100755
index 00000000000000..f2efdc00428020
--- /dev/null
+++ b/net/remote/cleanup.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -x
+! tmux list-sessions || tmux kill-session
+declare sudo=
+if sudo true; then
+  sudo="sudo -n"
+fi
+
+echo "pwd: $(pwd)"
+for pid in solana/*.pid; do
+  pgid=$(ps opgid= "$(cat "$pid")" | tr -d '[:space:]')
+  if [[ -n $pgid ]]; then
+    $sudo kill -- -"$pgid"
+  fi
+done
+if [[ -f solana/netem.cfg ]]; then
+  solana/scripts/netem.sh delete < solana/netem.cfg
+  rm -f solana/netem.cfg
+fi
+solana/scripts/net-shaper.sh force_cleanup
+for pattern in validator.sh boostrap-leader.sh solana- remote- iftop validator client node; do
+  echo "killing $pattern"
+  pkill -f $pattern
+done