perf-optimizer-loop: scrub internal names for public repo

Remove username, site-specific paths (/shared/aaji, /home/aaji,
rad-vultr-login), cluster-specific node names (lux-mi355x-*),
partition/account names (lux, vultr_lux), and the now-refuted
a-nodes-vs-b-nodes 17% performance gap narrative from all committed
files.

- Replace hardcoded paths with $AI4S_SHARED_DIR throughout scripts
  and agent prompt files; add AI4S_SHARED_DIR required-or-fail guard
  in run_optimizer_loop.sh
- Replace partition/account SBATCH directives with generic defaults
- Rewrite lessons #9, #10, #16, #18 in SKILL.md to drop cluster-
  specific node-class framing; preserve the methodological lesson
  (always check nodes_list before drawing class conclusions)
- Delete HANDOFF.md (ephemeral session artifact, not repo material);
  add HANDOFF.md to .gitignore
- Remove Authors/contacts block from HANDOFF.md before deletion

Co-Authored-By: Claude Sonnet 4 <noreply@anthropic.com>

Author: ashwinma

.cursor/skills/ai4science-perf-analysis/SKILL.md

@@ -62,6 +62,9 @@ vic_server.log
 
 # Local cluster config (site-specific, auto-created by agent)
 .cluster-config.yaml
+
+# Session-scoped agent handoff files (ephemeral, not for the repo)
+HANDOFF.md
 .claude/settings.local.json
 nul
 

.gitignore

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 # microbench_node_health.sh
 #
-# Per-node hardware/firmware/health survey for AMD MI355X (gfx950) nodes on
-# Vultr Lux. Runs four micro-tests in ~30 wall-clock seconds:
+# Per-node hardware/firmware/health survey for AMD MI355X (gfx950) nodes.
+# Runs four micro-tests in ~30 wall-clock seconds:
 #   1. host inventory                  - CPU, NUMA, kernel, NICs, PCIe link, mounts, /tmp dd
 #   2. GPU + driver inventory          - rocminfo, rocm-smi (vbios/fw/topo), PCIe link state
 #   3. CPU dual-NUMA STREAM            - COPY/SCALE/ADD/TRIAD, 128 threads, NUMA-interleaved
@@ -24,7 +24,7 @@
 #         and one summary line of pass/fail to stdout.
 #
 # Env vars (all optional, with defaults):
-#   OUT_DIR            (default: /shared/$USER/microbench-node-health)
+#   OUT_DIR            (default: $AI4S_SHARED_DIR/microbench-node-health)
 #   HG_SIF             (default: $AI4S_SHARED_DIR/images/pytorch_rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0.sif)
 #   AI4S_SHARED_DIR    (default: /shared/$USER)
 #   STREAM_THRESHOLD_COPY_GBPS    (default: 200)    - dual-NUMA COPY pass floor
@@ -33,8 +33,8 @@
 #
 # SBATCH directives:
 #SBATCH --job-name=node-health
-#SBATCH --partition=lux
-#SBATCH --account=vultr_lux
+#SBATCH --partition=gpu
+#SBATCH --account=default
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
 #SBATCH --cpus-per-task=128
@@ -47,11 +47,11 @@
 set -u
 
 H=$(hostname -s)
-OUT_DIR="${OUT_DIR:-/shared/$USER/microbench-node-health}"
+OUT_DIR="${OUT_DIR:-${AI4S_SHARED_DIR:-/tmp}/microbench-node-health}"
 HOST_OUT="$OUT_DIR/$H"
 mkdir -p "$HOST_OUT"
 
-AI4S_SHARED_DIR="${AI4S_SHARED_DIR:-/shared/$USER}"
+AI4S_SHARED_DIR="${AI4S_SHARED_DIR:-/tmp}"  # set AI4S_SHARED_DIR to your cluster's shared storage root
 HG_SIF="${HG_SIF:-${AI4S_SHARED_DIR}/images/pytorch_rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0.sif}"
 STREAM_THRESHOLD_COPY_GBPS="${STREAM_THRESHOLD_COPY_GBPS:-200}"
 STREAM_THRESHOLD_TRIAD_GBPS="${STREAM_THRESHOLD_TRIAD_GBPS:-200}"

material_science/models/HydraGNN/examples/microbench_node_health.sh

@@ -17,7 +17,7 @@
 #   n_iters_budget  Max iterations (recommend 5).
 #
 # Abort:
-#   Graceful: touch /shared/aaji/models/HydraGNN/perf-runs/loop-<uuid>/STOP
+#   Graceful: touch $AI4S_SHARED_DIR/models/HydraGNN/perf-runs/loop-<uuid>/STOP
 #   Emergency: scancel <jobid> + tmux kill-session -t hg-loop
 
 set -euo pipefail
@@ -44,7 +44,7 @@ RECIPE_DIR="${REPO_ROOT}/material_science/models/HydraGNN/recipes/perf-optimizer
 ORCH_PROMPT="${RECIPE_DIR}/agents/orchestrator.md"
 
 # Loop-dir convention matches recipes/perf-optimizer-loop/README.md
-AI4S_SHARED_DIR="${AI4S_SHARED_DIR:-/shared/aaji}"
+: "${AI4S_SHARED_DIR:?AI4S_SHARED_DIR must be set (e.g. export AI4S_SHARED_DIR=/shared/\$USER)}"
 HG_BASE="${AI4S_SHARED_DIR}/models/HydraGNN"
 PERF_RUNS_DIR="${HG_BASE}/perf-runs"
 LOOP_DIR="${PERF_RUNS_DIR}/loop-${LOOP_UUID}"
@@ -72,10 +72,10 @@ PREFLIGHT_NOTES=()
 
 # 1. cluster up
 if command -v sinfo > /dev/null 2>&1; then
-  _IDLE_OR_MIX=$(sinfo -p lux,rad -h -o '%t' 2>/dev/null | grep -cE '^(idle|mix|alloc)$' || true)
+  _IDLE_OR_MIX=$(sinfo -h -o '%t' 2>/dev/null | grep -cE '^(idle|mix|alloc)$' || true)
   if [[ "${_IDLE_OR_MIX:-0}" -eq 0 ]]; then
     PREFLIGHT_FAIL_REASON="cluster_down"
-    PREFLIGHT_NOTES+=("sinfo reports 0 usable nodes on partitions lux,rad")
+    PREFLIGHT_NOTES+=("sinfo reports 0 usable nodes — check partition names in .cluster-config.yaml")
   fi
 else
   PREFLIGHT_FAIL_REASON="no_slurm"
@@ -93,8 +93,8 @@ fi
 
 # 3. tools present
 for _bin in \
-    /shared/aaji/tools/omnistat-pr271/bin/omnistat-usermode \
-    /shared/aaji/tools/victoriametrics/victoria-metrics-prod; do
+    "${AI4S_SHARED_DIR}/tools/omnistat-pr271/bin/omnistat-usermode" \
+    "${AI4S_SHARED_DIR}/tools/victoriametrics/victoria-metrics-prod"; do
   if [[ -z "$PREFLIGHT_FAIL_REASON" && ! -x "$_bin" ]]; then
     PREFLIGHT_FAIL_REASON="tool_missing"
     PREFLIGHT_NOTES+=("missing: $_bin (run the perf-analysis launcher subagent first to install)")
@@ -104,7 +104,7 @@ done
 # 4. SIF + overlay
 for _f in \
     "${HG_BASE}/overlays/hydragnn-overlay.img" \
-    "${AI4S_SHARED_DIR}/images/pytorch_rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0.sif"; do
+    "${AI4S_SHARED_DIR}/images/pytorch_rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0.sif"; do  # set AI4S_SHARED_DIR to your cluster's shared storage root
   if [[ -z "$PREFLIGHT_FAIL_REASON" && ! -f "$_f" ]]; then
     PREFLIGHT_FAIL_REASON="image_missing"
     PREFLIGHT_NOTES+=("missing: $_f")
@@ -151,7 +151,7 @@ if [[ -n "$PREFLIGHT_FAIL_REASON" ]]; then
 fi
 
 _DISK_FREE=$(df -h "$PERF_RUNS_DIR" | awk 'NR==2 {print $4}')
-_log_status "PREFLIGHT_OK cluster=lux disk_free=${_DISK_FREE} claude_cli=ok api_egress=ok orch_prompt=ok"
+_log_status "PREFLIGHT_OK disk_free=${_DISK_FREE} claude_cli=ok api_egress=ok orch_prompt=ok"
 
 if [[ $PREFLIGHT_ONLY -eq 1 ]]; then
   _log_status "PREFLIGHT_ONLY mode: exiting before invoking orchestrator"

material_science/models/HydraGNN/examples/run_optimizer_loop.sh

@@ -8,21 +8,20 @@
 #   1. Wraps srun with omnistat-usermode --start/--stopexporters/--stopserver.
 #   2. Generates a per-job copy of gfm_mlip.json with a "Profile" block injected
 #      so HydraGNN's built-in torch.profiler captures rank-0 of node-0 only.
-#   3. Hard-codes the lux partition / vultr_lux account so this script works
-#      out-of-the-box on Lux. Override via SBATCH_PARTITION/SBATCH_ACCOUNT if
-#      submitting to a different cluster.
+#   3. Defaults to a cluster partition/account that can be overridden via
+#      SBATCH_PARTITION/SBATCH_ACCOUNT env vars for your cluster.
 #   4. Defaults are tuned for a quick perf run: --nodes=2, NUM_EPOCH=2,
 #      MAX_NUM_BATCH=30, time=00:30:00 — enough for the wait=5/warmup=3/active=3
 #      profiler schedule plus a clean epoch 0 for warm caches.
 #
 # Quick start:
-#   export AI4S_SHARED_DIR=/shared/aaji
+#   export AI4S_SHARED_DIR=/shared/$USER
 #   sbatch material_science/models/HydraGNN/examples/sbatch_train_perf_amd.sh
 #
 # Required:
 #   AI4S_SHARED_DIR — shared base path
-#   /shared/aaji/tools/omnistat-pr271/    (created by the launcher subagent)
-#   /shared/aaji/tools/victoriametrics/   (created by the launcher subagent)
+#   $AI4S_SHARED_DIR/tools/omnistat-pr271/    (created by the launcher subagent)
+#   $AI4S_SHARED_DIR/tools/victoriametrics/   (created by the launcher subagent)
 #
 # Optional env-var overrides (with defaults):
 #   HG_NUM_EPOCH=2
@@ -33,8 +32,8 @@
 #   OMNISTAT_USERMODE_INTERVAL=1   # seconds
 
 #SBATCH --job-name=hydragnn-perf
-#SBATCH --partition=lux
-#SBATCH --account=vultr_lux
+#SBATCH --partition=gpu
+#SBATCH --account=default
 #SBATCH --nodes=2
 #SBATCH --ntasks-per-node=8
 #SBATCH --gpus-per-node=8
@@ -71,7 +70,7 @@ HG_REPO_DIR="${HG_REPO_DIR:-${HG_BASE}/code/HydraGNN}"
 HYDRAGNN_MAX_NUM_BATCH="${HYDRAGNN_MAX_NUM_BATCH:-30}"
 PROFILE_TARGET_EPOCH="${PROFILE_TARGET_EPOCH:-1}"
 
-OMNISTAT_VENV="${OMNISTAT_VENV:-/shared/aaji/tools/omnistat-pr271}"
+OMNISTAT_VENV="${OMNISTAT_VENV:-${AI4S_SHARED_DIR}/tools/omnistat-pr271}"
 # Read the omnistat config from the in-repo recipe (single source of truth).
 # Override with OMNISTAT_TEMPLATE=/path/to/file if you need a custom probe config.
 # A staged copy under ${HG_BASE}/perf-runs/ is intentionally NOT used as the
@@ -87,7 +86,7 @@ OMNISTAT_USERMODE_INTERVAL="${OMNISTAT_USERMODE_INTERVAL:-1}"
 # See material_science/models/HydraGNN/recipes/perf-analysis/agents/launcher.md
 # for how to build libomnistat_trace.so on a compute node.
 OMNISTAT_KERNEL_TRACE="${OMNISTAT_KERNEL_TRACE:-0}"
-OMNISTAT_TRACE_LIB="${OMNISTAT_TRACE_LIB:-/shared/aaji/tools/omnistat-src/build-trace/libomnistat_trace.so}"
+OMNISTAT_TRACE_LIB="${OMNISTAT_TRACE_LIB:-${AI4S_SHARED_DIR}/tools/omnistat-src/build-trace/libomnistat_trace.so}"
 # IMPORTANT: the kernel-trace collector registers /kernel_trace on the SAME
 # Flask app as the other omnistat endpoints, so the tool library must POST to
 # the [omnistat.collectors] `port` (8101 here), NOT the library's own default
@@ -173,9 +172,9 @@ if [[ "$OMNISTAT_KERNEL_TRACE" == "1" ]]; then
     echo "ERROR: OMNISTAT_KERNEL_TRACE=1 but tool library not found:" >&2
     echo "       $OMNISTAT_TRACE_LIB" >&2
     echo "       Build it on a compute node:" >&2
-    echo "         salloc -p lux -A vultr_lux -N 1 --time=00:15:00 --gpus-per-node=1 \\" >&2
+    echo "         salloc -p <partition> -A <account> -N 1 --time=00:15:00 --gpus-per-node=1 \\" >&2
     echo "           apptainer exec --rocm \"\$HG_SIF\" bash -c '" >&2
-    echo "             cd /shared/aaji/tools/omnistat-src && \\" >&2
+    echo "             cd \$AI4S_SHARED_DIR/tools/omnistat-src && \\" >&2
     echo "             cmake -S rocprofiler-sdk/ -B build-trace/ -DBUILD_KERNEL_TRACE_LIB=ON && \\" >&2
     echo "             cmake --build build-trace/ -j 8'" >&2
     exit 2