Skip to content

Commit 5592dcf

Browse files
luisguzman-adfaluisguzman-adfa
committed
ADFA-5051: harden the live rebuild (proot .l2s. cleanup, rollback completeness, re-verify)
Fixes found by on-device testing of the self-update: - proot ".l2s." symlink-loop artifacts (better-sqlite3 build) made cp -a / rm -rf choke and left staging cruft -> copy node_modules with tar --exclude='*.l2s.*' + purge_staging(). - rollback was dist-only: source + package.json synced BEFORE the live verify, so a bad-live update rolled back the binary but reported the failed version. Reorder: swap dist -> verify -> only THEN advance source + package.json + vhost. A failed verify leaves zero drift. - re-verify the rolled-back build actually recovered (log healthy/degraded). - clean the dist backup on every path. Folds into the unreleased 1.2.0 (no API change, no version bump).
1 parent aa009c6 commit 5592dcf

1 file changed

Lines changed: 64 additions & 28 deletions

File tree

‎tools/rebuild-dashboard.sh‎

Lines changed: 64 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
11
#!/bin/sh
2-
# tools/rebuild-dashboard.sh [CLONE_DIR] — ADFA-5011
2+
# tools/rebuild-dashboard.sh [CLONE_DIR] — ADFA-5011 / ADFA-5051
33
#
44
# Rebuild ONLY the dash-node REST API from the on-device clone, without a rootfs rebuild.
5-
# Blue-green + verify-before-swap so the live API is never left in a broken state:
5+
# Blue-green + verify-before-commit so the live API is never left in a broken or misreporting state:
66
#
77
# 1. git fetch + reset --hard origin/<branch> (deterministic; the box clone isn't edited)
88
# 2. build in a STAGING dir (yarn install + build)
99
# 3. smoke-test the STAGED build on a temp port (tools/dashboard-smoketest.sh)
10-
# 4. only if it passes: back up live dist, atomically swap staged dist in, sync source + nginx,
11-
# restart dash-node + nginx, and re-verify LIVE
12-
# 5. if the live check fails: roll back to the backed-up dist and restart
10+
# 4. only if it passes: back up live dist, sync node_modules, atomically swap the dist in, restart
11+
# dash-node, and verify LIVE
12+
# 5a. live OK -> FINALIZE: only now advance source + package.json + nginx vhost to match the running
13+
# dist (so the reported version can never get ahead of the code), reload nginx, done.
14+
# 5b. live FAIL -> roll the dist back, restart, and RE-VERIFY the rollback recovered. Because source +
15+
# package.json were never advanced (5a), a failed update leaves NO version/source drift.
16+
#
17+
# ADFA-5051 also hardens: proot ".l2s." symlink-loop artifacts no longer break the node_modules copy or
18+
# the staging cleanup (see purge_staging), and the backup is cleaned on every path.
1319
#
1420
# If step 1-3 fail, the LIVE dashboard is never touched. Launched DETACHED (setsid) by
1521
# POST /api/system/dashboard/rebuild, so the `pdsm restart dash-node` in step 4 cannot kill
@@ -34,7 +40,24 @@ LOCK="/var/run/dash-rebuild.lock"
3440

3541
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG" 2>/dev/null; }
3642
set_status() { echo "$1" > "$STATUS" 2>/dev/null || true; }
37-
cleanup() { [ -n "${TESTPID:-}" ] && kill "$TESTPID" 2>/dev/null || true; rm -rf "$STAGE"; rmdir "$LOCK" 2>/dev/null || true; }
43+
# ADFA-5051: proot renders some native-build symlinks (e.g. in better-sqlite3/build) as ".l2s." loops
44+
# that `rm -rf` can't recurse into (ELOOP -> "Directory not empty"), but a direct unlink removes them.
45+
# Sweep them first, then remove the tree — so staging is always cleanly removable.
46+
purge_staging() {
47+
[ -d "$STAGE" ] || return 0
48+
find "$STAGE" -name '*.l2s.*' -exec rm -f {} + 2>/dev/null || true
49+
rm -rf "$STAGE" 2>/dev/null || true
50+
}
51+
# ADFA-5051: poll the LIVE API's smoke test until it passes (~30s) or give up (1 = not healthy).
52+
verify_live() {
53+
_i=1
54+
while [ "$_i" -le 15 ]; do
55+
if sh "$SMOKE" "http://127.0.0.1:4000/api" >>"$LOG" 2>&1; then return 0; fi
56+
sleep 2; _i=$((_i + 1))
57+
done
58+
return 1
59+
}
60+
cleanup() { [ -n "${TESTPID:-}" ] && kill "$TESTPID" 2>/dev/null || true; purge_staging; rmdir "$LOCK" 2>/dev/null || true; }
3861
fail() { log "FAIL: $*"; set_status "error"; cleanup; exit 1; }
3962

4063
# Single-flight: mkdir is atomic.
@@ -55,9 +78,14 @@ git -C "$CLONE_DIR" reset --hard "origin/$BRANCH" >>"$LOG" 2>&1 || fail "git res
5578

5679
# 2) build in staging. Reuse live node_modules to speed the install; build fresh dist.
5780
log "staging build"
58-
rm -rf "$STAGE"; mkdir -p "$STAGE" || fail "mkdir staging"
81+
purge_staging; mkdir -p "$STAGE" || fail "mkdir staging"
5982
( cd "$SRC" && tar --exclude=node_modules --exclude=dist -cf - . ) | ( cd "$STAGE" && tar -xf - ) || fail "copy source to staging"
60-
[ -d "$LIVE/node_modules" ] && cp -a "$LIVE/node_modules" "$STAGE/node_modules"
83+
# Warm the install from the live node_modules to speed `yarn install`. Use tar (not cp -a) and EXCLUDE
84+
# proot's ".l2s." loop artifacts so the copy never trips on them (cp -a would ELOOP) and staging stays
85+
# cleanly removable; yarn reconciles anything missing.
86+
if [ -d "$LIVE/node_modules" ]; then
87+
( cd "$LIVE" && tar --exclude='*.l2s.*' -cf - node_modules ) | ( cd "$STAGE" && tar -xf - ) || true
88+
fi
6189
( cd "$STAGE" && yarn install >>"$LOG" 2>&1 && yarn build >>"$LOG" 2>&1 ) || fail "yarn install/build (offline or build error) — live untouched"
6290
[ -f "$STAGE/dist/server.js" ] || fail "no dist/server.js after build — live untouched"
6391

@@ -71,43 +99,51 @@ RC=$?
7199
kill "$TESTPID" 2>/dev/null || true; wait "$TESTPID" 2>/dev/null || true; TESTPID=""
72100
[ "$RC" -eq 0 ] || fail "staged smoke test failed (rc=$RC) — NOT promoting; live untouched"
73101

74-
# 4) promote: back up live dist, swap staged in, sync source + nginx, restart.
102+
# 4) promote — swap the dist FIRST and verify it live; only advance source + package.json (and the
103+
# nginx vhost) AFTER the live check passes. That way a failed verify rolls back the dist and the
104+
# version/source were NEVER touched, so the box can't report a version it isn't actually running.
75105
log "staged build passed — promoting"
76106
rm -rf "$BACKUP"
77107
[ -d "$LIVE/dist" ] && cp -a "$LIVE/dist" "$BACKUP"
78-
# Sync source (so the next build matches) — additive tar after dropping pure-source subdirs;
79-
# runtime state (node_modules, *.sqlite3 job storage, books/catalog.db) is left in place.
80-
for d in sockets views public test; do rm -rf "$LIVE/$d"; done
81-
( cd "$STAGE" && tar --exclude=node_modules --exclude=dist -cf - . ) | ( cd "$LIVE" && tar -xf - ) || fail "sync source to live"
82-
cp -a "$STAGE/node_modules/." "$LIVE/node_modules/" 2>/dev/null || true
108+
# node_modules is additive (new deps added, old ones remain), so the old dist tolerates it on rollback;
109+
# safe to sync before the swap so the new dist has its dependencies. tar+exclude avoids the ".l2s." loops.
110+
( cd "$STAGE" && tar --exclude='*.l2s.*' -cf - node_modules ) | ( cd "$LIVE" && tar -xf - ) || true
83111
# The dist swap is the near-atomic, restart-critical step (dash-node runs dist/server.js).
84112
rm -rf "$LIVE/dist" && cp -a "$STAGE/dist" "$LIVE/dist" || fail "dist swap"
85-
# nginx reads /etc/nginx/conf.d, not /library/dashboard, so mirror the vhost.
86-
[ -f "$LIVE/dash-node-nginx.conf" ] && { cp -f "$LIVE/dash-node-nginx.conf" "$NGINX_CONF_DIR/dash-node-nginx.conf"; chmod 0600 "$NGINX_CONF_DIR/dash-node-nginx.conf"; }
87113

88-
log "restart dash-node + nginx"
114+
log "restart dash-node"
89115
/usr/local/bin/pdsm restart dash-node >>"$LOG" 2>&1 || log "warn: pdsm restart dash-node returned non-zero"
90-
/usr/local/bin/pdsm restart nginx >>"$LOG" 2>&1 || log "warn: pdsm restart nginx returned non-zero"
91116

92-
# 5) verify LIVE; roll back the dist if it doesn't come up.
117+
# 5) verify LIVE. On success, finalize (source + package.json + nginx); on failure, roll the dist back.
93118
log "verifying live :4000"
94-
ok=0
95-
i=1
96-
while [ "$i" -le 15 ]; do
97-
if sh "$SMOKE" "http://127.0.0.1:4000/api" >>"$LOG" 2>&1; then ok=1; break; fi
98-
sleep 2; i=$((i + 1))
99-
done
100-
if [ "$ok" -eq 1 ]; then
101-
log "live OK — rebuild complete"
119+
if verify_live; then
120+
log "live OK — finalizing (source + package.json + nginx)"
121+
# Now safe to advance the source so the reported version + the next build match the running dist.
122+
# Additive tar after dropping the pure-source subdirs; runtime state (node_modules, *.sqlite3 job
123+
# storage, books/catalog.db) is left in place.
124+
for d in sockets views public test; do rm -rf "$LIVE/$d"; done
125+
( cd "$STAGE" && tar --exclude=node_modules --exclude=dist -cf - . ) | ( cd "$LIVE" && tar -xf - ) || log "warn: source sync incomplete"
126+
# nginx reads /etc/nginx/conf.d, not /library/dashboard, so mirror the vhost then reload nginx.
127+
[ -f "$LIVE/dash-node-nginx.conf" ] && { cp -f "$LIVE/dash-node-nginx.conf" "$NGINX_CONF_DIR/dash-node-nginx.conf"; chmod 0600 "$NGINX_CONF_DIR/dash-node-nginx.conf"; }
128+
/usr/local/bin/pdsm restart nginx >>"$LOG" 2>&1 || log "warn: pdsm restart nginx returned non-zero"
129+
log "rebuild complete"
102130
rm -rf "$BACKUP"
103131
set_status "done"
104132
else
105133
log "live check FAILED after swap — rolling back dist"
106134
if [ -d "$BACKUP" ]; then
107135
rm -rf "$LIVE/dist" && cp -a "$BACKUP" "$LIVE/dist"
108136
/usr/local/bin/pdsm restart dash-node >>"$LOG" 2>&1 || true
109-
log "rolled back to previous dist"
137+
# Confirm the rolled-back build actually recovered — don't just assume it did.
138+
if verify_live; then
139+
log "rolled back — verified healthy on the previous build (source + version were never advanced)"
140+
else
141+
log "ROLLBACK VERIFY FAILED — dashboard may be degraded; manual check needed"
142+
fi
143+
else
144+
log "no backup available to roll back to — dashboard may be degraded"
110145
fi
146+
rm -rf "$BACKUP"
111147
set_status "error"
112148
fi
113149
cleanup

0 commit comments

Comments
 (0)