From 2144dbf46f2924f752960d2ae847819db55c53fe Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 17:59:16 -0500 Subject: [PATCH 1/8] refactor(scripts): unify update.sh flag scheme, add FR checkpoint Single grammar across update.sh and the three sub-scripts. `./scripts/update.sh` (no args) now updates all sources incrementally from each source's last checkpoint. Flag groups: source selection (--source), mode (--force / --deploy-only), phase control (--skip-deploy / --skip-highlights / --skip-search), source- scoping (--titles for eCFR/USC; --from/--to/--days for FR), utility (--dry-run, --verbose, --help). Removed legacy prefixes: --ecfr-titles, --ecfr-all, --ecfr-skip-highlights --fr-days, --fr-from, --fr-to --usc-force, --usc-skip-highlights update-ecfr.sh --all Each prints a migration hint and exits 1. New FR checkpoint at downloads/fr/.fr-state.json ({ lastRun, lastDate }): default invocation resumes from lastDate; bootstrap (no checkpoint) errors with a hint requiring --from or --days, since FR has no inherent "all". eCFR/USC bootstrap (missing checkpoint) now logs explicitly and runs a full first-run automatically. Search-index strategy under --force: Default / single-source --force: incremental (deploy.sh --search-docker --source X) --force on all three sources: single full reindex at the end (no --source) Other: - --help now uses awk (BSD/GNU sed compatible) instead of GNU-only sed. - --dry-run forwards to each sub-script and uses `|| true` so one source's bootstrap-error doesn't halt the multi-source preview. - eCFR title-granularity safety guard skipped in `titles` mode (single- title runs only refresh one file and legitimately leave the rest alone). --- scripts/update-ecfr.sh | 189 +++++++++++++----- scripts/update-fr.sh | 225 +++++++++++++++++----- scripts/update-usc.sh | 79 +++++++- scripts/update.sh | 427 +++++++++++++++++++++++++++++++---------- 4 files changed, 718 insertions(+), 202 deletions(-) diff --git a/scripts/update-ecfr.sh b/scripts/update-ecfr.sh index 7446c19..4686921 100755 --- a/scripts/update-ecfr.sh +++ b/scripts/update-ecfr.sh @@ -3,22 +3,31 @@ # # Usage: # ./scripts/update-ecfr.sh # Incremental: only changed titles -# ./scripts/update-ecfr.sh --titles 1,17 # Explicit titles -# ./scripts/update-ecfr.sh --all # Force full reconvert -# ./scripts/update-ecfr.sh --skip-deploy # Local only (no VPS push) -# ./scripts/update-ecfr.sh --deploy-only # Push existing output + reindex +# ./scripts/update-ecfr.sh --titles 1,17 # Explicit titles (skip change-detection) +# ./scripts/update-ecfr.sh --force # Force reconvert all 50 titles +# ./scripts/update-ecfr.sh --skip-deploy # Local only (no VPS push, no search) +# ./scripts/update-ecfr.sh --skip-search # Skip search reindex (still rsync) # ./scripts/update-ecfr.sh --skip-highlights # Skip highlight generation +# ./scripts/update-ecfr.sh --deploy-only # Push existing output + reindex +# ./scripts/update-ecfr.sh --dry-run # Print plan, exit 0 +# +# Modes: +# incremental Default. ecfr-changed-titles.ts compares API metadata vs checkpoint. +# bootstrap No checkpoint at downloads/ecfr/.ecfr-titles-state.json. +# ecfr-changed-titles.ts returns all titles; pipeline performs full bootstrap. +# titles Explicit --titles spec. Bypasses change-detection. +# force --force: download + convert all 50 titles regardless of checkpoint. # # Steps: -# 1. Detect changed titles via eCFR API metadata -# 2. Download changed title XML from eCFR API -# 3. Convert changed titles to Markdown at all granularities (section, title, chapter, part) -# 4. Generate highlights for changed sections (mtime-based, skippable) -# 5. Regenerate eCFR nav JSON -# 6. Regenerate sitemaps -# 7. Save checkpoint -# 8. Rsync content (all granularities) + nav + sitemaps to VPS -# 9. Incremental search index on VPS +# 1. Detect changed titles via eCFR API metadata (or use --titles / --force). +# 2. Download changed title XML from eCFR API. +# 3. Convert changed titles to Markdown at all granularities (section, title, chapter, part). +# 4. Generate highlights for changed sections (mtime-based, skippable). +# 5. Regenerate eCFR nav JSON. +# 6. Regenerate sitemaps (deferred when LEXBUILD_DEFER_SITEMAP=1). +# 7. Save checkpoint. +# 8. Rsync content (all granularities) + nav + sitemaps to VPS. +# 9. Incremental search index via Docker. # # Requires: # - Built CLI: pnpm turbo build (or at least @lexbuild/ecfr + @lexbuild/cli) @@ -40,13 +49,17 @@ fi CONTENT_DEST="${CONTENT_DEST:-/srv/lexbuild/content}" NAV_DEST="${NAV_DEST:-/srv/lexbuild/nav}" +ECFR_CHECKPOINT_PATH="$REPO_ROOT/downloads/ecfr/.ecfr-titles-state.json" + # --- Parse arguments --- TITLES="" -ALL=false +FORCE=false SKIP_DEPLOY=false DEPLOY_ONLY=false SKIP_HIGHLIGHTS=false +SKIP_SEARCH=false +DRY_RUN=false while [[ $# -gt 0 ]]; do case "$1" in @@ -54,8 +67,8 @@ while [[ $# -gt 0 ]]; do TITLES="$2" shift 2 ;; - --all) - ALL=true + --force) + FORCE=true shift ;; --skip-deploy) @@ -70,29 +83,87 @@ while [[ $# -gt 0 ]]; do SKIP_HIGHLIGHTS=true shift ;; + --skip-search) + SKIP_SEARCH=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --all) + echo "Error: --all has been removed. Use --force instead." >&2 + echo " ./scripts/update-ecfr.sh --force" >&2 + exit 1 + ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 + exit 1 +fi + +if [ -n "$TITLES" ] && [ "$FORCE" = true ]; then + echo "Error: --titles and --force are mutually exclusive." >&2 exit 1 fi +# --- Resolve mode --- + +MODE="" +if [ "$DEPLOY_ONLY" = true ]; then + MODE="deploy-only" +elif [ "$FORCE" = true ]; then + MODE="force" +elif [ -n "$TITLES" ]; then + MODE="titles" +elif [ ! -f "$ECFR_CHECKPOINT_PATH" ]; then + MODE="bootstrap" +else + MODE="incremental" +fi + +# --- Print plan and exit on --dry-run --- + +print_plan() { + echo "==> eCFR update plan" + echo " Mode: $MODE" + if [ -n "$TITLES" ]; then + echo " Titles: $TITLES" + fi + echo " Force: $FORCE" + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Skip highlights: $SKIP_HIGHLIGHTS" + echo " Checkpoint: $ECFR_CHECKPOINT_PATH" + if [ -f "$ECFR_CHECKPOINT_PATH" ]; then + echo " Checkpoint exists: yes" + else + echo " Checkpoint exists: no (bootstrap)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + exit 0 +fi + CLI="node packages/cli/dist/index.js" # --- Preflight checks --- if [ ! -f "packages/cli/dist/index.js" ]; then - echo "Error: CLI not built. Run: pnpm turbo build" + echo "Error: CLI not built. Run: pnpm turbo build" >&2 exit 1 fi @@ -104,7 +175,7 @@ if [ "$SKIP_DEPLOY" = false ] && [ "$DEPLOY_ONLY" = false ] && [ -z "${VPS_HOST: fi if [ "$DEPLOY_ONLY" = true ] && [ -z "${VPS_HOST:-}" ]; then - echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." + echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." >&2 exit 1 fi @@ -119,33 +190,42 @@ if [ "$DEPLOY_ONLY" = false ]; then PIPELINE_MARKER="$(mktemp -t lexbuild-ecfr-update.XXXXXX)" trap 'rm -f "$PIPELINE_MARKER"' EXIT - # Step 1: Detect changed titles (unless --titles or --all specified) + # Step 1: Detect titles to process CURRENCY_DATE="" - if [ -n "$TITLES" ]; then - echo "==> eCFR update for titles: $TITLES" - elif [ "$ALL" = true ]; then - echo "==> eCFR full update (all titles)" - TITLES="all" - else - echo "--- Step 1/7: Detecting changed eCFR titles" - CHANGE_JSON=$(npx tsx scripts/ecfr-changed-titles.ts --json) - TITLES=$(echo "$CHANGE_JSON" | node -e " - const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); - process.stdout.write(d.changedTitles.join(',')); - ") - CURRENCY_DATE=$(echo "$CHANGE_JSON" | node -e " - const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); - process.stdout.write(d.currencyDate || ''); - ") - - if [ -z "$TITLES" ]; then - echo " No eCFR titles changed since last run. Nothing to do." - exit 0 - fi - echo " Changed titles: $TITLES" - echo "" - fi + case "$MODE" in + titles) + echo "==> eCFR update for titles: $TITLES" + ;; + force) + echo "==> eCFR force update (all titles)" + TITLES="all" + ;; + bootstrap) + echo "==> eCFR bootstrap: no checkpoint at $ECFR_CHECKPOINT_PATH" + echo " Performing full first-run download (all titles)." + TITLES="all" + ;; + incremental) + echo "--- Step 1/7: Detecting changed eCFR titles" + CHANGE_JSON=$(npx tsx scripts/ecfr-changed-titles.ts --json) + TITLES=$(echo "$CHANGE_JSON" | node -e " + const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); + process.stdout.write(d.changedTitles.join(',')); + ") + CURRENCY_DATE=$(echo "$CHANGE_JSON" | node -e " + const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); + process.stdout.write(d.currencyDate || ''); + ") + + if [ -z "$TITLES" ]; then + echo " No eCFR titles changed since last run. Nothing to do." + exit 0 + fi + echo " Changed titles: $TITLES" + echo "" + ;; + esac # Build CLI download/convert args if [ "$TITLES" = "all" ]; then @@ -200,7 +280,12 @@ if [ "$DEPLOY_ONLY" = false ]; then # emitted ~300 bytes of frontmatter with no body). A healthy eCFR title # corpus is ~1.3GB across 49 titles; if we're under 10MB total, something # broke and we should not overwrite production with stubs. - if [ -d "output-title/ecfr" ]; then + # + # Skipped in `titles` mode: a single-title run only refreshes one file and + # legitimately leaves the rest of output-title/ecfr untouched (preserved by + # writeFileIfChanged), so total bytes there have no meaning for that run's + # health. Force/incremental modes still get the check. + if [ "$MODE" != "titles" ] && [ -d "output-title/ecfr" ]; then TITLE_BYTES=$(find output-title/ecfr -name "*.md" -type f -exec wc -c {} + 2>/dev/null | awk 'END {print $1}') TITLE_BYTES="${TITLE_BYTES:-0}" if [ "$TITLE_BYTES" -lt 10000000 ]; then @@ -297,8 +382,12 @@ echo "" # restart-storms under memory pressure). The deploy script handles: local # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. -echo "--- Step 9/9: Building and shipping search index via local Docker" -"$SCRIPT_DIR/deploy.sh" --search-docker --source ecfr +if [ "$SKIP_SEARCH" = true ]; then + echo "--- Skipping search index step (--skip-search)" +else + echo "--- Step 9/9: Building and shipping search index via local Docker" + "$SCRIPT_DIR/deploy.sh" --search-docker --source ecfr +fi echo "" echo "==> eCFR update complete" diff --git a/scripts/update-fr.sh b/scripts/update-fr.sh index 4785a08..ea11070 100755 --- a/scripts/update-fr.sh +++ b/scripts/update-fr.sh @@ -2,24 +2,34 @@ # update-fr.sh — Download, convert, and deploy new Federal Register documents. # # Usage: -# ./scripts/update-fr.sh # Yesterday's documents (default) -# ./scripts/update-fr.sh --days 3 # Last 3 days -# ./scripts/update-fr.sh --from 2026-03-25 --to 2026-04-01 # Explicit date range -# ./scripts/update-fr.sh --skip-deploy # Local only (download + convert + generate, no VPS push) -# ./scripts/update-fr.sh --deploy-only # Push existing output + reindex (no download/convert) +# ./scripts/update-fr.sh # Incremental from .fr-state.json +# ./scripts/update-fr.sh --days 3 # Last 3 days (writes checkpoint) +# ./scripts/update-fr.sh --from 2026-03-25 # Explicit start (--to defaults to today) +# ./scripts/update-fr.sh --from 2026-03-25 --to 2026-04-01 +# ./scripts/update-fr.sh --force --from 2026-01-01 # Force redownload (requires --from) +# ./scripts/update-fr.sh --skip-deploy # Local only (no rsync, no search) +# ./scripts/update-fr.sh --skip-search # Skip search reindex (still rsync) +# ./scripts/update-fr.sh --deploy-only # Push existing output + reindex +# ./scripts/update-fr.sh --dry-run # Print plan, exit 0 # -# Steps: -# 1. Download FR documents via API (JSON + XML per document) -# 2. Convert new XML to Markdown (date-filtered, not full reconvert) -# 3. Regenerate FR nav JSON -# 4. Regenerate sitemaps -# 5. Rsync content + nav + sitemaps to VPS -# 6. Incremental search index on VPS (only new/changed FR docs) +# Modes: +# incremental Default. --from = lastDate from .fr-state.json, --to = today. +# bootstrap No checkpoint. Requires --from or --days from the caller. +# window --from / --to / --days specified explicitly. Writes checkpoint at end. +# force --force + --from. Redownloads and reconverts the window even if XML on disk. +# +# Checkpoint: +# downloads/fr/.fr-state.json — { lastRun: ISO8601, lastDate: YYYY-MM-DD } # -# Requires: -# - Built CLI: pnpm turbo build (or at least @lexbuild/fr + @lexbuild/cli) -# - For deploy: SSH access to VPS, scripts/.deploy.env with VPS_HOST -# - For search: Meilisearch running on VPS (PM2-managed) +# Steps: +# 1. Resolve date range (mode-aware). +# 2. Download FR documents via API (JSON + XML per document). +# 3. Convert new XML to Markdown (date-filtered). +# 4. Regenerate FR nav JSON. +# 5. Regenerate sitemaps (deferred when LEXBUILD_DEFER_SITEMAP=1). +# 6. Write checkpoint (.fr-state.json). +# 7. Rsync content + nav + sitemaps to VPS (unless --skip-deploy). +# 8. Incremental search index via Docker (unless --skip-search or --skip-deploy). set -euo pipefail @@ -36,13 +46,18 @@ fi CONTENT_DEST="${CONTENT_DEST:-/srv/lexbuild/content}" NAV_DEST="${NAV_DEST:-/srv/lexbuild/nav}" +CHECKPOINT_PATH="$REPO_ROOT/downloads/fr/.fr-state.json" + # --- Parse arguments --- -DAYS=1 +DAYS="" FROM="" TO="" +FORCE=false SKIP_DEPLOY=false DEPLOY_ONLY=false +SKIP_SEARCH=false +DRY_RUN=false while [[ $# -gt 0 ]]; do case "$1" in @@ -58,6 +73,10 @@ while [[ $# -gt 0 ]]; do TO="$2" shift 2 ;; + --force) + FORCE=true + shift + ;; --skip-deploy) SKIP_DEPLOY=true shift @@ -66,31 +85,133 @@ while [[ $# -gt 0 ]]; do DEPLOY_ONLY=true shift ;; + --skip-search) + SKIP_SEARCH=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 exit 1 fi -# Compute date range -if [ -n "$FROM" ]; then - DATE_FROM="$FROM" - DATE_TO="${TO:-$(date +%Y-%m-%d)}" +if [ -n "$DAYS" ] && { [ -n "$FROM" ] || [ -n "$TO" ]; }; then + echo "Error: --days is mutually exclusive with --from / --to." >&2 + exit 1 +fi + +# --- Resolve mode + date range --- + +# read_checkpoint_date prints lastDate from .fr-state.json, or empty if missing/invalid +read_checkpoint_date() { + if [ ! -f "$CHECKPOINT_PATH" ]; then + return 0 + fi + node -e " + try { + const d = JSON.parse(require('fs').readFileSync('$CHECKPOINT_PATH', 'utf-8')); + if (typeof d.lastDate === 'string' && /^\\d{4}-\\d{2}-\\d{2}$/.test(d.lastDate)) { + process.stdout.write(d.lastDate); + } + } catch (_) { /* missing or malformed: silent */ } + " 2>/dev/null +} + +CHECKPOINT_DATE="$(read_checkpoint_date)" +TODAY="$(date +%Y-%m-%d)" + +# Validate --force preconditions early +if [ "$FORCE" = true ] && [ -z "$FROM" ] && [ -z "$DAYS" ]; then + if [ "$DRY_RUN" = true ]; then + echo "==> FR update plan" + echo " Mode: force (would error)" + echo " Reason: --force on FR requires --from YYYY-MM-DD or --days N." + echo " FR has no inherent 'all' (decades of documents)." + exit 0 + fi + echo "Error: --force on FR requires --from YYYY-MM-DD or --days N." >&2 + echo " FR has no inherent 'all' (decades of documents). Specify a window." >&2 + exit 2 +fi + +# Determine mode +MODE="" +if [ "$DEPLOY_ONLY" = true ]; then + MODE="deploy-only" +elif [ "$FORCE" = true ]; then + MODE="force" +elif [ -n "$DAYS" ] || [ -n "$FROM" ]; then + MODE="window" +elif [ -n "$CHECKPOINT_DATE" ]; then + MODE="incremental" else - # macOS date: -v-Nd for N days ago + MODE="bootstrap" +fi + +# Bootstrap requires explicit --from or --days. In --dry-run we still print the +# would-error plan so the orchestrator's multi-source preview stays informative. +if [ "$MODE" = "bootstrap" ]; then + if [ "$DRY_RUN" = true ]; then + echo "==> FR update plan" + echo " Mode: bootstrap (would error)" + echo " Reason: No checkpoint at $CHECKPOINT_PATH." + echo " Run with --from YYYY-MM-DD or --days N to bootstrap." + exit 0 + fi + echo "Error: FR has no checkpoint at $CHECKPOINT_PATH." >&2 + echo " Run with --from YYYY-MM-DD or --days N to bootstrap." >&2 + exit 2 +fi + +# Compute effective DATE_FROM / DATE_TO +if [ -n "$DAYS" ]; then DATE_FROM="$(date -v-${DAYS}d +%Y-%m-%d)" - DATE_TO="$(date +%Y-%m-%d)" + DATE_TO="${TO:-$TODAY}" +elif [ -n "$FROM" ]; then + DATE_FROM="$FROM" + DATE_TO="${TO:-$TODAY}" +elif [ "$MODE" = "incremental" ]; then + DATE_FROM="$CHECKPOINT_DATE" + DATE_TO="$TODAY" +fi + +# --- Print plan and exit on --dry-run --- + +print_plan() { + echo "==> FR update plan" + echo " Mode: $MODE" + if [ "$MODE" != "deploy-only" ]; then + echo " Window: $DATE_FROM → $DATE_TO" + fi + echo " Force: $FORCE" + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Checkpoint: $CHECKPOINT_PATH" + if [ -n "$CHECKPOINT_DATE" ]; then + echo " Last run date: $CHECKPOINT_DATE" + else + echo " Last run date: (none)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + exit 0 fi CLI="node packages/cli/dist/index.js" @@ -98,7 +219,7 @@ CLI="node packages/cli/dist/index.js" # --- Preflight checks --- if [ ! -f "packages/cli/dist/index.js" ]; then - echo "Error: CLI not built. Run: pnpm turbo build --filter=@lexbuild/fr --filter=@lexbuild/cli" + echo "Error: CLI not built. Run: pnpm turbo build" >&2 exit 1 fi @@ -110,14 +231,14 @@ if [ "$SKIP_DEPLOY" = false ] && [ "$DEPLOY_ONLY" = false ] && [ -z "${VPS_HOST: fi if [ "$DEPLOY_ONLY" = true ] && [ -z "${VPS_HOST:-}" ]; then - echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." + echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." >&2 exit 1 fi -# --- Step 1–4: Local pipeline (skip if --deploy-only) --- +# --- Step 1–6: Local pipeline (skip if --deploy-only) --- if [ "$DEPLOY_ONLY" = false ]; then - echo "==> FR update for $DATE_FROM to $DATE_TO" + echo "==> FR update ($MODE: $DATE_FROM → $DATE_TO)" echo "" # Pipeline-start marker: used after convert to verify that downloaded XML @@ -128,14 +249,14 @@ if [ "$DEPLOY_ONLY" = false ]; then trap 'rm -f "$PIPELINE_MARKER"' EXIT # Step 1: Download - echo "--- Step 1/4: Downloading FR documents ($DATE_FROM to $DATE_TO)" + echo "--- Step 1/6: Downloading FR documents ($DATE_FROM to $DATE_TO)" $CLI download-fr --from "$DATE_FROM" --to "$DATE_TO" echo "" NEW_XML_COUNT=$(find downloads/fr -name "*.xml" -newer "$PIPELINE_MARKER" 2>/dev/null | wc -l | tr -d ' ') # Step 2: Convert (date-filtered — only converts files in the date range) - echo "--- Step 2/4: Converting FR documents ($DATE_FROM to $DATE_TO)" + echo "--- Step 2/6: Converting FR documents ($DATE_FROM to $DATE_TO)" $CLI convert-fr --all --from "$DATE_FROM" --to "$DATE_TO" echo "" @@ -155,7 +276,7 @@ if [ "$DEPLOY_ONLY" = false ]; then fi # Step 3: Regenerate FR nav - echo "--- Step 3/4: Generating FR nav JSON" + echo "--- Step 3/6: Generating FR nav JSON" ( cd apps/astro && npx tsx scripts/generate-nav.ts --source fr ) || exit 1 echo "" @@ -163,24 +284,38 @@ if [ "$DEPLOY_ONLY" = false ]; then # Skipped when LEXBUILD_DEFER_SITEMAP=1 (set by update.sh to avoid # regenerating the full sitemap index once per source). if [ "${LEXBUILD_DEFER_SITEMAP:-}" != "1" ]; then - echo "--- Step 4/4: Generating sitemaps" + echo "--- Step 4/6: Generating sitemaps" ( cd apps/astro && npx tsx scripts/generate-sitemap.ts ) || exit 1 echo "" else - echo "--- Step 4/4: Skipping sitemap (deferred to update.sh)" + echo "--- Step 4/6: Skipping sitemap (deferred to update.sh)" echo "" fi + + # Step 5: Write checkpoint with today's date as the new resume point. + # Using DATE_TO (which is today on default runs, or the explicit --to value) + # ensures the next default invocation resumes from a contiguous date. + echo "--- Step 5/6: Writing FR checkpoint ($DATE_TO)" + mkdir -p "$(dirname "$CHECKPOINT_PATH")" + node -e " + const fs = require('fs'); + fs.writeFileSync('$CHECKPOINT_PATH', JSON.stringify({ + lastRun: new Date().toISOString(), + lastDate: '$DATE_TO', + }, null, 2) + '\\n'); + " + echo "" fi -# --- Step 5–6: Deploy to VPS (skip if --skip-deploy) --- +# --- Step 7–8: Deploy to VPS (skip if --skip-deploy) --- if [ "$SKIP_DEPLOY" = true ]; then echo "==> Local pipeline complete (--skip-deploy). Files ready in output/fr/" exit 0 fi -# Step 5: Rsync content + nav + sitemaps -echo "--- Step 5/6: Syncing to VPS" +# Step 7: Rsync content + nav + sitemaps +echo "--- Step 6/6 (deploy): Syncing to VPS" if [ -d "output/fr" ]; then echo " FR documents" @@ -206,14 +341,18 @@ if [ "${LEXBUILD_DEFER_SITEMAP:-}" != "1" ]; then fi echo "" -# Step 6: Build search index locally in Docker, ship the data directory to VPS. +# Step 8: Build search index locally in Docker, ship the data directory to VPS. # Delegating to deploy.sh --search-docker --source fr avoids running heavy # bulk upserts against the single production Meilisearch (caused PM2 # restart-storms under memory pressure). The deploy script handles: local # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. -echo "--- Step 6/6: Building and shipping search index via local Docker" -"$SCRIPT_DIR/deploy.sh" --search-docker --source fr +if [ "$SKIP_SEARCH" = true ]; then + echo "--- Skipping search index step (--skip-search)" +else + echo "--- Building and shipping search index via local Docker" + "$SCRIPT_DIR/deploy.sh" --search-docker --source fr +fi echo "" -echo "==> FR update complete ($DATE_FROM to $DATE_TO)" +echo "==> FR update complete ($DATE_FROM → $DATE_TO)" diff --git a/scripts/update-usc.sh b/scripts/update-usc.sh index 8f76a03..9c0f1c3 100755 --- a/scripts/update-usc.sh +++ b/scripts/update-usc.sh @@ -2,11 +2,18 @@ # update-usc.sh — Check for new USC release point, download, convert, and deploy. # # Usage: -# ./scripts/update-usc.sh # Check for new release point -# ./scripts/update-usc.sh --force # Force full reconvert -# ./scripts/update-usc.sh --skip-deploy # Local only (no VPS push) -# ./scripts/update-usc.sh --deploy-only # Push existing output + reindex +# ./scripts/update-usc.sh # Incremental: check for new release point +# ./scripts/update-usc.sh --force # Force full redownload + reconvert +# ./scripts/update-usc.sh --skip-deploy # Local only (no VPS push, no search) +# ./scripts/update-usc.sh --skip-search # Skip search reindex (still rsync) # ./scripts/update-usc.sh --skip-highlights # Skip highlight generation +# ./scripts/update-usc.sh --deploy-only # Push existing output + reindex +# ./scripts/update-usc.sh --dry-run # Print plan, exit 0 +# +# Modes: +# incremental Default. Compares latest OLRC release point to .usc-release-point. +# bootstrap No checkpoint at downloads/usc/.usc-release-point. Treated as new release point. +# force --force: skip release-point check, always download + reconvert. # # Steps: # 1. Check latest OLRC release point against stored checkpoint @@ -44,6 +51,8 @@ FORCE=false SKIP_DEPLOY=false DEPLOY_ONLY=false SKIP_HIGHLIGHTS=false +SKIP_SEARCH=false +DRY_RUN=false LATEST="" while [[ $# -gt 0 ]]; do @@ -64,26 +73,69 @@ while [[ $# -gt 0 ]]; do SKIP_HIGHLIGHTS=true shift ;; + --skip-search) + SKIP_SEARCH=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 exit 1 fi CLI="node packages/cli/dist/index.js" CHECKPOINT="downloads/usc/.usc-release-point" +# --- Resolve mode --- + +MODE="" +if [ "$DEPLOY_ONLY" = true ]; then + MODE="deploy-only" +elif [ "$FORCE" = true ]; then + MODE="force" +elif [ ! -f "$CHECKPOINT" ]; then + MODE="bootstrap" +else + MODE="incremental" +fi + +# --- Print plan and exit on --dry-run --- + +print_plan() { + echo "==> USC update plan" + echo " Mode: $MODE" + echo " Force: $FORCE" + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Skip highlights: $SKIP_HIGHLIGHTS" + echo " Checkpoint: $CHECKPOINT" + if [ -f "$CHECKPOINT" ]; then + echo " Stored RP: $(cat "$CHECKPOINT")" + else + echo " Stored RP: (none — bootstrap)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + exit 0 +fi + # --- Preflight checks --- if [ ! -f "packages/cli/dist/index.js" ]; then @@ -137,6 +189,9 @@ if [ "$DEPLOY_ONLY" = false ]; then if [ -n "$STORED" ] && [ "$LATEST" != "$STORED" ]; then echo " New release point: $LATEST (was $STORED)" + elif [ -z "$STORED" ]; then + echo " Bootstrap: no checkpoint at $CHECKPOINT — performing full first-run download." + echo " Release point: $LATEST" else echo " Release point: $LATEST" fi @@ -261,8 +316,12 @@ echo "" # restart-storms under memory pressure). The deploy script handles: local # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. -echo "--- Step 9/9: Building and shipping search index via local Docker" -"$SCRIPT_DIR/deploy.sh" --search-docker --source usc +if [ "$SKIP_SEARCH" = true ]; then + echo "--- Skipping search index step (--skip-search)" +else + echo "--- Step 9/9: Building and shipping search index via local Docker" + "$SCRIPT_DIR/deploy.sh" --search-docker --source usc +fi echo "" if [ -n "$LATEST" ]; then diff --git a/scripts/update.sh b/scripts/update.sh index c7ff70d..2c4674b 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,39 +1,80 @@ #!/usr/bin/env bash # update.sh — Unified content update orchestrator for all sources. # -# Usage: -# ./scripts/update.sh # All sources incrementally -# ./scripts/update.sh --source ecfr # One source -# ./scripts/update.sh --source ecfr,fr # Multiple sources -# ./scripts/update.sh --skip-deploy # All sources, local only -# ./scripts/update.sh --deploy-only # Push existing output for all sources +# Default behavior: +# ./scripts/update.sh Update all sources (eCFR → FR → USC) incrementally +# from each source's last checkpoint. Painless. # -# Source-specific pass-through args: -# --ecfr-titles 1,17 Pass --titles 1,17 to update-ecfr.sh -# --ecfr-all Pass --all to update-ecfr.sh -# --ecfr-skip-highlights Pass --skip-highlights to update-ecfr.sh -# --fr-days 3 Pass --days 3 to update-fr.sh -# --fr-from YYYY-MM-DD Pass --from to update-fr.sh -# --fr-to YYYY-MM-DD Pass --to to update-fr.sh -# --usc-force Pass --force to update-usc.sh -# --usc-skip-highlights Pass --skip-highlights to update-usc.sh +# Common usage: +# ./scripts/update.sh --source fr Only FR, incremental +# ./scripts/update.sh --source ecfr,fr Multi-source, incremental +# ./scripts/update.sh --source ecfr --titles 1,17 eCFR titles 1, 17 only +# ./scripts/update.sh --source fr --days 7 FR last 7 days +# ./scripts/update.sh --source usc --force USC full redownload + reconvert +# ./scripts/update.sh --force --from 2026-01-01 All sources, full rebuild (FR requires --from) +# ./scripts/update.sh --skip-deploy All sources incrementally, local only +# ./scripts/update.sh --deploy-only Push existing local output + reindex +# ./scripts/update.sh --dry-run Print plan, exit 0 # -# Execution order: eCFR → FR → USC -# Each source's script handles its own change detection and early exit. +# Flags (logical groups): +# +# SOURCE SELECTION +# --source Comma-separated: usc, ecfr, fr, all (default: all) +# +# MODE (mutually exclusive) +# (default) Incremental from each source's checkpoint +# --force Full redownload + reconvert for selected source(s). +# FR force requires --from. +# --deploy-only Skip download/convert. Push existing output + reindex. +# +# PIPELINE PHASE CONTROL +# --skip-deploy Local pipeline only (no rsync, no search reindex) +# --skip-highlights Skip highlight generation (USC, eCFR) +# --skip-search Skip search reindex (still rsync content/nav/sitemaps) +# +# SOURCE-SPECIFIC SCOPING +# --titles eCFR/USC only. "1", "1-5", "1,3,8", "1-5,8,11" +# --from FR only. Override checkpoint-derived start date. +# --to FR only. Defaults to today. +# --days FR only. Last N days. Mutually exclusive with --from/--to. +# +# UTILITY +# --dry-run Print plan, exit 0. No network, no file writes. +# -v, --verbose Verbose output. +# -h, --help Show usage. +# +# Execution order: eCFR → FR → USC. Each source's sub-script handles its own +# change detection and early-exits if nothing's new. Sitemap regeneration is +# deferred to a single post-run step to avoid 3x redundant work. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # --- Parse arguments --- SOURCES="" -SHARED_ARGS="" -SKIP_DEPLOY=false +FORCE=false DEPLOY_ONLY=false -ECFR_ARGS="" -FR_ARGS="" -USC_ARGS="" +SKIP_DEPLOY=false +SKIP_HIGHLIGHTS=false +SKIP_SEARCH=false +TITLES="" +FROM="" +TO="" +DAYS="" +DRY_RUN=false +VERBOSE=false + +# Migration helper for removed flags. +removed_flag() { + local old="$1" + local hint="$2" + echo "Error: $old has been removed in the unified flag scheme." >&2 + echo " Use: $hint" >&2 + exit 1 +} while [[ $# -gt 0 ]]; do case "$1" in @@ -41,86 +82,237 @@ while [[ $# -gt 0 ]]; do SOURCES="$2" shift 2 ;; - --skip-deploy) - SHARED_ARGS="$SHARED_ARGS $1" - SKIP_DEPLOY=true + --force) + FORCE=true shift ;; --deploy-only) - SHARED_ARGS="$SHARED_ARGS $1" DEPLOY_ONLY=true shift ;; - # eCFR pass-through - --ecfr-titles) - ECFR_ARGS="$ECFR_ARGS --titles $2" - shift 2 + --skip-deploy) + SKIP_DEPLOY=true + shift ;; - --ecfr-all) - ECFR_ARGS="$ECFR_ARGS --all" + --skip-highlights) + SKIP_HIGHLIGHTS=true shift ;; - --ecfr-skip-highlights) - ECFR_ARGS="$ECFR_ARGS --skip-highlights" + --skip-search) + SKIP_SEARCH=true shift ;; - # FR pass-through - --fr-days) - FR_ARGS="$FR_ARGS --days $2" + --titles) + TITLES="$2" shift 2 ;; - --fr-from) - FR_ARGS="$FR_ARGS --from $2" + --from) + FROM="$2" shift 2 ;; - --fr-to) - FR_ARGS="$FR_ARGS --to $2" + --to) + TO="$2" shift 2 ;; - # USC pass-through - --usc-force) - USC_ARGS="$USC_ARGS --force" + --days) + DAYS="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true shift ;; - --usc-skip-highlights) - USC_ARGS="$USC_ARGS --skip-highlights" + -v|--verbose) + VERBOSE=true shift ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; + + # --- Migration errors for removed flags --- + --ecfr-titles) + removed_flag "--ecfr-titles" "./scripts/update.sh --source ecfr --titles $2" + ;; + --ecfr-all) + removed_flag "--ecfr-all" "./scripts/update.sh --source ecfr --force" + ;; + --ecfr-skip-highlights) + removed_flag "--ecfr-skip-highlights" "./scripts/update.sh --source ecfr --skip-highlights" + ;; + --fr-days) + removed_flag "--fr-days" "./scripts/update.sh --source fr --days $2" + ;; + --fr-from) + removed_flag "--fr-from" "./scripts/update.sh --source fr --from $2" + ;; + --fr-to) + removed_flag "--fr-to" "./scripts/update.sh --source fr --to $2" + ;; + --usc-force) + removed_flag "--usc-force" "./scripts/update.sh --source usc --force" + ;; + --usc-skip-highlights) + removed_flag "--usc-skip-highlights" "./scripts/update.sh --source usc --skip-highlights" + ;; + *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done +# --- Validate --- + if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 + exit 1 +fi + +if [ "$FORCE" = true ] && [ "$DEPLOY_ONLY" = true ]; then + echo "Error: --force and --deploy-only are mutually exclusive." >&2 + exit 1 +fi + +if [ -n "$DAYS" ] && { [ -n "$FROM" ] || [ -n "$TO" ]; }; then + echo "Error: --days is mutually exclusive with --from / --to." >&2 exit 1 fi # Default: all sources -if [ -z "$SOURCES" ]; then +if [ -z "$SOURCES" ] || [ "$SOURCES" = "all" ]; then SOURCES="ecfr,fr,usc" fi +# Validate source names +IFS=',' read -ra SOURCE_LIST <<< "$SOURCES" +for src in "${SOURCE_LIST[@]}"; do + case "$src" in + usc|ecfr|fr) ;; + *) + echo "Error: unknown source '$src'. Valid: usc, ecfr, fr, all." >&2 + exit 1 + ;; + esac +done + should_run() { echo "$SOURCES" | grep -qw "$1" } -FAILED="" -SUCCEEDED="" +# Flag/source compatibility checks. +if [ -n "$TITLES" ]; then + if should_run "fr" && ! should_run "usc" && ! should_run "ecfr"; then + echo "Error: --titles requires --source to include usc or ecfr." >&2 + exit 1 + fi +fi -# Defer sitemap regeneration to a single post-run step below. Without this, -# each source script would regenerate the full sitemap index (covering every -# source's URLs) and rsync it — 3x redundant work on a full update run. -export LEXBUILD_DEFER_SITEMAP=1 +if [ -n "$DAYS" ] || [ -n "$FROM" ] || [ -n "$TO" ]; then + if ! should_run "fr"; then + echo "Error: --days / --from / --to require --source to include fr." >&2 + exit 1 + fi +fi -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +# FR force requires explicit window. +if [ "$FORCE" = true ] && should_run "fr" && [ -z "$FROM" ] && [ -z "$DAYS" ]; then + echo "Error: --force on FR requires --from YYYY-MM-DD or --days N." >&2 + echo " FR has no inherent 'all' (decades of documents). Specify a window." >&2 + exit 2 +fi + +# When --force runs against all sources, do a single full search reindex at the +# end instead of three per-source incremental indexes. We achieve this by +# passing --skip-search to each sub-script and calling deploy.sh --search-docker +# (no --source) once after sub-scripts complete. +RUN_FULL_SEARCH_AFTER=false +if [ "$FORCE" = true ] && [ "$SKIP_SEARCH" = false ] && [ "$SKIP_DEPLOY" = false ] && [ "$DEPLOY_ONLY" = false ]; then + # All three sources, force, want search → full rebuild + if should_run "ecfr" && should_run "fr" && should_run "usc"; then + RUN_FULL_SEARCH_AFTER=true + fi +fi + +# --- Print plan --- + +print_plan() { + echo "==> Update plan" + echo " Sources: $SOURCES" + if [ "$DEPLOY_ONLY" = true ]; then + echo " Mode: deploy-only" + elif [ "$FORCE" = true ]; then + echo " Mode: force (full rebuild)" + else + echo " Mode: incremental" + fi + if [ -n "$TITLES" ]; then echo " Titles: $TITLES"; fi + if [ -n "$DAYS" ]; then echo " Days: $DAYS"; fi + if [ -n "$FROM" ]; then echo " From: $FROM"; fi + if [ -n "$TO" ]; then echo " To: $TO"; fi + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Skip highlights: $SKIP_HIGHLIGHTS" + if [ "$RUN_FULL_SEARCH_AFTER" = true ]; then + echo " Search strategy: full reindex after sub-scripts (force on all sources)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + echo "" + echo " (--dry-run: forwarding --dry-run to each sub-script for plan details)" + echo "" + # Run each sub-script's dry-run; don't let one source's "would-error" halt the + # rest of the preview. set -e is inherited; use `|| true` to continue. + for src in "${SOURCE_LIST[@]}"; do + case "$src" in + ecfr) + # shellcheck disable=SC2086 + "$SCRIPT_DIR/update-ecfr.sh" --dry-run \ + $( [ -n "$TITLES" ] && echo "--titles $TITLES" ) \ + $( [ "$FORCE" = true ] && echo "--force" ) \ + $( [ "$SKIP_DEPLOY" = true ] && echo "--skip-deploy" ) \ + $( [ "$SKIP_SEARCH" = true ] && echo "--skip-search" ) \ + $( [ "$SKIP_HIGHLIGHTS" = true ] && echo "--skip-highlights" ) \ + $( [ "$DEPLOY_ONLY" = true ] && echo "--deploy-only" ) || true + ;; + fr) + # shellcheck disable=SC2086 + "$SCRIPT_DIR/update-fr.sh" --dry-run \ + $( [ -n "$DAYS" ] && echo "--days $DAYS" ) \ + $( [ -n "$FROM" ] && echo "--from $FROM" ) \ + $( [ -n "$TO" ] && echo "--to $TO" ) \ + $( [ "$FORCE" = true ] && echo "--force" ) \ + $( [ "$SKIP_DEPLOY" = true ] && echo "--skip-deploy" ) \ + $( [ "$SKIP_SEARCH" = true ] && echo "--skip-search" ) \ + $( [ "$DEPLOY_ONLY" = true ] && echo "--deploy-only" ) || true + ;; + usc) + # shellcheck disable=SC2086 + "$SCRIPT_DIR/update-usc.sh" --dry-run \ + $( [ "$FORCE" = true ] && echo "--force" ) \ + $( [ "$SKIP_DEPLOY" = true ] && echo "--skip-deploy" ) \ + $( [ "$SKIP_SEARCH" = true ] && echo "--skip-search" ) \ + $( [ "$SKIP_HIGHLIGHTS" = true ] && echo "--skip-highlights" ) \ + $( [ "$DEPLOY_ONLY" = true ] && echo "--deploy-only" ) || true + ;; + esac + echo "" + done + exit 0 +fi + +print_plan +echo "" + +# --- Run sources --- + +# Defer sitemap regeneration to a single post-run step below. +export LEXBUILD_DEFER_SITEMAP=1 # Load deploy config (for final sitemap rsync below) if [ -f "$SCRIPT_DIR/.deploy.env" ]; then @@ -128,51 +320,75 @@ if [ -f "$SCRIPT_DIR/.deploy.env" ]; then source "$SCRIPT_DIR/.deploy.env" fi -# --- Run sources --- +FAILED="" +SUCCEEDED="" -if should_run "ecfr"; then - echo "===== eCFR Update =====" +# When RUN_FULL_SEARCH_AFTER is true, suppress per-source search calls. +EFFECTIVE_SKIP_SEARCH="$SKIP_SEARCH" +if [ "$RUN_FULL_SEARCH_AFTER" = true ]; then + EFFECTIVE_SKIP_SEARCH=true +fi + +run_source() { + local src="$1" + local script="$2" + shift 2 + local args=("$@") + + echo "===== ${src} Update =====" echo "" - if "$SCRIPT_DIR/update-ecfr.sh" $ECFR_ARGS $SHARED_ARGS; then - SUCCEEDED="$SUCCEEDED ecfr" + if "$script" "${args[@]}"; then + SUCCEEDED="$SUCCEEDED $src" echo "" else - ECFR_EXIT=$? + local rc=$? echo "" - echo "WARNING: eCFR update failed (exit code $ECFR_EXIT)" - FAILED="$FAILED ecfr" + echo "WARNING: $src update failed (exit code $rc)" + FAILED="$FAILED $src" echo "" fi +} + +build_common_args() { + local args=() + [ "$FORCE" = true ] && args+=(--force) + [ "$SKIP_DEPLOY" = true ] && args+=(--skip-deploy) + [ "$EFFECTIVE_SKIP_SEARCH" = true ] && args+=(--skip-search) + [ "$SKIP_HIGHLIGHTS" = true ] && args+=(--skip-highlights) + [ "$DEPLOY_ONLY" = true ] && args+=(--deploy-only) + [ "$VERBOSE" = true ] && args+=(--verbose) + printf '%s\n' "${args[@]}" +} + +# eCFR +if should_run "ecfr"; then + ECFR_ARGS=() + [ -n "$TITLES" ] && ECFR_ARGS+=(--titles "$TITLES") + while IFS= read -r a; do [ -n "$a" ] && ECFR_ARGS+=("$a"); done < <(build_common_args) + run_source "eCFR" "$SCRIPT_DIR/update-ecfr.sh" "${ECFR_ARGS[@]}" fi +# FR if should_run "fr"; then - echo "===== FR Update =====" - echo "" - if "$SCRIPT_DIR/update-fr.sh" $FR_ARGS $SHARED_ARGS; then - SUCCEEDED="$SUCCEEDED fr" - echo "" - else - FR_EXIT=$? - echo "" - echo "WARNING: FR update failed (exit code $FR_EXIT)" - FAILED="$FAILED fr" - echo "" - fi + FR_ARGS=() + [ -n "$DAYS" ] && FR_ARGS+=(--days "$DAYS") + [ -n "$FROM" ] && FR_ARGS+=(--from "$FROM") + [ -n "$TO" ] && FR_ARGS+=(--to "$TO") + while IFS= read -r a; do [ -n "$a" ] && FR_ARGS+=("$a"); done < <(build_common_args) + # FR doesn't support --skip-highlights (no highlights step). Strip it. + filtered_fr=() + for arg in "${FR_ARGS[@]}"; do + [ "$arg" = "--skip-highlights" ] && continue + filtered_fr+=("$arg") + done + run_source "FR" "$SCRIPT_DIR/update-fr.sh" "${filtered_fr[@]}" fi +# USC if should_run "usc"; then - echo "===== USC Update =====" - echo "" - if "$SCRIPT_DIR/update-usc.sh" $USC_ARGS $SHARED_ARGS; then - SUCCEEDED="$SUCCEEDED usc" - echo "" - else - USC_EXIT=$? - echo "" - echo "WARNING: USC update failed (exit code $USC_EXIT)" - FAILED="$FAILED usc" - echo "" - fi + USC_ARGS=() + while IFS= read -r a; do [ -n "$a" ] && USC_ARGS+=("$a"); done < <(build_common_args) + run_source "USC" "$SCRIPT_DIR/update-usc.sh" "${USC_ARGS[@]}" fi # --- Post-run: regenerate sitemap index once, covering all sources --- @@ -185,20 +401,33 @@ if [ -n "$SUCCEEDED" ] && [ "$SKIP_DEPLOY" = false ]; then FAILED="$FAILED sitemap" } - if [ "$DEPLOY_ONLY" = true ] || [ "$SKIP_DEPLOY" = false ]; then - if [ -n "${VPS_HOST:-}" ]; then - SITEMAP_FILES=("$REPO_ROOT/apps/astro/public"/sitemap*.xml) - [ -f "$REPO_ROOT/apps/astro/public/robots.txt" ] && SITEMAP_FILES+=("$REPO_ROOT/apps/astro/public/robots.txt") - if [ ${#SITEMAP_FILES[@]} -gt 0 ] && [ -e "${SITEMAP_FILES[0]}" ]; then - echo " Syncing sitemaps to VPS" - rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/public/" || FAILED="$FAILED sitemap-rsync" - rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/dist/client/" || FAILED="$FAILED sitemap-rsync" - fi + if [ -n "${VPS_HOST:-}" ]; then + SITEMAP_FILES=("$REPO_ROOT/apps/astro/public"/sitemap*.xml) + [ -f "$REPO_ROOT/apps/astro/public/robots.txt" ] && SITEMAP_FILES+=("$REPO_ROOT/apps/astro/public/robots.txt") + if [ ${#SITEMAP_FILES[@]} -gt 0 ] && [ -e "${SITEMAP_FILES[0]}" ]; then + echo " Syncing sitemaps to VPS" + rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/public/" || FAILED="$FAILED sitemap-rsync" + rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/dist/client/" || FAILED="$FAILED sitemap-rsync" fi fi echo "" fi +# --- Post-run: full search reindex (only when --force on all sources) --- + +if [ "$RUN_FULL_SEARCH_AFTER" = true ] && [ -n "$SUCCEEDED" ]; then + echo "===== Full search reindex (Docker) =====" + echo "" + if "$SCRIPT_DIR/deploy.sh" --search-docker; then + echo "" + else + echo "" + echo "WARNING: full search reindex failed" + FAILED="$FAILED search-full" + echo "" + fi +fi + # --- Summary --- if [ -n "$FAILED" ]; then From 8d5805a425177e9a1a55faab2ffffa274a86c1a3 Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 17:59:23 -0500 Subject: [PATCH 2/8] docs: rewrite update-script references for new flag scheme - Root CLAUDE.md: rewrite the script tree and Build & Dev Commands section to reflect the unified grammar; add checkpoint-file docs and bootstrap rules. - Root README.md: rewrite the Incremental Updates section. - Root CHANGELOG.md: add Unreleased entry describing the rename. - apps/astro/src/content/docs/cli/commands.md: replace old flag examples in the Update Scripts subsection. - apps/astro/src/content/docs/guides/bulk-download.md: rewrite the Update Scripts subsection with checkpoint + bootstrap details. - apps/astro/src/content/docs/project/changelog.md: mirror Unreleased entry. --- CHANGELOG.md | 10 ++++ CLAUDE.md | 54 +++++++++++++------ README.md | 27 +++++++--- apps/astro/src/content/docs/cli/commands.md | 17 +++--- .../src/content/docs/guides/bulk-download.md | 34 ++++++++---- .../src/content/docs/project/changelog.md | 7 +++ 6 files changed, 110 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6707304..f10da3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). +## [Unreleased] + +### Changed + +- Unify the update-script flag scheme. `./scripts/update.sh` (no args) now updates all sources incrementally from each source's last checkpoint. Source restriction lives on `--source`; source-scoping flags (`--titles`, `--days`, `--from`, `--to`) live at the top level instead of per-source prefixes. Old prefixes (`--ecfr-titles`, `--ecfr-all`, `--ecfr-skip-highlights`, `--fr-days`, `--fr-from`, `--fr-to`, `--usc-force`, `--usc-skip-highlights`) are removed; running them prints a migration hint and exits 1. `update-ecfr.sh --all` similarly removed in favor of `--force`. +- Add `--skip-search`, `--dry-run`, and consistent `--force` semantics to all four scripts. +- `update-fr.sh` now persists a JSON checkpoint at `downloads/fr/.fr-state.json` (`{ lastRun, lastDate }`). Default invocations resume from `lastDate`. Bootstrap (no checkpoint) errors with a hint requiring `--from` or `--days`, since FR has no inherent "all". +- eCFR/USC bootstrap (missing `.ecfr-titles-state.json` / `.usc-release-point`) now logs the bootstrap explicitly and falls back to a full first-run automatically. +- When `--force` runs against all three sources, the orchestrator now performs a single full `deploy.sh --search-docker` reindex at the end instead of three per-source incremental indexes. + ## [1.25.0] ### Added diff --git a/CLAUDE.md b/CLAUDE.md index f6fac5e..8b20b8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,19 +19,22 @@ lexbuild/ │ └── api/ # @lexbuild/api — Data API (Hono, SQLite, Meilisearch proxy) ├── scripts/ │ ├── deploy.sh # Production deploy (code, content, or full remote pipeline) -│ ├── update.sh # Unified incremental content update (all sources) -│ ├── update-ecfr.sh # eCFR incremental update (auto-detects changed titles) -│ ├── update-fr.sh # FR incremental update (date-range based) -│ ├── update-usc.sh # USC incremental update (release point detection) +│ ├── update.sh # Unified content update orchestrator (incremental by default) +│ ├── update-ecfr.sh # eCFR sub-script (change-detection via API metadata) +│ ├── update-fr.sh # FR sub-script (checkpoint-based date window) +│ ├── update-usc.sh # USC sub-script (release-point detection) │ ├── ecfr-changed-titles.ts # eCFR change detection helper (API metadata vs checkpoint) │ ├── setup-secrets.sh # Initialize ~/.lexbuild-secrets on VPS │ └── .deploy.env.example # Template for .deploy.env (VPS_HOST config) ├── downloads/ │ ├── usc/ -│ │ └── xml/ # Full USC XML files (usc01.xml ... usc54.xml) — gitignored +│ │ ├── xml/ # Full USC XML files (usc01.xml ... usc54.xml) — gitignored +│ │ └── .usc-release-point # USC checkpoint (latest OLRC release point ID) │ ├── ecfr/ -│ │ └── xml/ # Full eCFR XML files (ECFR-title1.xml ... ECFR-title50.xml) — gitignored -│ └── fr/ # FR XML + JSON files (YYYY/MM/doc-number.xml/.json) — gitignored +│ │ ├── xml/ # Full eCFR XML files (ECFR-title1.xml ... ECFR-title50.xml) — gitignored +│ │ └── .ecfr-titles-state.json # eCFR checkpoint (per-title amendment dates) +│ └── fr/ # FR XML + JSON files (YYYY/MM/doc-number.xml/.json) — gitignored +│ └── .fr-state.json # FR checkpoint ({ lastRun, lastDate }) ├── fixtures/ │ ├── fragments/ # Small synthetic XML snippets for unit tests │ └── expected/ # Expected output snapshots for integration tests @@ -129,18 +132,35 @@ pnpm turbo build:api --filter=@lexbuild/api # Production build ./scripts/deploy.sh --search-docker --source fr # Incremental: index one source into existing volume ./scripts/deploy.sh --search-docker-seed # Seed Docker volume from VPS (recover after volume loss) -# Incremental content updates (from monorepo root) -# Search indexing runs locally in Docker, not on the VPS — each update script's -# final step delegates to `deploy.sh --search-docker --source `. -./scripts/update.sh # All sources incrementally -./scripts/update.sh --source ecfr # One source -./scripts/update.sh --skip-deploy # Local only -./scripts/update-ecfr.sh # eCFR only (auto-detects changed titles) -./scripts/update-ecfr.sh --titles 1,17 # Specific eCFR titles -./scripts/update-fr.sh --days 3 # FR last 3 days -./scripts/update-usc.sh # USC (checks for new release point) +# Content updates (from monorepo root) +# Default behavior is incremental from each source's checkpoint. Search indexing +# runs locally in Docker; each sub-script delegates to `deploy.sh --search-docker +# --source ` after the local pipeline. +./scripts/update.sh # All sources, incremental from checkpoints +./scripts/update.sh --source fr # One source, incremental +./scripts/update.sh --source ecfr,fr # Multi-source, incremental +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only (skip change-detection) +./scripts/update.sh --source fr --days 7 # FR last 7 days +./scripts/update.sh --source usc --force # USC full redownload + reconvert +./scripts/update.sh --force --from 2026-01-01 # All sources, full rebuild (FR force requires --from) +./scripts/update.sh --skip-deploy # Local only (no rsync, no search) +./scripts/update.sh --skip-search # Rsync content/nav, but skip search reindex +./scripts/update.sh --deploy-only # Push existing local output + reindex +./scripts/update.sh --dry-run # Print plan, exit 0 ``` +Sub-scripts (`update-ecfr.sh`, `update-fr.sh`, `update-usc.sh`) accept the same flag grammar +minus `--source`. Run any of them with `--help` for the full list. + +**Checkpoints** (gitignored, in `downloads//`): +- `usc/.usc-release-point` — latest OLRC release point ID (plain text). +- `ecfr/.ecfr-titles-state.json` — per-title `latestAmendedOn` snapshot. +- `fr/.fr-state.json` — `{ lastRun, lastDate }`. Default `update-fr.sh` uses `lastDate` as `--from`. + +If a checkpoint is missing, eCFR/USC bootstrap into a full first-run automatically. FR bootstrap +errors with a hint; you must specify `--from YYYY-MM-DD` or `--days N` because FR has no inherent +"all" (decades of documents). + See `packages/cli/CLAUDE.md` for full command options. See `apps/astro/CLAUDE.md` for content pipeline scripts. ### CI / Release diff --git a/README.md b/README.md index ab0545a..ddf3903 100644 --- a/README.md +++ b/README.md @@ -131,18 +131,33 @@ lexbuild enrich-fr --from 2000-01-01 ### Incremental Updates -Update scripts handle change detection, download, convert, and deploy in one command: +A single script handles change detection, download, convert, and deploy across every source. The default is incremental from each source's last checkpoint: ```bash -# Update all sources (auto-detects changes) +# Painless: update everything from each source's checkpoint +./scripts/update.sh + +# Restrict to one source (or several) +./scripts/update.sh --source fr +./scripts/update.sh --source ecfr,fr + +# Source-scoping flags +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only +./scripts/update.sh --source fr --days 7 # FR last 7 days + +# Force a full redownload + reconvert +./scripts/update.sh --source usc --force +./scripts/update.sh --force --from 2026-01-01 # All sources (FR force requires --from) + +# Local only (no rsync to VPS, no search reindex) ./scripts/update.sh --skip-deploy -# Update individual sources -./scripts/update-ecfr.sh --skip-deploy -./scripts/update-fr.sh --days 3 --skip-deploy -./scripts/update-usc.sh --skip-deploy +# Preview without running +./scripts/update.sh --dry-run ``` +Each source has a checkpoint in `downloads//`. eCFR/USC bootstrap automatically into a full first-run if their checkpoint is missing; FR errors with a hint and requires `--from`. Run `./scripts/update.sh --help` (or any sub-script with `--help`) for the full grammar. + `update-usc.sh` and `update-ecfr.sh` convert every granularity in one parse using the `--granularities` flag (see below), so the convert step no longer scales with the number of output granularities. --- diff --git a/apps/astro/src/content/docs/cli/commands.md b/apps/astro/src/content/docs/cli/commands.md index 082175c..8514e81 100644 --- a/apps/astro/src/content/docs/cli/commands.md +++ b/apps/astro/src/content/docs/cli/commands.md @@ -127,17 +127,20 @@ lexbuild convert-fr --all ## Update Scripts -For routine updates, wrapper scripts handle the full pipeline (detect changes, download, convert, generate artifacts, deploy): +A single orchestrator handles change detection, download, convert, and deploy across all sources. Default is incremental from each source's last checkpoint: ```bash -./scripts/update.sh # All sources -./scripts/update.sh --source ecfr # One source -./scripts/update-ecfr.sh --skip-deploy # eCFR, local only -./scripts/update-fr.sh --days 3 # FR, last 3 days -./scripts/update-usc.sh # USC, checks release point +./scripts/update.sh # All sources, incremental from checkpoints +./scripts/update.sh --source fr # One source +./scripts/update.sh --source ecfr,fr # Multi-source +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only +./scripts/update.sh --source fr --days 7 # FR last 7 days +./scripts/update.sh --source usc --force # USC full redownload + reconvert +./scripts/update.sh --skip-deploy # Local pipeline only +./scripts/update.sh --dry-run # Print plan, exit 0 ``` -Each script auto-detects what changed and only processes updates. `update-usc.sh` and `update-ecfr.sh` convert all granularities in one call using `--granularities` (see above), so the convert step parses the XML once per title rather than once per granularity. See [Incremental Updates](/docs/guides/bulk-download#incremental-updates) for details. +`update-usc.sh` and `update-ecfr.sh` convert all granularities in one call using `--granularities` (see above), so the convert step parses the XML once per title rather than once per granularity. See [Incremental Updates](/docs/guides/bulk-download#incremental-updates) for details on checkpoints and bootstrap behavior. ## Getting Help diff --git a/apps/astro/src/content/docs/guides/bulk-download.md b/apps/astro/src/content/docs/guides/bulk-download.md index f588962..0e04299 100644 --- a/apps/astro/src/content/docs/guides/bulk-download.md +++ b/apps/astro/src/content/docs/guides/bulk-download.md @@ -143,24 +143,40 @@ lexbuild convert-fr --from 2026-03-01 ### Update Scripts -For streamlined incremental updates, wrapper scripts handle the full pipeline (detect changes, download, convert, generate artifacts, deploy): +A single orchestrator handles change detection, download, convert, and deploy across every source. Default is incremental from each source's last checkpoint: ```bash -# All sources — auto-detects changes, downloads, converts, deploys +# All sources, incremental from each source's checkpoint ./scripts/update.sh -# Individual sources -./scripts/update-ecfr.sh # Only changed titles (via API metadata) -./scripts/update-fr.sh --days 3 # Last 3 days -./scripts/update-usc.sh # Checks OLRC release point +# Restrict to specific sources +./scripts/update.sh --source fr +./scripts/update.sh --source ecfr,fr -# Local only (no VPS deploy) +# Source-scoping +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only +./scripts/update.sh --source fr --days 7 # FR last 7 days + +# Force a full redownload + reconvert +./scripts/update.sh --source usc --force +./scripts/update.sh --force --from 2026-01-01 # All sources (FR force requires --from) + +# Local only (no VPS deploy, no search reindex) ./scripts/update.sh --skip-deploy + +# Preview without running +./scripts/update.sh --dry-run ``` -The eCFR script compares `latestAmendedOn` dates from the eCFR API against a local checkpoint to detect which titles have new amendments. The USC script checks for new OLRC release points. The FR script uses date-range filtering. +Checkpoints live in `downloads//`: + +- **eCFR** (`.ecfr-titles-state.json`) snapshots each title's `latestAmendedOn` date. The script compares against the live eCFR API to detect which titles have new amendments. +- **USC** (`.usc-release-point`) stores the latest OLRC release point ID; the pipeline runs only when the API returns a newer one. +- **FR** (`.fr-state.json`) stores `lastRun` and `lastDate`. Default invocations use `lastDate` as the `--from` argument and update `lastDate` to today after a successful run. + +If a checkpoint is missing, eCFR/USC bootstrap into a full first-run automatically. FR has no inherent "all" (decades of documents), so a missing checkpoint requires explicit `--from YYYY-MM-DD` or `--days N`. -All converters use `writeFileIfChanged()` internally, so unchanged sections keep their original file timestamps. This means downstream tools (Shiki highlighting, Meilisearch indexing) automatically skip reprocessing unchanged content. +All converters use `writeFileIfChanged()` internally, so unchanged sections keep their original file timestamps. Downstream tools (Shiki highlighting, Meilisearch indexing) automatically skip reprocessing unchanged content. ## Output Granularity diff --git a/apps/astro/src/content/docs/project/changelog.md b/apps/astro/src/content/docs/project/changelog.md index 2f70edb..2e133a6 100644 --- a/apps/astro/src/content/docs/project/changelog.md +++ b/apps/astro/src/content/docs/project/changelog.md @@ -12,6 +12,13 @@ For the complete changelog, see [CHANGELOG.md on GitHub](https://github.com/chri ## Recent Releases +### Unreleased + +- Unified update-script flag scheme. `./scripts/update.sh` (no args) now updates all sources incrementally from each source's last checkpoint. Source restriction via `--source`; source-scoping flags (`--titles`, `--days`, `--from`, `--to`) live at the top level. Old prefixes (`--ecfr-titles`, `--fr-days`, `--usc-force`, etc.) are removed and print migration hints +- New `--skip-search`, `--dry-run`, and consistent `--force` semantics across all four scripts +- New FR checkpoint at `downloads/fr/.fr-state.json` (`{ lastRun, lastDate }`). Default `update-fr.sh` resumes from `lastDate`. Missing checkpoint errors with a hint requiring `--from` or `--days` +- eCFR/USC bootstrap (missing checkpoint) now logs explicitly and runs a full first-run automatically + ### 1.17.2 - Added **Data API** (`apps/api/`) -- Hono-based REST API serving U.S. legal content from SQLite with Meilisearch search proxy From 205170d99e308c1e81bb340f903e8513fc9484d8 Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 19:29:01 -0500 Subject: [PATCH 3/8] fix(scripts): bash 3.2 safety, FR backfill regression, --titles validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-review findings addressed: CRITICAL - update.sh: empty-array expansion crashed default ./scripts/update.sh under bash 3.2 (macOS) + set -u. `printf "%s\n" "${args[@]}"` and call sites expanding "${ECFR_ARGS[@]}" etc. all hit `args[@]: unbound variable`. Fixed via the bash 3.2-safe ${arr[@]+"${arr[@]}"} pattern; build_common_args now emits one flag per echo line instead of via an intermediate array. - update.sh: removed_flag arms referenced $2 unconditionally, so bare --ecfr-titles/--fr-days/--fr-from/--fr-to crashed with `$2: unbound variable` before the migration hint could print. Fixed with ${2:-}. DATA-LOSS - update-fr.sh: backfill (e.g. --from 2020-01-01 --to 2020-12-31) on a machine whose checkpoint already pointed at 2026-04-20 would overwrite the cursor with the older date and trigger a multi-year redownload on the next default run. Now writes max(CHECKPOINT_DATE, DATE_TO). ACCURACY / VALIDATION - update.sh: --titles validation now requires --source to include ecfr. USC sub-script doesn't accept --titles; previously --source usc --titles X passed through silently and ran the full USC pipeline with no filtering. - update.sh: help text and docs updated from "eCFR/USC only" to "eCFR only". - update-usc.sh: header step 9 now says "via local Docker (shipped to VPS)" matching update-ecfr.sh / update-fr.sh and the inline rationale. - update-fr.sh: step numbering aligned with the 8-step header (was N/6). ROBUSTNESS - update-fr.sh: checkpoint write now atomic (tmp + rename) so a partial- write crash can't corrupt .fr-state.json into something read_checkpoint_date fails to parse. - update-fr.sh: read_checkpoint_date now warns on stderr when the file exists but is malformed/missing-lastDate, instead of silently routing the user to a misleading "no checkpoint" bootstrap-error. - update-fr.sh: node -e snippets now pass paths via env vars instead of string-substitution, eliminating a single-quote injection risk. - update.sh: failure-summary line now goes to stderr; sub-script failure warning also routes to stderr. VERIFICATION - /bin/bash (3.2) ./scripts/update.sh --source ecfr --skip-deploy --skip-search → exit 0 (was: bash unbound-variable crash) - ./scripts/update.sh --ecfr-titles → migration error, exit 1 (was: crash) - ./scripts/update.sh --source usc --titles 1,17 → "--titles requires --source to include ecfr", exit 1 (was: silent full-USC run) - max-date logic unit-tested: backfill preserves checkpoint, forward advances it, bootstrap adopts date_to. - Malformed checkpoint produces explicit "Warning: ... is malformed" before bootstrap fallthrough. --- scripts/update-fr.sh | 94 +++++++++++++++++++++++++++++-------------- scripts/update-usc.sh | 2 +- scripts/update.sh | 55 ++++++++++++++----------- 3 files changed, 96 insertions(+), 55 deletions(-) diff --git a/scripts/update-fr.sh b/scripts/update-fr.sh index ea11070..4e745e4 100755 --- a/scripts/update-fr.sh +++ b/scripts/update-fr.sh @@ -117,19 +117,37 @@ fi # --- Resolve mode + date range --- -# read_checkpoint_date prints lastDate from .fr-state.json, or empty if missing/invalid +# read_checkpoint_date prints lastDate from .fr-state.json on stdout. +# Empty stdout means: file is missing OR malformed in some way. The two cases +# are distinguished for the user by emitting a Warning to stderr when the +# file exists but can't be parsed (so a corrupt checkpoint isn't surfaced as +# a misleading "no checkpoint" bootstrap-error). The path is passed via env +# rather than substituted into the JS source to avoid single-quote injection. read_checkpoint_date() { if [ ! -f "$CHECKPOINT_PATH" ]; then return 0 fi - node -e " + local result rc + result=$(CHECKPOINT_PATH="$CHECKPOINT_PATH" node -e ' try { - const d = JSON.parse(require('fs').readFileSync('$CHECKPOINT_PATH', 'utf-8')); - if (typeof d.lastDate === 'string' && /^\\d{4}-\\d{2}-\\d{2}$/.test(d.lastDate)) { + const d = JSON.parse(require("fs").readFileSync(process.env.CHECKPOINT_PATH, "utf-8")); + if (typeof d.lastDate === "string" && /^\d{4}-\d{2}-\d{2}$/.test(d.lastDate)) { process.stdout.write(d.lastDate); + } else { + process.stderr.write("lastDate missing or wrong shape"); + process.exit(3); } - } catch (_) { /* missing or malformed: silent */ } - " 2>/dev/null + } catch (e) { + process.stderr.write(e.message); + process.exit(3); + } + ' 2>&1) + rc=$? + if [ $rc -ne 0 ]; then + echo "Warning: $CHECKPOINT_PATH exists but is malformed ($result). Treating as missing." >&2 + return 0 + fi + printf '%s' "$result" } CHECKPOINT_DATE="$(read_checkpoint_date)" @@ -235,7 +253,7 @@ if [ "$DEPLOY_ONLY" = true ] && [ -z "${VPS_HOST:-}" ]; then exit 1 fi -# --- Step 1–6: Local pipeline (skip if --deploy-only) --- +# --- Steps 2–6: Local pipeline (skip if --deploy-only) --- if [ "$DEPLOY_ONLY" = false ]; then echo "==> FR update ($MODE: $DATE_FROM → $DATE_TO)" @@ -248,15 +266,15 @@ if [ "$DEPLOY_ONLY" = false ]; then PIPELINE_MARKER="$(mktemp -t lexbuild-fr-update.XXXXXX)" trap 'rm -f "$PIPELINE_MARKER"' EXIT - # Step 1: Download - echo "--- Step 1/6: Downloading FR documents ($DATE_FROM to $DATE_TO)" + # Step 2: Download + echo "--- Step 2/8: Downloading FR documents ($DATE_FROM to $DATE_TO)" $CLI download-fr --from "$DATE_FROM" --to "$DATE_TO" echo "" NEW_XML_COUNT=$(find downloads/fr -name "*.xml" -newer "$PIPELINE_MARKER" 2>/dev/null | wc -l | tr -d ' ') - # Step 2: Convert (date-filtered — only converts files in the date range) - echo "--- Step 2/6: Converting FR documents ($DATE_FROM to $DATE_TO)" + # Step 3: Convert (date-filtered — only converts files in the date range) + echo "--- Step 3/8: Converting FR documents ($DATE_FROM to $DATE_TO)" $CLI convert-fr --all --from "$DATE_FROM" --to "$DATE_TO" echo "" @@ -275,39 +293,55 @@ if [ "$DEPLOY_ONLY" = false ]; then echo "" fi - # Step 3: Regenerate FR nav - echo "--- Step 3/6: Generating FR nav JSON" + # Step 4: Regenerate FR nav + echo "--- Step 4/8: Generating FR nav JSON" ( cd apps/astro && npx tsx scripts/generate-nav.ts --source fr ) || exit 1 echo "" - # Step 4: Regenerate sitemaps (full rebuild to update sitemap index). + # Step 5: Regenerate sitemaps (full rebuild to update sitemap index). # Skipped when LEXBUILD_DEFER_SITEMAP=1 (set by update.sh to avoid # regenerating the full sitemap index once per source). if [ "${LEXBUILD_DEFER_SITEMAP:-}" != "1" ]; then - echo "--- Step 4/6: Generating sitemaps" + echo "--- Step 5/8: Generating sitemaps" ( cd apps/astro && npx tsx scripts/generate-sitemap.ts ) || exit 1 echo "" else - echo "--- Step 4/6: Skipping sitemap (deferred to update.sh)" + echo "--- Step 5/8: Skipping sitemap (deferred to update.sh)" echo "" fi - # Step 5: Write checkpoint with today's date as the new resume point. - # Using DATE_TO (which is today on default runs, or the explicit --to value) - # ensures the next default invocation resumes from a contiguous date. - echo "--- Step 5/6: Writing FR checkpoint ($DATE_TO)" + # Step 6: Write checkpoint as the new resume point. + # + # New lastDate = max(CHECKPOINT_DATE, DATE_TO). Without the max guard, a + # historical backfill (e.g. `--from 2020-01-01 --to 2020-12-31` on a machine + # whose checkpoint is already 2026-04-20) would regress the resume cursor to + # 2020-12-31 and trigger a multi-year redownload on the next default run. + # Atomic write (tmp + rename) prevents a partial-write crash from corrupting + # .fr-state.json into something read_checkpoint_date can't parse. + # + # Intentionally inside the DEPLOY_ONLY=false block: --deploy-only pushes + # existing output without ingesting anything new, so we must not advance + # the cursor past data we haven't actually downloaded. + NEW_LAST_DATE="$DATE_TO" + if [ -n "$CHECKPOINT_DATE" ] && [ "$CHECKPOINT_DATE" \> "$NEW_LAST_DATE" ]; then + NEW_LAST_DATE="$CHECKPOINT_DATE" + fi + echo "--- Step 6/8: Writing FR checkpoint (lastDate=$NEW_LAST_DATE)" mkdir -p "$(dirname "$CHECKPOINT_PATH")" - node -e " - const fs = require('fs'); - fs.writeFileSync('$CHECKPOINT_PATH', JSON.stringify({ + CHECKPOINT_PATH="$CHECKPOINT_PATH" NEW_LAST_DATE="$NEW_LAST_DATE" node -e ' + const fs = require("fs"); + const path = process.env.CHECKPOINT_PATH; + const tmp = path + ".tmp." + process.pid; + fs.writeFileSync(tmp, JSON.stringify({ lastRun: new Date().toISOString(), - lastDate: '$DATE_TO', - }, null, 2) + '\\n'); - " + lastDate: process.env.NEW_LAST_DATE, + }, null, 2) + "\n"); + fs.renameSync(tmp, path); + ' || { echo "Error: failed to write FR checkpoint to $CHECKPOINT_PATH" >&2; exit 1; } echo "" fi -# --- Step 7–8: Deploy to VPS (skip if --skip-deploy) --- +# --- Steps 7–8: Deploy to VPS (skip if --skip-deploy) --- if [ "$SKIP_DEPLOY" = true ]; then echo "==> Local pipeline complete (--skip-deploy). Files ready in output/fr/" @@ -315,7 +349,7 @@ if [ "$SKIP_DEPLOY" = true ]; then fi # Step 7: Rsync content + nav + sitemaps -echo "--- Step 6/6 (deploy): Syncing to VPS" +echo "--- Step 7/8: Syncing to VPS" if [ -d "output/fr" ]; then echo " FR documents" @@ -348,9 +382,9 @@ echo "" # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. if [ "$SKIP_SEARCH" = true ]; then - echo "--- Skipping search index step (--skip-search)" + echo "--- Step 8/8: Skipping search index step (--skip-search)" else - echo "--- Building and shipping search index via local Docker" + echo "--- Step 8/8: Building and shipping search index via local Docker" "$SCRIPT_DIR/deploy.sh" --search-docker --source fr fi diff --git a/scripts/update-usc.sh b/scripts/update-usc.sh index 9c0f1c3..cf77c35 100755 --- a/scripts/update-usc.sh +++ b/scripts/update-usc.sh @@ -24,7 +24,7 @@ # 6. Regenerate sitemaps # 7. Save new release point # 8. Rsync content (all granularities) + nav + sitemaps to VPS -# 9. Incremental search index on VPS +# 9. Incremental search index via local Docker (shipped to VPS) # # Requires: # - Built CLI: pnpm turbo build (or at least @lexbuild/usc + @lexbuild/cli) diff --git a/scripts/update.sh b/scripts/update.sh index 2c4674b..49b7cae 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -33,7 +33,7 @@ # --skip-search Skip search reindex (still rsync content/nav/sitemaps) # # SOURCE-SPECIFIC SCOPING -# --titles eCFR/USC only. "1", "1-5", "1,3,8", "1-5,8,11" +# --titles eCFR only. "1", "1-5", "1,3,8", "1-5,8,11" # --from FR only. Override checkpoint-derived start date. # --to FR only. Defaults to today. # --days FR only. Last N days. Mutually exclusive with --from/--to. @@ -132,8 +132,11 @@ while [[ $# -gt 0 ]]; do ;; # --- Migration errors for removed flags --- + # ${2:-} keeps `set -u` from killing the shell when the user + # types a value-taking removed flag without a value (e.g. bare + # `--ecfr-titles`). The migration hint should fire even in that case. --ecfr-titles) - removed_flag "--ecfr-titles" "./scripts/update.sh --source ecfr --titles $2" + removed_flag "--ecfr-titles" "./scripts/update.sh --source ecfr --titles ${2:-}" ;; --ecfr-all) removed_flag "--ecfr-all" "./scripts/update.sh --source ecfr --force" @@ -142,13 +145,13 @@ while [[ $# -gt 0 ]]; do removed_flag "--ecfr-skip-highlights" "./scripts/update.sh --source ecfr --skip-highlights" ;; --fr-days) - removed_flag "--fr-days" "./scripts/update.sh --source fr --days $2" + removed_flag "--fr-days" "./scripts/update.sh --source fr --days ${2:-N}" ;; --fr-from) - removed_flag "--fr-from" "./scripts/update.sh --source fr --from $2" + removed_flag "--fr-from" "./scripts/update.sh --source fr --from ${2:-YYYY-MM-DD}" ;; --fr-to) - removed_flag "--fr-to" "./scripts/update.sh --source fr --to $2" + removed_flag "--fr-to" "./scripts/update.sh --source fr --to ${2:-YYYY-MM-DD}" ;; --usc-force) removed_flag "--usc-force" "./scripts/update.sh --source usc --force" @@ -204,11 +207,13 @@ should_run() { } # Flag/source compatibility checks. -if [ -n "$TITLES" ]; then - if should_run "fr" && ! should_run "usc" && ! should_run "ecfr"; then - echo "Error: --titles requires --source to include usc or ecfr." >&2 - exit 1 - fi +# --titles is currently eCFR-only — the USC sub-script doesn't accept --titles +# (USC pipeline operates on the full corpus per release point, not by title). +# A `--source usc --titles X` invocation would silently run the full USC +# pipeline with no filtering, which is the opposite of what the user asked for. +if [ -n "$TITLES" ] && ! should_run "ecfr"; then + echo "Error: --titles requires --source to include ecfr." >&2 + exit 1 fi if [ -n "$DAYS" ] || [ -n "$FROM" ] || [ -n "$TO" ]; then @@ -337,27 +342,29 @@ run_source() { echo "===== ${src} Update =====" echo "" - if "$script" "${args[@]}"; then + # ${args[@]+"${args[@]}"} keeps `set -u` happy on bash 3.2 when args is empty. + if "$script" ${args[@]+"${args[@]}"}; then SUCCEEDED="$SUCCEEDED $src" echo "" else local rc=$? echo "" - echo "WARNING: $src update failed (exit code $rc)" + echo "WARNING: $src update failed (exit code $rc)" >&2 FAILED="$FAILED $src" echo "" fi } +# Emit one common-flag arg per line on stdout. Empty output means no common +# flags are set (the caller's `while IFS= read` loop will simply not iterate). build_common_args() { - local args=() - [ "$FORCE" = true ] && args+=(--force) - [ "$SKIP_DEPLOY" = true ] && args+=(--skip-deploy) - [ "$EFFECTIVE_SKIP_SEARCH" = true ] && args+=(--skip-search) - [ "$SKIP_HIGHLIGHTS" = true ] && args+=(--skip-highlights) - [ "$DEPLOY_ONLY" = true ] && args+=(--deploy-only) - [ "$VERBOSE" = true ] && args+=(--verbose) - printf '%s\n' "${args[@]}" + [ "$FORCE" = true ] && echo --force + [ "$SKIP_DEPLOY" = true ] && echo --skip-deploy + [ "$EFFECTIVE_SKIP_SEARCH" = true ] && echo --skip-search + [ "$SKIP_HIGHLIGHTS" = true ] && echo --skip-highlights + [ "$DEPLOY_ONLY" = true ] && echo --deploy-only + [ "$VERBOSE" = true ] && echo --verbose + return 0 } # eCFR @@ -365,7 +372,7 @@ if should_run "ecfr"; then ECFR_ARGS=() [ -n "$TITLES" ] && ECFR_ARGS+=(--titles "$TITLES") while IFS= read -r a; do [ -n "$a" ] && ECFR_ARGS+=("$a"); done < <(build_common_args) - run_source "eCFR" "$SCRIPT_DIR/update-ecfr.sh" "${ECFR_ARGS[@]}" + run_source "eCFR" "$SCRIPT_DIR/update-ecfr.sh" ${ECFR_ARGS[@]+"${ECFR_ARGS[@]}"} fi # FR @@ -377,18 +384,18 @@ if should_run "fr"; then while IFS= read -r a; do [ -n "$a" ] && FR_ARGS+=("$a"); done < <(build_common_args) # FR doesn't support --skip-highlights (no highlights step). Strip it. filtered_fr=() - for arg in "${FR_ARGS[@]}"; do + for arg in ${FR_ARGS[@]+"${FR_ARGS[@]}"}; do [ "$arg" = "--skip-highlights" ] && continue filtered_fr+=("$arg") done - run_source "FR" "$SCRIPT_DIR/update-fr.sh" "${filtered_fr[@]}" + run_source "FR" "$SCRIPT_DIR/update-fr.sh" ${filtered_fr[@]+"${filtered_fr[@]}"} fi # USC if should_run "usc"; then USC_ARGS=() while IFS= read -r a; do [ -n "$a" ] && USC_ARGS+=("$a"); done < <(build_common_args) - run_source "USC" "$SCRIPT_DIR/update-usc.sh" "${USC_ARGS[@]}" + run_source "USC" "$SCRIPT_DIR/update-usc.sh" ${USC_ARGS[@]+"${USC_ARGS[@]}"} fi # --- Post-run: regenerate sitemap index once, covering all sources --- From 2ec63ad979dffac19f400bef4711231938369d35 Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 20:14:52 -0500 Subject: [PATCH 4/8] refactor(scripts): inline common-args array in update.sh dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace build_common_args() function — which printed flags as text on stdout to be parsed back into arrays via `while IFS= read -r` — with a plain top-level COMMON_ARGS=() array. Each per-source dispatch then appends its own scoping flags plus COMMON_ARGS via standard array concatenation. Drops --skip-highlights from the common bag and appends it inline only for eCFR/USC, which removes the post-build filtered_fr loop that existed solely to strip --skip-highlights before invoking update-fr.sh (FR has no highlights step). Pure refactor: dry-run output, migration errors, --titles validation, and bash 3.2 compatibility verified identical to before. --- scripts/update.sh | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/scripts/update.sh b/scripts/update.sh index 49b7cae..6bf5991 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -355,46 +355,39 @@ run_source() { fi } -# Emit one common-flag arg per line on stdout. Empty output means no common -# flags are set (the caller's `while IFS= read` loop will simply not iterate). -build_common_args() { - [ "$FORCE" = true ] && echo --force - [ "$SKIP_DEPLOY" = true ] && echo --skip-deploy - [ "$EFFECTIVE_SKIP_SEARCH" = true ] && echo --skip-search - [ "$SKIP_HIGHLIGHTS" = true ] && echo --skip-highlights - [ "$DEPLOY_ONLY" = true ] && echo --deploy-only - [ "$VERBOSE" = true ] && echo --verbose - return 0 -} +# Build the array of flags shared by every sub-script. --skip-highlights is +# not included here because FR doesn't accept it; eCFR/USC append it themselves. +COMMON_ARGS=() +[ "$FORCE" = true ] && COMMON_ARGS+=(--force) +[ "$SKIP_DEPLOY" = true ] && COMMON_ARGS+=(--skip-deploy) +[ "$EFFECTIVE_SKIP_SEARCH" = true ] && COMMON_ARGS+=(--skip-search) +[ "$DEPLOY_ONLY" = true ] && COMMON_ARGS+=(--deploy-only) +[ "$VERBOSE" = true ] && COMMON_ARGS+=(--verbose) # eCFR if should_run "ecfr"; then ECFR_ARGS=() [ -n "$TITLES" ] && ECFR_ARGS+=(--titles "$TITLES") - while IFS= read -r a; do [ -n "$a" ] && ECFR_ARGS+=("$a"); done < <(build_common_args) + [ "$SKIP_HIGHLIGHTS" = true ] && ECFR_ARGS+=(--skip-highlights) + ECFR_ARGS+=(${COMMON_ARGS[@]+"${COMMON_ARGS[@]}"}) run_source "eCFR" "$SCRIPT_DIR/update-ecfr.sh" ${ECFR_ARGS[@]+"${ECFR_ARGS[@]}"} fi -# FR +# FR — no --skip-highlights support (no highlights step in FR pipeline). if should_run "fr"; then FR_ARGS=() [ -n "$DAYS" ] && FR_ARGS+=(--days "$DAYS") [ -n "$FROM" ] && FR_ARGS+=(--from "$FROM") [ -n "$TO" ] && FR_ARGS+=(--to "$TO") - while IFS= read -r a; do [ -n "$a" ] && FR_ARGS+=("$a"); done < <(build_common_args) - # FR doesn't support --skip-highlights (no highlights step). Strip it. - filtered_fr=() - for arg in ${FR_ARGS[@]+"${FR_ARGS[@]}"}; do - [ "$arg" = "--skip-highlights" ] && continue - filtered_fr+=("$arg") - done - run_source "FR" "$SCRIPT_DIR/update-fr.sh" ${filtered_fr[@]+"${filtered_fr[@]}"} + FR_ARGS+=(${COMMON_ARGS[@]+"${COMMON_ARGS[@]}"}) + run_source "FR" "$SCRIPT_DIR/update-fr.sh" ${FR_ARGS[@]+"${FR_ARGS[@]}"} fi # USC if should_run "usc"; then USC_ARGS=() - while IFS= read -r a; do [ -n "$a" ] && USC_ARGS+=("$a"); done < <(build_common_args) + [ "$SKIP_HIGHLIGHTS" = true ] && USC_ARGS+=(--skip-highlights) + USC_ARGS+=(${COMMON_ARGS[@]+"${COMMON_ARGS[@]}"}) run_source "USC" "$SCRIPT_DIR/update-usc.sh" ${USC_ARGS[@]+"${USC_ARGS[@]}"} fi From 52164c7de1a57817746f8e5b60d4d33cd692a293 Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 20:32:32 -0500 Subject: [PATCH 5/8] fix(scripts): wire --verbose through sub-scripts to convert-* CLI Pre-existing bug: update.sh parsed -v/--verbose, set VERBOSE=true, and forwarded --verbose to every sub-script. None of the three sub-scripts implemented --verbose, so they rejected it as "Unknown option" and exited 1, marking each source as FAILED in the orchestrator's summary. Fix: - update-ecfr.sh, update-fr.sh, update-usc.sh now accept -v/--verbose and pass --verbose through to their convert-{source} CLI invocation. The download-* commands don't accept --verbose so they're unchanged. - Each sub-script's header docstring documents the new flag. The --verbose flag thus does what the help text always claimed: shells out to the verbose code path in convert-fr / convert-ecfr / convert-usc, which all accepted -v/--verbose at the CLI layer all along. --- scripts/update-ecfr.sh | 10 +++++++++- scripts/update-fr.sh | 11 ++++++++++- scripts/update-usc.sh | 10 +++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/update-ecfr.sh b/scripts/update-ecfr.sh index 4686921..f8a50ed 100755 --- a/scripts/update-ecfr.sh +++ b/scripts/update-ecfr.sh @@ -10,6 +10,7 @@ # ./scripts/update-ecfr.sh --skip-highlights # Skip highlight generation # ./scripts/update-ecfr.sh --deploy-only # Push existing output + reindex # ./scripts/update-ecfr.sh --dry-run # Print plan, exit 0 +# ./scripts/update-ecfr.sh -v / --verbose # Pass --verbose through to convert-ecfr # # Modes: # incremental Default. ecfr-changed-titles.ts compares API metadata vs checkpoint. @@ -60,6 +61,7 @@ DEPLOY_ONLY=false SKIP_HIGHLIGHTS=false SKIP_SEARCH=false DRY_RUN=false +VERBOSE=false while [[ $# -gt 0 ]]; do case "$1" in @@ -91,6 +93,10 @@ while [[ $# -gt 0 ]]; do DRY_RUN=true shift ;; + -v|--verbose) + VERBOSE=true + shift + ;; --all) echo "Error: --all has been removed. Use --force instead." >&2 echo " ./scripts/update-ecfr.sh --force" >&2 @@ -253,8 +259,10 @@ if [ "$DEPLOY_ONLY" = false ]; then # emits section + title + chapter + part from one pass of the XML, writing # each to its own output dir. writeFileIfChanged preserves mtimes. echo "--- Step 3/7: Converting eCFR titles ($TITLE_ARG) at all granularities" + VERBOSE_ARG="" + [ "$VERBOSE" = true ] && VERBOSE_ARG="--verbose" # shellcheck disable=SC2086 - $CLI convert-ecfr $TITLE_ARG $CURRENCY_ARG \ + $CLI convert-ecfr $TITLE_ARG $CURRENCY_ARG $VERBOSE_ARG \ --granularities section,title,chapter,part \ --output ./output \ --output-title ./output-title \ diff --git a/scripts/update-fr.sh b/scripts/update-fr.sh index 4e745e4..9b77fc8 100755 --- a/scripts/update-fr.sh +++ b/scripts/update-fr.sh @@ -11,6 +11,7 @@ # ./scripts/update-fr.sh --skip-search # Skip search reindex (still rsync) # ./scripts/update-fr.sh --deploy-only # Push existing output + reindex # ./scripts/update-fr.sh --dry-run # Print plan, exit 0 +# ./scripts/update-fr.sh -v / --verbose # Pass --verbose through to convert-fr # # Modes: # incremental Default. --from = lastDate from .fr-state.json, --to = today. @@ -58,6 +59,7 @@ SKIP_DEPLOY=false DEPLOY_ONLY=false SKIP_SEARCH=false DRY_RUN=false +VERBOSE=false while [[ $# -gt 0 ]]; do case "$1" in @@ -93,6 +95,10 @@ while [[ $# -gt 0 ]]; do DRY_RUN=true shift ;; + -v|--verbose) + VERBOSE=true + shift + ;; --help|-h) awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 @@ -275,7 +281,10 @@ if [ "$DEPLOY_ONLY" = false ]; then # Step 3: Convert (date-filtered — only converts files in the date range) echo "--- Step 3/8: Converting FR documents ($DATE_FROM to $DATE_TO)" - $CLI convert-fr --all --from "$DATE_FROM" --to "$DATE_TO" + VERBOSE_ARG="" + [ "$VERBOSE" = true ] && VERBOSE_ARG="--verbose" + # shellcheck disable=SC2086 + $CLI convert-fr --all --from "$DATE_FROM" --to "$DATE_TO" $VERBOSE_ARG echo "" # Sanity check: downloaded XML must yield converted Markdown. writeFileIfChanged diff --git a/scripts/update-usc.sh b/scripts/update-usc.sh index cf77c35..8c0a0a3 100755 --- a/scripts/update-usc.sh +++ b/scripts/update-usc.sh @@ -9,6 +9,7 @@ # ./scripts/update-usc.sh --skip-highlights # Skip highlight generation # ./scripts/update-usc.sh --deploy-only # Push existing output + reindex # ./scripts/update-usc.sh --dry-run # Print plan, exit 0 +# ./scripts/update-usc.sh -v / --verbose # Pass --verbose through to convert-usc # # Modes: # incremental Default. Compares latest OLRC release point to .usc-release-point. @@ -53,6 +54,7 @@ DEPLOY_ONLY=false SKIP_HIGHLIGHTS=false SKIP_SEARCH=false DRY_RUN=false +VERBOSE=false LATEST="" while [[ $# -gt 0 ]]; do @@ -81,6 +83,10 @@ while [[ $# -gt 0 ]]; do DRY_RUN=true shift ;; + -v|--verbose) + VERBOSE=true + shift + ;; --help|-h) awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 @@ -210,8 +216,10 @@ if [ "$DEPLOY_ONLY" = false ]; then # $CLI is intentionally unquoted so its embedded spaces word-split into # "node ... dist/index.js" args. shellcheck disable=SC2086 echo "--- Step 3/7: Converting USC titles at all granularities" + VERBOSE_ARG="" + [ "$VERBOSE" = true ] && VERBOSE_ARG="--verbose" # shellcheck disable=SC2086 - $CLI convert-usc --all \ + $CLI convert-usc --all $VERBOSE_ARG \ --granularities section,title,chapter \ --output ./output \ --output-title ./output-title \ From be8d5aca9535034d065c238c53a8c0fd3eff777a Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 20:35:30 -0500 Subject: [PATCH 6/8] docs: document --verbose flag for update scripts Adds --verbose / -v to the update-script command tables in: - CLAUDE.md (Build & Dev Commands section) - README.md (Incremental Updates section) - apps/astro/src/content/docs/cli/commands.md - apps/astro/src/content/docs/guides/bulk-download.md Plus a Fixed entry in the root CHANGELOG and the Astro project changelog mirror, noting that --verbose was previously broken and now works. --- CHANGELOG.md | 4 ++++ CLAUDE.md | 1 + README.md | 3 +++ apps/astro/src/content/docs/cli/commands.md | 1 + apps/astro/src/content/docs/guides/bulk-download.md | 3 +++ apps/astro/src/content/docs/project/changelog.md | 1 + 6 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f10da3f..21eaf26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Conventional Commits](https://www.conventionalcommi - eCFR/USC bootstrap (missing `.ecfr-titles-state.json` / `.usc-release-point`) now logs the bootstrap explicitly and falls back to a full first-run automatically. - When `--force` runs against all three sources, the orchestrator now performs a single full `deploy.sh --search-docker` reindex at the end instead of three per-source incremental indexes. +### Fixed + +- `--verbose` / `-v` on `update.sh` and the three sub-scripts now actually works. Previously the orchestrator parsed it and forwarded `--verbose` to each sub-script, but the sub-scripts rejected it as "Unknown option" and exited 1. Each sub-script now accepts `-v` / `--verbose` and threads `--verbose` through to its `convert-*` CLI invocation. + ## [1.25.0] ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 8b20b8b..049d012 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,6 +147,7 @@ pnpm turbo build:api --filter=@lexbuild/api # Production build ./scripts/update.sh --skip-search # Rsync content/nav, but skip search reindex ./scripts/update.sh --deploy-only # Push existing local output + reindex ./scripts/update.sh --dry-run # Print plan, exit 0 +./scripts/update.sh -v / --verbose # Pass --verbose through to each convert step ``` Sub-scripts (`update-ecfr.sh`, `update-fr.sh`, `update-usc.sh`) accept the same flag grammar diff --git a/README.md b/README.md index ddf3903..e528fad 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,9 @@ A single script handles change detection, download, convert, and deploy across e # Preview without running ./scripts/update.sh --dry-run + +# Verbose convert output +./scripts/update.sh -v ``` Each source has a checkpoint in `downloads//`. eCFR/USC bootstrap automatically into a full first-run if their checkpoint is missing; FR errors with a hint and requires `--from`. Run `./scripts/update.sh --help` (or any sub-script with `--help`) for the full grammar. diff --git a/apps/astro/src/content/docs/cli/commands.md b/apps/astro/src/content/docs/cli/commands.md index 8514e81..99c91b3 100644 --- a/apps/astro/src/content/docs/cli/commands.md +++ b/apps/astro/src/content/docs/cli/commands.md @@ -138,6 +138,7 @@ A single orchestrator handles change detection, download, convert, and deploy ac ./scripts/update.sh --source usc --force # USC full redownload + reconvert ./scripts/update.sh --skip-deploy # Local pipeline only ./scripts/update.sh --dry-run # Print plan, exit 0 +./scripts/update.sh -v # Pass --verbose through to each convert step ``` `update-usc.sh` and `update-ecfr.sh` convert all granularities in one call using `--granularities` (see above), so the convert step parses the XML once per title rather than once per granularity. See [Incremental Updates](/docs/guides/bulk-download#incremental-updates) for details on checkpoints and bootstrap behavior. diff --git a/apps/astro/src/content/docs/guides/bulk-download.md b/apps/astro/src/content/docs/guides/bulk-download.md index 0e04299..a933697 100644 --- a/apps/astro/src/content/docs/guides/bulk-download.md +++ b/apps/astro/src/content/docs/guides/bulk-download.md @@ -166,6 +166,9 @@ A single orchestrator handles change detection, download, convert, and deploy ac # Preview without running ./scripts/update.sh --dry-run + +# Verbose convert output +./scripts/update.sh -v ``` Checkpoints live in `downloads//`: diff --git a/apps/astro/src/content/docs/project/changelog.md b/apps/astro/src/content/docs/project/changelog.md index 2e133a6..9aaaa56 100644 --- a/apps/astro/src/content/docs/project/changelog.md +++ b/apps/astro/src/content/docs/project/changelog.md @@ -18,6 +18,7 @@ For the complete changelog, see [CHANGELOG.md on GitHub](https://github.com/chri - New `--skip-search`, `--dry-run`, and consistent `--force` semantics across all four scripts - New FR checkpoint at `downloads/fr/.fr-state.json` (`{ lastRun, lastDate }`). Default `update-fr.sh` resumes from `lastDate`. Missing checkpoint errors with a hint requiring `--from` or `--days` - eCFR/USC bootstrap (missing checkpoint) now logs explicitly and runs a full first-run automatically +- Fixed: `--verbose` / `-v` on the update scripts now actually works — sub-scripts accept the flag and pass `--verbose` through to their `convert-*` CLI step (previously exited 1 with "Unknown option") ### 1.17.2 From ea3c77716423f1e0ea2c7d3b95ec52c78e75f243 Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 20:37:32 -0500 Subject: [PATCH 7/8] docs(claude): add three shell-script pitfalls to CLAUDE.md Captured from this session's PR-review findings: - macOS bash 3.2 + set -u crashes empty-array expansion (fix: ${arr[@]+...}) - set -u + $2 in value-taking case arms crashes before user-error handling (fix: ${2:-}) - BSD sed vs GNU sed brace blocks differ; use awk for cross-platform multi-line text extraction All three apply to every script in scripts/ since they all use set -euo pipefail. --- CLAUDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 049d012..deed0fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -293,6 +293,9 @@ Note: identifiers use `/us/cfr/` (content type) not `/us/ecfr/` (data source). B ## Common Pitfalls +- **macOS ships bash 3.2 + `set -u` crashes empty-array expansion**: `/usr/bin/env bash` resolves to `/bin/bash` 3.2.57 on macOS. `printf '%s\n' "${arr[@]}"` and `"${arr[@]}"` at call sites fire `bash: arr[@]: unbound variable` when the array is empty. Use the bash 3.2-safe pattern: `printf '%s\n' "${arr[@]+"${arr[@]}"}"` (and same at call sites). Affects every script in `scripts/` since they all use `set -euo pipefail`. +- **`set -u` + `$2` in value-taking case arms**: a case arm like `--foo) some_helper "hint $2" ;;` crashes with `bash: $2: unbound variable` if the user runs `script --foo` (no value) — *before* the helper's error-handling runs. Use `${2:-}`. The migration helpers in `scripts/update.sh` are the live example. +- **macOS BSD sed vs GNU sed brace blocks**: `sed -n '2,/^$/{ s/^# //; p }'` (multi-command brace block) breaks on BSD sed with "extra characters at the end of p command". Use `awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}'` for cross-platform multi-line extraction. The update scripts use this pattern for `--help`. - **Hono v4 HTTPException bypasses middleware catch blocks**: In Hono v4, `HTTPException` is intercepted at the compose layer before middleware try-catch runs. Use `app.onError()` (not middleware) to handle `HTTPException`. The Data API configures this in `apps/api/src/app.ts`. - **USC snapshot fixtures use a `lexbuild@__VERSION__` placeholder**: `fixtures/expected/*.md` store the generator field as `lexbuild@__VERSION__`. The snapshot test (`packages/usc/src/snapshot.test.ts`) normalizes the live `generator: "lexbuild@X.Y.Z"` line to the same placeholder before comparison via a `normalizeGenerator()` helper, so version bumps do NOT churn the fixtures — no sed bump needed on release. (Historical note: before this normalization landed, the changesets Version Packages PR would fail CI because it bumped `package.json` but not the fixtures; the documented recovery was either to update fixtures in that PR or do a fully manual version bump.) - **Changeset `major` with lockstep versioning bumps ALL packages**: All 6 published packages are in the `fixed` array. A `major` changeset on any one package bumps every package to the next major (e.g., 1.x → 2.0.0). Prefer `minor` or `patch` unless all packages genuinely have breaking changes. From 4a72e41c348652cfa5db392c2cae171c76593068 Mon Sep 17 00:00:00 2001 From: Chris Thomas Date: Fri, 24 Apr 2026 20:47:15 -0500 Subject: [PATCH 8/8] fix(scripts): address Copilot PR review (#2, #3, #4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three valid findings from Copilot's review of PR #132: #2 — full search reindex no longer hard-fails on local-only runs update.sh's RUN_FULL_SEARCH_AFTER path called `deploy.sh --search-docker` unconditionally on `--force` across all three sources. deploy.sh hard-errors when VPS_HOST is unset, while sub-scripts auto-fall-back to local-only in that case. The orchestrator now mirrors that auto-fallback: prints "Skipping full search reindex" and continues when VPS_HOST is unset. #3 — `--source ecfr, fr` (with space after comma) now works Previous parsing left a leading space on the second entry, which then failed the source validator. Now strips surrounding whitespace per entry before validation, matching the user-friendly comma-list shape. #4 — bare value-taking flags print a friendly error `./scripts/update.sh --source` (or --titles, --from, --to, --days) under set -u previously crashed with `bash: $2: unbound variable` before any help could print. Added a `require_value` helper to update.sh, update-fr.sh, and update-ecfr.sh; bare invocations now print "Error: --foo requires a value." and exit 1. False-positive (verified): Copilot's first finding claimed read_checkpoint_date's `result=$(node -e ...)` + `rc=$?` pattern would be killed by set -e on Node failure before the warning could run. Tested directly: bash does NOT trigger set -e on assignments containing failing command substitutions. The malformed- checkpoint warning works as written. --- scripts/update-ecfr.sh | 16 ++++++++++- scripts/update-fr.sh | 20 ++++++++++++-- scripts/update.sh | 63 ++++++++++++++++++++++++++++++++---------- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/scripts/update-ecfr.sh b/scripts/update-ecfr.sh index f8a50ed..477182c 100755 --- a/scripts/update-ecfr.sh +++ b/scripts/update-ecfr.sh @@ -63,10 +63,24 @@ SKIP_SEARCH=false DRY_RUN=false VERBOSE=false +# require_value asserts that a value-taking flag was given a value. Without +# this, `script --titles` (no value) would crash under set -u with +# `bash: $2: unbound variable` before any user-facing error could print. +require_value() { + local flag="$1" + local val="${2:-}" + if [ -z "$val" ]; then + echo "Error: $flag requires a value." >&2 + echo " Run with --help for usage." >&2 + exit 1 + fi + printf '%s' "$val" +} + while [[ $# -gt 0 ]]; do case "$1" in --titles) - TITLES="$2" + TITLES="$(require_value --titles "${2:-}")" shift 2 ;; --force) diff --git a/scripts/update-fr.sh b/scripts/update-fr.sh index 9b77fc8..c1f6e79 100755 --- a/scripts/update-fr.sh +++ b/scripts/update-fr.sh @@ -61,18 +61,32 @@ SKIP_SEARCH=false DRY_RUN=false VERBOSE=false +# require_value asserts that a value-taking flag was given a value. Without +# this, `script --days` (no value) would crash under set -u with +# `bash: $2: unbound variable` before any user-facing error could print. +require_value() { + local flag="$1" + local val="${2:-}" + if [ -z "$val" ]; then + echo "Error: $flag requires a value." >&2 + echo " Run with --help for usage." >&2 + exit 1 + fi + printf '%s' "$val" +} + while [[ $# -gt 0 ]]; do case "$1" in --days) - DAYS="$2" + DAYS="$(require_value --days "${2:-}")" shift 2 ;; --from) - FROM="$2" + FROM="$(require_value --from "${2:-}")" shift 2 ;; --to) - TO="$2" + TO="$(require_value --to "${2:-}")" shift 2 ;; --force) diff --git a/scripts/update.sh b/scripts/update.sh index 6bf5991..bdaa002 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -76,10 +76,24 @@ removed_flag() { exit 1 } +# require_value asserts that a value-taking flag was given a value. Without +# this, `script --source` (no value) would crash under set -u with +# `bash: $2: unbound variable` before any user-facing error could print. +require_value() { + local flag="$1" + local val="${2:-}" + if [ -z "$val" ]; then + echo "Error: $flag requires a value." >&2 + echo " Run with --help for usage." >&2 + exit 1 + fi + printf '%s' "$val" +} + while [[ $# -gt 0 ]]; do case "$1" in --source) - SOURCES="$2" + SOURCES="$(require_value --source "${2:-}")" shift 2 ;; --force) @@ -103,19 +117,19 @@ while [[ $# -gt 0 ]]; do shift ;; --titles) - TITLES="$2" + TITLES="$(require_value --titles "${2:-}")" shift 2 ;; --from) - FROM="$2" + FROM="$(require_value --from "${2:-}")" shift 2 ;; --to) - TO="$2" + TO="$(require_value --to "${2:-}")" shift 2 ;; --days) - DAYS="$2" + DAYS="$(require_value --days "${2:-}")" shift 2 ;; --dry-run) @@ -190,17 +204,25 @@ if [ -z "$SOURCES" ] || [ "$SOURCES" = "all" ]; then SOURCES="ecfr,fr,usc" fi -# Validate source names -IFS=',' read -ra SOURCE_LIST <<< "$SOURCES" -for src in "${SOURCE_LIST[@]}"; do +# Validate source names. Trim surrounding whitespace per entry so that the +# common `--source ecfr, fr` shape (with a space after the comma) works +# instead of failing with "unknown source ' fr'". +IFS=',' read -ra RAW_SOURCE_LIST <<< "$SOURCES" +SOURCE_LIST=() +for src in "${RAW_SOURCE_LIST[@]}"; do + src="${src#"${src%%[![:space:]]*}"}" + src="${src%"${src##*[![:space:]]}"}" case "$src" in - usc|ecfr|fr) ;; + usc|ecfr|fr) + SOURCE_LIST+=("$src") + ;; *) echo "Error: unknown source '$src'. Valid: usc, ecfr, fr, all." >&2 exit 1 ;; esac done +SOURCES="$(IFS=,; echo "${SOURCE_LIST[*]}")" should_run() { echo "$SOURCES" | grep -qw "$1" @@ -414,17 +436,28 @@ if [ -n "$SUCCEEDED" ] && [ "$SKIP_DEPLOY" = false ]; then fi # --- Post-run: full search reindex (only when --force on all sources) --- +# +# `deploy.sh --search-docker` (no --source) hard-errors on missing VPS_HOST +# because it ultimately rsyncs the prebuilt index to the VPS. Sub-scripts +# auto-fall-back to local-only when VPS_HOST is unset; mirror that here so +# a local-only `./scripts/update.sh --force` doesn't fail at the very end. if [ "$RUN_FULL_SEARCH_AFTER" = true ] && [ -n "$SUCCEEDED" ]; then - echo "===== Full search reindex (Docker) =====" - echo "" - if "$SCRIPT_DIR/deploy.sh" --search-docker; then + if [ -z "${VPS_HOST:-}" ]; then + echo "===== Skipping full search reindex =====" + echo " VPS_HOST is not set; local-only update completed without remote reindex." echo "" else + echo "===== Full search reindex (Docker) =====" echo "" - echo "WARNING: full search reindex failed" - FAILED="$FAILED search-full" - echo "" + if "$SCRIPT_DIR/deploy.sh" --search-docker; then + echo "" + else + echo "" + echo "WARNING: full search reindex failed" >&2 + FAILED="$FAILED search-full" + echo "" + fi fi fi