diff --git a/CHANGELOG.md b/CHANGELOG.md index 6707304..21eaf26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). +## [Unreleased] + +### Changed + +- Unify the update-script flag scheme. `./scripts/update.sh` (no args) now updates all sources incrementally from each source's last checkpoint. Source restriction lives on `--source`; source-scoping flags (`--titles`, `--days`, `--from`, `--to`) live at the top level instead of per-source prefixes. Old prefixes (`--ecfr-titles`, `--ecfr-all`, `--ecfr-skip-highlights`, `--fr-days`, `--fr-from`, `--fr-to`, `--usc-force`, `--usc-skip-highlights`) are removed; running them prints a migration hint and exits 1. `update-ecfr.sh --all` similarly removed in favor of `--force`. +- Add `--skip-search`, `--dry-run`, and consistent `--force` semantics to all four scripts. +- `update-fr.sh` now persists a JSON checkpoint at `downloads/fr/.fr-state.json` (`{ lastRun, lastDate }`). Default invocations resume from `lastDate`. Bootstrap (no checkpoint) errors with a hint requiring `--from` or `--days`, since FR has no inherent "all". +- eCFR/USC bootstrap (missing `.ecfr-titles-state.json` / `.usc-release-point`) now logs the bootstrap explicitly and falls back to a full first-run automatically. +- When `--force` runs against all three sources, the orchestrator now performs a single full `deploy.sh --search-docker` reindex at the end instead of three per-source incremental indexes. + +### Fixed + +- `--verbose` / `-v` on `update.sh` and the three sub-scripts now actually works. Previously the orchestrator parsed it and forwarded `--verbose` to each sub-script, but the sub-scripts rejected it as "Unknown option" and exited 1. Each sub-script now accepts `-v` / `--verbose` and threads `--verbose` through to its `convert-*` CLI invocation. + ## [1.25.0] ### Added diff --git a/CLAUDE.md b/CLAUDE.md index f6fac5e..deed0fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,19 +19,22 @@ lexbuild/ │ └── api/ # @lexbuild/api — Data API (Hono, SQLite, Meilisearch proxy) ├── scripts/ │ ├── deploy.sh # Production deploy (code, content, or full remote pipeline) -│ ├── update.sh # Unified incremental content update (all sources) -│ ├── update-ecfr.sh # eCFR incremental update (auto-detects changed titles) -│ ├── update-fr.sh # FR incremental update (date-range based) -│ ├── update-usc.sh # USC incremental update (release point detection) +│ ├── update.sh # Unified content update orchestrator (incremental by default) +│ ├── update-ecfr.sh # eCFR sub-script (change-detection via API metadata) +│ ├── update-fr.sh # FR sub-script (checkpoint-based date window) +│ ├── update-usc.sh # USC sub-script (release-point detection) │ ├── ecfr-changed-titles.ts # eCFR change detection helper (API metadata vs checkpoint) │ ├── setup-secrets.sh # Initialize ~/.lexbuild-secrets on VPS │ └── .deploy.env.example # Template for .deploy.env (VPS_HOST config) ├── downloads/ │ ├── usc/ -│ │ └── xml/ # Full USC XML files (usc01.xml ... usc54.xml) — gitignored +│ │ ├── xml/ # Full USC XML files (usc01.xml ... usc54.xml) — gitignored +│ │ └── .usc-release-point # USC checkpoint (latest OLRC release point ID) │ ├── ecfr/ -│ │ └── xml/ # Full eCFR XML files (ECFR-title1.xml ... ECFR-title50.xml) — gitignored -│ └── fr/ # FR XML + JSON files (YYYY/MM/doc-number.xml/.json) — gitignored +│ │ ├── xml/ # Full eCFR XML files (ECFR-title1.xml ... ECFR-title50.xml) — gitignored +│ │ └── .ecfr-titles-state.json # eCFR checkpoint (per-title amendment dates) +│ └── fr/ # FR XML + JSON files (YYYY/MM/doc-number.xml/.json) — gitignored +│ └── .fr-state.json # FR checkpoint ({ lastRun, lastDate }) ├── fixtures/ │ ├── fragments/ # Small synthetic XML snippets for unit tests │ └── expected/ # Expected output snapshots for integration tests @@ -129,18 +132,36 @@ pnpm turbo build:api --filter=@lexbuild/api # Production build ./scripts/deploy.sh --search-docker --source fr # Incremental: index one source into existing volume ./scripts/deploy.sh --search-docker-seed # Seed Docker volume from VPS (recover after volume loss) -# Incremental content updates (from monorepo root) -# Search indexing runs locally in Docker, not on the VPS — each update script's -# final step delegates to `deploy.sh --search-docker --source `. -./scripts/update.sh # All sources incrementally -./scripts/update.sh --source ecfr # One source -./scripts/update.sh --skip-deploy # Local only -./scripts/update-ecfr.sh # eCFR only (auto-detects changed titles) -./scripts/update-ecfr.sh --titles 1,17 # Specific eCFR titles -./scripts/update-fr.sh --days 3 # FR last 3 days -./scripts/update-usc.sh # USC (checks for new release point) +# Content updates (from monorepo root) +# Default behavior is incremental from each source's checkpoint. Search indexing +# runs locally in Docker; each sub-script delegates to `deploy.sh --search-docker +# --source ` after the local pipeline. +./scripts/update.sh # All sources, incremental from checkpoints +./scripts/update.sh --source fr # One source, incremental +./scripts/update.sh --source ecfr,fr # Multi-source, incremental +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only (skip change-detection) +./scripts/update.sh --source fr --days 7 # FR last 7 days +./scripts/update.sh --source usc --force # USC full redownload + reconvert +./scripts/update.sh --force --from 2026-01-01 # All sources, full rebuild (FR force requires --from) +./scripts/update.sh --skip-deploy # Local only (no rsync, no search) +./scripts/update.sh --skip-search # Rsync content/nav, but skip search reindex +./scripts/update.sh --deploy-only # Push existing local output + reindex +./scripts/update.sh --dry-run # Print plan, exit 0 +./scripts/update.sh -v / --verbose # Pass --verbose through to each convert step ``` +Sub-scripts (`update-ecfr.sh`, `update-fr.sh`, `update-usc.sh`) accept the same flag grammar +minus `--source`. Run any of them with `--help` for the full list. + +**Checkpoints** (gitignored, in `downloads//`): +- `usc/.usc-release-point` — latest OLRC release point ID (plain text). +- `ecfr/.ecfr-titles-state.json` — per-title `latestAmendedOn` snapshot. +- `fr/.fr-state.json` — `{ lastRun, lastDate }`. Default `update-fr.sh` uses `lastDate` as `--from`. + +If a checkpoint is missing, eCFR/USC bootstrap into a full first-run automatically. FR bootstrap +errors with a hint; you must specify `--from YYYY-MM-DD` or `--days N` because FR has no inherent +"all" (decades of documents). + See `packages/cli/CLAUDE.md` for full command options. See `apps/astro/CLAUDE.md` for content pipeline scripts. ### CI / Release @@ -272,6 +293,9 @@ Note: identifiers use `/us/cfr/` (content type) not `/us/ecfr/` (data source). B ## Common Pitfalls +- **macOS ships bash 3.2 + `set -u` crashes empty-array expansion**: `/usr/bin/env bash` resolves to `/bin/bash` 3.2.57 on macOS. `printf '%s\n' "${arr[@]}"` and `"${arr[@]}"` at call sites fire `bash: arr[@]: unbound variable` when the array is empty. Use the bash 3.2-safe pattern: `printf '%s\n' "${arr[@]+"${arr[@]}"}"` (and same at call sites). Affects every script in `scripts/` since they all use `set -euo pipefail`. +- **`set -u` + `$2` in value-taking case arms**: a case arm like `--foo) some_helper "hint $2" ;;` crashes with `bash: $2: unbound variable` if the user runs `script --foo` (no value) — *before* the helper's error-handling runs. Use `${2:-}`. The migration helpers in `scripts/update.sh` are the live example. +- **macOS BSD sed vs GNU sed brace blocks**: `sed -n '2,/^$/{ s/^# //; p }'` (multi-command brace block) breaks on BSD sed with "extra characters at the end of p command". Use `awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}'` for cross-platform multi-line extraction. The update scripts use this pattern for `--help`. - **Hono v4 HTTPException bypasses middleware catch blocks**: In Hono v4, `HTTPException` is intercepted at the compose layer before middleware try-catch runs. Use `app.onError()` (not middleware) to handle `HTTPException`. The Data API configures this in `apps/api/src/app.ts`. - **USC snapshot fixtures use a `lexbuild@__VERSION__` placeholder**: `fixtures/expected/*.md` store the generator field as `lexbuild@__VERSION__`. The snapshot test (`packages/usc/src/snapshot.test.ts`) normalizes the live `generator: "lexbuild@X.Y.Z"` line to the same placeholder before comparison via a `normalizeGenerator()` helper, so version bumps do NOT churn the fixtures — no sed bump needed on release. (Historical note: before this normalization landed, the changesets Version Packages PR would fail CI because it bumped `package.json` but not the fixtures; the documented recovery was either to update fixtures in that PR or do a fully manual version bump.) - **Changeset `major` with lockstep versioning bumps ALL packages**: All 6 published packages are in the `fixed` array. A `major` changeset on any one package bumps every package to the next major (e.g., 1.x → 2.0.0). Prefer `minor` or `patch` unless all packages genuinely have breaking changes. diff --git a/README.md b/README.md index ab0545a..e528fad 100644 --- a/README.md +++ b/README.md @@ -131,18 +131,36 @@ lexbuild enrich-fr --from 2000-01-01 ### Incremental Updates -Update scripts handle change detection, download, convert, and deploy in one command: +A single script handles change detection, download, convert, and deploy across every source. The default is incremental from each source's last checkpoint: ```bash -# Update all sources (auto-detects changes) +# Painless: update everything from each source's checkpoint +./scripts/update.sh + +# Restrict to one source (or several) +./scripts/update.sh --source fr +./scripts/update.sh --source ecfr,fr + +# Source-scoping flags +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only +./scripts/update.sh --source fr --days 7 # FR last 7 days + +# Force a full redownload + reconvert +./scripts/update.sh --source usc --force +./scripts/update.sh --force --from 2026-01-01 # All sources (FR force requires --from) + +# Local only (no rsync to VPS, no search reindex) ./scripts/update.sh --skip-deploy -# Update individual sources -./scripts/update-ecfr.sh --skip-deploy -./scripts/update-fr.sh --days 3 --skip-deploy -./scripts/update-usc.sh --skip-deploy +# Preview without running +./scripts/update.sh --dry-run + +# Verbose convert output +./scripts/update.sh -v ``` +Each source has a checkpoint in `downloads//`. eCFR/USC bootstrap automatically into a full first-run if their checkpoint is missing; FR errors with a hint and requires `--from`. Run `./scripts/update.sh --help` (or any sub-script with `--help`) for the full grammar. + `update-usc.sh` and `update-ecfr.sh` convert every granularity in one parse using the `--granularities` flag (see below), so the convert step no longer scales with the number of output granularities. --- diff --git a/apps/astro/src/content/docs/cli/commands.md b/apps/astro/src/content/docs/cli/commands.md index 082175c..99c91b3 100644 --- a/apps/astro/src/content/docs/cli/commands.md +++ b/apps/astro/src/content/docs/cli/commands.md @@ -127,17 +127,21 @@ lexbuild convert-fr --all ## Update Scripts -For routine updates, wrapper scripts handle the full pipeline (detect changes, download, convert, generate artifacts, deploy): +A single orchestrator handles change detection, download, convert, and deploy across all sources. Default is incremental from each source's last checkpoint: ```bash -./scripts/update.sh # All sources -./scripts/update.sh --source ecfr # One source -./scripts/update-ecfr.sh --skip-deploy # eCFR, local only -./scripts/update-fr.sh --days 3 # FR, last 3 days -./scripts/update-usc.sh # USC, checks release point +./scripts/update.sh # All sources, incremental from checkpoints +./scripts/update.sh --source fr # One source +./scripts/update.sh --source ecfr,fr # Multi-source +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only +./scripts/update.sh --source fr --days 7 # FR last 7 days +./scripts/update.sh --source usc --force # USC full redownload + reconvert +./scripts/update.sh --skip-deploy # Local pipeline only +./scripts/update.sh --dry-run # Print plan, exit 0 +./scripts/update.sh -v # Pass --verbose through to each convert step ``` -Each script auto-detects what changed and only processes updates. `update-usc.sh` and `update-ecfr.sh` convert all granularities in one call using `--granularities` (see above), so the convert step parses the XML once per title rather than once per granularity. See [Incremental Updates](/docs/guides/bulk-download#incremental-updates) for details. +`update-usc.sh` and `update-ecfr.sh` convert all granularities in one call using `--granularities` (see above), so the convert step parses the XML once per title rather than once per granularity. See [Incremental Updates](/docs/guides/bulk-download#incremental-updates) for details on checkpoints and bootstrap behavior. ## Getting Help diff --git a/apps/astro/src/content/docs/guides/bulk-download.md b/apps/astro/src/content/docs/guides/bulk-download.md index f588962..a933697 100644 --- a/apps/astro/src/content/docs/guides/bulk-download.md +++ b/apps/astro/src/content/docs/guides/bulk-download.md @@ -143,24 +143,43 @@ lexbuild convert-fr --from 2026-03-01 ### Update Scripts -For streamlined incremental updates, wrapper scripts handle the full pipeline (detect changes, download, convert, generate artifacts, deploy): +A single orchestrator handles change detection, download, convert, and deploy across every source. Default is incremental from each source's last checkpoint: ```bash -# All sources — auto-detects changes, downloads, converts, deploys +# All sources, incremental from each source's checkpoint ./scripts/update.sh -# Individual sources -./scripts/update-ecfr.sh # Only changed titles (via API metadata) -./scripts/update-fr.sh --days 3 # Last 3 days -./scripts/update-usc.sh # Checks OLRC release point +# Restrict to specific sources +./scripts/update.sh --source fr +./scripts/update.sh --source ecfr,fr -# Local only (no VPS deploy) +# Source-scoping +./scripts/update.sh --source ecfr --titles 1,17 # eCFR titles 1, 17 only +./scripts/update.sh --source fr --days 7 # FR last 7 days + +# Force a full redownload + reconvert +./scripts/update.sh --source usc --force +./scripts/update.sh --force --from 2026-01-01 # All sources (FR force requires --from) + +# Local only (no VPS deploy, no search reindex) ./scripts/update.sh --skip-deploy + +# Preview without running +./scripts/update.sh --dry-run + +# Verbose convert output +./scripts/update.sh -v ``` -The eCFR script compares `latestAmendedOn` dates from the eCFR API against a local checkpoint to detect which titles have new amendments. The USC script checks for new OLRC release points. The FR script uses date-range filtering. +Checkpoints live in `downloads//`: + +- **eCFR** (`.ecfr-titles-state.json`) snapshots each title's `latestAmendedOn` date. The script compares against the live eCFR API to detect which titles have new amendments. +- **USC** (`.usc-release-point`) stores the latest OLRC release point ID; the pipeline runs only when the API returns a newer one. +- **FR** (`.fr-state.json`) stores `lastRun` and `lastDate`. Default invocations use `lastDate` as the `--from` argument and update `lastDate` to today after a successful run. + +If a checkpoint is missing, eCFR/USC bootstrap into a full first-run automatically. FR has no inherent "all" (decades of documents), so a missing checkpoint requires explicit `--from YYYY-MM-DD` or `--days N`. -All converters use `writeFileIfChanged()` internally, so unchanged sections keep their original file timestamps. This means downstream tools (Shiki highlighting, Meilisearch indexing) automatically skip reprocessing unchanged content. +All converters use `writeFileIfChanged()` internally, so unchanged sections keep their original file timestamps. Downstream tools (Shiki highlighting, Meilisearch indexing) automatically skip reprocessing unchanged content. ## Output Granularity diff --git a/apps/astro/src/content/docs/project/changelog.md b/apps/astro/src/content/docs/project/changelog.md index 2f70edb..9aaaa56 100644 --- a/apps/astro/src/content/docs/project/changelog.md +++ b/apps/astro/src/content/docs/project/changelog.md @@ -12,6 +12,14 @@ For the complete changelog, see [CHANGELOG.md on GitHub](https://github.com/chri ## Recent Releases +### Unreleased + +- Unified update-script flag scheme. `./scripts/update.sh` (no args) now updates all sources incrementally from each source's last checkpoint. Source restriction via `--source`; source-scoping flags (`--titles`, `--days`, `--from`, `--to`) live at the top level. Old prefixes (`--ecfr-titles`, `--fr-days`, `--usc-force`, etc.) are removed and print migration hints +- New `--skip-search`, `--dry-run`, and consistent `--force` semantics across all four scripts +- New FR checkpoint at `downloads/fr/.fr-state.json` (`{ lastRun, lastDate }`). Default `update-fr.sh` resumes from `lastDate`. Missing checkpoint errors with a hint requiring `--from` or `--days` +- eCFR/USC bootstrap (missing checkpoint) now logs explicitly and runs a full first-run automatically +- Fixed: `--verbose` / `-v` on the update scripts now actually works — sub-scripts accept the flag and pass `--verbose` through to their `convert-*` CLI step (previously exited 1 with "Unknown option") + ### 1.17.2 - Added **Data API** (`apps/api/`) -- Hono-based REST API serving U.S. legal content from SQLite with Meilisearch search proxy diff --git a/scripts/update-ecfr.sh b/scripts/update-ecfr.sh index 7446c19..477182c 100755 --- a/scripts/update-ecfr.sh +++ b/scripts/update-ecfr.sh @@ -3,22 +3,32 @@ # # Usage: # ./scripts/update-ecfr.sh # Incremental: only changed titles -# ./scripts/update-ecfr.sh --titles 1,17 # Explicit titles -# ./scripts/update-ecfr.sh --all # Force full reconvert -# ./scripts/update-ecfr.sh --skip-deploy # Local only (no VPS push) -# ./scripts/update-ecfr.sh --deploy-only # Push existing output + reindex +# ./scripts/update-ecfr.sh --titles 1,17 # Explicit titles (skip change-detection) +# ./scripts/update-ecfr.sh --force # Force reconvert all 50 titles +# ./scripts/update-ecfr.sh --skip-deploy # Local only (no VPS push, no search) +# ./scripts/update-ecfr.sh --skip-search # Skip search reindex (still rsync) # ./scripts/update-ecfr.sh --skip-highlights # Skip highlight generation +# ./scripts/update-ecfr.sh --deploy-only # Push existing output + reindex +# ./scripts/update-ecfr.sh --dry-run # Print plan, exit 0 +# ./scripts/update-ecfr.sh -v / --verbose # Pass --verbose through to convert-ecfr +# +# Modes: +# incremental Default. ecfr-changed-titles.ts compares API metadata vs checkpoint. +# bootstrap No checkpoint at downloads/ecfr/.ecfr-titles-state.json. +# ecfr-changed-titles.ts returns all titles; pipeline performs full bootstrap. +# titles Explicit --titles spec. Bypasses change-detection. +# force --force: download + convert all 50 titles regardless of checkpoint. # # Steps: -# 1. Detect changed titles via eCFR API metadata -# 2. Download changed title XML from eCFR API -# 3. Convert changed titles to Markdown at all granularities (section, title, chapter, part) -# 4. Generate highlights for changed sections (mtime-based, skippable) -# 5. Regenerate eCFR nav JSON -# 6. Regenerate sitemaps -# 7. Save checkpoint -# 8. Rsync content (all granularities) + nav + sitemaps to VPS -# 9. Incremental search index on VPS +# 1. Detect changed titles via eCFR API metadata (or use --titles / --force). +# 2. Download changed title XML from eCFR API. +# 3. Convert changed titles to Markdown at all granularities (section, title, chapter, part). +# 4. Generate highlights for changed sections (mtime-based, skippable). +# 5. Regenerate eCFR nav JSON. +# 6. Regenerate sitemaps (deferred when LEXBUILD_DEFER_SITEMAP=1). +# 7. Save checkpoint. +# 8. Rsync content (all granularities) + nav + sitemaps to VPS. +# 9. Incremental search index via Docker. # # Requires: # - Built CLI: pnpm turbo build (or at least @lexbuild/ecfr + @lexbuild/cli) @@ -40,22 +50,41 @@ fi CONTENT_DEST="${CONTENT_DEST:-/srv/lexbuild/content}" NAV_DEST="${NAV_DEST:-/srv/lexbuild/nav}" +ECFR_CHECKPOINT_PATH="$REPO_ROOT/downloads/ecfr/.ecfr-titles-state.json" + # --- Parse arguments --- TITLES="" -ALL=false +FORCE=false SKIP_DEPLOY=false DEPLOY_ONLY=false SKIP_HIGHLIGHTS=false +SKIP_SEARCH=false +DRY_RUN=false +VERBOSE=false + +# require_value asserts that a value-taking flag was given a value. Without +# this, `script --titles` (no value) would crash under set -u with +# `bash: $2: unbound variable` before any user-facing error could print. +require_value() { + local flag="$1" + local val="${2:-}" + if [ -z "$val" ]; then + echo "Error: $flag requires a value." >&2 + echo " Run with --help for usage." >&2 + exit 1 + fi + printf '%s' "$val" +} while [[ $# -gt 0 ]]; do case "$1" in --titles) - TITLES="$2" + TITLES="$(require_value --titles "${2:-}")" shift 2 ;; - --all) - ALL=true + --force) + FORCE=true shift ;; --skip-deploy) @@ -70,29 +99,91 @@ while [[ $# -gt 0 ]]; do SKIP_HIGHLIGHTS=true shift ;; + --skip-search) + SKIP_SEARCH=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + --all) + echo "Error: --all has been removed. Use --force instead." >&2 + echo " ./scripts/update-ecfr.sh --force" >&2 + exit 1 + ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 + exit 1 +fi + +if [ -n "$TITLES" ] && [ "$FORCE" = true ]; then + echo "Error: --titles and --force are mutually exclusive." >&2 exit 1 fi +# --- Resolve mode --- + +MODE="" +if [ "$DEPLOY_ONLY" = true ]; then + MODE="deploy-only" +elif [ "$FORCE" = true ]; then + MODE="force" +elif [ -n "$TITLES" ]; then + MODE="titles" +elif [ ! -f "$ECFR_CHECKPOINT_PATH" ]; then + MODE="bootstrap" +else + MODE="incremental" +fi + +# --- Print plan and exit on --dry-run --- + +print_plan() { + echo "==> eCFR update plan" + echo " Mode: $MODE" + if [ -n "$TITLES" ]; then + echo " Titles: $TITLES" + fi + echo " Force: $FORCE" + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Skip highlights: $SKIP_HIGHLIGHTS" + echo " Checkpoint: $ECFR_CHECKPOINT_PATH" + if [ -f "$ECFR_CHECKPOINT_PATH" ]; then + echo " Checkpoint exists: yes" + else + echo " Checkpoint exists: no (bootstrap)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + exit 0 +fi + CLI="node packages/cli/dist/index.js" # --- Preflight checks --- if [ ! -f "packages/cli/dist/index.js" ]; then - echo "Error: CLI not built. Run: pnpm turbo build" + echo "Error: CLI not built. Run: pnpm turbo build" >&2 exit 1 fi @@ -104,7 +195,7 @@ if [ "$SKIP_DEPLOY" = false ] && [ "$DEPLOY_ONLY" = false ] && [ -z "${VPS_HOST: fi if [ "$DEPLOY_ONLY" = true ] && [ -z "${VPS_HOST:-}" ]; then - echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." + echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." >&2 exit 1 fi @@ -119,33 +210,42 @@ if [ "$DEPLOY_ONLY" = false ]; then PIPELINE_MARKER="$(mktemp -t lexbuild-ecfr-update.XXXXXX)" trap 'rm -f "$PIPELINE_MARKER"' EXIT - # Step 1: Detect changed titles (unless --titles or --all specified) + # Step 1: Detect titles to process CURRENCY_DATE="" - if [ -n "$TITLES" ]; then - echo "==> eCFR update for titles: $TITLES" - elif [ "$ALL" = true ]; then - echo "==> eCFR full update (all titles)" - TITLES="all" - else - echo "--- Step 1/7: Detecting changed eCFR titles" - CHANGE_JSON=$(npx tsx scripts/ecfr-changed-titles.ts --json) - TITLES=$(echo "$CHANGE_JSON" | node -e " - const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); - process.stdout.write(d.changedTitles.join(',')); - ") - CURRENCY_DATE=$(echo "$CHANGE_JSON" | node -e " - const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); - process.stdout.write(d.currencyDate || ''); - ") - - if [ -z "$TITLES" ]; then - echo " No eCFR titles changed since last run. Nothing to do." - exit 0 - fi - echo " Changed titles: $TITLES" - echo "" - fi + case "$MODE" in + titles) + echo "==> eCFR update for titles: $TITLES" + ;; + force) + echo "==> eCFR force update (all titles)" + TITLES="all" + ;; + bootstrap) + echo "==> eCFR bootstrap: no checkpoint at $ECFR_CHECKPOINT_PATH" + echo " Performing full first-run download (all titles)." + TITLES="all" + ;; + incremental) + echo "--- Step 1/7: Detecting changed eCFR titles" + CHANGE_JSON=$(npx tsx scripts/ecfr-changed-titles.ts --json) + TITLES=$(echo "$CHANGE_JSON" | node -e " + const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); + process.stdout.write(d.changedTitles.join(',')); + ") + CURRENCY_DATE=$(echo "$CHANGE_JSON" | node -e " + const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8')); + process.stdout.write(d.currencyDate || ''); + ") + + if [ -z "$TITLES" ]; then + echo " No eCFR titles changed since last run. Nothing to do." + exit 0 + fi + echo " Changed titles: $TITLES" + echo "" + ;; + esac # Build CLI download/convert args if [ "$TITLES" = "all" ]; then @@ -173,8 +273,10 @@ if [ "$DEPLOY_ONLY" = false ]; then # emits section + title + chapter + part from one pass of the XML, writing # each to its own output dir. writeFileIfChanged preserves mtimes. echo "--- Step 3/7: Converting eCFR titles ($TITLE_ARG) at all granularities" + VERBOSE_ARG="" + [ "$VERBOSE" = true ] && VERBOSE_ARG="--verbose" # shellcheck disable=SC2086 - $CLI convert-ecfr $TITLE_ARG $CURRENCY_ARG \ + $CLI convert-ecfr $TITLE_ARG $CURRENCY_ARG $VERBOSE_ARG \ --granularities section,title,chapter,part \ --output ./output \ --output-title ./output-title \ @@ -200,7 +302,12 @@ if [ "$DEPLOY_ONLY" = false ]; then # emitted ~300 bytes of frontmatter with no body). A healthy eCFR title # corpus is ~1.3GB across 49 titles; if we're under 10MB total, something # broke and we should not overwrite production with stubs. - if [ -d "output-title/ecfr" ]; then + # + # Skipped in `titles` mode: a single-title run only refreshes one file and + # legitimately leaves the rest of output-title/ecfr untouched (preserved by + # writeFileIfChanged), so total bytes there have no meaning for that run's + # health. Force/incremental modes still get the check. + if [ "$MODE" != "titles" ] && [ -d "output-title/ecfr" ]; then TITLE_BYTES=$(find output-title/ecfr -name "*.md" -type f -exec wc -c {} + 2>/dev/null | awk 'END {print $1}') TITLE_BYTES="${TITLE_BYTES:-0}" if [ "$TITLE_BYTES" -lt 10000000 ]; then @@ -297,8 +404,12 @@ echo "" # restart-storms under memory pressure). The deploy script handles: local # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. -echo "--- Step 9/9: Building and shipping search index via local Docker" -"$SCRIPT_DIR/deploy.sh" --search-docker --source ecfr +if [ "$SKIP_SEARCH" = true ]; then + echo "--- Skipping search index step (--skip-search)" +else + echo "--- Step 9/9: Building and shipping search index via local Docker" + "$SCRIPT_DIR/deploy.sh" --search-docker --source ecfr +fi echo "" echo "==> eCFR update complete" diff --git a/scripts/update-fr.sh b/scripts/update-fr.sh index 4785a08..c1f6e79 100755 --- a/scripts/update-fr.sh +++ b/scripts/update-fr.sh @@ -2,24 +2,35 @@ # update-fr.sh — Download, convert, and deploy new Federal Register documents. # # Usage: -# ./scripts/update-fr.sh # Yesterday's documents (default) -# ./scripts/update-fr.sh --days 3 # Last 3 days -# ./scripts/update-fr.sh --from 2026-03-25 --to 2026-04-01 # Explicit date range -# ./scripts/update-fr.sh --skip-deploy # Local only (download + convert + generate, no VPS push) -# ./scripts/update-fr.sh --deploy-only # Push existing output + reindex (no download/convert) +# ./scripts/update-fr.sh # Incremental from .fr-state.json +# ./scripts/update-fr.sh --days 3 # Last 3 days (writes checkpoint) +# ./scripts/update-fr.sh --from 2026-03-25 # Explicit start (--to defaults to today) +# ./scripts/update-fr.sh --from 2026-03-25 --to 2026-04-01 +# ./scripts/update-fr.sh --force --from 2026-01-01 # Force redownload (requires --from) +# ./scripts/update-fr.sh --skip-deploy # Local only (no rsync, no search) +# ./scripts/update-fr.sh --skip-search # Skip search reindex (still rsync) +# ./scripts/update-fr.sh --deploy-only # Push existing output + reindex +# ./scripts/update-fr.sh --dry-run # Print plan, exit 0 +# ./scripts/update-fr.sh -v / --verbose # Pass --verbose through to convert-fr # -# Steps: -# 1. Download FR documents via API (JSON + XML per document) -# 2. Convert new XML to Markdown (date-filtered, not full reconvert) -# 3. Regenerate FR nav JSON -# 4. Regenerate sitemaps -# 5. Rsync content + nav + sitemaps to VPS -# 6. Incremental search index on VPS (only new/changed FR docs) +# Modes: +# incremental Default. --from = lastDate from .fr-state.json, --to = today. +# bootstrap No checkpoint. Requires --from or --days from the caller. +# window --from / --to / --days specified explicitly. Writes checkpoint at end. +# force --force + --from. Redownloads and reconverts the window even if XML on disk. +# +# Checkpoint: +# downloads/fr/.fr-state.json — { lastRun: ISO8601, lastDate: YYYY-MM-DD } # -# Requires: -# - Built CLI: pnpm turbo build (or at least @lexbuild/fr + @lexbuild/cli) -# - For deploy: SSH access to VPS, scripts/.deploy.env with VPS_HOST -# - For search: Meilisearch running on VPS (PM2-managed) +# Steps: +# 1. Resolve date range (mode-aware). +# 2. Download FR documents via API (JSON + XML per document). +# 3. Convert new XML to Markdown (date-filtered). +# 4. Regenerate FR nav JSON. +# 5. Regenerate sitemaps (deferred when LEXBUILD_DEFER_SITEMAP=1). +# 6. Write checkpoint (.fr-state.json). +# 7. Rsync content + nav + sitemaps to VPS (unless --skip-deploy). +# 8. Incremental search index via Docker (unless --skip-search or --skip-deploy). set -euo pipefail @@ -36,28 +47,52 @@ fi CONTENT_DEST="${CONTENT_DEST:-/srv/lexbuild/content}" NAV_DEST="${NAV_DEST:-/srv/lexbuild/nav}" +CHECKPOINT_PATH="$REPO_ROOT/downloads/fr/.fr-state.json" + # --- Parse arguments --- -DAYS=1 +DAYS="" FROM="" TO="" +FORCE=false SKIP_DEPLOY=false DEPLOY_ONLY=false +SKIP_SEARCH=false +DRY_RUN=false +VERBOSE=false + +# require_value asserts that a value-taking flag was given a value. Without +# this, `script --days` (no value) would crash under set -u with +# `bash: $2: unbound variable` before any user-facing error could print. +require_value() { + local flag="$1" + local val="${2:-}" + if [ -z "$val" ]; then + echo "Error: $flag requires a value." >&2 + echo " Run with --help for usage." >&2 + exit 1 + fi + printf '%s' "$val" +} while [[ $# -gt 0 ]]; do case "$1" in --days) - DAYS="$2" + DAYS="$(require_value --days "${2:-}")" shift 2 ;; --from) - FROM="$2" + FROM="$(require_value --from "${2:-}")" shift 2 ;; --to) - TO="$2" + TO="$(require_value --to "${2:-}")" shift 2 ;; + --force) + FORCE=true + shift + ;; --skip-deploy) SKIP_DEPLOY=true shift @@ -66,31 +101,155 @@ while [[ $# -gt 0 ]]; do DEPLOY_ONLY=true shift ;; + --skip-search) + SKIP_SEARCH=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 exit 1 fi -# Compute date range -if [ -n "$FROM" ]; then - DATE_FROM="$FROM" - DATE_TO="${TO:-$(date +%Y-%m-%d)}" +if [ -n "$DAYS" ] && { [ -n "$FROM" ] || [ -n "$TO" ]; }; then + echo "Error: --days is mutually exclusive with --from / --to." >&2 + exit 1 +fi + +# --- Resolve mode + date range --- + +# read_checkpoint_date prints lastDate from .fr-state.json on stdout. +# Empty stdout means: file is missing OR malformed in some way. The two cases +# are distinguished for the user by emitting a Warning to stderr when the +# file exists but can't be parsed (so a corrupt checkpoint isn't surfaced as +# a misleading "no checkpoint" bootstrap-error). The path is passed via env +# rather than substituted into the JS source to avoid single-quote injection. +read_checkpoint_date() { + if [ ! -f "$CHECKPOINT_PATH" ]; then + return 0 + fi + local result rc + result=$(CHECKPOINT_PATH="$CHECKPOINT_PATH" node -e ' + try { + const d = JSON.parse(require("fs").readFileSync(process.env.CHECKPOINT_PATH, "utf-8")); + if (typeof d.lastDate === "string" && /^\d{4}-\d{2}-\d{2}$/.test(d.lastDate)) { + process.stdout.write(d.lastDate); + } else { + process.stderr.write("lastDate missing or wrong shape"); + process.exit(3); + } + } catch (e) { + process.stderr.write(e.message); + process.exit(3); + } + ' 2>&1) + rc=$? + if [ $rc -ne 0 ]; then + echo "Warning: $CHECKPOINT_PATH exists but is malformed ($result). Treating as missing." >&2 + return 0 + fi + printf '%s' "$result" +} + +CHECKPOINT_DATE="$(read_checkpoint_date)" +TODAY="$(date +%Y-%m-%d)" + +# Validate --force preconditions early +if [ "$FORCE" = true ] && [ -z "$FROM" ] && [ -z "$DAYS" ]; then + if [ "$DRY_RUN" = true ]; then + echo "==> FR update plan" + echo " Mode: force (would error)" + echo " Reason: --force on FR requires --from YYYY-MM-DD or --days N." + echo " FR has no inherent 'all' (decades of documents)." + exit 0 + fi + echo "Error: --force on FR requires --from YYYY-MM-DD or --days N." >&2 + echo " FR has no inherent 'all' (decades of documents). Specify a window." >&2 + exit 2 +fi + +# Determine mode +MODE="" +if [ "$DEPLOY_ONLY" = true ]; then + MODE="deploy-only" +elif [ "$FORCE" = true ]; then + MODE="force" +elif [ -n "$DAYS" ] || [ -n "$FROM" ]; then + MODE="window" +elif [ -n "$CHECKPOINT_DATE" ]; then + MODE="incremental" else - # macOS date: -v-Nd for N days ago + MODE="bootstrap" +fi + +# Bootstrap requires explicit --from or --days. In --dry-run we still print the +# would-error plan so the orchestrator's multi-source preview stays informative. +if [ "$MODE" = "bootstrap" ]; then + if [ "$DRY_RUN" = true ]; then + echo "==> FR update plan" + echo " Mode: bootstrap (would error)" + echo " Reason: No checkpoint at $CHECKPOINT_PATH." + echo " Run with --from YYYY-MM-DD or --days N to bootstrap." + exit 0 + fi + echo "Error: FR has no checkpoint at $CHECKPOINT_PATH." >&2 + echo " Run with --from YYYY-MM-DD or --days N to bootstrap." >&2 + exit 2 +fi + +# Compute effective DATE_FROM / DATE_TO +if [ -n "$DAYS" ]; then DATE_FROM="$(date -v-${DAYS}d +%Y-%m-%d)" - DATE_TO="$(date +%Y-%m-%d)" + DATE_TO="${TO:-$TODAY}" +elif [ -n "$FROM" ]; then + DATE_FROM="$FROM" + DATE_TO="${TO:-$TODAY}" +elif [ "$MODE" = "incremental" ]; then + DATE_FROM="$CHECKPOINT_DATE" + DATE_TO="$TODAY" +fi + +# --- Print plan and exit on --dry-run --- + +print_plan() { + echo "==> FR update plan" + echo " Mode: $MODE" + if [ "$MODE" != "deploy-only" ]; then + echo " Window: $DATE_FROM → $DATE_TO" + fi + echo " Force: $FORCE" + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Checkpoint: $CHECKPOINT_PATH" + if [ -n "$CHECKPOINT_DATE" ]; then + echo " Last run date: $CHECKPOINT_DATE" + else + echo " Last run date: (none)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + exit 0 fi CLI="node packages/cli/dist/index.js" @@ -98,7 +257,7 @@ CLI="node packages/cli/dist/index.js" # --- Preflight checks --- if [ ! -f "packages/cli/dist/index.js" ]; then - echo "Error: CLI not built. Run: pnpm turbo build --filter=@lexbuild/fr --filter=@lexbuild/cli" + echo "Error: CLI not built. Run: pnpm turbo build" >&2 exit 1 fi @@ -110,14 +269,14 @@ if [ "$SKIP_DEPLOY" = false ] && [ "$DEPLOY_ONLY" = false ] && [ -z "${VPS_HOST: fi if [ "$DEPLOY_ONLY" = true ] && [ -z "${VPS_HOST:-}" ]; then - echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." + echo "Error: --deploy-only requires VPS_HOST. Set it in scripts/.deploy.env." >&2 exit 1 fi -# --- Step 1–4: Local pipeline (skip if --deploy-only) --- +# --- Steps 2–6: Local pipeline (skip if --deploy-only) --- if [ "$DEPLOY_ONLY" = false ]; then - echo "==> FR update for $DATE_FROM to $DATE_TO" + echo "==> FR update ($MODE: $DATE_FROM → $DATE_TO)" echo "" # Pipeline-start marker: used after convert to verify that downloaded XML @@ -127,16 +286,19 @@ if [ "$DEPLOY_ONLY" = false ]; then PIPELINE_MARKER="$(mktemp -t lexbuild-fr-update.XXXXXX)" trap 'rm -f "$PIPELINE_MARKER"' EXIT - # Step 1: Download - echo "--- Step 1/4: Downloading FR documents ($DATE_FROM to $DATE_TO)" + # Step 2: Download + echo "--- Step 2/8: Downloading FR documents ($DATE_FROM to $DATE_TO)" $CLI download-fr --from "$DATE_FROM" --to "$DATE_TO" echo "" NEW_XML_COUNT=$(find downloads/fr -name "*.xml" -newer "$PIPELINE_MARKER" 2>/dev/null | wc -l | tr -d ' ') - # Step 2: Convert (date-filtered — only converts files in the date range) - echo "--- Step 2/4: Converting FR documents ($DATE_FROM to $DATE_TO)" - $CLI convert-fr --all --from "$DATE_FROM" --to "$DATE_TO" + # Step 3: Convert (date-filtered — only converts files in the date range) + echo "--- Step 3/8: Converting FR documents ($DATE_FROM to $DATE_TO)" + VERBOSE_ARG="" + [ "$VERBOSE" = true ] && VERBOSE_ARG="--verbose" + # shellcheck disable=SC2086 + $CLI convert-fr --all --from "$DATE_FROM" --to "$DATE_TO" $VERBOSE_ARG echo "" # Sanity check: downloaded XML must yield converted Markdown. writeFileIfChanged @@ -154,33 +316,63 @@ if [ "$DEPLOY_ONLY" = false ]; then echo "" fi - # Step 3: Regenerate FR nav - echo "--- Step 3/4: Generating FR nav JSON" + # Step 4: Regenerate FR nav + echo "--- Step 4/8: Generating FR nav JSON" ( cd apps/astro && npx tsx scripts/generate-nav.ts --source fr ) || exit 1 echo "" - # Step 4: Regenerate sitemaps (full rebuild to update sitemap index). + # Step 5: Regenerate sitemaps (full rebuild to update sitemap index). # Skipped when LEXBUILD_DEFER_SITEMAP=1 (set by update.sh to avoid # regenerating the full sitemap index once per source). if [ "${LEXBUILD_DEFER_SITEMAP:-}" != "1" ]; then - echo "--- Step 4/4: Generating sitemaps" + echo "--- Step 5/8: Generating sitemaps" ( cd apps/astro && npx tsx scripts/generate-sitemap.ts ) || exit 1 echo "" else - echo "--- Step 4/4: Skipping sitemap (deferred to update.sh)" + echo "--- Step 5/8: Skipping sitemap (deferred to update.sh)" echo "" fi + + # Step 6: Write checkpoint as the new resume point. + # + # New lastDate = max(CHECKPOINT_DATE, DATE_TO). Without the max guard, a + # historical backfill (e.g. `--from 2020-01-01 --to 2020-12-31` on a machine + # whose checkpoint is already 2026-04-20) would regress the resume cursor to + # 2020-12-31 and trigger a multi-year redownload on the next default run. + # Atomic write (tmp + rename) prevents a partial-write crash from corrupting + # .fr-state.json into something read_checkpoint_date can't parse. + # + # Intentionally inside the DEPLOY_ONLY=false block: --deploy-only pushes + # existing output without ingesting anything new, so we must not advance + # the cursor past data we haven't actually downloaded. + NEW_LAST_DATE="$DATE_TO" + if [ -n "$CHECKPOINT_DATE" ] && [ "$CHECKPOINT_DATE" \> "$NEW_LAST_DATE" ]; then + NEW_LAST_DATE="$CHECKPOINT_DATE" + fi + echo "--- Step 6/8: Writing FR checkpoint (lastDate=$NEW_LAST_DATE)" + mkdir -p "$(dirname "$CHECKPOINT_PATH")" + CHECKPOINT_PATH="$CHECKPOINT_PATH" NEW_LAST_DATE="$NEW_LAST_DATE" node -e ' + const fs = require("fs"); + const path = process.env.CHECKPOINT_PATH; + const tmp = path + ".tmp." + process.pid; + fs.writeFileSync(tmp, JSON.stringify({ + lastRun: new Date().toISOString(), + lastDate: process.env.NEW_LAST_DATE, + }, null, 2) + "\n"); + fs.renameSync(tmp, path); + ' || { echo "Error: failed to write FR checkpoint to $CHECKPOINT_PATH" >&2; exit 1; } + echo "" fi -# --- Step 5–6: Deploy to VPS (skip if --skip-deploy) --- +# --- Steps 7–8: Deploy to VPS (skip if --skip-deploy) --- if [ "$SKIP_DEPLOY" = true ]; then echo "==> Local pipeline complete (--skip-deploy). Files ready in output/fr/" exit 0 fi -# Step 5: Rsync content + nav + sitemaps -echo "--- Step 5/6: Syncing to VPS" +# Step 7: Rsync content + nav + sitemaps +echo "--- Step 7/8: Syncing to VPS" if [ -d "output/fr" ]; then echo " FR documents" @@ -206,14 +398,18 @@ if [ "${LEXBUILD_DEFER_SITEMAP:-}" != "1" ]; then fi echo "" -# Step 6: Build search index locally in Docker, ship the data directory to VPS. +# Step 8: Build search index locally in Docker, ship the data directory to VPS. # Delegating to deploy.sh --search-docker --source fr avoids running heavy # bulk upserts against the single production Meilisearch (caused PM2 # restart-storms under memory pressure). The deploy script handles: local # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. -echo "--- Step 6/6: Building and shipping search index via local Docker" -"$SCRIPT_DIR/deploy.sh" --search-docker --source fr +if [ "$SKIP_SEARCH" = true ]; then + echo "--- Step 8/8: Skipping search index step (--skip-search)" +else + echo "--- Step 8/8: Building and shipping search index via local Docker" + "$SCRIPT_DIR/deploy.sh" --search-docker --source fr +fi echo "" -echo "==> FR update complete ($DATE_FROM to $DATE_TO)" +echo "==> FR update complete ($DATE_FROM → $DATE_TO)" diff --git a/scripts/update-usc.sh b/scripts/update-usc.sh index 8f76a03..8c0a0a3 100755 --- a/scripts/update-usc.sh +++ b/scripts/update-usc.sh @@ -2,11 +2,19 @@ # update-usc.sh — Check for new USC release point, download, convert, and deploy. # # Usage: -# ./scripts/update-usc.sh # Check for new release point -# ./scripts/update-usc.sh --force # Force full reconvert -# ./scripts/update-usc.sh --skip-deploy # Local only (no VPS push) -# ./scripts/update-usc.sh --deploy-only # Push existing output + reindex +# ./scripts/update-usc.sh # Incremental: check for new release point +# ./scripts/update-usc.sh --force # Force full redownload + reconvert +# ./scripts/update-usc.sh --skip-deploy # Local only (no VPS push, no search) +# ./scripts/update-usc.sh --skip-search # Skip search reindex (still rsync) # ./scripts/update-usc.sh --skip-highlights # Skip highlight generation +# ./scripts/update-usc.sh --deploy-only # Push existing output + reindex +# ./scripts/update-usc.sh --dry-run # Print plan, exit 0 +# ./scripts/update-usc.sh -v / --verbose # Pass --verbose through to convert-usc +# +# Modes: +# incremental Default. Compares latest OLRC release point to .usc-release-point. +# bootstrap No checkpoint at downloads/usc/.usc-release-point. Treated as new release point. +# force --force: skip release-point check, always download + reconvert. # # Steps: # 1. Check latest OLRC release point against stored checkpoint @@ -17,7 +25,7 @@ # 6. Regenerate sitemaps # 7. Save new release point # 8. Rsync content (all granularities) + nav + sitemaps to VPS -# 9. Incremental search index on VPS +# 9. Incremental search index via local Docker (shipped to VPS) # # Requires: # - Built CLI: pnpm turbo build (or at least @lexbuild/usc + @lexbuild/cli) @@ -44,6 +52,9 @@ FORCE=false SKIP_DEPLOY=false DEPLOY_ONLY=false SKIP_HIGHLIGHTS=false +SKIP_SEARCH=false +DRY_RUN=false +VERBOSE=false LATEST="" while [[ $# -gt 0 ]]; do @@ -64,26 +75,73 @@ while [[ $# -gt 0 ]]; do SKIP_HIGHLIGHTS=true shift ;; + --skip-search) + SKIP_SEARCH=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 exit 1 fi CLI="node packages/cli/dist/index.js" CHECKPOINT="downloads/usc/.usc-release-point" +# --- Resolve mode --- + +MODE="" +if [ "$DEPLOY_ONLY" = true ]; then + MODE="deploy-only" +elif [ "$FORCE" = true ]; then + MODE="force" +elif [ ! -f "$CHECKPOINT" ]; then + MODE="bootstrap" +else + MODE="incremental" +fi + +# --- Print plan and exit on --dry-run --- + +print_plan() { + echo "==> USC update plan" + echo " Mode: $MODE" + echo " Force: $FORCE" + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Skip highlights: $SKIP_HIGHLIGHTS" + echo " Checkpoint: $CHECKPOINT" + if [ -f "$CHECKPOINT" ]; then + echo " Stored RP: $(cat "$CHECKPOINT")" + else + echo " Stored RP: (none — bootstrap)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + exit 0 +fi + # --- Preflight checks --- if [ ! -f "packages/cli/dist/index.js" ]; then @@ -137,6 +195,9 @@ if [ "$DEPLOY_ONLY" = false ]; then if [ -n "$STORED" ] && [ "$LATEST" != "$STORED" ]; then echo " New release point: $LATEST (was $STORED)" + elif [ -z "$STORED" ]; then + echo " Bootstrap: no checkpoint at $CHECKPOINT — performing full first-run download." + echo " Release point: $LATEST" else echo " Release point: $LATEST" fi @@ -155,8 +216,10 @@ if [ "$DEPLOY_ONLY" = false ]; then # $CLI is intentionally unquoted so its embedded spaces word-split into # "node ... dist/index.js" args. shellcheck disable=SC2086 echo "--- Step 3/7: Converting USC titles at all granularities" + VERBOSE_ARG="" + [ "$VERBOSE" = true ] && VERBOSE_ARG="--verbose" # shellcheck disable=SC2086 - $CLI convert-usc --all \ + $CLI convert-usc --all $VERBOSE_ARG \ --granularities section,title,chapter \ --output ./output \ --output-title ./output-title \ @@ -261,8 +324,12 @@ echo "" # restart-storms under memory pressure). The deploy script handles: local # Docker Meilisearch, incremental indexing, tar+scp of the LMDB data dir, # and the atomic PM2 swap on the VPS. -echo "--- Step 9/9: Building and shipping search index via local Docker" -"$SCRIPT_DIR/deploy.sh" --search-docker --source usc +if [ "$SKIP_SEARCH" = true ]; then + echo "--- Skipping search index step (--skip-search)" +else + echo "--- Step 9/9: Building and shipping search index via local Docker" + "$SCRIPT_DIR/deploy.sh" --search-docker --source usc +fi echo "" if [ -n "$LATEST" ]; then diff --git a/scripts/update.sh b/scripts/update.sh index c7ff70d..bdaa002 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,126 +1,345 @@ #!/usr/bin/env bash # update.sh — Unified content update orchestrator for all sources. # -# Usage: -# ./scripts/update.sh # All sources incrementally -# ./scripts/update.sh --source ecfr # One source -# ./scripts/update.sh --source ecfr,fr # Multiple sources -# ./scripts/update.sh --skip-deploy # All sources, local only -# ./scripts/update.sh --deploy-only # Push existing output for all sources +# Default behavior: +# ./scripts/update.sh Update all sources (eCFR → FR → USC) incrementally +# from each source's last checkpoint. Painless. # -# Source-specific pass-through args: -# --ecfr-titles 1,17 Pass --titles 1,17 to update-ecfr.sh -# --ecfr-all Pass --all to update-ecfr.sh -# --ecfr-skip-highlights Pass --skip-highlights to update-ecfr.sh -# --fr-days 3 Pass --days 3 to update-fr.sh -# --fr-from YYYY-MM-DD Pass --from to update-fr.sh -# --fr-to YYYY-MM-DD Pass --to to update-fr.sh -# --usc-force Pass --force to update-usc.sh -# --usc-skip-highlights Pass --skip-highlights to update-usc.sh +# Common usage: +# ./scripts/update.sh --source fr Only FR, incremental +# ./scripts/update.sh --source ecfr,fr Multi-source, incremental +# ./scripts/update.sh --source ecfr --titles 1,17 eCFR titles 1, 17 only +# ./scripts/update.sh --source fr --days 7 FR last 7 days +# ./scripts/update.sh --source usc --force USC full redownload + reconvert +# ./scripts/update.sh --force --from 2026-01-01 All sources, full rebuild (FR requires --from) +# ./scripts/update.sh --skip-deploy All sources incrementally, local only +# ./scripts/update.sh --deploy-only Push existing local output + reindex +# ./scripts/update.sh --dry-run Print plan, exit 0 # -# Execution order: eCFR → FR → USC -# Each source's script handles its own change detection and early exit. +# Flags (logical groups): +# +# SOURCE SELECTION +# --source Comma-separated: usc, ecfr, fr, all (default: all) +# +# MODE (mutually exclusive) +# (default) Incremental from each source's checkpoint +# --force Full redownload + reconvert for selected source(s). +# FR force requires --from. +# --deploy-only Skip download/convert. Push existing output + reindex. +# +# PIPELINE PHASE CONTROL +# --skip-deploy Local pipeline only (no rsync, no search reindex) +# --skip-highlights Skip highlight generation (USC, eCFR) +# --skip-search Skip search reindex (still rsync content/nav/sitemaps) +# +# SOURCE-SPECIFIC SCOPING +# --titles eCFR only. "1", "1-5", "1,3,8", "1-5,8,11" +# --from FR only. Override checkpoint-derived start date. +# --to FR only. Defaults to today. +# --days FR only. Last N days. Mutually exclusive with --from/--to. +# +# UTILITY +# --dry-run Print plan, exit 0. No network, no file writes. +# -v, --verbose Verbose output. +# -h, --help Show usage. +# +# Execution order: eCFR → FR → USC. Each source's sub-script handles its own +# change detection and early-exits if nothing's new. Sitemap regeneration is +# deferred to a single post-run step to avoid 3x redundant work. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" # --- Parse arguments --- SOURCES="" -SHARED_ARGS="" -SKIP_DEPLOY=false +FORCE=false DEPLOY_ONLY=false -ECFR_ARGS="" -FR_ARGS="" -USC_ARGS="" +SKIP_DEPLOY=false +SKIP_HIGHLIGHTS=false +SKIP_SEARCH=false +TITLES="" +FROM="" +TO="" +DAYS="" +DRY_RUN=false +VERBOSE=false + +# Migration helper for removed flags. +removed_flag() { + local old="$1" + local hint="$2" + echo "Error: $old has been removed in the unified flag scheme." >&2 + echo " Use: $hint" >&2 + exit 1 +} + +# require_value asserts that a value-taking flag was given a value. Without +# this, `script --source` (no value) would crash under set -u with +# `bash: $2: unbound variable` before any user-facing error could print. +require_value() { + local flag="$1" + local val="${2:-}" + if [ -z "$val" ]; then + echo "Error: $flag requires a value." >&2 + echo " Run with --help for usage." >&2 + exit 1 + fi + printf '%s' "$val" +} while [[ $# -gt 0 ]]; do case "$1" in --source) - SOURCES="$2" + SOURCES="$(require_value --source "${2:-}")" shift 2 ;; - --skip-deploy) - SHARED_ARGS="$SHARED_ARGS $1" - SKIP_DEPLOY=true + --force) + FORCE=true shift ;; --deploy-only) - SHARED_ARGS="$SHARED_ARGS $1" DEPLOY_ONLY=true shift ;; - # eCFR pass-through - --ecfr-titles) - ECFR_ARGS="$ECFR_ARGS --titles $2" - shift 2 + --skip-deploy) + SKIP_DEPLOY=true + shift ;; - --ecfr-all) - ECFR_ARGS="$ECFR_ARGS --all" + --skip-highlights) + SKIP_HIGHLIGHTS=true shift ;; - --ecfr-skip-highlights) - ECFR_ARGS="$ECFR_ARGS --skip-highlights" + --skip-search) + SKIP_SEARCH=true shift ;; - # FR pass-through - --fr-days) - FR_ARGS="$FR_ARGS --days $2" + --titles) + TITLES="$(require_value --titles "${2:-}")" shift 2 ;; - --fr-from) - FR_ARGS="$FR_ARGS --from $2" + --from) + FROM="$(require_value --from "${2:-}")" shift 2 ;; - --fr-to) - FR_ARGS="$FR_ARGS --to $2" + --to) + TO="$(require_value --to "${2:-}")" shift 2 ;; - # USC pass-through - --usc-force) - USC_ARGS="$USC_ARGS --force" + --days) + DAYS="$(require_value --days "${2:-}")" + shift 2 + ;; + --dry-run) + DRY_RUN=true shift ;; - --usc-skip-highlights) - USC_ARGS="$USC_ARGS --skip-highlights" + -v|--verbose) + VERBOSE=true shift ;; --help|-h) - sed -n '2,/^$/{ s/^# //; s/^#$//; p }' "$0" + awk 'NR==1{next} /^$/{exit} {sub(/^# ?/, ""); print}' "$0" exit 0 ;; + + # --- Migration errors for removed flags --- + # ${2:-} keeps `set -u` from killing the shell when the user + # types a value-taking removed flag without a value (e.g. bare + # `--ecfr-titles`). The migration hint should fire even in that case. + --ecfr-titles) + removed_flag "--ecfr-titles" "./scripts/update.sh --source ecfr --titles ${2:-}" + ;; + --ecfr-all) + removed_flag "--ecfr-all" "./scripts/update.sh --source ecfr --force" + ;; + --ecfr-skip-highlights) + removed_flag "--ecfr-skip-highlights" "./scripts/update.sh --source ecfr --skip-highlights" + ;; + --fr-days) + removed_flag "--fr-days" "./scripts/update.sh --source fr --days ${2:-N}" + ;; + --fr-from) + removed_flag "--fr-from" "./scripts/update.sh --source fr --from ${2:-YYYY-MM-DD}" + ;; + --fr-to) + removed_flag "--fr-to" "./scripts/update.sh --source fr --to ${2:-YYYY-MM-DD}" + ;; + --usc-force) + removed_flag "--usc-force" "./scripts/update.sh --source usc --force" + ;; + --usc-skip-highlights) + removed_flag "--usc-skip-highlights" "./scripts/update.sh --source usc --skip-highlights" + ;; + *) - echo "Unknown option: $1" - echo "Run with --help for usage." + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 exit 1 ;; esac done +# --- Validate --- + if [ "$SKIP_DEPLOY" = true ] && [ "$DEPLOY_ONLY" = true ]; then - echo "Error: --skip-deploy and --deploy-only are mutually exclusive." + echo "Error: --skip-deploy and --deploy-only are mutually exclusive." >&2 + exit 1 +fi + +if [ "$FORCE" = true ] && [ "$DEPLOY_ONLY" = true ]; then + echo "Error: --force and --deploy-only are mutually exclusive." >&2 + exit 1 +fi + +if [ -n "$DAYS" ] && { [ -n "$FROM" ] || [ -n "$TO" ]; }; then + echo "Error: --days is mutually exclusive with --from / --to." >&2 exit 1 fi # Default: all sources -if [ -z "$SOURCES" ]; then +if [ -z "$SOURCES" ] || [ "$SOURCES" = "all" ]; then SOURCES="ecfr,fr,usc" fi +# Validate source names. Trim surrounding whitespace per entry so that the +# common `--source ecfr, fr` shape (with a space after the comma) works +# instead of failing with "unknown source ' fr'". +IFS=',' read -ra RAW_SOURCE_LIST <<< "$SOURCES" +SOURCE_LIST=() +for src in "${RAW_SOURCE_LIST[@]}"; do + src="${src#"${src%%[![:space:]]*}"}" + src="${src%"${src##*[![:space:]]}"}" + case "$src" in + usc|ecfr|fr) + SOURCE_LIST+=("$src") + ;; + *) + echo "Error: unknown source '$src'. Valid: usc, ecfr, fr, all." >&2 + exit 1 + ;; + esac +done +SOURCES="$(IFS=,; echo "${SOURCE_LIST[*]}")" + should_run() { echo "$SOURCES" | grep -qw "$1" } -FAILED="" -SUCCEEDED="" +# Flag/source compatibility checks. +# --titles is currently eCFR-only — the USC sub-script doesn't accept --titles +# (USC pipeline operates on the full corpus per release point, not by title). +# A `--source usc --titles X` invocation would silently run the full USC +# pipeline with no filtering, which is the opposite of what the user asked for. +if [ -n "$TITLES" ] && ! should_run "ecfr"; then + echo "Error: --titles requires --source to include ecfr." >&2 + exit 1 +fi -# Defer sitemap regeneration to a single post-run step below. Without this, -# each source script would regenerate the full sitemap index (covering every -# source's URLs) and rsync it — 3x redundant work on a full update run. -export LEXBUILD_DEFER_SITEMAP=1 +if [ -n "$DAYS" ] || [ -n "$FROM" ] || [ -n "$TO" ]; then + if ! should_run "fr"; then + echo "Error: --days / --from / --to require --source to include fr." >&2 + exit 1 + fi +fi -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +# FR force requires explicit window. +if [ "$FORCE" = true ] && should_run "fr" && [ -z "$FROM" ] && [ -z "$DAYS" ]; then + echo "Error: --force on FR requires --from YYYY-MM-DD or --days N." >&2 + echo " FR has no inherent 'all' (decades of documents). Specify a window." >&2 + exit 2 +fi + +# When --force runs against all sources, do a single full search reindex at the +# end instead of three per-source incremental indexes. We achieve this by +# passing --skip-search to each sub-script and calling deploy.sh --search-docker +# (no --source) once after sub-scripts complete. +RUN_FULL_SEARCH_AFTER=false +if [ "$FORCE" = true ] && [ "$SKIP_SEARCH" = false ] && [ "$SKIP_DEPLOY" = false ] && [ "$DEPLOY_ONLY" = false ]; then + # All three sources, force, want search → full rebuild + if should_run "ecfr" && should_run "fr" && should_run "usc"; then + RUN_FULL_SEARCH_AFTER=true + fi +fi + +# --- Print plan --- + +print_plan() { + echo "==> Update plan" + echo " Sources: $SOURCES" + if [ "$DEPLOY_ONLY" = true ]; then + echo " Mode: deploy-only" + elif [ "$FORCE" = true ]; then + echo " Mode: force (full rebuild)" + else + echo " Mode: incremental" + fi + if [ -n "$TITLES" ]; then echo " Titles: $TITLES"; fi + if [ -n "$DAYS" ]; then echo " Days: $DAYS"; fi + if [ -n "$FROM" ]; then echo " From: $FROM"; fi + if [ -n "$TO" ]; then echo " To: $TO"; fi + echo " Skip deploy: $SKIP_DEPLOY" + echo " Skip search: $SKIP_SEARCH" + echo " Skip highlights: $SKIP_HIGHLIGHTS" + if [ "$RUN_FULL_SEARCH_AFTER" = true ]; then + echo " Search strategy: full reindex after sub-scripts (force on all sources)" + fi +} + +if [ "$DRY_RUN" = true ]; then + print_plan + echo "" + echo " (--dry-run: forwarding --dry-run to each sub-script for plan details)" + echo "" + # Run each sub-script's dry-run; don't let one source's "would-error" halt the + # rest of the preview. set -e is inherited; use `|| true` to continue. + for src in "${SOURCE_LIST[@]}"; do + case "$src" in + ecfr) + # shellcheck disable=SC2086 + "$SCRIPT_DIR/update-ecfr.sh" --dry-run \ + $( [ -n "$TITLES" ] && echo "--titles $TITLES" ) \ + $( [ "$FORCE" = true ] && echo "--force" ) \ + $( [ "$SKIP_DEPLOY" = true ] && echo "--skip-deploy" ) \ + $( [ "$SKIP_SEARCH" = true ] && echo "--skip-search" ) \ + $( [ "$SKIP_HIGHLIGHTS" = true ] && echo "--skip-highlights" ) \ + $( [ "$DEPLOY_ONLY" = true ] && echo "--deploy-only" ) || true + ;; + fr) + # shellcheck disable=SC2086 + "$SCRIPT_DIR/update-fr.sh" --dry-run \ + $( [ -n "$DAYS" ] && echo "--days $DAYS" ) \ + $( [ -n "$FROM" ] && echo "--from $FROM" ) \ + $( [ -n "$TO" ] && echo "--to $TO" ) \ + $( [ "$FORCE" = true ] && echo "--force" ) \ + $( [ "$SKIP_DEPLOY" = true ] && echo "--skip-deploy" ) \ + $( [ "$SKIP_SEARCH" = true ] && echo "--skip-search" ) \ + $( [ "$DEPLOY_ONLY" = true ] && echo "--deploy-only" ) || true + ;; + usc) + # shellcheck disable=SC2086 + "$SCRIPT_DIR/update-usc.sh" --dry-run \ + $( [ "$FORCE" = true ] && echo "--force" ) \ + $( [ "$SKIP_DEPLOY" = true ] && echo "--skip-deploy" ) \ + $( [ "$SKIP_SEARCH" = true ] && echo "--skip-search" ) \ + $( [ "$SKIP_HIGHLIGHTS" = true ] && echo "--skip-highlights" ) \ + $( [ "$DEPLOY_ONLY" = true ] && echo "--deploy-only" ) || true + ;; + esac + echo "" + done + exit 0 +fi + +print_plan +echo "" + +# --- Run sources --- + +# Defer sitemap regeneration to a single post-run step below. +export LEXBUILD_DEFER_SITEMAP=1 # Load deploy config (for final sitemap rsync below) if [ -f "$SCRIPT_DIR/.deploy.env" ]; then @@ -128,51 +347,70 @@ if [ -f "$SCRIPT_DIR/.deploy.env" ]; then source "$SCRIPT_DIR/.deploy.env" fi -# --- Run sources --- +FAILED="" +SUCCEEDED="" -if should_run "ecfr"; then - echo "===== eCFR Update =====" +# When RUN_FULL_SEARCH_AFTER is true, suppress per-source search calls. +EFFECTIVE_SKIP_SEARCH="$SKIP_SEARCH" +if [ "$RUN_FULL_SEARCH_AFTER" = true ]; then + EFFECTIVE_SKIP_SEARCH=true +fi + +run_source() { + local src="$1" + local script="$2" + shift 2 + local args=("$@") + + echo "===== ${src} Update =====" echo "" - if "$SCRIPT_DIR/update-ecfr.sh" $ECFR_ARGS $SHARED_ARGS; then - SUCCEEDED="$SUCCEEDED ecfr" + # ${args[@]+"${args[@]}"} keeps `set -u` happy on bash 3.2 when args is empty. + if "$script" ${args[@]+"${args[@]}"}; then + SUCCEEDED="$SUCCEEDED $src" echo "" else - ECFR_EXIT=$? + local rc=$? echo "" - echo "WARNING: eCFR update failed (exit code $ECFR_EXIT)" - FAILED="$FAILED ecfr" + echo "WARNING: $src update failed (exit code $rc)" >&2 + FAILED="$FAILED $src" echo "" fi +} + +# Build the array of flags shared by every sub-script. --skip-highlights is +# not included here because FR doesn't accept it; eCFR/USC append it themselves. +COMMON_ARGS=() +[ "$FORCE" = true ] && COMMON_ARGS+=(--force) +[ "$SKIP_DEPLOY" = true ] && COMMON_ARGS+=(--skip-deploy) +[ "$EFFECTIVE_SKIP_SEARCH" = true ] && COMMON_ARGS+=(--skip-search) +[ "$DEPLOY_ONLY" = true ] && COMMON_ARGS+=(--deploy-only) +[ "$VERBOSE" = true ] && COMMON_ARGS+=(--verbose) + +# eCFR +if should_run "ecfr"; then + ECFR_ARGS=() + [ -n "$TITLES" ] && ECFR_ARGS+=(--titles "$TITLES") + [ "$SKIP_HIGHLIGHTS" = true ] && ECFR_ARGS+=(--skip-highlights) + ECFR_ARGS+=(${COMMON_ARGS[@]+"${COMMON_ARGS[@]}"}) + run_source "eCFR" "$SCRIPT_DIR/update-ecfr.sh" ${ECFR_ARGS[@]+"${ECFR_ARGS[@]}"} fi +# FR — no --skip-highlights support (no highlights step in FR pipeline). if should_run "fr"; then - echo "===== FR Update =====" - echo "" - if "$SCRIPT_DIR/update-fr.sh" $FR_ARGS $SHARED_ARGS; then - SUCCEEDED="$SUCCEEDED fr" - echo "" - else - FR_EXIT=$? - echo "" - echo "WARNING: FR update failed (exit code $FR_EXIT)" - FAILED="$FAILED fr" - echo "" - fi + FR_ARGS=() + [ -n "$DAYS" ] && FR_ARGS+=(--days "$DAYS") + [ -n "$FROM" ] && FR_ARGS+=(--from "$FROM") + [ -n "$TO" ] && FR_ARGS+=(--to "$TO") + FR_ARGS+=(${COMMON_ARGS[@]+"${COMMON_ARGS[@]}"}) + run_source "FR" "$SCRIPT_DIR/update-fr.sh" ${FR_ARGS[@]+"${FR_ARGS[@]}"} fi +# USC if should_run "usc"; then - echo "===== USC Update =====" - echo "" - if "$SCRIPT_DIR/update-usc.sh" $USC_ARGS $SHARED_ARGS; then - SUCCEEDED="$SUCCEEDED usc" - echo "" - else - USC_EXIT=$? - echo "" - echo "WARNING: USC update failed (exit code $USC_EXIT)" - FAILED="$FAILED usc" - echo "" - fi + USC_ARGS=() + [ "$SKIP_HIGHLIGHTS" = true ] && USC_ARGS+=(--skip-highlights) + USC_ARGS+=(${COMMON_ARGS[@]+"${COMMON_ARGS[@]}"}) + run_source "USC" "$SCRIPT_DIR/update-usc.sh" ${USC_ARGS[@]+"${USC_ARGS[@]}"} fi # --- Post-run: regenerate sitemap index once, covering all sources --- @@ -185,20 +423,44 @@ if [ -n "$SUCCEEDED" ] && [ "$SKIP_DEPLOY" = false ]; then FAILED="$FAILED sitemap" } - if [ "$DEPLOY_ONLY" = true ] || [ "$SKIP_DEPLOY" = false ]; then - if [ -n "${VPS_HOST:-}" ]; then - SITEMAP_FILES=("$REPO_ROOT/apps/astro/public"/sitemap*.xml) - [ -f "$REPO_ROOT/apps/astro/public/robots.txt" ] && SITEMAP_FILES+=("$REPO_ROOT/apps/astro/public/robots.txt") - if [ ${#SITEMAP_FILES[@]} -gt 0 ] && [ -e "${SITEMAP_FILES[0]}" ]; then - echo " Syncing sitemaps to VPS" - rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/public/" || FAILED="$FAILED sitemap-rsync" - rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/dist/client/" || FAILED="$FAILED sitemap-rsync" - fi + if [ -n "${VPS_HOST:-}" ]; then + SITEMAP_FILES=("$REPO_ROOT/apps/astro/public"/sitemap*.xml) + [ -f "$REPO_ROOT/apps/astro/public/robots.txt" ] && SITEMAP_FILES+=("$REPO_ROOT/apps/astro/public/robots.txt") + if [ ${#SITEMAP_FILES[@]} -gt 0 ] && [ -e "${SITEMAP_FILES[0]}" ]; then + echo " Syncing sitemaps to VPS" + rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/public/" || FAILED="$FAILED sitemap-rsync" + rsync -avz "${SITEMAP_FILES[@]}" "${VPS_HOST}:~/lexbuild/apps/astro/dist/client/" || FAILED="$FAILED sitemap-rsync" fi fi echo "" fi +# --- Post-run: full search reindex (only when --force on all sources) --- +# +# `deploy.sh --search-docker` (no --source) hard-errors on missing VPS_HOST +# because it ultimately rsyncs the prebuilt index to the VPS. Sub-scripts +# auto-fall-back to local-only when VPS_HOST is unset; mirror that here so +# a local-only `./scripts/update.sh --force` doesn't fail at the very end. + +if [ "$RUN_FULL_SEARCH_AFTER" = true ] && [ -n "$SUCCEEDED" ]; then + if [ -z "${VPS_HOST:-}" ]; then + echo "===== Skipping full search reindex =====" + echo " VPS_HOST is not set; local-only update completed without remote reindex." + echo "" + else + echo "===== Full search reindex (Docker) =====" + echo "" + if "$SCRIPT_DIR/deploy.sh" --search-docker; then + echo "" + else + echo "" + echo "WARNING: full search reindex failed" >&2 + FAILED="$FAILED search-full" + echo "" + fi + fi +fi + # --- Summary --- if [ -n "$FAILED" ]; then