diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index ef71380..963e119 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -10,19 +10,33 @@ concurrency: group: ${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true -permissions: - contents: read +permissions: {} jobs: megalinter: name: MegaLinter runs-on: ubuntu-latest + permissions: + # write is required so the auto-fix step can push linter fixes back to the + # branch. Scoped to this job rather than the workflow. Fixes are only ever + # pushed to branches in this repository — see FIX_BRANCH below. + contents: write + env: + # Branch that fixes are checked out from and pushed to. Empty for pull + # requests from forks, whose token is read-only and whose head branch does + # not exist here; empty makes actions/checkout use its default ref and + # disables the commit step, so a fork PR still gets linted, just not fixed. + FIX_BRANCH: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.head_ref || github.ref_name) || '' }} steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 - persist-credentials: false + ref: ${{ env.FIX_BRANCH }} + # Credentials are deliberately persisted so the auto-fix step can push. + # This is not the usual hardening default; it is required for + # APPLY_FIXES_MODE: commit to work. + persist-credentials: true # zizmor: ignore[artipacked] - name: MegaLinter id: ml @@ -30,6 +44,42 @@ jobs: env: VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Apply fixes on every event and commit them straight to the branch. + # Which linters may fix is controlled by APPLY_FIXES in .mega-linter.yml. + APPLY_FIXES_EVENT: all + APPLY_FIXES_MODE: commit + + - name: Commit and push applied linter fixes + # !cancelled() is load-bearing: the MegaLinter step exits non-zero + # whenever any linter reports an error (here, the security scanners + # always do), and by default that skips every later step — so without + # this the fixes would never be pushed. + # has_updated_sources is 1 only when a linter actually rewrote a file. + # + # This push uses the default GITHUB_TOKEN, and pushes made with it do + # not trigger new workflow runs. That rules out an auto-fix loop by + # construction — but it also means the fix commit itself is never + # linted, and on a PR it becomes the head commit without checks having + # run against it. Verified: the run created for the first auto-fix + # commit sat in `action_required` and never executed. + # To get the fix commit validated, push with a GitHub App token or PAT + # instead, which does trigger workflows (and then converges after one + # extra run, since the second finds nothing left to fix). + if: > + !cancelled() && + steps.ml.outputs.has_updated_sources == 1 && + env.FIX_BRANCH != '' + uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0 + with: + branch: ${{ env.FIX_BRANCH }} + commit_message: 'style: apply MegaLinter auto-fixes' + commit_user_name: megalinter-bot + commit_user_email: megalinter-bot@users.noreply.github.com + # Without this the author defaults to github.actor, which attributes + # a machine-generated commit to whoever happened to push. + commit_author: megalinter-bot + # Only ever commit real source fixes — never MegaLinter's own output. + file_pattern: ':!megalinter-reports' - name: Archive MegaLinter reports if: always() diff --git a/.gitignore b/.gitignore index 59ff4a6..b6c4368 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ dist/ # Test coverage coverage/ +# MegaLinter run output (regenerated every CI run, uploaded as an artifact) +megalinter-reports/ + # Logs *.log npm-debug.log* diff --git a/AGENT.md b/AGENT.md index 3bd1315..f925cfa 100644 --- a/AGENT.md +++ b/AGENT.md @@ -1,24 +1,39 @@ # Agent Guide: Media Clusterer ## Tech Stack + - **Frontend**: Vite (v8+), TypeScript, Vanilla CSS -- **AI**: Transformers.js (v4.2.0, Multimodal Nomic embeddings) +- **AI**: Three interchangeable embedding backends selected by `settings.modelVariant`: + - `sapiens2-*` (**default** `sapiens2-fp16`) — `src/sapiens2.ts`, raw onnxruntime-web + - `nomic` — Transformers.js (v4.2.0) multimodal Nomic embeddings + - `chrome-ai` — `src/chromeAI.ts`, Chrome Prompt API caption → `nomic-embed-text` vector + + All three produce 768-dim L2-normalized vectors. - **Projections**: DruidJS (UMAP, t-SNE, PCA, Isomap, LLE, MDS, Sammon, TriMap) - **Database**: IndexedDB (for vector caching) - **Deployment**: Cloudflare Pages +## Documentation + +- **Architecture decisions** live in `docs/adr/` — see `docs/adr/README.md` for the convention. + Write an ADR when a change is hard to reverse or constrains later work. +- **Feature plans** live in root-level `*_PLAN.md` files. + ## Core Workflows ### Build & Deploy + - Build: `npm run build` - Deploy: `npm run deploy` (requires `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID` in `.env`) - Type-check: `npm run type-check` ### Testing + - Run all tests: `npm test` - Coverage: `npm run test:coverage` ## Coding Standards + - **Types**: Always use strict TypeScript. Define interfaces in `src/types.ts`. Avoid `any` and `unknown` in 99% of circumstances; prefer precise types or generics. - **Logic**: Use the `IProjection` interface for dimensionality reduction algorithms. - **State**: Centralized in the `state` object in `src/app.ts`. @@ -27,15 +42,18 @@ ## Architecture ### Application Modes + 1. **AI Mode** (default): Loads embeddings, runs projections, enables semantic search 2. **Viewer-Only Mode**: Skips AI, arranges by folder/date grid, no search ### State Management + - All state in `state` object (phase, files, vectors, points, clusters, thumbnails, settings) - URL state managed via History API (`#folder:...`, `#dt:...`) - Session persistence via localStorage (resume capability) ### Resource Management + - Object URLs created lazily via `URL.createObjectURL(file)` in `lazyDecodeThumbnail` - Resources MUST be cleaned up before processing new files: - Close ImageBitmaps: `bmp?.close()` @@ -43,6 +61,7 @@ - Clear caches: `thumbDecoding`, `thumbnailLRU` ### Navigation + - **Folder breadcrumbs**: Click path segments → `navigateToFolder(path)` - **Datetime breadcrumbs**: Click datetime parts → `filterByDateTime(...)` - **Back/Forward**: Handled via `popstate` event listener @@ -51,21 +70,25 @@ ## Key Functions ### File Processing + - `collectImages(dirHandle, sampleSize, basePath)` - Walk directory tree, apply reservoir sampling if sampleSize > 0 - `processFiles(files)` - Main entry point, handles both AI and viewer modes - `lazyDecodeThumbnail(idx)` - Create object URL and decode thumbnail lazily ### Navigation + - `run(dirHandle, basePath)` - Load and process files from directory - `navigateToFolder(targetPath)` - Navigate to subfolder (reuses currentDirHandle) - `filterByDateTime(granularity, year, month, day, hour, minute)` - Filter and rescan full folder ### Modal + - `openFileModal(index)` - Show media with metadata in footer - `closeModal()` - Hide modal, pause video - `navigateModal(direction)` - Arrow buttons for grid navigation ## Deployment Details + - **Platform**: Cloudflare Pages - **Project Name**: `media-clusterer` - **Output Dir**: `dist/` @@ -73,6 +96,7 @@ - **Preview Branches**: PR branches get `https://.media-clusterer.pages.dev` ## GitHub Actions + - Runs on push to `main` and PRs to `main` - Type-check, tests, build, deploy to Cloudflare Pages - Comments on PR with preview URL and commit info diff --git a/IMPROVEMENT_PLAN.md b/IMPROVEMENT_PLAN.md index 50c6fab..7ce31eb 100644 --- a/IMPROVEMENT_PLAN.md +++ b/IMPROVEMENT_PLAN.md @@ -46,7 +46,9 @@ Secondary main-thread costs found alongside: batchSize=1 that is one IndexedDB transaction **per image**. `cacheStats` cursor-scans the whole store on every refresh (`src/db.ts:144`). - Chrome-AI captions write `localStorage` synchronously per image - (`src/app.ts:1390,1424`). + (`src/app.ts:1390,1424`). **Addressed by `VIDEO_LM_PLAN.md` milestone M5**, which moves + captions into an IndexedDB `captions` store; multi-frame video captions are longer and + must not land on this path. - Render-loop churn: a fresh `Int32Array(pts.length)` and a `Set` are allocated **every frame** (`src/app.ts:677,690`); `new Image()` is constructed inside the draw loop on cache miss (`src/app.ts:758`); the O(n) nearest-point-to-center scan is duplicated three diff --git a/VIDEO_LM_PLAN.md b/VIDEO_LM_PLAN.md new file mode 100644 index 0000000..f2f621d --- /dev/null +++ b/VIDEO_LM_PLAN.md @@ -0,0 +1,402 @@ +# Plan: Video Language Model Integration + +Implementation plan for [ADR-0001](docs/adr/0001-small-video-language-model.md), which decides to adopt +SmolVLM2 as a small, video-native, app-controlled replacement for the Gemini Nano captioner and as the +source of the clustering vector. Written 2026-08-01. +This document is a plan only — no changes are implemented by the PR that introduces it. + +Milestones are `M1`…`M6`, in dependency order. `M2` is deliberately shippable on its own. + +## Context + +**Current Problem:** + +- Captioning via Chrome's Prompt API crashes 8 GB Chromebooks. Gemini Nano's floor is >4 GB VRAM, or + 16 GB RAM + 4 cores on CPU, plus 22 GB free disk — and the app cannot request anything smaller because + the browser owns the model (`src/chromeAI.ts`, `src/app.ts:1019-1101`). +- Videos are treated as one still. `extractVideoFrame()` (`src/app.ts:504-555`) seeks once to + `min(1.0, duration/2)` at 224 px; that frame is the entire representation of the video + (`src/app.ts:1476-1507`). +- The caption prompt is image-specific in its wording (`src/chromeAI.ts:52-56`) — it asks about "dominant + colors" and "lighting", never about motion or change. +- The `chrome-ai` path downloads two models to produce one vector: Nano for the caption, then 134 MB of + `nomic-embed-text-v1.5` to vectorise it (`src/app.ts:1054-1082`, `src/app.ts:1590-1594`). +- Captions are written with a synchronous `localStorage.setItem` per file inside the embed hot loop + (`src/app.ts:1585`), with no eviction and only a `console.warn` on quota exceed — already flagged as + `IMPROVEMENT_PLAN.md` P1-1. + +**Desired Outcome:** + +- Captioning and embedding fit in roughly 1 GB on an 8 GB Chromebook, with the app choosing the tier. +- Videos are described from several frames, with a prompt that asks about action over time. +- One model produces both the caption and the 768-d clustering vector. +- Semantic text search becomes opt-in rather than a mandatory 134 MB download. +- Captions live in IndexedDB, off the synchronous hot path. + +## Implementation Approach + +### Libraries to Add + +**None.** `@huggingface/transformers` is already a dependency at `^4.2.0` (`package.json`), and SmolVLM +support landed upstream in transformers.js 3.4.0. No new packages, no version bump. + +### Model facts that drive every sizing decision + +Verified against the Hub for `HuggingFaceTB/SmolVLM2-256M-Video-Instruct`: + +| Fact | Value | Why it matters | +|-------------------------------------------|----------------------------------------------|-------------------------------------------------------------------------------------------------| +| `vision_config.hidden_size` | **768** | Exactly the app's existing vector width — pooled features are a drop-in | +| `vision_config.image_size` / `patch_size` | 512 / 16 | 1024 patches per tile | +| `scale_factor` / `pixel_shuffle_factor` | 4 | 1024 patches → **64 visual tokens per frame** into the LLM | +| `text_config` | 576 hidden, 30 layers, vocab 49280 | Tiny decoder; 4 frames ≈ 256 tokens of visual prefill | +| `video_sampling` | `fps 1`, `max_frames 64`, `longest_edge 512` | The model's native video protocol; we use 4 frames, far below the ceiling | +| `do_image_splitting` default | `true`, `size.longest_edge 2048` | **Must be set to `false`** — tiling multiplies memory and is the easiest way to blow the budget | + +ONNX file sizes, which set the tier download figures: + +| File (q4f16) | 256M | 500M | +|------------------------|------------|------------| +| `vision_encoder` | 55 MB | 58 MB | +| `embed_tokens` | 57 MB | 95 MB | +| `decoder_model_merged` | 77 MB | 205 MB | +| **Total** | **189 MB** | **358 MB** | + +For scale, the app's current embedders cost 116 MB (`sapiens2-int8`), 229 MB (`sapiens2-fp16`), 380 MB +(`nomic`). Tier A at 55 MB is the cheapest embedder the app has ever shipped. + +> ⚠️ **Confirm before building on it (M2, day one).** `vision_encoder.onnx` at fp32 is 374 MB ≈ 93.6 M +> parameters ≈ the bare SigLIP-base tower, which implies the export stops *before* the pixel-shuffle +> connector and therefore emits 768-d patch embeddings. That is inferred from file size, not read off the +> graph. Load the model, print `session.outputNames` and the output dims, and check. If it is instead the +> post-connector tensor it will be 576-d — nothing breaks, because each variant already writes under its +> own cache prefix; the only change is the `visionDim` field on the tier descriptor. + +### Tiers + +| Tier | `ModelVariant` | Download | Produces | Requires | +|------|-------------------|----------|--------------------|--------------------| +| A | `smolvlm2-vision` | 55 MB | 768-d vectors | WASM is fine | +| B | `smolvlm2-256m` | 189 MB | Captions + vectors | WebGPU recommended | +| C | `smolvlm2-500m` | 358 MB | Captions + vectors | WebGPU | + +The tier is a data row, not a code branch — see "Key Design Decisions". + +### Files to Modify + +1. **`src/types.ts`** + - Extend `ModelVariant` with `'smolvlm2-vision' | 'smolvlm2-256m' | 'smolvlm2-500m'`. + - Add `VlmTier`, and the worker request/response message union. + - Add `framesPerVideo: number` to `Settings`. + +2. **`src/app.ts`** + - New branch in `loadModelOnce()` (`:1013`) for the `smolvlm2-*` variants. + - New branch in `embedAll()` (`:1433`) alongside the existing `isSapiens2` / `isChromeAI` paths + (`:1442-1443`, `:1566-1595`); new cache prefixes in the `cachePrefix` chain (`:1450-1456`). + - Generalise `extractVideoFrame()` → `extractVideoFrames(file, n, px)` (`:504-555`); resolution is a + parameter so the 224 px thumbnail path is not promoted to 512 px. Add `state.searchVectors`. + - Caption read/write sites move to IndexedDB (`:1551-1553`, `:1585`, `:2489-2493`, `:2536-2540`). + +3. **`src/db.ts`** + - `DB_VERSION` 1 → 2 (`:11`), new `captions` object store, `captionGetBatch` / `captionPutBatch` + mirroring `cacheGetBatch` / `cachePutBatch` (`:62`, `:111`). + +4. **`src/hardware.ts`** + - Add `pickVlmTier()` next to the existing `computeOptimalBatchSize()` / `getMemoryPressure()`. + +5. **`src/modelFallback.ts`** + - Add SmolVLM2 repo/file lists to `modelDownloadUrls()` (`:42`). `buildUploadCache()` (`:70`) needs no + change — it already matches on the path after `/resolve//`. + +6. **`index.html`** + - Three new `