diff --git a/controller/app/src/main/java/org/iiab/controller/kolibri/data/LocalTreeSource.java b/controller/app/src/main/java/org/iiab/controller/kolibri/data/LocalTreeSource.java index 6d82b9143..2f59e7266 100644 --- a/controller/app/src/main/java/org/iiab/controller/kolibri/data/LocalTreeSource.java +++ b/controller/app/src/main/java/org/iiab/controller/kolibri/data/LocalTreeSource.java @@ -46,8 +46,10 @@ public final class LocalTreeSource implements TreeSource { private static final String TAG = "K2Go-Kolibri"; - /** Endpoint under the box API base: {@code /k2go-api/kolibri/tree/}. */ - private static final String TREE_PATH = "/kolibri/tree/"; + // Endpoint under the box API base: /k2go-api/kolibri/subtree/. Deliberately NOT + // /kolibri/tree/:channelId — that one is the web wizard's granular tree (counts, no bytes). + // This one returns Studio-shaped JSON with byte sizes, so StudioCatalogMapper parses it (ADFA-5094). + private static final String TREE_PATH = "/kolibri/subtree/"; /** Box API base ({@code …:8085/k2go-api}); overridable so a test can point at a mock. */ private static final String DEFAULT_BASE = BoxEndpoints.API; diff --git a/controller/app/src/test/java/org/iiab/controller/kolibri/data/LocalTreeSourceTest.java b/controller/app/src/test/java/org/iiab/controller/kolibri/data/LocalTreeSourceTest.java index cc1ac4ea5..6970a1c78 100644 --- a/controller/app/src/test/java/org/iiab/controller/kolibri/data/LocalTreeSourceTest.java +++ b/controller/app/src/test/java/org/iiab/controller/kolibri/data/LocalTreeSourceTest.java @@ -30,7 +30,7 @@ * the same treatment {@link StudioTreeSourceTest} gives the internet source. * *

Two things are specific to the local source and worth pinning: it addresses - * the box's {@code /k2go-api/kolibri/tree/} path, and a 404 — the ordinary case + * the box's {@code /k2go-api/kolibri/subtree/} path, and a 404 — the ordinary case * where a channel's metadata is not on the box — comes back as {@code null} so * {@link FallbackTreeSource} moves on to Studio, rather than as an error. The * JSON parsing itself is Studio's mapper and is exercised there; here the focus @@ -84,7 +84,7 @@ public void readsATreeAndHitsTheBoxPath() throws Exception { assertEquals(800L, t.subtreeBytes()); RecordedRequest req = server.takeRequest(); - assertEquals("/k2go-api/kolibri/tree/" + CH, req.getPath()); + assertEquals("/k2go-api/kolibri/subtree/" + CH, req.getPath()); assertEquals("GET", req.getMethod()); assertEquals("application/json", req.getHeader("Accept")); } @@ -125,6 +125,6 @@ public void aBaseUrlWithoutATrailingSlashWorksToo() throws Exception { String noSlash = server.url("/k2go-api").toString(); server.enqueue(json(treeJson())); assertNotNull(new LocalTreeSource(noSlash).fetchTree(CH)); - assertEquals("/k2go-api/kolibri/tree/" + CH, server.takeRequest().getPath()); + assertEquals("/k2go-api/kolibri/subtree/" + CH, server.takeRequest().getPath()); } } diff --git a/controller/docs/ADR-5094-offline-tree-depth.md b/controller/docs/ADR-5094-offline-tree-depth.md index c54ccac8d..71739dc68 100644 --- a/controller/docs/ADR-5094-offline-tree-depth.md +++ b/controller/docs/ADR-5094-offline-tree-depth.md @@ -25,11 +25,22 @@ So the tree can live on the box at a fraction of the content's size. onto the box without downloading content. The box then knows the whole tree offline; the content is fetched later, subtree by subtree, as it is today. -**D2 — Box serves the tree, Studio-shaped (PR3).** The box answers -`GET /k2go-api/kolibri/tree/:nodeId` with the **same JSON as Studio's `contentnode_tree`**. -Not a new shape: `StudioCatalogMapper.tree(JSONObject)` then parses box and Studio -responses with no branching, and the device side stays a thin read. The box owns the -translation from Kolibri's own model into that shape. +**D2 — Box serves the tree, Studio-shaped, from the local DB (PR3).** The box answers +`GET /k2go-api/kolibri/subtree/:nodeId` with the **same JSON as Studio's `contentnode_tree`**, +built from the local Kolibri content database. `StudioCatalogMapper.tree(JSONObject)` then +parses box and Studio responses with no branching, and the device side stays a thin read. + +*Corrected after reading the box (PR3 exploration):* +- **The path is `/kolibri/subtree/:nodeId`, not `/kolibri/tree/:nodeId`.** A + `GET /kolibri/tree/:channelId` already exists and serves the web wizard a *granular* tree + (`contentnode_granular`: resource counts, camelCase, **no byte sizes**). Reusing it would + collide and give the app the wrong shape, so the app path is a distinct, nodeId-keyed one. +- **Byte sizes come from raw SQL, not a Kolibri API.** The granular endpoint carries no + bytes, and `importexportsizeview` reports only a selection's *outstanding* transfer, not + per-node structural size. So the box sums `content_localfile.file_size` over each node's + MPTT (`lft`/`rght`) range and emits it in Studio's `files[].file_size` field. This is what + gives the picker byte sizes at every level offline — the reason for choosing this over an + app-side cache of the granular tree. **D3 — App routes local-first (app, PR2, this PR).** `fetchTree` prefers the box and falls back to Studio. Concretely: a `TreeSource` seam with two implementations — `LocalTreeSource` @@ -39,7 +50,7 @@ Every source keeps the "never throws, null on failure" contract, so a box miss channel not imported — falls through to Studio silently. **D4 — Two independent PRs.** PR2 is the app side (this). PR3 is the box/dashboard side -(D1, D2). They are independent: with only PR2 merged the box has no `/tree` endpoint, so +(D1, D2). They are independent: with only PR2 merged the box has no `/subtree` endpoint, so `LocalTreeSource` always misses and behaviour is exactly what it was before — the tree comes from Studio. Nothing to gate, nothing to break; PR3 lights the local path up. diff --git a/static/dashboard/routes.ts b/static/dashboard/routes.ts index 0c074b4d1..570a4a629 100644 --- a/static/dashboard/routes.ts +++ b/static/dashboard/routes.ts @@ -13,8 +13,8 @@ import { searchCatalog, listLibrary, removeBook, listLanguages, getCalibreSessio import { parseBox, parseEstimate } from './sockets/maps.socket'; import { preflight, listInstalledChannels, browseRemoteChannels, resolveIdentifier, - browseChannelTree, estimateSelection, deleteChannel, getKolibriTask, verifyCredentials, - ChannelNotInstalledError, + browseChannelTree, buildLocalSubtree, estimateSelection, deleteChannel, getKolibriTask, + verifyCredentials, ChannelNotInstalledError, } from './sockets/kolibri.query'; import { checkReadiness, KolibriAuthError, KolibriApiError, login as kolibriLogin } from './sockets/kolibri.session'; import { @@ -472,6 +472,26 @@ apiRouter.get('/kolibri/tree/:channelId', async (req: Request, res: Response): P } }); +// Studio-shaped subtree WITH byte sizes, built from the local DB (ADFA-5094). Distinct from +// /kolibri/tree/:channelId above (granular, for the web wizard, no bytes): the app's +// LocalTreeSource parses this with the same mapper it uses for Studio, so the picker shows +// sizes at every level offline. Node-id keyed; 404 when the channel's metadata is not imported, +// which the app reads as "fall back to Studio". +apiRouter.get('/kolibri/subtree/:nodeId', (req: Request, res: Response): void => { + const nodeId = String(req.params.nodeId); + if (!/^[0-9a-f]{32}$/.test(nodeId)) { + res.status(400).json({ error: 'invalid node id' }); + return; + } + try { + const tree = buildLocalSubtree(nodeId); + if (!tree) { res.status(404).json({ error: 'channel not imported' }); return; } + res.json(tree); + } catch (e: any) { + res.status(500).json({ error: e?.message || 'subtree build failed' }); + } +}); + // What a selection still has to transfer, plus free space, for the consent step. // Body: { channelId, nodeIds?, excludeNodeIds? } apiRouter.post('/kolibri/estimate', async (req: Request, res: Response): Promise => { diff --git a/static/dashboard/sockets/kolibri.query.ts b/static/dashboard/sockets/kolibri.query.ts index 63d9e699a..5847e792c 100644 --- a/static/dashboard/sockets/kolibri.query.ts +++ b/static/dashboard/sockets/kolibri.query.ts @@ -226,6 +226,135 @@ export async function browseChannelTree( return toTreeNode(node); } +// ---- Offline subtree, Studio-shaped, with byte sizes (ADFA-5094) ------------------ +// The app's LocalTreeSource reads this and parses it with the SAME mapper it uses for +// Studio, so the wire shape here is Studio's public contentnode_tree — not the granular +// TreeNode above (which the web wizard uses and which carries no bytes). +// +// Why raw SQL and not a Kolibri API: the granular endpoint has no byte sizes, and +// importexportsizeview reports only a selection's OUTSTANDING transfer, never per-node +// structural size. The per-node bytes live in content_localfile.file_size, so they are +// summed here over each node's MPTT (lft/rght) range. That is what lets the picker show +// sizes at every level with no network. +// +// NOTE (verification): the SQL runs against a live Kolibri database, which cannot be +// exercised in CI — it is validated on a box with a metadata-imported channel. If a column +// or the schema differs, this throws, the route returns 500, and LocalTreeSource falls back +// to Studio: a wrong query degrades to today's behaviour, it does not break the picker. + +interface StudioFile { file_size: number; preset: string; } +interface StudioNode { + id: string; + title: string; + kind: string; + is_leaf: boolean; + lft: number; + rght: number; + files: StudioFile[]; + children: { more: unknown | null; results: StudioNode[] }; +} + +interface CnRow { + id: string; + parent_id: string | null; + title: string; + kind: string; + lft: number; + rght: number; +} +interface FileRow { node_id: string; preset: string; file_size: number; } + +// A generous ceiling that keeps the JSON well under LocalTreeSource's 8 MB cap while +// covering ordinary channels whole. A subtree larger than this is truncated and its root +// is marked incomplete, so the mapper reports the size as unknown rather than under-counting. +const SUBTREE_NODE_CAP = 20000; + +/** + * One channel subtree rooted at {@code nodeId}, in Studio's contentnode_tree shape, built + * from the local content database. Returns {@code null} when the node is not present — i.e. + * the channel's metadata has not been imported — which the app reads as "fall back to Studio". + * + * The channel is scoped by the root node's own channel_id + MPTT range, so a node id alone + * is enough (as with Studio), and no channelId has to be threaded from the caller. + */ +export function buildLocalSubtree(nodeId: string): StudioNode | null { + if (!fs.existsSync(MAIN_DB)) return null; + const db = new Database(MAIN_DB, { readonly: true }); + try { + const root = db.prepare( + `SELECT id, parent_id, title, kind, lft, rght, channel_id + FROM content_contentnode WHERE id = ?`, + ).get(nodeId) as (CnRow & { channel_id: string }) | undefined; + if (!root) return null; + + const rows = db.prepare( + `SELECT id, parent_id, title, kind, lft, rght + FROM content_contentnode + WHERE channel_id = ? AND lft >= ? AND rght <= ? + ORDER BY lft + LIMIT ?`, + ).all(root.channel_id, root.lft, root.rght, SUBTREE_NODE_CAP) as CnRow[]; + + // Real content files only: thumbnails and supplementary files are excluded so the + // sizes match a normal import, the same exclusion StudioCatalogMapper makes. + // No LIMIT on the subquery (unlike the rows query): it must cover every node the rows + // query kept — an ORDER-BY-less LIMIT here could select a different subset and leave a + // built node without its files, under-counting its size. Files for nodes past the cap + // are simply never looked up. + const files = db.prepare( + `SELECT f.contentnode_id AS node_id, f.preset AS preset, lf.file_size AS file_size + FROM content_file f + JOIN content_localfile lf ON lf.id = f.local_file_id + WHERE f.thumbnail = 0 AND f.supplementary = 0 + AND f.contentnode_id IN ( + SELECT id FROM content_contentnode + WHERE channel_id = ? AND lft >= ? AND rght <= ?)`, + ).all(root.channel_id, root.lft, root.rght) as FileRow[]; + + const filesByNode = new Map(); + for (const f of files) { + const arr = filesByNode.get(f.node_id) ?? []; + arr.push({ file_size: f.file_size ?? 0, preset: f.preset ?? '' }); + filesByNode.set(f.node_id, arr); + } + + const byId = new Map(); + for (const r of rows) { + byId.set(r.id, { + id: r.id, + title: r.title ?? '', + kind: r.kind ?? '', + is_leaf: r.kind !== 'topic', + lft: r.lft, + rght: r.rght, + files: filesByNode.get(r.id) ?? [], + children: { more: null, results: [] }, + }); + } + for (const r of rows) { + if (r.id === root.id || !r.parent_id) continue; + byId.get(r.parent_id)?.children.results.push(byId.get(r.id)!); + } + + const rootNode = byId.get(root.id) ?? null; + // Truncation correctness. Rows are ordered by lft and capped, so the dropped nodes are + // exactly those with lft beyond the last one kept. Any node whose range (lft..rght) + // extends past that cutoff therefore has a missing descendant, so mark it incomplete — + // the mapper then reports its subtree size as unknown instead of silently under-counting. + // Over-marking is safe; under-marking is the bug. Nodes fully within the cutoff keep a + // complete child set and an exact size. + if (rows.length >= SUBTREE_NODE_CAP) { + const cutoffLft = rows[rows.length - 1].lft; + for (const node of byId.values()) { + if (node.rght > cutoffLft) node.children.more = { cursor: 'truncated' }; + } + } + return rootNode; + } finally { + db.close(); + } +} + export interface SelectionSize { resourceCount: number; fileSize: number;