ADFA-4838: maps + books runners (migrate to the REST engine like kiwix)

Port the remaining live-content types off the Phase 1 socket handlers onto the durable
job engine, so all three run the same way.

- Generalize the job payload: items are now arbitrary (strings for kiwix ZIMs; objects
  for maps/books). RunnerContext exposes items (raw) + ids (string convenience); routes
  accept { items } or { ids }.
- maps.exec.ts: tile-extract.py extract, reusing the socket handler's box validation;
  structured progress (parses a % when the script prints one).
- books.exec.ts: Calibre-Web auth/CSRF + Gutenberg fetch + upload, per-book progress.

Deletion (maps/books) stays a separate op for now. Typechecks (tsc --noEmit).

Author: luisguzman-adfa

static/dashboard/routes.ts

@@ -35,10 +35,13 @@ export const apiRouter: Router = express.Router();
 apiRouter.post('/:type/download', (req: Request, res: Response): void => {
     const type = String(req.params.type);
     if (!isType(type)) { res.status(404).json({ error: 'unknown type' }); return; }
-    const body = req.body as { ids?: unknown };
-    const ids = Array.isArray(body?.ids) ? body.ids.map((x) => String(x)) : [];
-    if (ids.length === 0) { res.status(400).json({ error: 'ids required' }); return; }
-    res.status(202).json(toApi(jobs.create(type, ids)));
+    // kiwix sends { ids: ["file.zim"] }; maps/books send { items: [ {...} ] }.
+    const body = req.body as { ids?: unknown; items?: unknown };
+    const items: unknown[] = Array.isArray(body?.items)
+        ? body.items
+        : Array.isArray(body?.ids) ? body.ids : [];
+    if (items.length === 0) { res.status(400).json({ error: 'items (or ids) required' }); return; }
+    res.status(202).json(toApi(jobs.create(type, items)));
 });
 
 // Poll one job's structured status.

static/dashboard/server.ts

@@ -14,6 +14,8 @@ import { handleBooksEvents } from './sockets/books.socket';
 // the shared JobManager; the router exposes them over /api.
 import { jobs } from './sockets/jobs';
 import './sockets/kiwix.exec';
+import './sockets/maps.exec';
+import './sockets/books.exec';
 import { apiRouter } from './routes';
 
 const app = express();

static/dashboard/sockets/books.exec.ts

@@ -0,0 +1,120 @@
+// sockets/books.exec.ts — ADFA-4838
+//
+// Books runner for the durable job engine: for each requested Gutenberg book, download
+// the EPUB and upload it to the local Calibre-Web. Ported from the Phase 1 books.socket
+// download_books_batch handler (auth/CSRF + fetch + upload), made durable and reporting
+// structured per-book progress. A job item is { id, title, url }.
+import { jobs, RunnerContext } from './jobs';
+import fs from 'fs';
+import path from 'path';
+
+const CALIBRE_WEB_LOCAL_URL = 'http://127.0.0.1:8083';
+const TMP_DIR = '/tmp/books_downloader/';
+const SYSTEM_USER_AGENT = 'K2Go Dashboard/1.0 (https://github.com/appdevforall/KnowledgeToGo)';
+// Defaults match the Phase 1 handler; a credential override can be added later if needed.
+const CALIBRE_WEB_USER = 'Admin';
+const CALIBRE_WEB_PASS = 'changeme';
+
+interface BookItem { id?: string; title?: string; url?: string; }
+
+/** Authenticate against Calibre-Web and return a usable cookie + fresh CSRF token. */
+async function getCalibreSession(): Promise<{ cookie: string; csrfToken: string }> {
+    const loginPageRes = await fetch(`${CALIBRE_WEB_LOCAL_URL}/login`);
+    const initialCookies = loginPageRes.headers.getSetCookie().map((c) => c.split(';')[0]).join('; ');
+    const loginHtml = await loginPageRes.text();
+
+    const csrfMatch = loginHtml.match(/name="csrf_token" value="(.*?)"/);
+    if (!csrfMatch) throw new Error('Could not find CSRF token on login page');
+    const csrfToken = csrfMatch[1];
+
+    const loginData = new URLSearchParams();
+    loginData.append('csrf_token', csrfToken);
+    loginData.append('username', CALIBRE_WEB_USER);
+    loginData.append('password', CALIBRE_WEB_PASS);
+
+    const authRes = await fetch(`${CALIBRE_WEB_LOCAL_URL}/login`, {
+        method: 'POST',
+        headers: {
+            Cookie: initialCookies,
+            'Content-Type': 'application/x-www-form-urlencoded',
+            Referer: `${CALIBRE_WEB_LOCAL_URL}/login`,
+        },
+        body: loginData,
+        redirect: 'manual',
+    });
+
+    if (authRes.status !== 302 && authRes.status !== 303) {
+        throw new Error('Invalid Calibre-Web credentials');
+    }
+
+    const authCookieString = authRes.headers.getSetCookie().map((c) => c.split(';')[0]).join('; ');
+
+    const homePageRes = await fetch(`${CALIBRE_WEB_LOCAL_URL}/`, { headers: { Cookie: authCookieString } });
+    const homeHtml = await homePageRes.text();
+    const finalCsrfMatch =
+        homeHtml.match(/name="csrf_token"\s+value="([^"]+)"/i) ||
+        homeHtml.match(/value="([^"]+)"\s+name="csrf_token"/i);
+    const finalCsrfToken = finalCsrfMatch ? finalCsrfMatch[1] : csrfToken;
+
+    return { cookie: authCookieString, csrfToken: finalCsrfToken };
+}
+
+const booksRunner: (ctx: RunnerContext) => Promise<void> = async (ctx) => {
+    if (!fs.existsSync(TMP_DIR)) fs.mkdirSync(TMP_DIR, { recursive: true });
+
+    const books = ctx.items
+        .map((x) => x as BookItem)
+        .filter((b) => b && typeof b.url === 'string' && typeof b.id === 'string');
+    if (books.length === 0) throw new Error('no books requested');
+
+    ctx.update({ phase: 'processing', percent: 0 });
+
+    let session: { cookie: string; csrfToken: string };
+    try {
+        session = await getCalibreSession();
+    } catch (e) {
+        throw new Error('Calibre-Web authentication failed');
+    }
+
+    let done = 0;
+    for (const book of books) {
+        ctx.throwIfCanceled();
+        const id = String(book.id);
+        const title = String(book.title ?? id);
+        const url = String(book.url);
+        ctx.update({ phase: 'processing', detail: title, percent: Math.round((done / books.length) * 100) });
+
+        const tmp = path.join(TMP_DIR, `pg_${id}.epub`);
+        try {
+            const response = await fetch(url, {
+                headers: { 'User-Agent': SYSTEM_USER_AGENT, Accept: 'application/epub+zip' },
+            });
+            if (!response.ok) throw new Error(`HTTP ${response.status} from Gutenberg`);
+
+            const fileBuffer = await response.arrayBuffer();
+            fs.writeFileSync(tmp, Buffer.from(fileBuffer));
+
+            const form = new FormData();
+            form.append('csrf_token', session.csrfToken);
+            form.append('btn-upload', new Blob([fileBuffer], { type: 'application/epub+zip' }), `${title}.epub`);
+
+            const uploadRes = await fetch(`${CALIBRE_WEB_LOCAL_URL}/upload`, {
+                method: 'POST',
+                headers: { Cookie: session.cookie, Referer: `${CALIBRE_WEB_LOCAL_URL}/` },
+                body: form,
+            });
+            if (!uploadRes.ok) throw new Error(`Calibre-Web rejected upload: ${uploadRes.status}`);
+        } finally {
+            if (fs.existsSync(tmp)) fs.unlinkSync(tmp);
+        }
+
+        done++;
+        ctx.update({ phase: 'processing', percent: Math.round((done / books.length) * 100) });
+    }
+
+    ctx.update({ phase: 'done', percent: 100 });
+};
+
+jobs.registerRunner('books', booksRunner);
+
+export { booksRunner };

static/dashboard/sockets/jobs.ts

@@ -42,6 +42,9 @@ export interface JobUpdate {
 /** Passed to a runner: report progress, check cancellation, spawn tracked children. */
 export interface RunnerContext {
     readonly job: Job;
+    /** Raw request items (strings for kiwix; objects for maps/books). */
+    readonly items: unknown[];
+    /** Convenience view: just the string items (kiwix ZIM filenames). */
     readonly ids: string[];
     update(patch: JobUpdate): void;
     isCanceled(): boolean;
@@ -88,12 +91,13 @@ class JobManager {
         this.runners.set(type, runner);
     }
 
-    /** Create + start a job. Returns the persisted row immediately (phase 'queued'). */
-    create(type: JobType, ids: string[]): Job {
+    /** Create + start a job. Returns the persisted row immediately (phase 'queued').
+     *  `items` are strings for kiwix (ZIM filenames) or objects for maps/books. */
+    create(type: JobType, items: unknown[]): Job {
         const now = Date.now();
         const id = `${type}-${now}-${Math.random().toString(36).slice(2, 8)}`;
         const job: Job = {
-            id, type, target: JSON.stringify(ids), phase: 'queued',
+            id, type, target: JSON.stringify(items), phase: 'queued',
             percent: -1, speed: 0, detail: null, error: null, created: now, updated: now,
         };
         this.db.prepare(
@@ -159,11 +163,12 @@ class JobManager {
         const rt = { canceled: false, procs: new Set<ChildProcess>() };
         this.runtime.set(job.id, rt);
 
-        let ids: string[] = [];
-        try { ids = JSON.parse(job.target) as string[]; } catch { ids = []; }
+        let items: unknown[] = [];
+        try { const parsed = JSON.parse(job.target); items = Array.isArray(parsed) ? parsed : []; } catch { items = []; }
+        const ids = items.filter((x): x is string => typeof x === 'string');
 
         const ctx: RunnerContext = {
-            job, ids,
+            job, items, ids,
             update: (p) => this.patch(job.id, p),
             isCanceled: () => rt.canceled,
             throwIfCanceled: () => { if (rt.canceled) throw new CanceledError(); },

static/dashboard/sockets/maps.exec.ts

@@ -0,0 +1,55 @@
+// sockets/maps.exec.ts — ADFA-4838
+//
+// Maps runner for the durable job engine: extract a tile region via tile-extract.py.
+// Ported from the Phase 1 maps.socket handler; reuses its box validation. A job item
+// is { name, box, noninteractive? }; deletion stays a separate op (not a long job).
+import { jobs, RunnerContext, CanceledError } from './jobs';
+import { parseBox } from './maps.socket';
+import path from 'path';
+
+const SCRIPTS_DIR = '/opt/iiab/maps/tile-extract/';
+const EXTRACT_SCRIPT = path.join(SCRIPTS_DIR, 'tile-extract.py');
+// name: letters/digits/hyphen/underscore, 1..34 (same rule as the socket handler).
+const NAME_RE = /^[A-Za-z0-9_-]{1,34}$/;
+
+interface MapItem { name?: string; box?: string; noninteractive?: boolean; }
+
+const mapsRunner: (ctx: RunnerContext) => Promise<void> = async (ctx) => {
+    const item = (ctx.items[0] ?? {}) as MapItem;
+    const name = String(item.name ?? '');
+    const rawBox = String(item.box ?? '');
+    if (!NAME_RE.test(name)) throw new Error('invalid region name (A-Z a-z 0-9 _ -, length 1-34)');
+    const parsed = parseBox(rawBox);
+    if (!parsed.ok) throw new Error(parsed.error);
+
+    ctx.update({ phase: 'processing', percent: -1, detail: name });
+    const args = [EXTRACT_SCRIPT, 'extract', name, parsed.box, 'noninteractive'];
+
+    await new Promise<void>((resolve, reject) => {
+        const p = ctx.spawn('sudo', args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
+        const onData = (buf: Buffer) => {
+            const text = buf.toString();
+            ctx.log(text.trim());
+            // tile-extract prints progress lines; surface a % if present, else stay indeterminate.
+            const re = /(\d+)\s*%/g;
+            let m: RegExpExecArray | null;
+            let last = -1;
+            while ((m = re.exec(text)) !== null) last = parseInt(m[1], 10);
+            if (last >= 0) ctx.update({ phase: 'processing', percent: last });
+        };
+        p.stdout?.on('data', onData);
+        p.stderr?.on('data', onData);
+        p.on('error', reject);
+        p.on('exit', (code, signal) => {
+            if (signal === 'SIGKILL' || ctx.isCanceled()) return reject(new CanceledError());
+            if (code === 0) resolve();
+            else reject(new Error(`tile-extract exited with code ${code}`));
+        });
+    });
+
+    ctx.update({ phase: 'done', percent: 100 });
+};
+
+jobs.registerRunner('maps', mapsRunner);
+
+export { mapsRunner };