Merge pull request #630 from appdevforall/feat/K2GO-443-forgejo-job-engine

K2GO-443 feat(forgejo): repo refresh on the durable job engine

Author: luisguzman-adfa

controller/app/src/main/java/org/appdevforall/k2go/forgejo/presentation/ForgejoRepoRefresh.java

@@ -3,25 +3,26 @@
  * Name        : ForgejoRepoRefresh.java
  * Author      : AppDevForAll
  * Copyright   : Copyright (c) 2026 AppDevForAll
- * Description : K2GO-422. The single "Update repos" flow, shared by the module detail button and the
- *               module action sheet row so neither duplicates it. It gates like the dashboard update
- *               (needs internet, then metered consent), shows minimal inline progress (a description, a
- *               live one-line output tail, an indeterminate bar and a Cancel) injected right after the
- *               trigger view, runs the box refresh on an IO thread, and reports the outcome in a snackbar.
+ * Description : K2GO-422 / K2GO-443. The single "Update repos" flow, shared by the module detail button
+ *               and the module action sheet row. It gates like the dashboard update (needs internet, then
+ *               metered consent), then drives the durable job engine (type "forgejo") through the shared
+ *               RestContentClient: a determinate bar (repo N of M) with the current repo name and a Cancel,
+ *               injected right after the trigger view.
  *
- *               Lifecycle: there is NO persistent app-side state. The box refresh job is detached
- *               (setsid), so a host that goes away mid-run (the sheet dismissed, the fragment detached)
- *               just drops the UI updates (guarded by View.isAttachedToWindow()); the box finishes on its
- *               own and the next /forgejo/status read reflects reality. The only state is the box's own
- *               status/pid files, which the box manages. So there is nothing here to leak or to clear.
+ *               Forgejo is a git operation (fetch + fast-forward/merge + authenticated push per seeded
+ *               example repo), not a file download, so there is no speed and NO pause/resume: the progress
+ *               is repo-count, and a retry re-runs the idempotent refresh (the engine owns that).
+ *
+ *               Lifecycle: no persistent app-side state here. The box job is durable and runs on, so a host
+ *               that goes away just drops the UI updates (guarded by View.isAttachedToWindow()); the client
+ *               polls to a terminal state and tears itself down. A terminal state (done / failed /
+ *               cancelled) removes the inline UI and reports it in a snackbar.
  * ============================================================================
  */
 package org.appdevforall.k2go.forgejo.presentation;
 
 import android.app.Activity;
 import android.content.Context;
-import android.os.Handler;
-import android.os.Looper;
 import android.text.TextUtils;
 import android.view.Gravity;
 import android.view.View;
@@ -35,14 +36,19 @@
 import com.google.android.material.progressindicator.LinearProgressIndicator;
 
 import org.appdevforall.k2go.R;
-import org.appdevforall.k2go.forgejo.data.ForgejoSeedClient;
-import org.appdevforall.k2go.util.AppExecutors;
+import org.appdevforall.k2go.content.RestContentClient;
 import org.appdevforall.k2go.util.Snackbars;
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 public final class ForgejoRepoRefresh {
 
     private ForgejoRepoRefresh() {}
 
+    // The box runner reads the repo set itself and ignores the job items, so a single sentinel satisfies
+    // POST /forgejo/download (which requires a non-empty items/ids) and keys the start-or-attach / guard.
+    private static final String SENTINEL = "repos";
+
     /**
      * Gate (internet, then metered consent) then run the refresh with progress injected right after
      * {@code trigger}. The trigger stays in place (only disabled) as a visible anchor for the snackbar.
@@ -60,7 +66,6 @@ private static void run(@NonNull View trigger) {
         final ViewGroup parent = (ViewGroup) trigger.getParent();
         if (parent == null || !trigger.isAttachedToWindow()) return;   // host went away during the gate
         final Context ctx = trigger.getContext();
-        final Handler main = new Handler(Looper.getMainLooper());
         final float d = ctx.getResources().getDisplayMetrics().density;
         final int side = Math.round(20 * d);
 
@@ -71,14 +76,13 @@ private static void run(@NonNull View trigger) {
         plp.leftMargin = side; plp.rightMargin = side; plp.topMargin = Math.round(8 * d);
         progress.setLayoutParams(plp);
 
-        // Description of what is happening (static), then a live line that tails the box output one line at
-        // a time (the per-repo outcomes advance like the proot rows: it advances, not a log box).
         final TextView label = new TextView(ctx);
         label.setText(R.string.k2go_forgejo_updating);
         label.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_BodySmall);
         label.setTextColor(ContextCompat.getColor(ctx, R.color.k2go_muted));
         progress.addView(label);
 
+        // The current repo being refreshed (from the job's detail), advancing one at a time.
         final TextView liveLine = new TextView(ctx);
         liveLine.setTextAppearance(com.google.android.material.R.style.TextAppearance_Material3_BodySmall);
         liveLine.setTextColor(ContextCompat.getColor(ctx, R.color.k2go_muted));
@@ -90,7 +94,6 @@ private static void run(@NonNull View trigger) {
         liveLine.setLayoutParams(llp);
         progress.addView(liveLine);
 
-        // The bar and Cancel share one line: the bar takes the width, Cancel sits beside it.
         final LinearLayout barLine = new LinearLayout(ctx);
         barLine.setOrientation(LinearLayout.HORIZONTAL);
         barLine.setGravity(Gravity.CENTER_VERTICAL);
@@ -100,7 +103,7 @@ private static void run(@NonNull View trigger) {
         barLine.setLayoutParams(barLineLp);
 
         final LinearProgressIndicator bar = new LinearProgressIndicator(ctx);
-        bar.setIndeterminate(true);
+        bar.setIndeterminate(true);   // becomes determinate once the first per-repo percent arrives
         LinearLayout.LayoutParams blp = new LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f);
         bar.setLayoutParams(blp);
         barLine.addView(bar);
@@ -118,43 +121,94 @@ private static void run(@NonNull View trigger) {
         parent.addView(progress, parent.indexOfChild(trigger) + 1);
         trigger.setEnabled(false);   // stays in place as an anchor; re-enabled when the refresh settles
 
+        final RestContentClient client = new RestContentClient("forgejo");
+        final boolean[] settled = { false };   // one terminal cleanup (done / error / cancel)
+        // Per-repo outcome tally the runner carries in the final detail (K2GO_SUMMARY changed problems
+        // total); -1 = unknown (an older box). Kept so "done" can still say "some blocked" / "up to date".
+        final int[] summary = { -1, -1, -1 };
+
         cancel.setOnClickListener(cv -> {
             cancel.setEnabled(false);
             label.setText(R.string.k2go_forgejo_update_cancelling);
-            AppExecutors.get().io().execute(() -> new ForgejoSeedClient().cancelRefresh());
+            client.cancel();   // cancel() tears down without a listener callback, so settle here
+            terminal(settled, trigger, parent, progress, R.string.k2go_forgejo_update_cancelled);
         });
 
-        AppExecutors.get().io().execute(() -> {
-            final ForgejoSeedClient client = new ForgejoSeedClient();
-            final ForgejoSeedClient.Result r = client.refresh(rawLine -> {
-                // Tail one line at a time, trimmed and without the org prefix, so it reads cleanly.
-                final String shown = rawLine.trim().replace("AppDevForAll/", "");
-                main.post(() -> { if (liveLine.isAttachedToWindow()) liveLine.setText(shown); });
-            });
-            // The box reports per-repo outcome counts on the refresh status (advanced / could-not / total).
-            final int changed = client.lastChanged();
-            final int problems = client.lastProblems();
-            final int total = client.lastTotal();
-            main.post(() -> {
-                if (!trigger.isAttachedToWindow()) return;
-                parent.removeView(progress);
-                trigger.setEnabled(true);
-                Snackbars.make(trigger, ctx.getString(messageFor(r, changed, problems, total))).show();
-            });
+        // The status line shows the percent and the current repo together (git progress is repo-count, so
+        // the percent is coarse: 0/33/66 for 3 repos). Percent arrives via onProgress, the repo via onLog;
+        // both feed render() so the line stays "<pct>%  <repo>".
+        final int[] pct = { -1 };
+        final String[] repo = { "" };
+        final Runnable render = () -> {
+            if (!liveLine.isAttachedToWindow()) return;
+            liveLine.setText(pct[0] >= 0 ? pct[0] + "%  " + repo[0] : repo[0]);
+        };
+
+        client.start(sentinelBody(), new RestContentClient.Listener() {
+            @Override public void onProgress(int percent, String speed) {
+                if (!liveLine.isAttachedToWindow()) return;
+                bar.setIndeterminate(percent < 0);
+                if (percent >= 0) { bar.setProgressCompat(percent, true); pct[0] = percent; render.run(); }
+            }
+            @Override public void onIndexing() {
+                if (liveLine.isAttachedToWindow()) bar.setIndeterminate(true);
+            }
+            @Override public void onLog(String line) {
+                final String t = line.trim();
+                if (t.startsWith("K2GO_SUMMARY")) {   // app<->runner token: the final outcome tally, not a repo
+                    String[] p = t.split("\\s+");
+                    if (p.length >= 4) {
+                        try {
+                            summary[0] = Integer.parseInt(p[1]);
+                            summary[1] = Integer.parseInt(p[2]);
+                            summary[2] = Integer.parseInt(p[3]);
+                        } catch (NumberFormatException ignore) { /* leave unknown */ }
+                    }
+                    return;
+                }
+                // Otherwise the job detail is the current repo ("owner/name"); drop the org prefix.
+                repo[0] = t.replace("AppDevForAll/", "");
+                render.run();
+            }
+            @Override public void onDone() {
+                terminal(settled, trigger, parent, progress, messageFor(summary[0], summary[1], summary[2]));
+            }
+            @Override public void onError(String message) {
+                terminal(settled, trigger, parent, progress, R.string.k2go_forgejo_update_failed);
+            }
         });
     }
 
-    /** Map the refresh outcome to a user message covering every state. */
-    private static int messageFor(ForgejoSeedClient.Result r, int changed, int problems, int total) {
-        if (r == ForgejoSeedClient.Result.CANCELLED) return R.string.k2go_forgejo_update_cancelled;
-        if (r != ForgejoSeedClient.Result.DONE) return R.string.k2go_forgejo_update_failed;   // box unreachable
+    private static JSONObject sentinelBody() {
+        try { return new JSONObject().put("ids", new JSONArray().put(SENTINEL)); }
+        catch (Exception e) { return new JSONObject(); }
+    }
+
+    /**
+     * Map the per-repo outcome tally to a user message. The job finished (this is onDone), so the only
+     * question is what happened per repo. -1 counts = unknown (an older box without the summary) -> the
+     * generic "updated". changed = repos that advanced; problems = conflict or a fetch/push failure.
+     */
+    private static int messageFor(int changed, int problems, int total) {
         if (problems > 0) {
-            // some repos could not be updated (conflict or a fetch/push failure); reconcile in the web UI
+            // some repos could not be updated (reconcile in the web UI); all vs some depends on the rest
             return (total - problems > 0) ? R.string.k2go_forgejo_update_some_failed
                                           : R.string.k2go_forgejo_update_all_failed;
         }
         if (changed > 0) return R.string.k2go_forgejo_update_done;    // at least one repo advanced
-        if (changed == 0) return R.string.k2go_forgejo_update_none;   // nothing to update
+        if (changed == 0 && total >= 0) return R.string.k2go_forgejo_update_none;   // nothing to update
         return R.string.k2go_forgejo_update_done;                     // unknown counts (older box)
     }
+
+    /** Remove the inline UI, re-enable the trigger, and report the outcome once (guarded). */
+    private static void terminal(boolean[] settled, @NonNull View trigger, @NonNull ViewGroup parent,
+                                 @NonNull View progress, int msgRes) {
+        if (settled[0]) return;
+        settled[0] = true;
+        if (progress.getParent() == parent) parent.removeView(progress);
+        trigger.setEnabled(true);
+        if (trigger.isAttachedToWindow()) {
+            Snackbars.make(trigger, trigger.getContext().getString(msgRes)).show();
+        }
+    }
 }

static/dashboard/CHANGELOG.md

@@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i
 (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this
 file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`.
 
+- **1.3.13** - Forgejo repo refresh on the durable job engine (K2GO-443). `forgejo` is now a job type: `POST /forgejo/download` plus `GET /forgejo/jobs/:id` (structured `{phase, percent, detail}`) and retry/cancel over the generic `/:type/*` surface. The runner (`sockets/forgejo.exec.ts`) wraps the existing box orchestration (`static/forgejo/orchestration` -> `refresh_forgejo`) and reports per-repo progress (repo N of M + the current repo name) parsed from a new `K2GO_PROGRESS` marker the orchestration emits. Forgejo is a git operation (fetch + fast-forward/merge + authenticated push per seeded example repo), not a file download, so there is NO aria2 and NO pause/resume; retry re-runs the idempotent refresh. The seed (install) path is unchanged. The older `POST /forgejo/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443)
 - **1.3.12** - Add-ons gallery download on the durable job engine (K2GO-443). `code-addons` is now a job type: `POST /code-addons/download` plus `GET /code-addons/jobs/:id` (structured `{phase, percent, speed, detail}`) and pause/resume/retry/cancel over the generic `/:type/*` surface, like build-assets. The runner (`sockets/code_addons.exec.ts`) downloads only the heavy add-on binaries (.cgp + source tarballs) with aria2c (resilient: `--continue` resume, survives a network change) using the shared `downloadWithAria2` helper; the mirror stages the small files (shell, catalog, icons, pages) with its Cloudflare clean + catalog base rewrite and prints the aria2 input for the heavy ones (`mirror_addons.py --print-aria2-input`), then verifies them (`--finalize-only`), and the runner swaps the staged tree in atomically. The older `POST /addons/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443)
 - **1.3.11** - Build-assets download on the durable job engine (K2GO-443). `code-assets` is now a job type: `POST /code-assets/download` plus `GET /code-assets/jobs/:id` (structured `{phase, percent, speed, detail}`) and pause/resume/retry/cancel over the generic `/:type/*` surface, like kiwix/maps. The runner (`sockets/code_assets.exec.ts`) downloads the build assets with aria2c (resilient: `--continue` resume, survives a full interface loss via the outer retry loop) using the shared `downloadWithAria2` helper, then the mirror verifies each file against its published `.md5` and writes the browse page (`mirror_code_assets.py --finalize-only`), and the runner swaps the staged tree in atomically. The older `POST /code-assets/refresh` (wrapper) stays for now. Localhost-only. (K2GO-443)
 - **1.3.10** - Code on the Go build-assets refresh (K2GO-437). New `POST /code-assets/refresh` re-mirrors the Code on the Go build assets into `/library/www/code-assets` LIVE (box up, no runrole) through a detached wrapper (`tools/code-assets-refresh.sh`) that mirrors into a staging dir and swaps it in only on success (a failed or cancelled refresh never serves a half-mirror), `GET /code-assets/refresh/status` returns `{state, lines, downloaded, reused, failed, upToDate}` to poll (counts parsed from the mirror's final log line; `upToDate` when nothing changed), and `POST /code-assets/refresh/cancel` stops a running refresh (SIGKILL to the wrapper process group; safe because the live tree is only replaced after a clean run). The wrapper reuses the role's `mirror_code_assets.py` (manifest-driven, per-file `.md5` verify and reuse, generates the browse page), so install (bake) and refresh (live) share one mechanism. Localhost-only. (K2GO-437)

static/dashboard/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dashboard-console",
-  "version": "1.3.12",
+  "version": "1.3.13",
   "description": "",
   "main": "index.js",
   "scripts": {

static/dashboard/routes.ts

@@ -40,7 +40,7 @@ const ZIMS_DIR = '/library/zims/content/';
 const KIWIX_INDEXER = '/usr/bin/iiab-make-kiwix-lib';
 const ZIM_NAME_RE = /^[A-Za-z0-9._-]{1,150}\.zim$/;
 
-const VALID_TYPES: JobType[] = ['kiwix', 'maps', 'books', 'kolibri', 'basemaps', 'code-assets', 'code-addons'];
+const VALID_TYPES: JobType[] = ['kiwix', 'maps', 'books', 'kolibri', 'basemaps', 'code-assets', 'code-addons', 'forgejo'];
 function isType(t: string): t is JobType {
     return (VALID_TYPES as string[]).includes(t);
 }
@@ -1117,11 +1117,11 @@ apiRouter.post('/:type/download', (req: Request, res: Response): void => {
         ? body.items
         : Array.isArray(body?.ids) ? body.ids : [];
     if (items.length === 0) { res.status(400).json({ error: 'items (or ids) required' }); return; }
-    // K2GO-443: these content types stage into ONE shared tree (/library/www/<type>.new), unlike kiwix's
-    // independent files, so only one job per such type may run at a time. The app re-attaches via
-    // start-or-attach; this is the hard guard behind it (two concurrent jobs would corrupt the staging).
-    const SINGLE_TREE_TYPES = ['code-assets', 'code-addons'];
-    if (SINGLE_TREE_TYPES.includes(type)
+    // K2GO-443: these types must run one job at a time: code-assets / code-addons stage into ONE shared
+    // tree (/library/www/<type>.new), and forgejo fetches+pushes the same repo set (two concurrent runs
+    // would race the staging / the repos). The app re-attaches via start-or-attach; this is the hard guard.
+    const SINGLE_RUN_TYPES = ['code-assets', 'code-addons', 'forgejo'];
+    if (SINGLE_RUN_TYPES.includes(type)
         && jobs.list(type).some((j) =>
             ['queued', 'downloading', 'indexing', 'processing', 'paused'].includes(j.phase))) {
         res.status(409).json({ error: `a ${type} job is already running` });

static/dashboard/server.ts

@@ -12,6 +12,7 @@ import './sockets/books.exec';
 import './sockets/kolibri.exec';
 import './sockets/code_assets.exec';   // K2GO-443: build-assets runner (aria2 job engine)
 import './sockets/code_addons.exec';   // K2GO-443: add-ons gallery runner (aria2 job engine)
+import './sockets/forgejo.exec';        // K2GO-443: forgejo repo-refresh runner (git, per-repo progress)
 import { apiRouter } from './routes';
 import { startServiceHeal } from './sockets/service-heal';
 import { startLogRotation, stopLogRotation } from './sockets/log-rotate';