Skip to content

Commit 4b38dd2

Browse files
luisguzman-adfaluisguzman-adfa
authored
Merge pull request #404 from appdevforall/feat/ADFA-5094-offline-catalog
ADFA-5094: catalog freshness service (weekly publish + on-device refresh)
2 parents c10da6f + 386e691 commit 4b38dd2

49 files changed

Lines changed: 976 additions & 4 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/publish-catalogs.yml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: Publish content catalogs
2+
3+
# ADFA-5094 (ADR-5094): generate the Kolibri/Kiwix catalogs once a week off the build and
4+
# publish them + a versioned manifest to Cloudflare R2, so the app can refresh the catalog
5+
# without an APK release. Reuses the R2 pattern the release build uses for update.json
6+
# (android-release-build.yml): aws s3 cp with the CLOUDFLARE_* vars, fixed keys that overwrite
7+
# so a URL always resolves to the latest. Content stays on the Nginx mirror; only the catalog
8+
# metadata is published here, served from k2go-download.appdevforall.org/catalogs/.
9+
on:
10+
schedule:
11+
# Weekly, Monday 06:17 UTC. Off-peak, odd minute (GitHub throttles round-hour crons).
12+
- cron: '17 6 * * 1'
13+
workflow_dispatch: {}
14+
15+
jobs:
16+
publish-catalogs:
17+
name: Generate catalogs & upload to Cloudflare R2
18+
runs-on: ubuntu-latest
19+
defaults:
20+
run:
21+
working-directory: ./controller/app
22+
steps:
23+
- name: Checkout Code
24+
uses: actions/checkout@v5
25+
with:
26+
submodules: recursive
27+
28+
- name: Set up Python
29+
uses: actions/setup-python@v5
30+
with:
31+
python-version: '3.x'
32+
33+
# The generators query Studio/Kiwix once, here, on a runner with bandwidth — not from
34+
# every release build (ADR-4954 D1). Never fail the job: a blocked fetch keeps the
35+
# committed asset, and we simply skip its upload below.
36+
- name: Generate catalogs
37+
run: |
38+
python3 tools/build_kolibri_catalog.py || echo "::warning::kolibri generator failed; its upload will be skipped"
39+
python3 tools/build_kiwix_catalog.py || echo "::warning::kiwix generator failed; its upload will be skipped"
40+
41+
- name: Build manifests and upload to Cloudflare R2
42+
env:
43+
AWS_ACCESS_KEY_ID: ${{ vars.CLOUDFLARE_KEY_ID }}
44+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }}
45+
AWS_DEFAULT_REGION: auto
46+
R2_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
47+
BUCKET_NAME: "iiaboa-apk-repo"
48+
BASE_URL: "https://k2go-download.appdevforall.org/catalogs"
49+
run: |
50+
set -euo pipefail
51+
ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
52+
GENERATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
53+
VERSION="$(date -u +%Y.%m.%d)"
54+
55+
# publish <name> <file> <content-type> <items-mode>
56+
# items-mode "jsonl" -> extract per-channel {id,version} for later per-item delta /
57+
# the "a newer version exists" signal; "none" -> file-level hash only (e.g. Kiwix CSV).
58+
publish() {
59+
name="$1"; file="$2"; ctype="$3"; items_mode="$4"
60+
if [ ! -f "$file" ]; then
61+
echo "::warning::$file not found; skipping $name (kept whatever is already published)"
62+
return 0
63+
fi
64+
base="$(basename "$file")"
65+
hash="sha256:$(sha256sum "$file" | cut -d' ' -f1)"
66+
items='[]'
67+
if [ "$items_mode" = "jsonl" ]; then
68+
items="$(grep -v '"catalog"' "$file" | jq -c '{id, version}' | jq -sc '.')"
69+
fi
70+
jq -n \
71+
--arg catalog "$name" --arg version "$VERSION" --arg generated "$GENERATED" \
72+
--arg hash "$hash" --arg url "$BASE_URL/$base" --argjson items "$items" \
73+
'{catalog:$catalog, version:$version, generated:$generated, hash:$hash, url:$url, items:$items}' \
74+
> "$name.manifest.json"
75+
echo "----- $name.manifest.json -----"; cat "$name.manifest.json"
76+
77+
# Upload the catalog first, then the manifest, so the manifest never points at a
78+
# missing file. Fixed keys -> overwrite -> the URL always resolves to the latest.
79+
aws s3 cp "$file" "s3://$BUCKET_NAME/catalogs/$base" \
80+
--endpoint-url "$ENDPOINT" --content-type "$ctype"
81+
aws s3 cp "$name.manifest.json" "s3://$BUCKET_NAME/catalogs/$name.manifest.json" \
82+
--endpoint-url "$ENDPOINT" --content-type "application/json"
83+
}
84+
85+
publish kolibri src/main/assets/kolibri_catalog.jsonl "application/x-ndjson" jsonl
86+
publish kiwix src/main/assets/kiwix_catalog.csv "text/csv" none

‎controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogManifestClient.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* ============================================================================
3+
* Name : CatalogManifestClient.java
4+
* Author : AppDevForAll
5+
* Copyright : Copyright (c) 2026 AppDevForAll
6+
* Description : ADFA-5094 (ADR-5094). Fetches a catalog manifest with a
7+
* conditional GET (ETag / If-None-Match -> 304) and downloads the
8+
* catalog body, mirroring the OTA update client's HTTP. Blocking;
9+
* call from an IO thread. Never throws.
10+
* ============================================================================
11+
*/
12+
package org.iiab.controller.catalog.data;
13+
14+
import android.util.Log;
15+
16+
import org.iiab.controller.catalog.domain.CatalogManifest;
17+
18+
import java.io.BufferedReader;
19+
import java.io.File;
20+
import java.io.FileOutputStream;
21+
import java.io.InputStream;
22+
import java.io.InputStreamReader;
23+
import java.net.HttpURLConnection;
24+
import java.net.URL;
25+
26+
public final class CatalogManifestClient {
27+
28+
private static final String TAG = "K2Go-Catalog";
29+
private static final int CONNECT_TIMEOUT_MS = 5000;
30+
private static final int READ_TIMEOUT_MS = 8000;
31+
32+
public enum Status { OK, NOT_MODIFIED, FAILED }
33+
34+
/** Outcome of a manifest fetch. {@code manifest}/{@code etag} are set only on {@code OK}. */
35+
public static final class Result {
36+
public final Status status;
37+
public final CatalogManifest manifest;
38+
public final String etag;
39+
40+
private Result(Status status, CatalogManifest manifest, String etag) {
41+
this.status = status;
42+
this.manifest = manifest;
43+
this.etag = etag;
44+
}
45+
46+
static Result ok(CatalogManifest m, String etag) { return new Result(Status.OK, m, etag); }
47+
static Result notModified() { return new Result(Status.NOT_MODIFIED, null, null); }
48+
static Result failed() { return new Result(Status.FAILED, null, null); }
49+
}
50+
51+
/** GET the manifest; sends {@code If-None-Match} when {@code knownEtag} is set. */
52+
public Result fetchManifest(String url, String knownEtag) {
53+
HttpURLConnection conn = null;
54+
try {
55+
conn = (HttpURLConnection) new URL(url).openConnection();
56+
conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
57+
conn.setReadTimeout(READ_TIMEOUT_MS);
58+
conn.setRequestMethod("GET");
59+
if (knownEtag != null && !knownEtag.isEmpty()) {
60+
conn.setRequestProperty("If-None-Match", knownEtag);
61+
}
62+
int code = conn.getResponseCode();
63+
if (code == HttpURLConnection.HTTP_NOT_MODIFIED) {
64+
return Result.notModified();
65+
}
66+
if (code != HttpURLConnection.HTTP_OK) {
67+
Log.w(TAG, "manifest fetch HTTP " + code + " for " + url);
68+
return Result.failed();
69+
}
70+
String etag = conn.getHeaderField("ETag");
71+
StringBuilder body = new StringBuilder();
72+
try (BufferedReader r = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
73+
String line;
74+
while ((line = r.readLine()) != null) {
75+
body.append(line);
76+
}
77+
}
78+
CatalogManifest m = CatalogManifestParser.parse(body.toString());
79+
return m == null ? Result.failed() : Result.ok(m, etag);
80+
} catch (Throwable t) {
81+
Log.w(TAG, "manifest fetch failed: " + t.getMessage());
82+
return Result.failed();
83+
} finally {
84+
if (conn != null) {
85+
conn.disconnect();
86+
}
87+
}
88+
}
89+
90+
/**
91+
* Download the catalog body to {@code dest} via a temp file swapped in on success, so a
92+
* partial download never replaces a good overlay. Returns true on success.
93+
*/
94+
public boolean downloadTo(String url, File dest) {
95+
HttpURLConnection conn = null;
96+
File tmp = new File(dest.getAbsolutePath() + ".tmp");
97+
try {
98+
conn = (HttpURLConnection) new URL(url).openConnection();
99+
conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
100+
conn.setReadTimeout(READ_TIMEOUT_MS);
101+
conn.setRequestMethod("GET");
102+
if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
103+
return false;
104+
}
105+
try (InputStream in = conn.getInputStream();
106+
FileOutputStream out = new FileOutputStream(tmp)) {
107+
byte[] buf = new byte[8192];
108+
int n;
109+
while ((n = in.read(buf)) != -1) {
110+
out.write(buf, 0, n);
111+
}
112+
}
113+
return tmp.renameTo(dest);
114+
} catch (Throwable t) {
115+
Log.w(TAG, "catalog download failed: " + t.getMessage());
116+
tmp.delete();
117+
return false;
118+
} finally {
119+
if (conn != null) {
120+
conn.disconnect();
121+
}
122+
}
123+
}
124+
}

‎controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogManifestParser.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* ============================================================================
3+
* Name : CatalogManifestParser.java
4+
* Author : AppDevForAll
5+
* Copyright : Copyright (c) 2026 AppDevForAll
6+
* Description : ADFA-5094 (ADR-5094). Parses the manifest JSON the weekly
7+
* workflow publishes into a CatalogManifest. Never throws — a
8+
* malformed manifest yields null, and the caller keeps the current
9+
* catalog.
10+
* ============================================================================
11+
*/
12+
package org.iiab.controller.catalog.data;
13+
14+
import org.iiab.controller.catalog.domain.CatalogManifest;
15+
import org.json.JSONArray;
16+
import org.json.JSONObject;
17+
18+
import java.util.ArrayList;
19+
import java.util.List;
20+
21+
public final class CatalogManifestParser {
22+
23+
private CatalogManifestParser() {
24+
}
25+
26+
/** @return the parsed manifest, or null if the input is missing or malformed. */
27+
public static CatalogManifest parse(String json) {
28+
if (json == null || json.isEmpty()) {
29+
return null;
30+
}
31+
try {
32+
JSONObject o = new JSONObject(json);
33+
List<CatalogManifest.Item> items = new ArrayList<>();
34+
JSONArray arr = o.optJSONArray("items");
35+
if (arr != null) {
36+
for (int i = 0; i < arr.length(); i++) {
37+
JSONObject it = arr.optJSONObject(i);
38+
if (it == null) {
39+
continue;
40+
}
41+
String id = it.optString("id", "");
42+
if (!id.isEmpty()) {
43+
items.add(new CatalogManifest.Item(id, it.optInt("version", 0)));
44+
}
45+
}
46+
}
47+
return new CatalogManifest(
48+
o.optString("catalog", ""),
49+
o.optString("version", ""),
50+
o.optString("generated", ""),
51+
o.optString("hash", ""),
52+
o.optString("url", ""),
53+
items);
54+
} catch (Throwable t) {
55+
return null;
56+
}
57+
}
58+
}

‎controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogOverlay.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* ============================================================================
3+
* Name : CatalogOverlay.java
4+
* Author : AppDevForAll
5+
* Copyright : Copyright (c) 2026 AppDevForAll
6+
* Description : ADFA-5094 (ADR-5094). Where a pulled catalog lands on the
7+
* device. The refresh worker writes the overlay here and the
8+
* catalog source reads it (when present and newer) in place of the
9+
* APK-bundled asset. Same path convention on both sides.
10+
* ============================================================================
11+
*/
12+
package org.iiab.controller.catalog.data;
13+
14+
import android.content.Context;
15+
16+
import java.io.File;
17+
18+
public final class CatalogOverlay {
19+
20+
private static final String DIR = "catalogs";
21+
22+
private CatalogOverlay() {
23+
}
24+
25+
/** {@code filesDir/catalogs}, created if missing. */
26+
public static File dir(Context ctx) {
27+
File d = new File(ctx.getApplicationContext().getFilesDir(), DIR);
28+
if (!d.exists()) {
29+
d.mkdirs();
30+
}
31+
return d;
32+
}
33+
34+
/** The overlay file for a catalog whose bundled asset is named {@code basename}. */
35+
public static File file(Context ctx, String basename) {
36+
return new File(dir(ctx), basename);
37+
}
38+
}

‎controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogRefreshScheduler.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
* ============================================================================
3+
* Name : CatalogRefreshScheduler.java
4+
* Author : AppDevForAll
5+
* Copyright : Copyright (c) 2026 AppDevForAll
6+
* Description : ADFA-5094 (ADR-5094). Enqueues the catalog refresh: a weekly,
7+
* network-constrained periodic job, plus an opportunistic one-shot
8+
* (e.g. when the picker opens). The worker's own TTL gate keeps the
9+
* one-shot cheap. Unique per catalog, KEEP so relaunches don't
10+
* reset the schedule.
11+
* ============================================================================
12+
*/
13+
package org.iiab.controller.catalog.data;
14+
15+
import android.content.Context;
16+
17+
import androidx.work.Constraints;
18+
import androidx.work.Data;
19+
import androidx.work.ExistingPeriodicWorkPolicy;
20+
import androidx.work.ExistingWorkPolicy;
21+
import androidx.work.NetworkType;
22+
import androidx.work.OneTimeWorkRequest;
23+
import androidx.work.PeriodicWorkRequest;
24+
import androidx.work.WorkManager;
25+
26+
import java.util.concurrent.TimeUnit;
27+
28+
public final class CatalogRefreshScheduler {
29+
30+
private CatalogRefreshScheduler() {
31+
}
32+
33+
private static Data input(String name, String manifestUrl, String basename) {
34+
return new Data.Builder()
35+
.putString(CatalogRefreshWorker.KEY_NAME, name)
36+
.putString(CatalogRefreshWorker.KEY_MANIFEST_URL, manifestUrl)
37+
.putString(CatalogRefreshWorker.KEY_BASENAME, basename)
38+
.build();
39+
}
40+
41+
private static Constraints connected() {
42+
return new Constraints.Builder()
43+
.setRequiredNetworkType(NetworkType.CONNECTED)
44+
.build();
45+
}
46+
47+
/** Weekly periodic refresh, enqueued once per catalog (KEEP). Safe to call on every launch. */
48+
public static void scheduleWeekly(Context ctx, String name, String manifestUrl, String basename) {
49+
PeriodicWorkRequest req = new PeriodicWorkRequest.Builder(
50+
CatalogRefreshWorker.class, 7, TimeUnit.DAYS)
51+
.setConstraints(connected())
52+
.setInputData(input(name, manifestUrl, basename))
53+
.build();
54+
WorkManager.getInstance(ctx.getApplicationContext())
55+
.enqueueUniquePeriodicWork("catalog-refresh-" + name,
56+
ExistingPeriodicWorkPolicy.KEEP, req);
57+
}
58+
59+
/** Opportunistic one-shot; the worker's TTL gate no-ops it when still fresh. */
60+
public static void refreshNow(Context ctx, String name, String manifestUrl, String basename) {
61+
OneTimeWorkRequest req = new OneTimeWorkRequest.Builder(CatalogRefreshWorker.class)
62+
.setConstraints(connected())
63+
.setInputData(input(name, manifestUrl, basename))
64+
.build();
65+
WorkManager.getInstance(ctx.getApplicationContext())
66+
.enqueueUniqueWork("catalog-refresh-now-" + name,
67+
ExistingWorkPolicy.KEEP, req);
68+
}
69+
}

0 commit comments

Comments
 (0)