|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# maven-offline: the download machine (K2GO-437). |
| 4 | +# |
| 5 | +# Builds one offline Maven2 repository under out/repo that holds every dependency |
| 6 | +# the builds of Knowledge to Go, Code on the Go, and the add-ons need. A device or |
| 7 | +# a laptop on the K2Go network then builds those projects with the network off. |
| 8 | +# |
| 9 | +# Pipeline: resolve (per project, isolated Gradle home) -> reshape (Gradle cache to |
| 10 | +# Maven2 layout) -> extras (R8/D8, aapt2, brotli4j) -> checksums -> report -> smoke. |
| 11 | +# |
| 12 | +# Run on an online host. Linux is the canonical producer host. See README.md. |
| 13 | + |
| 14 | +set -euo pipefail |
| 15 | + |
| 16 | +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 17 | +REPO_ROOT="$(cd "$HERE/../.." && pwd)" |
| 18 | +OUT="$HERE/out/repo" |
| 19 | +HOMES="$HERE/.gradle-homes" |
| 20 | +WORK="$HERE/.work" |
| 21 | +INIT="$HERE/gradle/resolve-all.init.gradle" |
| 22 | +PROJECTS_TSV="$HERE/config/projects.tsv" |
| 23 | +EXTRAS_TSV="$HERE/config/extra-artifacts.tsv" |
| 24 | +LOCAL_SRC="$HERE/config/sources.local.tsv" |
| 25 | + |
| 26 | +MAVEN_CENTRAL="https://repo1.maven.org/maven2" |
| 27 | +GOOGLE_MAVEN="https://dl.google.com/dl/android/maven2" |
| 28 | + |
| 29 | +DO_SMOKE=0 |
| 30 | +NO_RESOLVE=0 |
| 31 | +ONLY="" |
| 32 | + |
| 33 | +usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; } |
| 34 | + |
| 35 | +while [ $# -gt 0 ]; do |
| 36 | + case "$1" in |
| 37 | + --smoke) DO_SMOKE=1 ;; |
| 38 | + --no-resolve) NO_RESOLVE=1 ;; |
| 39 | + --out) OUT="${2:?--out requires a value}"; shift ;; |
| 40 | + --only) ONLY="${2:?--only requires a value}"; shift ;; |
| 41 | + -h|--help) usage; exit 0 ;; |
| 42 | + *) echo "unknown arg: $1" >&2; exit 2 ;; |
| 43 | + esac |
| 44 | + shift |
| 45 | +done |
| 46 | + |
| 47 | +log() { printf '>> %s\n' "$*"; } |
| 48 | + |
| 49 | +# gradlew wrapper for a project dir, picking the Windows launcher under Git Bash. |
| 50 | +gradlew_for() { |
| 51 | + case "$(uname -s)" in |
| 52 | + MINGW*|MSYS*|CYGWIN*) printf '%s/gradlew.bat' "$1" ;; |
| 53 | + *) printf '%s/gradlew' "$1" ;; |
| 54 | + esac |
| 55 | +} |
| 56 | + |
| 57 | +# Resolve a local path override for a project name (config/sources.local.tsv), else "". |
| 58 | +local_override() { |
| 59 | + [ -f "$LOCAL_SRC" ] || { printf ''; return; } |
| 60 | + awk -F'\t' -v n="$1" '!/^#/ && $1==n {print $2; exit}' "$LOCAL_SRC" |
| 61 | +} |
| 62 | + |
| 63 | +# Clone (shallow) or point at a local checkout; echoes the source dir. |
| 64 | +source_dir() { |
| 65 | + local name="$1" source="$2" ref="$3" ovr |
| 66 | + ovr="$(local_override "$name")" |
| 67 | + if [ -n "$ovr" ]; then printf '%s' "$ovr"; return; fi |
| 68 | + if [ "$source" = "self" ]; then printf '%s' "$REPO_ROOT"; return; fi |
| 69 | + local dst="$WORK/$name" |
| 70 | + if [ ! -d "$dst/.git" ]; then |
| 71 | + mkdir -p "$WORK" |
| 72 | + git clone --depth 1 --branch "$ref" "$source" "$dst" >&2 |
| 73 | + fi |
| 74 | + printf '%s' "$dst" |
| 75 | +} |
| 76 | + |
| 77 | +# Force-download one Gradle build's dependencies into an isolated home. |
| 78 | +resolve_build() { |
| 79 | + local gradle_root="$1" home="$2" gw |
| 80 | + gw="$(gradlew_for "$gradle_root")" |
| 81 | + # Make a missing wrapper a loud skip, not a swallowed failure: some add-ons in the |
| 82 | + # addons repo ship no per-build gradlew, which would otherwise drop their deps silently. |
| 83 | + if [ ! -f "$gw" ]; then |
| 84 | + log "WARN: no gradlew at $gradle_root; skipping (this build has no own wrapper)" |
| 85 | + return 0 |
| 86 | + fi |
| 87 | + log "resolve: $gradle_root (home ${home##*/})" |
| 88 | + ( cd "$gradle_root" && "$gw" --gradle-user-home "$home" \ |
| 89 | + --init-script "$INIT" --no-daemon --console=plain -q \ |
| 90 | + resolveAllDeps ) || log "resolve returned nonzero (lenient; continuing)" |
| 91 | +} |
| 92 | + |
| 93 | +# Copy one isolated home's module cache into out/repo as Maven2 layout. |
| 94 | +# Gradle cache: files-2.1/<group>/<artifact>/<version>/<hash>/<file> |
| 95 | +# Maven2: <group-with-slashes>/<artifact>/<version>/<file> |
| 96 | +reshape_home() { |
| 97 | + local home="$1" cache="$1/caches/modules-2/files-2.1" n=0 |
| 98 | + [ -d "$cache" ] || { log "no cache in ${home##*/}"; return; } |
| 99 | + while IFS= read -r f; do |
| 100 | + local rel="${f#"$cache"/}" |
| 101 | + local group="${rel%%/*}"; rel="${rel#*/}" |
| 102 | + local artifact="${rel%%/*}"; rel="${rel#*/}" |
| 103 | + local version="${rel%%/*}"; rel="${rel#*/}" |
| 104 | + local file="${rel##*/}" |
| 105 | + local dest="$OUT/${group//.//}/$artifact/$version" |
| 106 | + mkdir -p "$dest" |
| 107 | + [ -f "$dest/$file" ] || cp "$f" "$dest/$file" |
| 108 | + n=$((n+1)) |
| 109 | + done < <(find "$cache" -type f) |
| 110 | + log "reshaped ${home##*/}: $n files" |
| 111 | +} |
| 112 | + |
| 113 | +# Download <url> to <dest> (skip if present and non-empty). |
| 114 | +fetch() { |
| 115 | + local url="$1" dest="$2" |
| 116 | + [ -s "$dest" ] && return 0 |
| 117 | + mkdir -p "$(dirname "$dest")" |
| 118 | + curl -fsSL "$url" -o "$dest" && return 0 |
| 119 | + rm -f "$dest"; log "miss: $url"; return 1 |
| 120 | +} |
| 121 | + |
| 122 | +# Place one GAV artifact (jar + pom) from a base repo into out/repo. |
| 123 | +place_artifact() { |
| 124 | + local base="$1" group="$2" artifact="$3" version="$4" classifier="$5" ext="$6" |
| 125 | + local gpath="${group//.//}/$artifact/$version" |
| 126 | + local name="$artifact-$version"; [ "$classifier" != "-" ] && name="$name-$classifier" |
| 127 | + fetch "$base/$gpath/$name.$ext" "$OUT/$gpath/$name.$ext" || true |
| 128 | + fetch "$base/$gpath/$artifact-$version.pom" "$OUT/$gpath/$artifact-$version.pom" || true |
| 129 | +} |
| 130 | + |
| 131 | +# Extras a plain resolve misses: brotli4j natives (config) + aapt2/R8 (derived). |
| 132 | +fetch_extras() { |
| 133 | + log "extras: brotli4j natives" |
| 134 | + while IFS=$'\t' read -r group artifact version classifier ext hosts; do |
| 135 | + [ -z "${group:-}" ] && continue |
| 136 | + case "$group" in \#*) continue ;; esac |
| 137 | + place_artifact "$MAVEN_CENTRAL" "$group" "$artifact" "$version" "$classifier" "$ext" |
| 138 | + done < "$EXTRAS_TSV" |
| 139 | + fetch_aapt2 |
| 140 | +} |
| 141 | + |
| 142 | +# aapt2 (Linux + Windows) follows each project's AGP version. R8/D8 is NOT fetched here: |
| 143 | +# for AGP 9.x it ships inside com.android.tools.build:builder, which a resolve captures. |
| 144 | +fetch_aapt2() { |
| 145 | + # Data-driven: the AGP versions are the com.android.tools.build:gradle dirs the resolve |
| 146 | + # produced, so a project moving (e.g. K2Go 8.4 -> 8.8) needs no edit here. aapt2 is |
| 147 | + # fetched for every AGP present (a few MB each): over-fetching is safe, under-fetching |
| 148 | + # would break an offline build. |
| 149 | + local gdir="$OUT/com/android/tools/build/gradle" |
| 150 | + [ -d "$gdir" ] || { log "no AGP in repo yet; skipping aapt2"; return 0; } |
| 151 | + local meta="$WORK/aapt2-metadata.xml" |
| 152 | + fetch "$GOOGLE_MAVEN/com/android/tools/build/aapt2/maven-metadata.xml" "$meta" \ |
| 153 | + || { log "aapt2: metadata unavailable; skipping aapt2"; return 0; } |
| 154 | + local d agp ver |
| 155 | + for d in "$gdir"/*/; do |
| 156 | + agp="$(basename "$d")" |
| 157 | + ver="$(grep -oE "<version>${agp//./\\.}-[0-9]+</version>" "$meta" | sed -E 's:</?version>::g' | tail -1)" |
| 158 | + [ -z "$ver" ] && { log "aapt2: no published version for AGP $agp"; continue; } |
| 159 | + log "aapt2 for AGP $agp -> $ver (linux, windows)" |
| 160 | + place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" linux jar |
| 161 | + place_artifact "$GOOGLE_MAVEN" com.android.tools.build aapt2 "$ver" windows jar |
| 162 | + done |
| 163 | +} |
| 164 | + |
| 165 | +# Gradle Module Metadata (.module) can declare a file whose served `url` differs from the |
| 166 | +# cache `name` (KMP androidx -android AARs: cache name lifecycle-runtime-release.aar, url |
| 167 | +# lifecycle-runtime-android-<v>.aar). A Maven2 consumer reading the .module fetches by url, |
| 168 | +# so a reshape that keeps only the `name` 404s offline. Materialize a copy under each url. |
| 169 | +materialize_module_urls() { |
| 170 | + log "materialize GMM urls" |
| 171 | + python3 - "$OUT" <<'PY' |
| 172 | +import json, os, sys, shutil |
| 173 | +root = sys.argv[1]; made = 0 |
| 174 | +for dp, _, files in os.walk(root): |
| 175 | + for fn in files: |
| 176 | + if not fn.endswith('.module'): continue |
| 177 | + try: |
| 178 | + with open(os.path.join(dp, fn), encoding='utf-8') as f: mod = json.load(f) |
| 179 | + except Exception: continue |
| 180 | + for var in mod.get('variants', []): |
| 181 | + for fe in var.get('files', []): |
| 182 | + name, url = fe.get('name'), fe.get('url') |
| 183 | + if not name or not url or name == url: continue |
| 184 | + # url may be relative with ../ (GMM relocations point to a sibling version dir) |
| 185 | + src = os.path.join(dp, name) |
| 186 | + dst = os.path.normpath(os.path.join(dp, url)) |
| 187 | + if os.path.exists(src) and not os.path.exists(dst): |
| 188 | + os.makedirs(os.path.dirname(dst), exist_ok=True) |
| 189 | + shutil.copyfile(src, dst); made += 1 |
| 190 | +print(f"materialized {made} url-named copies") |
| 191 | +PY |
| 192 | +} |
| 193 | + |
| 194 | +# sha1 + md5 beside every artifact (Gradle validates .sha1 on download). |
| 195 | +write_checksums() { |
| 196 | + log "checksums" |
| 197 | + find "$OUT" -type f ! -name '*.sha1' ! -name '*.md5' | while IFS= read -r f; do |
| 198 | + [ -f "$f.sha1" ] || sha1sum "$f" | cut -d' ' -f1 > "$f.sha1" |
| 199 | + [ -f "$f.md5" ] || md5sum "$f" | cut -d' ' -f1 > "$f.md5" |
| 200 | + done |
| 201 | +} |
| 202 | + |
| 203 | +report() { |
| 204 | + log "repository: $OUT" |
| 205 | + log "size: $(du -sh "$OUT" | cut -f1) files: $(find "$OUT" -type f | wc -l)" |
| 206 | +} |
| 207 | + |
| 208 | +process_project() { |
| 209 | + local name="$1" source="$2" ref="$3" gradle_root="$4" mode="$5" |
| 210 | + [ -n "$ONLY" ] && [ "$ONLY" != "$name" ] && return |
| 211 | + local src; src="$(source_dir "$name" "$source" "$ref")" |
| 212 | + local root="$src/$gradle_root" |
| 213 | + if [ "$mode" = "multi" ]; then |
| 214 | + local sub |
| 215 | + for sub in "$root"/*/; do |
| 216 | + [ -e "$sub/settings.gradle" ] || [ -e "$sub/settings.gradle.kts" ] || continue |
| 217 | + resolve_build "${sub%/}" "$HOMES/$name-$(basename "$sub")" |
| 218 | + reshape_home "$HOMES/$name-$(basename "$sub")" |
| 219 | + done |
| 220 | + else |
| 221 | + resolve_build "$root" "$HOMES/$name" |
| 222 | + reshape_home "$HOMES/$name" |
| 223 | + fi |
| 224 | +} |
| 225 | + |
| 226 | +# Rebuild the repo from already-resolved homes, skipping Gradle. Lets you re-run the |
| 227 | +# reshape / extras / checksums after a tool change without re-downloading, and reuse a |
| 228 | +# resolve done elsewhere (drop its Gradle home under .gradle-homes/). |
| 229 | +reshape_all_homes() { |
| 230 | + local home |
| 231 | + for home in "$HOMES"/*/; do |
| 232 | + [ -d "$home/caches/modules-2/files-2.1" ] || continue |
| 233 | + reshape_home "${home%/}" |
| 234 | + done |
| 235 | +} |
| 236 | + |
| 237 | +main() { |
| 238 | + mkdir -p "$OUT" "$HOMES" |
| 239 | + if [ "$NO_RESOLVE" = 1 ]; then |
| 240 | + reshape_all_homes |
| 241 | + else |
| 242 | + while IFS=$'\t' read -r name source ref gradle_root mode; do |
| 243 | + [ -z "${name:-}" ] && continue |
| 244 | + case "$name" in \#*) continue ;; esac |
| 245 | + process_project "$name" "$source" "$ref" "$gradle_root" "$mode" |
| 246 | + done < "$PROJECTS_TSV" |
| 247 | + fi |
| 248 | + fetch_extras |
| 249 | + materialize_module_urls |
| 250 | + write_checksums |
| 251 | + report |
| 252 | + [ "$DO_SMOKE" = 1 ] && smoke_test |
| 253 | + log "done" |
| 254 | +} |
| 255 | + |
| 256 | +# Prove each root project resolves from out/repo with the network off. This is the |
| 257 | +# acceptance proof that the repository is complete. It reuses each project's already |
| 258 | +# downloaded Gradle distribution so --offline needs no network for the wrapper itself. |
| 259 | +smoke_test() { |
| 260 | + local offline_init="$HERE/gradle/offline-repo.init.gradle" |
| 261 | + local repo; repo="$(cd "$OUT" && pwd)" |
| 262 | + while IFS=$'\t' read -r name source ref gradle_root mode; do |
| 263 | + [ -z "${name:-}" ] && continue |
| 264 | + case "$name" in \#*) continue ;; esac |
| 265 | + [ -n "$ONLY" ] && [ "$ONLY" != "$name" ] && continue |
| 266 | + [ "$mode" = "multi" ] && continue # the add-ons share the trunk; smoke the roots |
| 267 | + local src root warm smoke gw |
| 268 | + src="$(source_dir "$name" "$source" "$ref")" |
| 269 | + root="$src/$gradle_root" |
| 270 | + warm="$HOMES/$name"; smoke="$HOMES/$name-offline" |
| 271 | + rm -rf "$smoke"; mkdir -p "$smoke" |
| 272 | + [ -d "$warm/wrapper" ] && cp -r "$warm/wrapper" "$smoke/wrapper" |
| 273 | + gw="$(gradlew_for "$root")" |
| 274 | + log "smoke (offline): $name" |
| 275 | + if ( cd "$root" && "$gw" --gradle-user-home "$smoke" --offline \ |
| 276 | + "-Dmavenoffline.repo=$repo" \ |
| 277 | + --init-script "$offline_init" --init-script "$INIT" \ |
| 278 | + --no-daemon --console=plain -q resolveAllDeps ); then |
| 279 | + log "smoke OK: $name resolves with the network off" |
| 280 | + else |
| 281 | + log "smoke FAIL: $name has missing artifacts in out/repo"; return 1 |
| 282 | + fi |
| 283 | + done < "$PROJECTS_TSV" |
| 284 | +} |
| 285 | + |
| 286 | +main |
0 commit comments