ADFA-4739: Kotlin docs DB pipeline + Build Kotlin Docs GitHub Action

.github/workflows/docdb-regression-test.yaml

@@ -98,7 +98,9 @@ jobs:
         echo "Extracting database from zip file..."
 
         # Install unzip if not available
-        sudo apt-get update -qq && sudo apt-get install -y unzip
+        # brotli: docdb_studio reads dictionary-compressed Content rows through the
+        # CLI (ADFA-5153); the downloaded production database is one of those.
+        sudo apt-get update -qq && sudo apt-get install -y unzip brotli
 
         # Extract the zip file
         if ! unzip -o documentation.zip; then

.gitignore

@@ -12,3 +12,11 @@ __pycache__/
 *$py.class
 *.db
 *.sqlite
+run_e2e_pipeline_test.local.sh
+grep_content_blobs.local.py
+
+# Timestamped safety backups written by populate_db.py /
+# insert_optimized_media.py / sync_kdoc_json_to_db.py before they modify a
+# database ("*.db" above does not match these - the timestamp comes last).
+*.db.backup-*
+*.db.bak.*

Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+# Builds the kotlin-stdlib/kotlin-test/kotlin-reflect API docs as JSON via the
+# kdoc-to-json Dokka plugin, against a full kotlin/ (https://github.com/JetBrains/kotlin)
+# repo checkout - freshly compiling and publishing the plugin from source
+# first, so every run picks up whatever's currently in
+# Dokka-plugin-kdoc2json/kdoc-to-json/src, not a jar left over from an
+# earlier run.
+#
+# Only generates the JSON output (dokkaGenerateModuleJson), not the default
+# HTML - JSON/latest/all-libs is the only thing this project's pipeline
+# (sync_kdoc_json_to_db.py) consumes. Use build-kotlin-stdlib.sh directly,
+# against libraries/tools/kotlin-stdlib-docs, if you also want the HTML
+# comparison output that test_kotlin_stdlib.sh checks against.
+#
+# The target kotlin-stdlib-docs project's build.gradle.kts is swapped out
+# for this directory's own (JSON-plugin-enabled) copy for the duration of
+# the build, then restored automatically on exit - the kotlin checkout is
+# left exactly as it was found, whether the build succeeds or fails.
+#
+# kotlin_big (a subproject of kotlin-stdlib-docs) extracts the actual
+# kotlin-stdlib/-reflect/-test binaries it documents from a Maven repo. Left to
+# its own devices it looks for "<kotlin-repo-root>/build/repo" at the checkout's
+# own defaultSnapshotVersion - i.e. artifacts that only exist if you have built
+# the entire kotlin repo locally first, and that are published nowhere public.
+# Against a plain `git clone --depth 1` that resolves to nothing and the build
+# fails before generating any docs. --kotlin-libs-version / --kotlin-libs-repo
+# point it at already-published artifacts instead, which is hours of CI cheaper
+# than building Kotlin just to document it.
+#
+# Only the final output path is written to stdout; every other message goes
+# to stderr, so this composes as:
+#   STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh <kotlin-repo-root>)"
+#
+# Usage:
+#   build-stdlib-json-docs.sh [options] <kotlin-repo-root> [output-dir]
+#
+# Options:
+#   --kotlin-libs-version V  Version of the kotlin-stdlib/-reflect/-test
+#                            artifacts to document (Gradle -PdeployVersion).
+#                            Should match <kotlin-repo-root>'s checked-out ref.
+#                            Default: unset, i.e. the checkout's own
+#                            defaultSnapshotVersion, which needs a local build
+#                            of the kotlin repo to exist.
+#   --kotlin-libs-repo URL   Maven repo to resolve them from (Gradle
+#                            -PkotlinLibsRepo). Default: unset. kotlin_big
+#                            already declares mavenCentral(), so a released
+#                            --kotlin-libs-version needs no repo override; this
+#                            is for a private or snapshot repo.
+set -euo pipefail
+
+log() { echo "$@" >&2; }
+
+usage() { log "Usage: $0 [--kotlin-libs-version V] [--kotlin-libs-repo URL] <path-to-kotlin-repo-root> [output-dir]"; }
+
+KOTLIN_LIBS_VERSION=""
+KOTLIN_LIBS_REPO=""
+POSITIONAL=()
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --kotlin-libs-version) KOTLIN_LIBS_VERSION="$2"; shift 2 ;;
+        --kotlin-libs-repo) KOTLIN_LIBS_REPO="$2"; shift 2 ;;
+        -h|--help) usage; exit 0 ;;
+        --) shift; POSITIONAL+=("$@"); break ;;
+        -*) log "error: unrecognized option '$1'"; usage; exit 1 ;;
+        *) POSITIONAL+=("$1"); shift ;;
+    esac
+done
+set -- ${POSITIONAL[@]+"${POSITIONAL[@]}"}
+
+if [ $# -lt 1 ]; then
+    usage
+    exit 1
+fi
+
+KOTLIN_ROOT="$(cd "$1" && pwd)"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PLUGIN_DIR="$(cd "$SCRIPT_DIR/../../kdoc-to-json" && pwd)"
+STDLIB_DOCS_DIR="$KOTLIN_ROOT/libraries/tools/kotlin-stdlib-docs"
+OUTPUT_ROOT="$(mkdir -p "${2:-$SCRIPT_DIR/build-output}" && cd "${2:-$SCRIPT_DIR/build-output}" && pwd)"
+JSON_OUTPUT_DIR="$OUTPUT_ROOT/json"
+
+# Passed through only when set, so an unset value falls through to the
+# build's own default rather than overriding it with an empty string.
+ARTIFACT_ARGS=()
+[ -n "$KOTLIN_LIBS_VERSION" ] && ARTIFACT_ARGS+=("-PdeployVersion=$KOTLIN_LIBS_VERSION")
+[ -n "$KOTLIN_LIBS_REPO" ] && ARTIFACT_ARGS+=("-PkotlinLibsRepo=$KOTLIN_LIBS_REPO")
+
+if [ ! -f "$KOTLIN_ROOT/gradle.properties" ]; then
+    log "error: '$KOTLIN_ROOT' doesn't look like a kotlin repo checkout (missing gradle.properties)."
+    exit 1
+fi
+if [ ! -f "$STDLIB_DOCS_DIR/settings.gradle.kts" ] || [ ! -x "$STDLIB_DOCS_DIR/gradlew" ]; then
+    log "error: '$STDLIB_DOCS_DIR' doesn't look like a kotlin-stdlib-docs project (missing settings.gradle.kts or gradlew)."
+    exit 1
+fi
+if [ ! -x "$PLUGIN_DIR/gradlew" ]; then
+    log "error: kdoc-to-json plugin project not found at '$PLUGIN_DIR' (missing gradlew)."
+    exit 1
+fi
+
+# The JSON-plugin-enabled build.gradle.kts we're about to install reads
+# dokka_version as a plain Gradle project property (-Pdokka_version=...)
+# rather than through this repo's own version catalog, so it has to be
+# supplied explicitly - pulled from the same catalog entry the rest of the
+# kotlin repo's Dokka usage is pinned to, so it never drifts out of sync.
+DOKKA_VERSION="$(grep -m1 '^dokka[[:space:]]*=' "$KOTLIN_ROOT/gradle/libs.versions.toml" | sed -E 's/^dokka[[:space:]]*=[[:space:]]*"([^"]*)".*/\1/')"
+if [ -z "$DOKKA_VERSION" ]; then
+    log "error: couldn't find a 'dokka = \"...\"' entry in $KOTLIN_ROOT/gradle/libs.versions.toml"
+    exit 1
+fi
+
+log "==> [1/2] Building and publishing a fresh copy of the kdoc-to-json plugin..."
+# Sent to stderr (fd 2), not left on stdout - a caller doing
+# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh ...)" must only capture the
+# final path this script echoes, not gradlew's own build console output.
+( cd "$PLUGIN_DIR" && ./gradlew clean publishToMavenLocal ) >&2
+
+log "==> Installing kdoc-to-json-enabled build.gradle.kts into $STDLIB_DOCS_DIR"
+ORIGINAL_BUILD_GRADLE="$(mktemp)"
+cp "$STDLIB_DOCS_DIR/build.gradle.kts" "$ORIGINAL_BUILD_GRADLE"
+restore_build_gradle() {
+    cp "$ORIGINAL_BUILD_GRADLE" "$STDLIB_DOCS_DIR/build.gradle.kts"
+    rm -f "$ORIGINAL_BUILD_GRADLE"
+}
+trap restore_build_gradle EXIT
+cp "$SCRIPT_DIR/build.gradle.kts" "$STDLIB_DOCS_DIR/build.gradle.kts"
+
+log "==> [2/2] Generating JSON documentation via kdoc-to-json (dokka $DOKKA_VERSION)..."
+log "    stdlib artifacts: ${KOTLIN_LIBS_VERSION:-(checkout default: needs a local kotlin build)}" \
+    "from ${KOTLIN_LIBS_REPO:-(mavenCentral + checkout default repo)}"
+# --refresh-dependencies forces Gradle to re-resolve the just-published
+# SNAPSHOT jar from mavenLocal() rather than serving a same-GAV copy it
+# cached from an earlier run of this same script.
+( cd "$STDLIB_DOCS_DIR" && ./gradlew dokkaGenerateModuleJson \
+    "-PdocsBuildDir=$JSON_OUTPUT_DIR" \
+    "-Pdokka_version=$DOKKA_VERSION" \
+    ${ARTIFACT_ARGS[@]+"${ARTIFACT_ARGS[@]}"} \
+    --refresh-dependencies ) >&2
+
+ALL_LIBS_DIR="$JSON_OUTPUT_DIR/latest/all-libs"
+if [ ! -d "$ALL_LIBS_DIR" ]; then
+    log "error: expected output at '$ALL_LIBS_DIR' but it wasn't created."
+    exit 1
+fi
+
+log "==> Done."
+echo "$ALL_LIBS_DIR"

Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts

@@ -46,14 +46,54 @@ allprojects {
         // 3. Maven Central (Keep this for standard standard stable libraries like Gson/Coroutines)
         mavenCentral()
 
-        // ALL REMOTE JETBRAINS SNAPSHOT SERVERS HAVE BEEN REMOVED!
+        // 4. Dokka's own dev-snapshot server - required by plugins:dokka-samples-transformer-plugin
+        // and plugins:dokka-version-filter-plugin (both included by kotlin-stdlib-docs'
+        // settings.gradle.kts and pulled onto the build graph by its dokka-convention plugin),
+        // which pin to a Dokka dev build rather than a Maven Central release. Same property +
+        // default kotlin-stdlib-docs' own settings.gradle.kts uses, so this only ever points
+        // wherever that project already expects it to.
+        maven(url = providers.gradleProperty("dokka_repository")
+            .getOrElse("https://redirector.kotlinlang.org/maven/dokka-dev"))
     }
 
     // --- ADDED THIS EXCLUSION BLOCK ---
     // Globally ignore the remote playground plugin since it is trapped on the dead 503 server.
     configurations.all {
         exclude(group = "org.jetbrains.dokka", module = "kotlin-playground-samples-plugin")
     }
+
+    // kotlin-stdlib-docs' own two Dokka plugin subprojects
+    // (plugins:dokka-samples-transformer-plugin, plugins:dokka-version-filter-plugin)
+    // each hardcode `kotlin { jvmToolchain(8) }`. Both are pulled onto the build
+    // graph by the dokka-convention plugin, and dokkaGenerateModuleJson below
+    // depends on dokkaGeneratePublicationHtml, so their dependencies have to
+    // resolve even though this build only ever wants JSON out. Gradle then needs a
+    // JDK 8 toolchain, and with no toolchain download repository configured it
+    // fails during task-graph resolution before a single doc is generated:
+    //   > Failed to calculate the value of task
+    //     ':plugins:dokka-samples-transformer-plugin:compileJava' property 'javaCompiler'.
+    //      > Cannot find a Java installation on your machine ... matching:
+    //        {languageVersion=8, ...}. Toolchain download repositories have not been configured.
+    // That only survives on a runner that happens to have an EOL JDK 8 lying
+    // around for auto-detection to find; it fails on any that doesn't, which
+    // includes the act container build-kotlin-docs-local.yaml is written for.
+    // These two plugins are only ever loaded in-process by Dokka, under the same
+    // JVM this build is already running on, so compiling them for that JVM instead
+    // is sufficient - and it means the pipeline needs exactly one JDK (the 17 that
+    // .github/workflows/build-kotlin-docs*.yaml installs), not two.
+    //
+    // Deliberately keyed off the running JVM rather than a hardcoded 17: whatever
+    // JDK Gradle is on is guaranteed to be present, so this can never ask for a
+    // toolchain that isn't installed, even if the workflow's java-version moves.
+    // Registered via plugins.withId + afterEvaluate so it runs after each
+    // subproject's own jvmToolchain(8) call rather than being overwritten by it.
+    plugins.withId("org.jetbrains.kotlin.jvm") {
+        afterEvaluate {
+            extensions.findByType(JavaPluginExtension::class.java)?.toolchain {
+                languageVersion.set(JavaLanguageVersion.of(JavaVersion.current().majorVersion))
+            }
+        }
+    }
 }
 
 val isTeamcityBuild = project.hasProperty("teamcity.version") ||
@@ -70,8 +110,38 @@ val defaultSnapshotVersion: String by rootProperties
 val kotlinLanguageVersion: String by rootProperties
 
 val githubRevision = if (isTeamcityBuild) project.property("githubRevision") else "master"
-val artifactsVersion by extra(if (isTeamcityBuild) project.property("deployVersion") as String else defaultSnapshotVersion)
-val artifactsRepo by extra(if (isTeamcityBuild) project.property("kotlinLibsRepo") as String else "$kotlin_root/build/repo")
+
+// Where kotlin_big pulls the kotlin-stdlib/-reflect/-test binaries it extracts
+// and documents from, and at what version.
+//
+// Upstream only honours -PkotlinLibsRepo/-PdeployVersion under TeamCity, and
+// otherwise hardcodes "$kotlin_root/build/repo" at defaultSnapshotVersion
+// (2.5.255-SNAPSHOT here) - i.e. the artifacts a *local build of the kotlin
+// repo itself* would have published. A plain `git clone --depth 1` has no such
+// build output, and that snapshot version is published nowhere public, so
+// :kotlin_big:extractStdlibCommonMain fails to resolve and the whole docs build
+// dies before generating anything:
+//   > Could not find org.jetbrains.kotlin:kotlin-stdlib:2.5.255-SNAPSHOT
+// Building the kotlin repo just to get them is hours of CI for artifacts that
+// already exist on Maven Central, so accept both as ordinary Gradle properties
+// regardless of TeamCity. kotlin_big already declares mavenCentral() alongside
+// artifactsRepo, so a released version resolves with no repo override at all -
+// -PkotlinLibsRepo is there for a private/snapshot repo (and to keep the pair
+// symmetric with upstream's own TeamCity path).
+//
+// Blank is treated as unset so a workflow input that defaults to '' falls
+// through to the upstream behaviour rather than resolving against an empty URL.
+fun overrideOrNull(name: String): String? =
+    (findProperty(name) as String?)?.takeIf { it.isNotBlank() }
+
+val artifactsVersion by extra(
+    overrideOrNull("deployVersion")
+        ?: if (isTeamcityBuild) project.property("deployVersion") as String else defaultSnapshotVersion
+)
+val artifactsRepo by extra(
+    overrideOrNull("kotlinLibsRepo")
+        ?: if (isTeamcityBuild) project.property("kotlinLibsRepo") as String else "$kotlin_root/build/repo"
+)
 val dokka_version: String by project
 
 println("# Parameters summary:")