From 1ec3d88f46bbec1104c0817925ccdfaac8fff11b Mon Sep 17 00:00:00 2001 From: JF Date: Wed, 7 Oct 2026 12:28:51 -0400 Subject: [PATCH] fix(javascript): vendor the pinned js-debug release without a GitHub API call (#867, #813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-js-debug.js asked api.github.com for the release's asset list before downloading, and that lookup is subject to the unauthenticated REST quota (60/h per IP). Shared CI runners and `docker build` (which carries no token) run out of it: the COBOL Host job's postinstall vendoring and the daily canary's image build both failed with 403. vendor-manifest.json pins the tag AND the asset name, so the pinned build now downloads straight from /releases/download// — served by github.com, outside the API quota, the way vendor-codelldb.js already does. 'latest' and version overrides still go through the API, and a direct download that fails (an upstream re-release under another name) falls back to it. The cobol-host job's install step also carries GITHUB_TOKEN, like the other jobs' vendor step. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 6 +++ changelog.d/867.fixed.md | 1 + .../scripts/build-js-debug.js | 53 ++++++++++++++----- .../scripts/lib/js-debug-helpers.d.ts | 16 ++++++ .../scripts/lib/js-debug-helpers.js | 38 +++++++++++++ .../tests/unit/build-js-debug.helpers.test.ts | 52 +++++++++++++++++- 6 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 changelog.d/867.fixed.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e09c333fb..a70f1fac6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -406,6 +406,12 @@ jobs: head -1 /tmp/cobc-version.txt | grep -E '^cobc \(GnuCOBOL\) 3\.1\.2([. ]|$)' - name: Install dependencies run: pnpm install --frozen-lockfile + # This job vendors in postinstall (it needs CodeLLDB and js-debug for + # the smoke); the pinned js-debug asset downloads without the API, but + # the token keeps a 'latest'/override build off the unauthenticated + # quota like the other jobs' vendor step (issue #813). + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build project run: pnpm build - name: Run COBOL host smoke and logpoints diff --git a/changelog.d/867.fixed.md b/changelog.d/867.fixed.md new file mode 100644 index 000000000..35649f475 --- /dev/null +++ b/changelog.d/867.fixed.md @@ -0,0 +1 @@ +**Installs and Docker builds no longer fail on the GitHub API rate limit** — vendoring the pinned js-debug release asked `api.github.com` for the release's asset list before downloading, and that lookup is subject to the unauthenticated quota of 60 requests per hour per IP: on a shared CI runner or inside a `docker build` (which has no token) it answered 403, failing the COBOL Host job's `pnpm install` and the daily published-artifact canary's image build. The pin in `vendor-manifest.json` already names the tag and the asset, so the vendoring now downloads it straight from `releases/download/`, which is outside the API quota — no token is needed for a pinned build. `JS_DEBUG_VERSION=latest` and version overrides still resolve through the API, and a direct download that fails falls back to it (#867, #813) diff --git a/packages/adapter-javascript/scripts/build-js-debug.js b/packages/adapter-javascript/scripts/build-js-debug.js index cf1ffa146..8191e662f 100644 --- a/packages/adapter-javascript/scripts/build-js-debug.js +++ b/packages/adapter-javascript/scripts/build-js-debug.js @@ -2,7 +2,9 @@ /** * Vendor Microsoft js-debug vsDebugServer.js into vendor/js-debug * - * - Fetches prebuilt artifact from GitHub releases (preferred) + * - Fetches prebuilt artifact from GitHub releases (preferred); the pinned + * release in vendor-manifest.json is downloaded directly from + * releases/download (no GitHub API call, so no rate limit — issues #867/#813) * - Optional build-from-source fallback when explicitly enabled * - Cross-platform (Windows/macOS/Linux), Node 18+ (uses global fetch) * - Deterministic output: @@ -11,8 +13,9 @@ * - vendor/js-debug/manifest.json * * Environment variables: - * - JS_DEBUG_VERSION: tag or 'latest' (default: 'latest') - * - GH_TOKEN: GitHub token to avoid API rate limits (optional) + * - JS_DEBUG_VERSION: tag or 'latest' (default: the pin in vendor-manifest.json) + * - GH_TOKEN: GitHub token to avoid API rate limits (optional; only the API + * path — 'latest' or a version override — is subject to them) * - JS_DEBUG_FORCE_REBUILD: 'true' to ignore cache and refetch * - JS_DEBUG_BUILD_FROM_SOURCE: 'true' to build from source if prebuilt fetch fails or is desired * @@ -31,7 +34,7 @@ import { spawn } from 'node:child_process'; import { extract as tarExtract } from 'tar'; import extractZip from 'extract-zip'; import { ensureDir, copy as fsxCopy } from 'fs-extra'; -import { selectBestAsset, normalizePath } from './lib/js-debug-helpers.js'; +import { selectBestAsset, normalizePath, pinnedAssetCandidate } from './lib/js-debug-helpers.js'; import { determineVendoringPlan } from './lib/vendor-strategy.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -559,18 +562,42 @@ async function main() { try { // Attempt prebuilt path first tmpDir = await makeTmpDir('js-debug-dl-'); - logInfo(`Fetching GitHub release '${VERSION}' for ${REPO_OWNER}/${REPO_NAME} ...`); - const release = await getRelease(VERSION); - if (release?.tag_name) { - resolvedVersion = release.tag_name; + + // The pinned release's asset is downloaded straight from + // github.com//releases/download/, which is outside the GitHub REST + // quota (60/h per IP unauthenticated — exhausted on shared CI runners and + // inside Docker builds, issues #867/#813). The API is asked only when the + // pin cannot name the asset, or when the direct download fails (an + // upstream re-release under another name), so the behaviour for 'latest' + // and version overrides is unchanged. + let best = null; + let archiveFile = null; + const pinned = pinnedAssetCandidate(PIN, VERSION); + if (pinned) { + archiveFile = path.join(tmpDir, `asset.${pinned.type === 'tgz' ? 'tgz' : 'zip'}`); + logInfo(`Pinned release ${VERSION}: downloading ${pinned.name} directly (no GitHub API call) ...`); + try { + await downloadWithRetries(pinned.url, archiveFile); + best = pinned; + } catch (err) { + logWarn(`Direct download of the pinned asset failed: ${(err && err.message) || err}. Falling back to the GitHub release API.`); + } } - const assets = Array.isArray(release?.assets) ? release.assets : []; - const best = selectBestAsset(assets); - const archiveFile = path.join(tmpDir, `asset.${best.type === 'tgz' ? 'tgz' : 'zip'}`); + if (!best) { + logInfo(`Fetching GitHub release '${VERSION}' for ${REPO_OWNER}/${REPO_NAME} ...`); + const release = await getRelease(VERSION); + if (release?.tag_name) { + resolvedVersion = release.tag_name; + } + + const assets = Array.isArray(release?.assets) ? release.assets : []; + best = selectBestAsset(assets); + archiveFile = path.join(tmpDir, `asset.${best.type === 'tgz' ? 'tgz' : 'zip'}`); - logInfo(`Selected asset: ${best.name} (${best.type}). Downloading...`); - await downloadWithRetries(best.url, archiveFile); + logInfo(`Selected asset: ${best.name} (${best.type}). Downloading...`); + await downloadWithRetries(best.url, archiveFile); + } // Supply-chain integrity gate: verify the downloaded archive against the // committed digest pin before extracting anything from it. GitHub release diff --git a/packages/adapter-javascript/scripts/lib/js-debug-helpers.d.ts b/packages/adapter-javascript/scripts/lib/js-debug-helpers.d.ts index 6687ff3ef..d33019d9d 100644 --- a/packages/adapter-javascript/scripts/lib/js-debug-helpers.d.ts +++ b/packages/adapter-javascript/scripts/lib/js-debug-helpers.d.ts @@ -35,3 +35,19 @@ export interface SelectedAsset { * @throws {Error} when no asset matches, naming the ones that were available */ export function selectBestAsset(assets: readonly ReleaseAsset[]): SelectedAsset; + +/** The `js-debug` entry of `vendor-manifest.json`, as the pin reader sees it. */ +export interface JsDebugPin { + version?: string; + upstream?: string; + /** asset file name → expected sha256 of the archive */ + assets?: Record; +} + +/** + * The asset a pinned vendoring downloads straight from + * `/releases/download//` — outside the GitHub REST + * quota (issues #867, #813). `null` when the API has to resolve the release + * (`latest`, a version override, or a pin that names no archive). + */ +export function pinnedAssetCandidate(pin: JsDebugPin, version: string): SelectedAsset | null; diff --git a/packages/adapter-javascript/scripts/lib/js-debug-helpers.js b/packages/adapter-javascript/scripts/lib/js-debug-helpers.js index facb00800..2c2b50c2a 100644 --- a/packages/adapter-javascript/scripts/lib/js-debug-helpers.js +++ b/packages/adapter-javascript/scripts/lib/js-debug-helpers.js @@ -101,3 +101,41 @@ export function selectBestAsset(assets) { return pick; } + +/** + * The asset a pinned vendoring can download without asking the GitHub API. + * + * `vendor-manifest.json` pins the release tag AND names the asset it expects, + * so for the pinned version the download URL is already known: + * `/releases/download//`. That URL is served by + * github.com (a redirect to the release CDN), not by api.github.com, so it + * is outside the unauthenticated REST quota of 60 requests per hour per IP — + * the one a shared CI runner or a Docker build has already used up when the + * `releases/tags/` lookup answers 403 (issues #867, #813). The API is + * only needed to resolve `latest` or a version override away from the pin. + * + * @param {{ version?: string, upstream?: string, assets?: Record }} pin + * The `js-debug` entry of vendor-manifest.json + * @param {string} version The requested version (`JS_DEBUG_VERSION` or the pin) + * @returns {{ url: string, name: string, type: 'tgz' | 'zip' } | null} + * The asset to download directly, or null when the API has to resolve it + */ +export function pinnedAssetCandidate(pin, version) { + const tag = pin?.version; + const upstream = String(pin?.upstream || '').replace(/\/+$/, ''); + const names = Object.keys(pin?.assets || {}); + if (!tag || !upstream || version !== tag || names.length === 0) { + return null; + } + const assets = names.map(name => ({ + name, + browser_download_url: `${upstream}/releases/download/${encodeURIComponent(tag)}/${encodeURIComponent(name)}` + })); + try { + return selectBestAsset(assets); + } catch { + // None of the pinned names is an archive the vendoring understands — + // let the API path report what the release actually offers. + return null; + } +} diff --git a/packages/adapter-javascript/tests/unit/build-js-debug.helpers.test.ts b/packages/adapter-javascript/tests/unit/build-js-debug.helpers.test.ts index 7549fef82..c760f1bbd 100644 --- a/packages/adapter-javascript/tests/unit/build-js-debug.helpers.test.ts +++ b/packages/adapter-javascript/tests/unit/build-js-debug.helpers.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest'; // Import ESM helper from JS file -import { selectBestAsset, normalizePath } from '../../scripts/lib/js-debug-helpers'; +import { selectBestAsset, normalizePath, pinnedAssetCandidate } from '../../scripts/lib/js-debug-helpers'; describe('js-debug helpers: normalizePath', () => { it('normalizes backslashes to forward slashes', () => { @@ -85,3 +85,53 @@ describe('js-debug helpers: selectBestAsset', () => { expect(() => selectBestAsset(assets)).toThrow(/No matching js-debug asset found/i); }); }); + +describe('js-debug helpers: pinnedAssetCandidate (issues #867, #813)', () => { + const pin = { + version: 'v1.112.0', + upstream: 'https://github.com/microsoft/vscode-js-debug', + assets: { + 'js-debug-dap-v1.112.0.tar.gz': '31eb1bd9792f62c32f7c22b66ce612e2e54a7664201a2d80bdb49cc4bf4ca925' + } + }; + + it('builds a releases/download URL for the pinned asset so no GitHub API call is needed', () => { + const candidate = pinnedAssetCandidate(pin, 'v1.112.0'); + expect(candidate).toEqual({ + name: 'js-debug-dap-v1.112.0.tar.gz', + url: 'https://github.com/microsoft/vscode-js-debug/releases/download/v1.112.0/js-debug-dap-v1.112.0.tar.gz', + type: 'tgz' + }); + }); + + it('returns null for "latest" — the release must be resolved through the API', () => { + expect(pinnedAssetCandidate(pin, 'latest')).toBeNull(); + }); + + it('returns null for a version override away from the pin', () => { + expect(pinnedAssetCandidate(pin, 'v1.111.0')).toBeNull(); + }); + + it('returns null when the pin names no assets', () => { + expect(pinnedAssetCandidate({ ...pin, assets: {} }, 'v1.112.0')).toBeNull(); + expect(pinnedAssetCandidate({ version: 'v1.112.0', upstream: pin.upstream }, 'v1.112.0')).toBeNull(); + }); + + it('applies the selectBestAsset preference when the pin names several assets', () => { + const several = { + ...pin, + assets: { + 'js-debug-dap-v1.112.0.zip': 'a', + 'js-debug-dap-v1.112.0.tar.gz': 'b' + } + }; + expect(pinnedAssetCandidate(several, 'v1.112.0')?.name).toBe('js-debug-dap-v1.112.0.tar.gz'); + }); + + it('derives the download host from the pin upstream, with a trailing slash tolerated', () => { + const forked = { ...pin, upstream: 'https://github.com/example/js-debug-fork/' }; + expect(pinnedAssetCandidate(forked, 'v1.112.0')?.url).toBe( + 'https://github.com/example/js-debug-fork/releases/download/v1.112.0/js-debug-dap-v1.112.0.tar.gz' + ); + }); +});