fix(ci): handle GitHub release API failures safely

.github/workflows/ci.yml

@@ -66,6 +66,10 @@ jobs:
         shell: bash
         run: scripts/test-read-current-local-artifact.sh
 
+      - name: Test GitHub API retry and 404 handling
+        shell: bash
+        run: scripts/test-github-api.sh
+
       - name: Test release probe fixture
         shell: bash
         run: scripts/test-probe-release.sh

.github/workflows/mirror.yml

@@ -368,6 +368,7 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
+          source scripts/github-api.sh
           release_assets=(
             assets/status.png
             release-manifest.json
@@ -474,7 +475,8 @@ jobs:
             create_args+=(--latest)
           fi
 
-          if existing_assets_json="$(gh release view "$RELEASE_TAG" --json assets --jq '.assets' 2>/dev/null)"; then
+          release_lookup_status=0
+          if existing_assets_json="$(github_release_assets_json_allow_404 "$RELEASE_TAG")"; then
             desired_asset_names=()
             for asset in "${release_assets[@]}"; do
               desired_asset_names+=("$(basename "$asset")")
@@ -560,6 +562,11 @@ jobs:
               echo "Release assets were reconciled without bulk clobbering existing downloads."
             fi
           else
+            release_lookup_status=$?
+            if [[ "$release_lookup_status" -ne "$GITHUB_API_NOT_FOUND_STATUS" ]]; then
+              echo "Unable to determine whether GitHub Release $RELEASE_TAG exists; refusing to create it." >&2
+              exit "$release_lookup_status"
+            fi
             if [[ "${#required_existing_asset_names[@]}" -gt 0 ]]; then
               echo "Release $RELEASE_TAG does not exist, but release-manifest.json depends on preserved assets that can only be reused from an existing Release:" >&2
               printf '  %s\n' "${required_existing_asset_names[@]}" >&2

scripts/emergency-publish-release.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/github-api.sh"
+
 tag="${1:?release tag is required}"
 title="${2:?release title is required}"
 notes_file="${3:?release notes file is required}"
@@ -97,7 +99,8 @@ lowercase() {
   tr '[:upper:]' '[:lower:]' <<<"$1"
 }
 
-if existing_assets="$(gh release view "$tag" --json assets --jq '.assets' 2>/dev/null)"; then
+release_lookup_status=0
+if existing_assets="$(github_release_assets_json_allow_404 "$tag")"; then
   existing_names_file="$(mktemp)"
   jq -r '.[].name' <<<"$existing_assets" | sort > "$existing_names_file"
   unexpected_existing_assets="$(comm -13 "$seen_names_file" "$existing_names_file")"
@@ -156,6 +159,11 @@ if existing_assets="$(gh release view "$tag" --json assets --jq '.assets' 2>/dev
     gh release upload "$tag" "$asset"
   done
 else
+  release_lookup_status=$?
+  if [[ "$release_lookup_status" -ne "$GITHUB_API_NOT_FOUND_STATUS" ]]; then
+    echo "Unable to determine whether GitHub prerelease $tag exists; refusing to create it." >&2
+    exit "$release_lookup_status"
+  fi
   gh release create "$tag" \
     --target "${GITHUB_SHA:-main}" \
     --title "$title" \

scripts/github-api.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+# Sourceable GitHub API helpers. A confirmed HTTP 404 is returned as a distinct
+# status so callers never confuse a transient API failure with a missing object.
+
+GITHUB_API_NOT_FOUND_STATUS=44
+
+github_api_error_is_transient() {
+  local error_file="$1"
+
+  grep -Eqi \
+    'HTTP (429|5[0-9][0-9])|timed out|timeout|connection (reset|refused)|temporary failure|unexpected EOF' \
+    "$error_file"
+}
+
+github_api_json_allow_404() {
+  local attempts="${GITHUB_API_ATTEMPTS:-3}"
+  local delay_seconds="${GITHUB_API_RETRY_DELAY_SECONDS:-2}"
+  local attempt
+  local error_file
+  local output
+  local status
+
+  if [[ ! "$attempts" =~ ^[1-9][0-9]*$ ]]; then
+    echo "GITHUB_API_ATTEMPTS must be a positive integer, got '$attempts'." >&2
+    return 2
+  fi
+  if [[ ! "$delay_seconds" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
+    echo "GITHUB_API_RETRY_DELAY_SECONDS must be a non-negative number, got '$delay_seconds'." >&2
+    return 2
+  fi
+
+  error_file="$(mktemp)"
+  for ((attempt = 1; attempt <= attempts; attempt++)); do
+    : > "$error_file"
+    if output="$(gh api "$@" 2>"$error_file")"; then
+      rm -f "$error_file"
+      printf '%s\n' "$output"
+      return 0
+    else
+      status=$?
+    fi
+
+    if grep -q 'HTTP 404' "$error_file"; then
+      rm -f "$error_file"
+      return "$GITHUB_API_NOT_FOUND_STATUS"
+    fi
+
+    cat "$error_file" >&2
+    if ((attempt < attempts)) && github_api_error_is_transient "$error_file"; then
+      echo "GitHub API request failed transiently; retrying ($attempt/$attempts)." >&2
+      sleep "$delay_seconds"
+      continue
+    fi
+
+    rm -f "$error_file"
+    return "$status"
+  done
+
+  rm -f "$error_file"
+  return 1
+}
+
+github_release_json_allow_404() {
+  local tag="$1"
+
+  github_api_json_allow_404 "repos/{owner}/{repo}/releases/tags/$tag"
+}
+
+github_release_assets_json_allow_404() {
+  local release_json
+  local status
+  local tag="$1"
+
+  if release_json="$(github_release_json_allow_404 "$tag")"; then
+    jq -c '.assets // []' <<<"$release_json"
+    return 0
+  else
+    status=$?
+  fi
+
+  return "$status"
+}

scripts/probe-release.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/github-api.sh"
+
 product_id="9PLM9XGG6VKS"
 package_identity="OpenAI.Codex"
 architecture="x64"
@@ -353,32 +355,6 @@ asset_size() {
   jq -r --arg name "$asset_name" '.[] | select(.name == $name) | .size' <<<"$assets_json" | head -n 1
 }
 
-github_api_json_allow_404() {
-  local err_file
-  local output
-  local status
-
-  err_file="$(mktemp)"
-  if output="$(gh api "$@" 2>"$err_file")"; then
-    rm -f "$err_file"
-    printf '%s\n' "$output"
-    return 0
-  else
-    # Capture gh's real exit status here: `$?` taken after the closing `fi` would
-    # be the if-statement's status (0 when the condition is false and there is no
-    # else), masking the failure.
-    status=$?
-  fi
-  if grep -q 'HTTP 404' "$err_file"; then
-    rm -f "$err_file"
-    return 1
-  fi
-
-  cat "$err_file" >&2
-  rm -f "$err_file"
-  return "$status"
-}
-
 latest_release_tag() {
   local release_json
   local status
@@ -391,7 +367,7 @@ latest_release_tag() {
     # condition with no else), not the helper's — capture it in the else branch.
     status=$?
   fi
-  if [[ "$status" -eq 1 ]]; then
+  if [[ "$status" -eq "$GITHUB_API_NOT_FOUND_STATUS" ]]; then
     printf ''
     return 0
   fi
@@ -401,7 +377,19 @@ latest_release_tag() {
 
 github_release_json() {
   local tag="$1"
-  gh api "repos/{owner}/{repo}/releases/tags/$tag"
+  local status
+
+  if github_release_json_allow_404 "$tag"; then
+    return 0
+  else
+    status=$?
+  fi
+  if [[ "$status" -eq "$GITHUB_API_NOT_FOUND_STATUS" ]]; then
+    echo "GitHub release '$tag' was not found." >&2
+    return 1
+  fi
+
+  return "$status"
 }
 
 release_assets_json() {

scripts/test-beta-publish-release.sh

@@ -59,6 +59,18 @@ cat > "$tmp_dir/bin/gh" <<'GH'
 set -euo pipefail
 
 printf '%s\n' "$*" >> "${GH_MOCK_LOG:?}"
+if [[ "${1:-}" == "api" ]]; then
+  if [[ "${GH_MOCK_API_ERROR:-}" == "503" ]]; then
+    echo 'gh: HTTP 503' >&2
+    exit 1
+  fi
+  if [[ ! -f "${GH_MOCK_STATE:?}" ]]; then
+    echo 'gh: Not Found (HTTP 404)' >&2
+    exit 1
+  fi
+  cat "$GH_MOCK_STATE"
+  exit 0
+fi
 [[ "${1:-}" == "release" ]] || { echo "unexpected gh command: $*" >&2; exit 1; }
 operation="${2:-}"
 tag="${3:-}"
@@ -139,6 +151,8 @@ publish() {
     GH_REPO=Wangnov/codex-app-mirror \
     GH_MOCK_STATE="$tmp_dir/release-state.json" \
     GH_MOCK_LOG="$tmp_dir/gh.log" \
+    GH_MOCK_API_ERROR="${GH_MOCK_API_ERROR:-}" \
+    GITHUB_API_RETRY_DELAY_SECONDS=0 \
     GITHUB_SHA=fixture-sha \
     bash "$repo_root/scripts/emergency-publish-release.sh" \
       "$tag" \
@@ -149,6 +163,21 @@ publish() {
       "$tmp_dir/artifacts"
 }
 
+# A transient lookup failure must never be treated as proof that the release is
+# absent, even when every retry is exhausted.
+set +e
+GH_MOCK_API_ERROR=503 publish >"$tmp_dir/503.log" 2>&1
+lookup_status=$?
+set -e
+if [[ "$lookup_status" -eq 0 ]] ||
+   [[ -f "$tmp_dir/release-state.json" ]] ||
+   grep -Fq "release create $tag" "$tmp_dir/gh.log"; then
+  echo "Expected GitHub API 503 to fail closed before prerelease creation." >&2
+  cat "$tmp_dir/503.log" >&2
+  exit 1
+fi
+: > "$tmp_dir/gh.log"
+
 (
   cd "$repo_root"
   publish > "$tmp_dir/first.log"

scripts/test-github-api.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+tmp_dir="$(mktemp -d)"
+cleanup() {
+  rm -rf "$tmp_dir"
+}
+trap cleanup EXIT
+
+mkdir -p "$tmp_dir/bin"
+counter_file="$tmp_dir/counter"
+
+cat > "$tmp_dir/bin/gh" <<'GH'
+#!/usr/bin/env bash
+set -euo pipefail
+
+[[ "${1:-}" == "api" ]] || { echo "unexpected gh invocation: $*" >&2; exit 2; }
+count="$(cat "${TEST_GH_COUNTER:?}")"
+count=$((count + 1))
+printf '%s' "$count" > "$TEST_GH_COUNTER"
+
+case "${TEST_GH_MODE:?}" in
+  recover)
+    if ((count < 3)); then
+      echo 'gh: HTTP 503' >&2
+      exit 1
+    fi
+    printf '{"tag_name":"codex-app-1.2.3"}\n'
+    ;;
+  persistent-503)
+    echo 'gh: HTTP 503' >&2
+    exit 1
+    ;;
+  not-found)
+    echo 'gh: Not Found (HTTP 404)' >&2
+    exit 1
+    ;;
+  release-assets)
+    printf '{"assets":[{"name":"asset.bin","size":3,"digest":"sha256:abc"}]}\n'
+    ;;
+  *)
+    echo "unexpected TEST_GH_MODE: $TEST_GH_MODE" >&2
+    exit 2
+    ;;
+esac
+GH
+chmod +x "$tmp_dir/bin/gh"
+
+source "$repo_root/scripts/github-api.sh"
+
+run_with_mode() {
+  local mode="$1"
+  shift
+
+  printf '0' > "$counter_file"
+  PATH="$tmp_dir/bin:$PATH" \
+    TEST_GH_MODE="$mode" \
+    TEST_GH_COUNTER="$counter_file" \
+    GITHUB_API_ATTEMPTS=3 \
+    GITHUB_API_RETRY_DELAY_SECONDS=0 \
+    "$@"
+}
+
+output="$(run_with_mode recover github_api_json_allow_404 'repos/{owner}/{repo}/releases/latest' 2>"$tmp_dir/recover.err")"
+[[ "$(jq -r .tag_name <<<"$output")" == "codex-app-1.2.3" ]]
+[[ "$(cat "$counter_file")" == "3" ]]
+[[ "$(grep -c 'retrying' "$tmp_dir/recover.err")" == "2" ]]
+
+set +e
+run_with_mode persistent-503 github_api_json_allow_404 'repos/{owner}/{repo}/releases/latest' >"$tmp_dir/503.out" 2>"$tmp_dir/503.err"
+status=$?
+set -e
+[[ "$status" -ne 0 && "$status" -ne "$GITHUB_API_NOT_FOUND_STATUS" ]]
+[[ "$(cat "$counter_file")" == "3" ]]
+[[ ! -s "$tmp_dir/503.out" ]]
+grep -Fq 'gh: HTTP 503' "$tmp_dir/503.err"
+
+set +e
+run_with_mode not-found github_api_json_allow_404 'repos/{owner}/{repo}/releases/latest' >"$tmp_dir/404.out" 2>"$tmp_dir/404.err"
+status=$?
+set -e
+[[ "$status" -eq "$GITHUB_API_NOT_FOUND_STATUS" ]]
+[[ "$(cat "$counter_file")" == "1" ]]
+[[ ! -s "$tmp_dir/404.out" ]]
+
+assets="$(run_with_mode release-assets github_release_assets_json_allow_404 codex-app-1.2.3)"
+[[ "$(jq -r '.[0].name' <<<"$assets")" == "asset.bin" ]]
+
+echo "GitHub API retry and 404 handling fixture PASS"