From 5da78a6613b07977af1443cb5d5161f04e9d03f4 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 16:16:32 -0800 Subject: [PATCH 01/14] Refactor --- .github/workflows/build.yml | 45 ++++++++++++ .github/workflows/fc-versions.yml | 116 ++++++++++++++---------------- README.md | 15 ++-- build.sh | 56 ++++++++++++--- 4 files changed, 153 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b2267b5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,45 @@ +name: Build Firecracker Versions + +on: + workflow_call: + outputs: + versions: + description: "JSON array of versions that were built" + value: ${{ jobs.prepare.outputs.versions }} + workflow_dispatch: + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.set-versions.outputs.versions }} + steps: + - uses: actions/checkout@v4 + + - name: Parse versions from file + id: set-versions + run: | + # Read versions, skip comments and empty lines, output as JSON array + versions=$(grep -v '^ *#' firecracker_versions.txt | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "versions=$versions" >> $GITHUB_OUTPUT + echo "Building versions: $versions" + + build: + needs: prepare + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + version: ${{ fromJson(needs.prepare.outputs.versions) }} + steps: + - uses: actions/checkout@v4 + + - name: Build Firecracker ${{ matrix.version }} + run: ./build.sh "${{ matrix.version }}" + + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: firecracker-${{ matrix.version }} + path: builds/ + retention-days: 7 diff --git a/.github/workflows/fc-versions.yml b/.github/workflows/fc-versions.yml index 1b9aab9..de354d6 100644 --- a/.github/workflows/fc-versions.yml +++ b/.github/workflows/fc-versions.yml @@ -2,68 +2,49 @@ name: FC Versions on: push: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true permissions: id-token: write contents: write jobs: + # Run parallel builds via reusable workflow + build: + uses: ./.github/workflows/build.yml + publish: - name: Build Firecracker and upload + name: Collect and upload builds + needs: build runs-on: ubuntu-22.04 steps: - name: Checkout repository uses: actions/checkout@v4 - - - uses: actions/create-github-app-token@v1 - id: app-token with: - app-id: ${{ vars.VERSION_BUMPER_APPID }} - private-key: ${{ secrets.VERSION_BUMPER_SECRET }} + fetch-depth: 0 - - name: Get the last release - id: last_release - continue-on-error: true - uses: cardinalby/git-get-release-action@v1 - env: - GITHUB_TOKEN: ${{ github.token }} + # Download all build artifacts + - name: Download all build artifacts + uses: actions/download-artifact@v4 with: - latest: true - prerelease: false - draft: false + path: builds + pattern: firecracker-* + merge-multiple: true - - name: Get next version - id: get-version - run: | - if [ "${{ steps.last_release.outputs.tag_name }}" == "" ]; then - echo "No previous release found, starting with v0.0.1" - echo "version=v0.0.1" >> $GITHUB_OUTPUT - else - version=${{ steps.last_release.outputs.tag_name }} - result=$(echo ${version} | awk -F. -v OFS=. '{$NF += 1 ; print}') - echo "version=$result" >> $GITHUB_OUTPUT - fi - - - name: Test next version - run: echo "Next version is ${{ steps.get-version.outputs.version }}" + - name: List downloaded builds + run: find builds -type f | head -20 - name: Setup Service Account + if: github.ref_name == 'main' uses: google-github-actions/auth@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - - name: Build firecrackers - run: sudo make build - - - name: Upload firecrackers as artifact - if: github.ref_name != 'main' - uses: actions/upload-artifact@v4 - with: - name: firecracker-${{ github.run_id }} - path: ./builds - retention-days: 7 - - name: Upload firecrackers to GCS if: github.ref_name == 'main' uses: "google-github-actions/upload-cloud-storage@v1" @@ -73,34 +54,41 @@ jobs: gzip: false parent: false - - name: Create Git tag + - name: Create releases for each Firecracker version if: github.ref_name == 'main' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git config user.name "github-actions" git config user.email "github-actions@github.com" - git tag ${{ steps.get-version.outputs.version }} - git push origin ${{ steps.get-version.outputs.version }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Prepare release assets - if: github.ref_name == 'main' - run: | - mkdir -p release-assets for dir in ./builds/*/; do - name=$(basename "$dir") - cp "$dir/vmlinux.bin" "release-assets/${name}.bin" - done - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + version_name=$(basename "$dir") + binary_path="$dir/firecracker" - - name: Upload Release Asset - if: github.ref_name == 'main' - uses: softprops/action-gh-release@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - name: Firecrackers ${{ steps.get-version.outputs.version }} - tag_name: ${{ steps.get-version.outputs.version }} - files: "./release-assets/*" + echo "Processing $version_name..." + + # Check if tag already exists + if git rev-parse "refs/tags/$version_name" >/dev/null 2>&1; then + echo "Tag $version_name already exists, skipping..." + continue + fi + + # Check if release already exists + if gh release view "$version_name" >/dev/null 2>&1; then + echo "Release $version_name already exists, skipping..." + continue + fi + + echo "Creating tag and release for $version_name..." + + # Create and push tag + git tag "$version_name" + git push origin "$version_name" + + # Create release with the binary + gh release create "$version_name" \ + --title "Firecracker $version_name" \ + --notes "Firecracker build: $version_name" \ + "$binary_path#${version_name}" + done diff --git a/README.md b/README.md index dcd39b1..2080515 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# fc-kernels +# fc-versions ## Overview @@ -8,22 +8,25 @@ This project automates the building of custom Firecracker. It supports building - Linux environment (for building firecracker) -## Building Kernels +## Building Firecrackers 1. **Configure firecracker versions:** - - Edit `firecracker_versions.txt` to specify which kernel versions to build (one per line, e.g., `-`). + - Edit `firecracker_versions.txt` to specify which firecracker versions to build (one per line, e.g., `-`). 2. **Build:** + ```sh make build # or directly ./build.sh ``` - The built kernels will be placed in `builds/vmlinux-/vmlinux.bin`. + + The built firecrackers will be placed in `builds/firecracker-/firecracker`. ## Development Workflow -- On every push, GitHub Actions will automatically build the kernels and save it as an artifact. + +- On every push, GitHub Actions will automatically build the firecrackers and save it as an artifact. ## License -This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details. \ No newline at end of file +This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details. diff --git a/build.sh b/build.sh index 1ff8c49..51939ee 100755 --- a/build.sh +++ b/build.sh @@ -2,32 +2,70 @@ set -euo pipefail +FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git" + function build_version { local version=$1 - echo "Starting build for Firecracker commit: $version" - echo "Checking out repo for Firecracker at commit: $version" - git checkout "${version}" + # Detect if the version is of the form tag_shorthash (e.g., v1.12.1_abcdef12) + if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then + local tag="${BASH_REMATCH[1]}" + local shorthash="${BASH_REMATCH[2]}" + echo "Starting build for Firecracker tag: $tag and shorthash: $shorthash" + + echo "Checking out repo at tag: $tag" + git checkout "$tag" + + # Find full hash from shorthash + fullhash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || true) + if [[ -z "$fullhash" ]]; then + echo "Error: Could not resolve hash $shorthash" + exit 1 + fi + + # Ensure that $fullhash is a descendant of $tag + if git merge-base --is-ancestor "$tag" "$fullhash"; then + echo "Shorthash $shorthash is AFTER tag $tag -- proceeding" + git checkout "$fullhash" + else + echo "Error: shorthash $shorthash is not a descendant of tag $tag" + exit 1 + fi + version_name="${tag}_$shorthash" + else + echo "Starting build for Firecracker at commit: $version" + echo "Checking out repo for Firecracker at commit: $version" + git checkout "${version}" + # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311 + version_name=$(git describe --tags --abbrev=0 "$(git rev-parse HEAD)")_$(git rev-parse --short HEAD) + fi - # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311 - version_name=$(git describe --tags --abbrev=0 $(git rev-parse HEAD))_$(git rev-parse --short HEAD) echo "Version name: $version_name" echo "Building Firecracker version: $version_name" - tools/devtool -y build --release + # Build only the firecracker binary, skip jailer and snapshot-editor for faster builds + tools/devtool -y build --release -- --bin firecracker echo "Copying finished build to builds directory" mkdir -p "../builds/${version_name}" cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker" } +# If a version is passed as argument, build only that version +# Otherwise, build all versions from firecracker_versions.txt +if [[ $# -ge 1 ]]; then + versions=("$@") +else + mapfile -t versions < <(grep -v '^ *#' firecracker_versions.txt | grep -v '^$') +fi + echo "Cloning the Firecracker repository" -git clone https://github.com/firecracker-microvm/firecracker.git firecracker +git clone $FIRECRACKER_REPO_URL firecracker cd firecracker -grep -v '^ *#' <../firecracker_versions.txt | while IFS= read -r version; do +for version in "${versions[@]}"; do build_version "$version" done cd .. -rm -rf firecracker \ No newline at end of file +rm -rf firecracker From 2707785be7ffd73b37addc7bf9b4a0109ab0630c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 16:23:48 -0800 Subject: [PATCH 02/14] Fix naming --- .github/workflows/build.yml | 9 +++++++-- build.sh | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2267b5..9270f3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,11 +35,16 @@ jobs: - uses: actions/checkout@v4 - name: Build Firecracker ${{ matrix.version }} - run: ./build.sh "${{ matrix.version }}" + id: build + run: | + ./build.sh "${{ matrix.version }}" + # Read the computed version name (with hash) + version_name=$(cat built_versions.txt | tail -1) + echo "version_name=$version_name" >> $GITHUB_OUTPUT - name: Upload build artifact uses: actions/upload-artifact@v4 with: - name: firecracker-${{ matrix.version }} + name: firecracker-${{ steps.build.outputs.version_name }} path: builds/ retention-days: 7 diff --git a/build.sh b/build.sh index 51939ee..519bf79 100755 --- a/build.sh +++ b/build.sh @@ -49,6 +49,9 @@ function build_version { echo "Copying finished build to builds directory" mkdir -p "../builds/${version_name}" cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker" + + # Write version name to file for CI to use + echo "$version_name" >> ../built_versions.txt } # If a version is passed as argument, build only that version From 1253e0b918397b58d074b5fcea669007731a2f08 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 18:00:34 -0800 Subject: [PATCH 03/14] Fix ci checks --- .github/workflows/build.yml | 12 +++++- .github/workflows/fc-versions.yml | 68 +++++++++++++++++++++++++++++-- build.sh | 8 ++-- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9270f3c..1873c0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,8 +38,9 @@ jobs: id: build run: | ./build.sh "${{ matrix.version }}" - # Read the computed version name (with hash) - version_name=$(cat built_versions.txt | tail -1) + # Read the computed version name (with hash) - format is "version_name:commit_hash" + version_info=$(cat built_versions.txt | tail -1) + version_name=$(echo "$version_info" | cut -d: -f1) echo "version_name=$version_name" >> $GITHUB_OUTPUT - name: Upload build artifact @@ -48,3 +49,10 @@ jobs: name: firecracker-${{ steps.build.outputs.version_name }} path: builds/ retention-days: 7 + + - name: Upload version info + uses: actions/upload-artifact@v4 + with: + name: version-info-${{ steps.build.outputs.version_name }} + path: built_versions.txt + retention-days: 7 diff --git a/.github/workflows/fc-versions.yml b/.github/workflows/fc-versions.yml index de354d6..4d717d1 100644 --- a/.github/workflows/fc-versions.yml +++ b/.github/workflows/fc-versions.yml @@ -38,15 +38,77 @@ jobs: - name: List downloaded builds run: find builds -type f | head -20 - - name: Setup Service Account + # Download version info to get commit hashes + - name: Download version info + uses: actions/download-artifact@v4 + with: + path: version-info + pattern: version-info-* + merge-multiple: true + + # Check CI status for all commits before publishing + - name: Check CI status for all versions + id: ci-check if: github.ref_name == 'main' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + FIRECRACKER_REPO="e2b-dev/firecracker" + all_passed=true + failed_versions="" + + # Read all version:hash pairs + for line in $(cat version-info/built_versions.txt 2>/dev/null || echo ""); do + version_name=$(echo "$line" | cut -d: -f1) + commit_hash=$(echo "$line" | cut -d: -f2) + + echo "Checking CI for $version_name (commit: $commit_hash)..." + + # Check combined commit status + status=$(gh api "/repos/${FIRECRACKER_REPO}/commits/${commit_hash}/status" --jq '.state' 2>/dev/null || echo "unknown") + + # Check check-runs (GitHub Actions) + check_conclusion=$(gh api "/repos/${FIRECRACKER_REPO}/commits/${commit_hash}/check-runs" --jq ' + if .total_count == 0 then "no_checks" + elif ([.check_runs[].conclusion] | all(. == "success" or . == "skipped" or . == null)) then "success" + elif ([.check_runs[].status] | any(. == "in_progress" or . == "queued")) then "pending" + else "failure" + end + ' 2>/dev/null || echo "unknown") + + echo " Status: $status, Check runs: $check_conclusion" + + if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then + echo " ❌ CI failed for $version_name" + all_passed=false + failed_versions="${failed_versions}${version_name} " + elif [[ "$status" == "pending" ]] || [[ "$check_conclusion" == "pending" ]]; then + echo " ⏳ CI still running for $version_name" + all_passed=false + failed_versions="${failed_versions}${version_name}(pending) " + else + echo " ✅ CI passed for $version_name" + fi + done + + if [[ "$all_passed" == "true" ]]; then + echo "All CI checks passed!" + echo "ci_passed=true" >> $GITHUB_OUTPUT + else + echo "CI checks failed or pending for: $failed_versions" + echo "ci_passed=false" >> $GITHUB_OUTPUT + echo "Skipping GCS upload and release creation." + fi + + - name: Setup Service Account + if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true' uses: google-github-actions/auth@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - name: Upload firecrackers to GCS - if: github.ref_name == 'main' + if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true' uses: "google-github-actions/upload-cloud-storage@v1" with: path: "./builds" @@ -55,7 +117,7 @@ jobs: parent: false - name: Create releases for each Firecracker version - if: github.ref_name == 'main' + if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/build.sh b/build.sh index 519bf79..0e9fa91 100755 --- a/build.sh +++ b/build.sh @@ -36,8 +36,10 @@ function build_version { echo "Starting build for Firecracker at commit: $version" echo "Checking out repo for Firecracker at commit: $version" git checkout "${version}" + + fullhash=$(git rev-parse HEAD) # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311 - version_name=$(git describe --tags --abbrev=0 "$(git rev-parse HEAD)")_$(git rev-parse --short HEAD) + version_name=$(git describe --tags --abbrev=0 "$fullhash")_$(git rev-parse --short HEAD) fi echo "Version name: $version_name" @@ -50,8 +52,8 @@ function build_version { mkdir -p "../builds/${version_name}" cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker" - # Write version name to file for CI to use - echo "$version_name" >> ../built_versions.txt + # Write version name and commit hash to file for CI to use + echo "${version_name}:${fullhash}" >> ../built_versions.txt } # If a version is passed as argument, build only that version From 7a35a2e6e408efd471ddaef6b132893a160d3a7f Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 18:03:03 -0800 Subject: [PATCH 04/14] Trigger CI check on PRs too --- .github/workflows/fc-versions.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/fc-versions.yml b/.github/workflows/fc-versions.yml index 4d717d1..913f5c5 100644 --- a/.github/workflows/fc-versions.yml +++ b/.github/workflows/fc-versions.yml @@ -46,10 +46,9 @@ jobs: pattern: version-info-* merge-multiple: true - # Check CI status for all commits before publishing + # Check CI status for all commits (runs on all branches for visibility) - name: Check CI status for all versions id: ci-check - if: github.ref_name == 'main' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | From 4477a5009392d945b0330ef1483f12f79e44dd13 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 18:13:45 -0800 Subject: [PATCH 05/14] Add extra checks --- .github/workflows/build.yml | 10 +-- .github/workflows/fc-versions.yml | 91 ++++++++----------------- scripts/check-fc-ci.sh | 106 ++++++++++++++++++++++++++++++ scripts/parse-versions.sh | 14 ++++ 4 files changed, 150 insertions(+), 71 deletions(-) create mode 100755 scripts/check-fc-ci.sh create mode 100755 scripts/parse-versions.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1873c0c..2b882a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,8 +19,7 @@ jobs: - name: Parse versions from file id: set-versions run: | - # Read versions, skip comments and empty lines, output as JSON array - versions=$(grep -v '^ *#' firecracker_versions.txt | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))') + versions=$(./scripts/parse-versions.sh firecracker_versions.txt) echo "versions=$versions" >> $GITHUB_OUTPUT echo "Building versions: $versions" @@ -49,10 +48,3 @@ jobs: name: firecracker-${{ steps.build.outputs.version_name }} path: builds/ retention-days: 7 - - - name: Upload version info - uses: actions/upload-artifact@v4 - with: - name: version-info-${{ steps.build.outputs.version_name }} - path: built_versions.txt - retention-days: 7 diff --git a/.github/workflows/fc-versions.yml b/.github/workflows/fc-versions.yml index 913f5c5..1a4ab49 100644 --- a/.github/workflows/fc-versions.yml +++ b/.github/workflows/fc-versions.yml @@ -17,9 +17,30 @@ jobs: build: uses: ./.github/workflows/build.yml + # Check CI status in parallel with builds (for faster feedback) + check-ci: + name: Check FC repo CI status + runs-on: ubuntu-latest + outputs: + ci_passed: ${{ steps.ci-check.outputs.ci_passed }} + steps: + - uses: actions/checkout@v4 + + - name: Check CI status for all versions + id: ci-check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + output=$(./scripts/check-fc-ci.sh firecracker_versions.txt) + echo "$output" + # Extract ci_passed from last line + ci_passed=$(echo "$output" | grep "^ci_passed=" | cut -d= -f2) + echo "ci_passed=$ci_passed" >> $GITHUB_OUTPUT + + # Collect artifacts and publish (waits for both build and CI check) publish: name: Collect and upload builds - needs: build + needs: [build, check-ci] runs-on: ubuntu-22.04 steps: - name: Checkout repository @@ -38,76 +59,22 @@ jobs: - name: List downloaded builds run: find builds -type f | head -20 - # Download version info to get commit hashes - - name: Download version info - uses: actions/download-artifact@v4 - with: - path: version-info - pattern: version-info-* - merge-multiple: true - - # Check CI status for all commits (runs on all branches for visibility) - - name: Check CI status for all versions - id: ci-check - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: CI check result run: | - FIRECRACKER_REPO="e2b-dev/firecracker" - all_passed=true - failed_versions="" - - # Read all version:hash pairs - for line in $(cat version-info/built_versions.txt 2>/dev/null || echo ""); do - version_name=$(echo "$line" | cut -d: -f1) - commit_hash=$(echo "$line" | cut -d: -f2) - - echo "Checking CI for $version_name (commit: $commit_hash)..." - - # Check combined commit status - status=$(gh api "/repos/${FIRECRACKER_REPO}/commits/${commit_hash}/status" --jq '.state' 2>/dev/null || echo "unknown") - - # Check check-runs (GitHub Actions) - check_conclusion=$(gh api "/repos/${FIRECRACKER_REPO}/commits/${commit_hash}/check-runs" --jq ' - if .total_count == 0 then "no_checks" - elif ([.check_runs[].conclusion] | all(. == "success" or . == "skipped" or . == null)) then "success" - elif ([.check_runs[].status] | any(. == "in_progress" or . == "queued")) then "pending" - else "failure" - end - ' 2>/dev/null || echo "unknown") - - echo " Status: $status, Check runs: $check_conclusion" - - if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then - echo " ❌ CI failed for $version_name" - all_passed=false - failed_versions="${failed_versions}${version_name} " - elif [[ "$status" == "pending" ]] || [[ "$check_conclusion" == "pending" ]]; then - echo " ⏳ CI still running for $version_name" - all_passed=false - failed_versions="${failed_versions}${version_name}(pending) " - else - echo " ✅ CI passed for $version_name" - fi - done - - if [[ "$all_passed" == "true" ]]; then - echo "All CI checks passed!" - echo "ci_passed=true" >> $GITHUB_OUTPUT - else - echo "CI checks failed or pending for: $failed_versions" - echo "ci_passed=false" >> $GITHUB_OUTPUT - echo "Skipping GCS upload and release creation." + echo "CI check passed: ${{ needs.check-ci.outputs.ci_passed }}" + if [[ "${{ needs.check-ci.outputs.ci_passed }}" != "true" ]]; then + echo "⚠️ CI checks did not pass - skipping GCS upload and release" fi - name: Setup Service Account - if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true' + if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true' uses: google-github-actions/auth@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} - name: Upload firecrackers to GCS - if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true' + if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true' uses: "google-github-actions/upload-cloud-storage@v1" with: path: "./builds" @@ -116,7 +83,7 @@ jobs: parent: false - name: Create releases for each Firecracker version - if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true' + if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/scripts/check-fc-ci.sh b/scripts/check-fc-ci.sh new file mode 100755 index 0000000..97ab0de --- /dev/null +++ b/scripts/check-fc-ci.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Check CI status for all Firecracker versions +# Outputs: ci_passed=true|false to stdout (for GitHub Actions) + +set -euo pipefail + +VERSIONS_FILE="${1:-firecracker_versions.txt}" +FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git" +FIRECRACKER_REPO_API="e2b-dev/firecracker" + +if [[ ! -f "$VERSIONS_FILE" ]]; then + echo "Error: $VERSIONS_FILE not found" >&2 + exit 1 +fi + +# Clone FC repo to resolve commit hashes +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Cloning FC repo to resolve commit hashes..." +git clone --bare "$FIRECRACKER_REPO_URL" "$TEMP_DIR/fc-repo" 2>/dev/null +cd "$TEMP_DIR/fc-repo" + +all_passed=true +failed_versions="" + +# Read versions from file +while IFS= read -r version || [[ -n "$version" ]]; do + # Skip comments and empty lines + [[ "$version" =~ ^[[:space:]]*# ]] && continue + [[ -z "$version" ]] && continue + + echo "Processing version: $version" + + # Resolve commit hash + if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then + # Format: tag_shorthash + tag="${BASH_REMATCH[1]}" + shorthash="${BASH_REMATCH[2]}" + commit_hash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || echo "") + version_name="${tag}_${shorthash}" + else + # Plain tag + commit_hash=$(git rev-parse --verify "${version}^{commit}" 2>/dev/null || echo "") + if [[ -n "$commit_hash" ]]; then + short_hash=$(git rev-parse --short "$commit_hash") + version_name="${version}_${short_hash}" + else + version_name="$version" + fi + fi + + if [[ -z "$commit_hash" ]]; then + echo " ⚠️ Could not resolve commit for $version" + continue + fi + + echo " Checking CI for $version_name (commit: $commit_hash)..." + + # Check combined commit status + status=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/status" --jq '.state' 2>/dev/null || echo "unknown") + + # Check check-runs (GitHub Actions) + # Order matters: check pending BEFORE success (null conclusion means in-progress) + check_conclusion=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/check-runs" --jq ' + if .total_count == 0 then "no_checks" + elif ([.check_runs[].status] | any(. == "in_progress" or . == "queued")) then "pending" + elif ([.check_runs[].conclusion] | any(. == "failure" or . == "cancelled" or . == "timed_out")) then "failure" + elif ([.check_runs[].conclusion] | all(. == "success" or . == "skipped" or . == "neutral")) then "success" + else "unknown" + end + ' 2>/dev/null || echo "unknown") + + echo " Status: $status, Check runs: $check_conclusion" + + # Treat unknown as failure (API errors should block release) + if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then + echo " ❌ CI failed for $version_name" + all_passed=false + failed_versions="${failed_versions}${version_name} " + elif [[ "$status" == "pending" ]] || [[ "$check_conclusion" == "pending" ]]; then + echo " ⏳ CI still running for $version_name" + all_passed=false + failed_versions="${failed_versions}${version_name}(pending) " + elif [[ "$status" == "unknown" ]] || [[ "$check_conclusion" == "unknown" ]]; then + echo " ⚠️ Could not verify CI status for $version_name (API error)" + all_passed=false + failed_versions="${failed_versions}${version_name}(unknown) " + elif [[ "$status" == "success" ]] || [[ "$check_conclusion" == "success" ]]; then + echo " ✅ CI passed for $version_name" + else + # Catch-all for unexpected states + echo " ⚠️ Unexpected CI state for $version_name: status=$status, check_conclusion=$check_conclusion" + all_passed=false + failed_versions="${failed_versions}${version_name}(unexpected) " + fi +done < "$OLDPWD/$VERSIONS_FILE" + +echo "" +if [[ "$all_passed" == "true" ]]; then + echo "All CI checks passed!" + echo "ci_passed=true" +else + echo "CI checks failed or pending for: $failed_versions" + echo "ci_passed=false" +fi diff --git a/scripts/parse-versions.sh b/scripts/parse-versions.sh new file mode 100755 index 0000000..9e3b389 --- /dev/null +++ b/scripts/parse-versions.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Parse versions from firecracker_versions.txt and output as JSON array + +set -euo pipefail + +VERSIONS_FILE="${1:-firecracker_versions.txt}" + +if [[ ! -f "$VERSIONS_FILE" ]]; then + echo "Error: $VERSIONS_FILE not found" >&2 + exit 1 +fi + +# Read versions, skip comments and empty lines, output as JSON array +grep -v '^ *#' "$VERSIONS_FILE" | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))' From 42a22f3bc276d27f5f1f8967ee68d0e3f1aa8332 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 18:17:22 -0800 Subject: [PATCH 06/14] Enhance CI status checks in script to include total counts and handle various states more robustly --- scripts/check-fc-ci.sh | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/check-fc-ci.sh b/scripts/check-fc-ci.sh index 97ab0de..498352a 100755 --- a/scripts/check-fc-ci.sh +++ b/scripts/check-fc-ci.sh @@ -57,37 +57,53 @@ while IFS= read -r version || [[ -n "$version" ]]; do echo " Checking CI for $version_name (commit: $commit_hash)..." - # Check combined commit status - status=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/status" --jq '.state' 2>/dev/null || echo "unknown") + # Check combined commit status (returns total_count of statuses) + status_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/status" 2>/dev/null || echo '{"state":"unknown","total_count":0}') + status=$(echo "$status_response" | jq -r '.state') + status_count=$(echo "$status_response" | jq -r '.total_count') # Check check-runs (GitHub Actions) # Order matters: check pending BEFORE success (null conclusion means in-progress) - check_conclusion=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/check-runs" --jq ' + check_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/check-runs" 2>/dev/null || echo '{"total_count":0}') + check_count=$(echo "$check_response" | jq -r '.total_count') + check_conclusion=$(echo "$check_response" | jq -r ' if .total_count == 0 then "no_checks" elif ([.check_runs[].status] | any(. == "in_progress" or . == "queued")) then "pending" elif ([.check_runs[].conclusion] | any(. == "failure" or . == "cancelled" or . == "timed_out")) then "failure" elif ([.check_runs[].conclusion] | all(. == "success" or . == "skipped" or . == "neutral")) then "success" else "unknown" end - ' 2>/dev/null || echo "unknown") + ') - echo " Status: $status, Check runs: $check_conclusion" + echo " Status: $status (count: $status_count), Check runs: $check_conclusion (count: $check_count)" - # Treat unknown as failure (API errors should block release) + # Handle the different cases if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then echo " ❌ CI failed for $version_name" all_passed=false failed_versions="${failed_versions}${version_name} " - elif [[ "$status" == "pending" ]] || [[ "$check_conclusion" == "pending" ]]; then + elif [[ "$check_conclusion" == "pending" ]]; then + # Only pending if there are actual check-runs in progress echo " ⏳ CI still running for $version_name" all_passed=false failed_versions="${failed_versions}${version_name}(pending) " - elif [[ "$status" == "unknown" ]] || [[ "$check_conclusion" == "unknown" ]]; then + elif [[ "$status" == "pending" ]] && [[ "$status_count" -gt 0 ]]; then + # Only pending if there are actual statuses pending + echo " ⏳ CI still running for $version_name" + all_passed=false + failed_versions="${failed_versions}${version_name}(pending) " + elif [[ "$status" == "unknown" ]] && [[ "$check_conclusion" == "unknown" ]]; then echo " ⚠️ Could not verify CI status for $version_name (API error)" all_passed=false failed_versions="${failed_versions}${version_name}(unknown) " elif [[ "$status" == "success" ]] || [[ "$check_conclusion" == "success" ]]; then echo " ✅ CI passed for $version_name" + elif [[ "$status_count" -eq 0 ]] && [[ "$check_count" -eq 0 ]]; then + # No CI configured for this commit - treat as acceptable + echo " ℹ️ No CI checks found for $version_name (assuming OK)" + elif [[ "$status" == "pending" ]] && [[ "$status_count" -eq 0 ]] && [[ "$check_conclusion" == "no_checks" ]]; then + # GitHub returns "pending" when there are no statuses - this is OK + echo " ℹ️ No CI checks found for $version_name (assuming OK)" else # Catch-all for unexpected states echo " ⚠️ Unexpected CI state for $version_name: status=$status, check_conclusion=$check_conclusion" From a45d039da38edc2ee3ecfe2cbdd0a4c4177eaee4 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 10 Dec 2025 18:29:38 -0800 Subject: [PATCH 07/14] Cleanup --- scripts/check-fc-ci.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/check-fc-ci.sh b/scripts/check-fc-ci.sh index 498352a..b55ed45 100755 --- a/scripts/check-fc-ci.sh +++ b/scripts/check-fc-ci.sh @@ -51,7 +51,9 @@ while IFS= read -r version || [[ -n "$version" ]]; do fi if [[ -z "$commit_hash" ]]; then - echo " ⚠️ Could not resolve commit for $version" + echo " ❌ Could not resolve commit for $version" + all_passed=false + failed_versions="${failed_versions}${version}(unresolved) " continue fi From 8718536fe9c38a54573c7810e2cda548bf31568c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:00:55 -0800 Subject: [PATCH 08/14] Refactor --- .github/workflows/build.yml | 50 +++++------------- .github/workflows/fc-versions.yml | 67 ++++++++++-------------- build.sh | 79 ++++++++--------------------- scripts/check-fc-ci.sh | 38 +------------- scripts/parse-versions-with-hash.sh | 63 +++++++++++++++++++++++ scripts/parse-versions.sh | 2 - 6 files changed, 126 insertions(+), 173 deletions(-) create mode 100755 scripts/parse-versions-with-hash.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b882a1..8c5725e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,49 +2,27 @@ name: Build Firecracker Versions on: workflow_call: - outputs: - versions: - description: "JSON array of versions that were built" - value: ${{ jobs.prepare.outputs.versions }} + inputs: + version: + required: true + type: string + hash: + required: true + type: string + version_name: + required: true + type: string workflow_dispatch: jobs: - prepare: - runs-on: ubuntu-latest - outputs: - versions: ${{ steps.set-versions.outputs.versions }} - steps: - - uses: actions/checkout@v4 - - - name: Parse versions from file - id: set-versions - run: | - versions=$(./scripts/parse-versions.sh firecracker_versions.txt) - echo "versions=$versions" >> $GITHUB_OUTPUT - echo "Building versions: $versions" - build: - needs: prepare runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - version: ${{ fromJson(needs.prepare.outputs.versions) }} steps: - uses: actions/checkout@v4 - - - name: Build Firecracker ${{ matrix.version }} - id: build - run: | - ./build.sh "${{ matrix.version }}" - # Read the computed version name (with hash) - format is "version_name:commit_hash" - version_info=$(cat built_versions.txt | tail -1) - version_name=$(echo "$version_info" | cut -d: -f1) - echo "version_name=$version_name" >> $GITHUB_OUTPUT - - - name: Upload build artifact - uses: actions/upload-artifact@v4 + - name: Build Firecracker ${{ inputs.version_name }} + run: ./build.sh "${{ inputs.version }}" "${{ inputs.hash }}" "${{ inputs.version_name }}" + - uses: actions/upload-artifact@v4 with: - name: firecracker-${{ steps.build.outputs.version_name }} + name: firecracker-${{ inputs.version_name }} path: builds/ retention-days: 7 diff --git a/.github/workflows/fc-versions.yml b/.github/workflows/fc-versions.yml index 1a4ab49..c2710b6 100644 --- a/.github/workflows/fc-versions.yml +++ b/.github/workflows/fc-versions.yml @@ -13,55 +13,61 @@ permissions: contents: write jobs: - # Run parallel builds via reusable workflow + prepare: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Parse versions and resolve hashes + id: set-matrix + run: | + versions_json=$(./scripts/parse-versions-with-hash.sh firecracker_versions.txt) + echo "matrix=$versions_json" >> $GITHUB_OUTPUT + build: + needs: prepare uses: ./.github/workflows/build.yml + strategy: + fail-fast: true + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix) }} + with: + version: ${{ matrix.version }} + hash: ${{ matrix.hash }} + version_name: ${{ matrix.version_name }} - # Check CI status in parallel with builds (for faster feedback) check-ci: - name: Check FC repo CI status runs-on: ubuntu-latest outputs: ci_passed: ${{ steps.ci-check.outputs.ci_passed }} steps: - uses: actions/checkout@v4 - - - name: Check CI status for all versions + - name: Check CI status id: ci-check env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | output=$(./scripts/check-fc-ci.sh firecracker_versions.txt) echo "$output" - # Extract ci_passed from last line ci_passed=$(echo "$output" | grep "^ci_passed=" | cut -d= -f2) echo "ci_passed=$ci_passed" >> $GITHUB_OUTPUT - # Collect artifacts and publish (waits for both build and CI check) publish: name: Collect and upload builds needs: [build, check-ci] runs-on: ubuntu-22.04 steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - # Download all build artifacts - - name: Download all build artifacts - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 with: path: builds pattern: firecracker-* merge-multiple: true - - - name: List downloaded builds - run: find builds -type f | head -20 - - name: CI check result run: | - echo "CI check passed: ${{ needs.check-ci.outputs.ci_passed }}" if [[ "${{ needs.check-ci.outputs.ci_passed }}" != "true" ]]; then echo "⚠️ CI checks did not pass - skipping GCS upload and release" fi @@ -82,41 +88,22 @@ jobs: gzip: false parent: false - - name: Create releases for each Firecracker version + - name: Create releases if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git config user.name "github-actions" git config user.email "github-actions@github.com" - for dir in ./builds/*/; do version_name=$(basename "$dir") - binary_path="$dir/firecracker" - - echo "Processing $version_name..." - - # Check if tag already exists - if git rev-parse "refs/tags/$version_name" >/dev/null 2>&1; then - echo "Tag $version_name already exists, skipping..." - continue - fi - - # Check if release already exists - if gh release view "$version_name" >/dev/null 2>&1; then - echo "Release $version_name already exists, skipping..." + if git rev-parse "refs/tags/$version_name" >/dev/null 2>&1 || gh release view "$version_name" >/dev/null 2>&1; then continue fi - - echo "Creating tag and release for $version_name..." - - # Create and push tag git tag "$version_name" git push origin "$version_name" - - # Create release with the binary gh release create "$version_name" \ --title "Firecracker $version_name" \ --notes "Firecracker build: $version_name" \ - "$binary_path#${version_name}" + "$dir/firecracker#${version_name}" done diff --git a/build.sh b/build.sh index 0e9fa91..7b09fb2 100755 --- a/build.sh +++ b/build.sh @@ -4,73 +4,34 @@ set -euo pipefail FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git" -function build_version { - local version=$1 - - # Detect if the version is of the form tag_shorthash (e.g., v1.12.1_abcdef12) - if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then - local tag="${BASH_REMATCH[1]}" - local shorthash="${BASH_REMATCH[2]}" - echo "Starting build for Firecracker tag: $tag and shorthash: $shorthash" +if [[ $# -lt 3 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi - echo "Checking out repo at tag: $tag" - git checkout "$tag" +version="$1" +fullhash="$2" +version_name="$3" - # Find full hash from shorthash - fullhash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || true) - if [[ -z "$fullhash" ]]; then - echo "Error: Could not resolve hash $shorthash" - exit 1 - fi +git clone $FIRECRACKER_REPO_URL firecracker +cd firecracker - # Ensure that $fullhash is a descendant of $tag - if git merge-base --is-ancestor "$tag" "$fullhash"; then - echo "Shorthash $shorthash is AFTER tag $tag -- proceeding" - git checkout "$fullhash" - else - echo "Error: shorthash $shorthash is not a descendant of tag $tag" - exit 1 - fi - version_name="${tag}_$shorthash" - else - echo "Starting build for Firecracker at commit: $version" - echo "Checking out repo for Firecracker at commit: $version" - git checkout "${version}" - - fullhash=$(git rev-parse HEAD) - # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311 - version_name=$(git describe --tags --abbrev=0 "$fullhash")_$(git rev-parse --short HEAD) +if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then + tag="${BASH_REMATCH[1]}" + git checkout "$tag" + if ! git merge-base --is-ancestor "$tag" "$fullhash"; then + echo "Error: shorthash is not a descendant of tag $tag" >&2 + exit 1 fi - - echo "Version name: $version_name" - - echo "Building Firecracker version: $version_name" - # Build only the firecracker binary, skip jailer and snapshot-editor for faster builds - tools/devtool -y build --release -- --bin firecracker - - echo "Copying finished build to builds directory" - mkdir -p "../builds/${version_name}" - cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker" - - # Write version name and commit hash to file for CI to use - echo "${version_name}:${fullhash}" >> ../built_versions.txt -} - -# If a version is passed as argument, build only that version -# Otherwise, build all versions from firecracker_versions.txt -if [[ $# -ge 1 ]]; then - versions=("$@") + git checkout "$fullhash" else - mapfile -t versions < <(grep -v '^ *#' firecracker_versions.txt | grep -v '^$') + git checkout "$fullhash" fi -echo "Cloning the Firecracker repository" -git clone $FIRECRACKER_REPO_URL firecracker -cd firecracker +tools/devtool -y build --release -- --bin firecracker -for version in "${versions[@]}"; do - build_version "$version" -done +mkdir -p "../builds/${version_name}" +cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker" cd .. rm -rf firecracker diff --git a/scripts/check-fc-ci.sh b/scripts/check-fc-ci.sh index b55ed45..e633827 100755 --- a/scripts/check-fc-ci.sh +++ b/scripts/check-fc-ci.sh @@ -1,6 +1,4 @@ #!/bin/bash -# Check CI status for all Firecracker versions -# Outputs: ci_passed=true|false to stdout (for GitHub Actions) set -euo pipefail @@ -13,34 +11,25 @@ if [[ ! -f "$VERSIONS_FILE" ]]; then exit 1 fi -# Clone FC repo to resolve commit hashes TEMP_DIR=$(mktemp -d) trap "rm -rf $TEMP_DIR" EXIT -echo "Cloning FC repo to resolve commit hashes..." git clone --bare "$FIRECRACKER_REPO_URL" "$TEMP_DIR/fc-repo" 2>/dev/null cd "$TEMP_DIR/fc-repo" all_passed=true failed_versions="" -# Read versions from file while IFS= read -r version || [[ -n "$version" ]]; do - # Skip comments and empty lines [[ "$version" =~ ^[[:space:]]*# ]] && continue [[ -z "$version" ]] && continue - echo "Processing version: $version" - - # Resolve commit hash if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then - # Format: tag_shorthash tag="${BASH_REMATCH[1]}" shorthash="${BASH_REMATCH[2]}" commit_hash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || echo "") version_name="${tag}_${shorthash}" else - # Plain tag commit_hash=$(git rev-parse --verify "${version}^{commit}" 2>/dev/null || echo "") if [[ -n "$commit_hash" ]]; then short_hash=$(git rev-parse --short "$commit_hash") @@ -57,15 +46,10 @@ while IFS= read -r version || [[ -n "$version" ]]; do continue fi - echo " Checking CI for $version_name (commit: $commit_hash)..." - - # Check combined commit status (returns total_count of statuses) status_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/status" 2>/dev/null || echo '{"state":"unknown","total_count":0}') status=$(echo "$status_response" | jq -r '.state') status_count=$(echo "$status_response" | jq -r '.total_count') - # Check check-runs (GitHub Actions) - # Order matters: check pending BEFORE success (null conclusion means in-progress) check_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/check-runs" 2>/dev/null || echo '{"total_count":0}') check_count=$(echo "$check_response" | jq -r '.total_count') check_conclusion=$(echo "$check_response" | jq -r ' @@ -77,20 +61,11 @@ while IFS= read -r version || [[ -n "$version" ]]; do end ') - echo " Status: $status (count: $status_count), Check runs: $check_conclusion (count: $check_count)" - - # Handle the different cases if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then echo " ❌ CI failed for $version_name" all_passed=false failed_versions="${failed_versions}${version_name} " - elif [[ "$check_conclusion" == "pending" ]]; then - # Only pending if there are actual check-runs in progress - echo " ⏳ CI still running for $version_name" - all_passed=false - failed_versions="${failed_versions}${version_name}(pending) " - elif [[ "$status" == "pending" ]] && [[ "$status_count" -gt 0 ]]; then - # Only pending if there are actual statuses pending + elif [[ "$check_conclusion" == "pending" ]] || ([[ "$status" == "pending" ]] && [[ "$status_count" -gt 0 ]]); then echo " ⏳ CI still running for $version_name" all_passed=false failed_versions="${failed_versions}${version_name}(pending) " @@ -101,13 +76,10 @@ while IFS= read -r version || [[ -n "$version" ]]; do elif [[ "$status" == "success" ]] || [[ "$check_conclusion" == "success" ]]; then echo " ✅ CI passed for $version_name" elif [[ "$status_count" -eq 0 ]] && [[ "$check_count" -eq 0 ]]; then - # No CI configured for this commit - treat as acceptable echo " ℹ️ No CI checks found for $version_name (assuming OK)" elif [[ "$status" == "pending" ]] && [[ "$status_count" -eq 0 ]] && [[ "$check_conclusion" == "no_checks" ]]; then - # GitHub returns "pending" when there are no statuses - this is OK echo " ℹ️ No CI checks found for $version_name (assuming OK)" else - # Catch-all for unexpected states echo " ⚠️ Unexpected CI state for $version_name: status=$status, check_conclusion=$check_conclusion" all_passed=false failed_versions="${failed_versions}${version_name}(unexpected) " @@ -115,10 +87,4 @@ while IFS= read -r version || [[ -n "$version" ]]; do done < "$OLDPWD/$VERSIONS_FILE" echo "" -if [[ "$all_passed" == "true" ]]; then - echo "All CI checks passed!" - echo "ci_passed=true" -else - echo "CI checks failed or pending for: $failed_versions" - echo "ci_passed=false" -fi +[[ "$all_passed" == "true" ]] && echo "ci_passed=true" || echo "ci_passed=false" diff --git a/scripts/parse-versions-with-hash.sh b/scripts/parse-versions-with-hash.sh new file mode 100755 index 0000000..f8c372f --- /dev/null +++ b/scripts/parse-versions-with-hash.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -euo pipefail + +VERSIONS_FILE="${1:-firecracker_versions.txt}" +FIRECRACKER_REPO_URL="${2:-https://github.com/e2b-dev/firecracker.git}" + +if [[ ! -f "$VERSIONS_FILE" ]]; then + echo "Error: $VERSIONS_FILE not found" >&2 + exit 1 +fi + +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +git clone --bare "$FIRECRACKER_REPO_URL" "$TEMP_DIR/fc-repo" 2>/dev/null +cd "$TEMP_DIR/fc-repo" + +versions_json="[" +first=true + +while IFS= read -r version || [[ -n "$version" ]]; do + [[ "$version" =~ ^[[:space:]]*# ]] && continue + [[ -z "$version" ]] && continue + + if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then + tag="${BASH_REMATCH[1]}" + shorthash="${BASH_REMATCH[2]}" + fullhash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || echo "") + if [[ -z "$fullhash" ]]; then + echo "Error: Could not resolve hash $shorthash for version $version" >&2 + exit 1 + fi + version_name="${tag}_${shorthash}" + else + fullhash=$(git rev-parse --verify "${version}^{commit}" 2>/dev/null || echo "") + if [[ -z "$fullhash" ]]; then + echo "Error: Could not resolve commit for version $version" >&2 + exit 1 + fi + if git rev-parse --verify "${version}^{tag}" >/dev/null 2>&1; then + short_hash=$(git rev-parse --short "$fullhash") + version_name="${version}_${short_hash}" + else + latest_tag=$(git describe --tags --abbrev=0 "$fullhash" 2>/dev/null || echo "") + if [[ -n "$latest_tag" ]]; then + short_hash=$(git rev-parse --short "$fullhash") + version_name="${latest_tag}_${short_hash}" + else + version_name="$version" + fi + fi + fi + + [[ "$first" == "true" ]] && first=false || versions_json+="," + versions_json+=$(jq -n \ + --arg version "$version" \ + --arg hash "$fullhash" \ + --arg version_name "$version_name" \ + '{version: $version, hash: $hash, version_name: $version_name}') +done < "$OLDPWD/$VERSIONS_FILE" + +echo "${versions_json}]" diff --git a/scripts/parse-versions.sh b/scripts/parse-versions.sh index 9e3b389..7d616cc 100755 --- a/scripts/parse-versions.sh +++ b/scripts/parse-versions.sh @@ -1,5 +1,4 @@ #!/bin/bash -# Parse versions from firecracker_versions.txt and output as JSON array set -euo pipefail @@ -10,5 +9,4 @@ if [[ ! -f "$VERSIONS_FILE" ]]; then exit 1 fi -# Read versions, skip comments and empty lines, output as JSON array grep -v '^ *#' "$VERSIONS_FILE" | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))' From 9b6acfef65b92c168478c3efeb90f4d99b95455c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:04:53 -0800 Subject: [PATCH 09/14] Cleanup --- scripts/parse-versions-with-hash.sh | 10 ++++------ scripts/parse-versions.sh | 12 ------------ 2 files changed, 4 insertions(+), 18 deletions(-) delete mode 100755 scripts/parse-versions.sh diff --git a/scripts/parse-versions-with-hash.sh b/scripts/parse-versions-with-hash.sh index f8c372f..96b529c 100755 --- a/scripts/parse-versions-with-hash.sh +++ b/scripts/parse-versions-with-hash.sh @@ -16,8 +16,7 @@ trap "rm -rf $TEMP_DIR" EXIT git clone --bare "$FIRECRACKER_REPO_URL" "$TEMP_DIR/fc-repo" 2>/dev/null cd "$TEMP_DIR/fc-repo" -versions_json="[" -first=true +versions=() while IFS= read -r version || [[ -n "$version" ]]; do [[ "$version" =~ ^[[:space:]]*# ]] && continue @@ -52,12 +51,11 @@ while IFS= read -r version || [[ -n "$version" ]]; do fi fi - [[ "$first" == "true" ]] && first=false || versions_json+="," - versions_json+=$(jq -n \ + versions+=("$(jq -n \ --arg version "$version" \ --arg hash "$fullhash" \ --arg version_name "$version_name" \ - '{version: $version, hash: $hash, version_name: $version_name}') + '{version: $version, hash: $hash, version_name: $version_name}')") done < "$OLDPWD/$VERSIONS_FILE" -echo "${versions_json}]" +printf '%s\n' "${versions[@]}" | jq -s -c '.' diff --git a/scripts/parse-versions.sh b/scripts/parse-versions.sh deleted file mode 100755 index 7d616cc..0000000 --- a/scripts/parse-versions.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -VERSIONS_FILE="${1:-firecracker_versions.txt}" - -if [[ ! -f "$VERSIONS_FILE" ]]; then - echo "Error: $VERSIONS_FILE not found" >&2 - exit 1 -fi - -grep -v '^ *#' "$VERSIONS_FILE" | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))' From 1d90ec2b46762598582a85960d8613585e37740c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:15:06 -0800 Subject: [PATCH 10/14] Cleanup --- .github/workflows/fc-versions.yml | 3 ++- scripts/check-fc-ci.sh | 41 ++++--------------------------- 2 files changed, 7 insertions(+), 37 deletions(-) diff --git a/.github/workflows/fc-versions.yml b/.github/workflows/fc-versions.yml index c2710b6..3e818b7 100644 --- a/.github/workflows/fc-versions.yml +++ b/.github/workflows/fc-versions.yml @@ -38,6 +38,7 @@ jobs: version_name: ${{ matrix.version_name }} check-ci: + needs: prepare runs-on: ubuntu-latest outputs: ci_passed: ${{ steps.ci-check.outputs.ci_passed }} @@ -48,7 +49,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - output=$(./scripts/check-fc-ci.sh firecracker_versions.txt) + output=$(./scripts/check-fc-ci.sh '${{ needs.prepare.outputs.matrix }}') echo "$output" ci_passed=$(echo "$output" | grep "^ci_passed=" | cut -d= -f2) echo "ci_passed=$ci_passed" >> $GITHUB_OUTPUT diff --git a/scripts/check-fc-ci.sh b/scripts/check-fc-ci.sh index e633827..b1e75e3 100755 --- a/scripts/check-fc-ci.sh +++ b/scripts/check-fc-ci.sh @@ -2,50 +2,19 @@ set -euo pipefail -VERSIONS_FILE="${1:-firecracker_versions.txt}" -FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git" FIRECRACKER_REPO_API="e2b-dev/firecracker" -if [[ ! -f "$VERSIONS_FILE" ]]; then - echo "Error: $VERSIONS_FILE not found" >&2 +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " >&2 exit 1 fi -TEMP_DIR=$(mktemp -d) -trap "rm -rf $TEMP_DIR" EXIT - -git clone --bare "$FIRECRACKER_REPO_URL" "$TEMP_DIR/fc-repo" 2>/dev/null -cd "$TEMP_DIR/fc-repo" +versions_json="$1" all_passed=true failed_versions="" -while IFS= read -r version || [[ -n "$version" ]]; do - [[ "$version" =~ ^[[:space:]]*# ]] && continue - [[ -z "$version" ]] && continue - - if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then - tag="${BASH_REMATCH[1]}" - shorthash="${BASH_REMATCH[2]}" - commit_hash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || echo "") - version_name="${tag}_${shorthash}" - else - commit_hash=$(git rev-parse --verify "${version}^{commit}" 2>/dev/null || echo "") - if [[ -n "$commit_hash" ]]; then - short_hash=$(git rev-parse --short "$commit_hash") - version_name="${version}_${short_hash}" - else - version_name="$version" - fi - fi - - if [[ -z "$commit_hash" ]]; then - echo " ❌ Could not resolve commit for $version" - all_passed=false - failed_versions="${failed_versions}${version}(unresolved) " - continue - fi - +while IFS='|' read -r version commit_hash version_name; do status_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/status" 2>/dev/null || echo '{"state":"unknown","total_count":0}') status=$(echo "$status_response" | jq -r '.state') status_count=$(echo "$status_response" | jq -r '.total_count') @@ -84,7 +53,7 @@ while IFS= read -r version || [[ -n "$version" ]]; do all_passed=false failed_versions="${failed_versions}${version_name}(unexpected) " fi -done < "$OLDPWD/$VERSIONS_FILE" +done < <(echo "$versions_json" | jq -r '.[] | "\(.version)|\(.hash)|\(.version_name)"') echo "" [[ "$all_passed" == "true" ]] && echo "ci_passed=true" || echo "ci_passed=false" From 2900ebe0513520bb00ea81cc9e341ff6b929e394 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:37:59 -0800 Subject: [PATCH 11/14] Cleanup --- Makefile | 7 ++++++- README.md | 4 +--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 4a5bacf..d51e847 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,12 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") .PHONY: build build: - ./build.sh + @versions_json=$$(./scripts/parse-versions-with-hash.sh firecracker_versions.txt); \ + echo "$$versions_json" | jq -r '.[] | "\(.version)|\(.hash)|\(.version_name)"' | \ + while IFS='|' read -r version hash version_name; do \ + echo "Building $$version_name..."; \ + ./build.sh "$$version" "$$hash" "$$version_name"; \ + done .PHONY: upload upload: diff --git a/README.md b/README.md index 2080515..11d0b6d 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,9 @@ This project automates the building of custom Firecracker. It supports building ```sh make build - # or directly - ./build.sh ``` - The built firecrackers will be placed in `builds/firecracker-/firecracker`. + The built firecrackers will be placed in `builds//firecracker`. ## Development Workflow From a8651b44272fa5ab43a833d462fdafece2c9e2b0 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:51:20 -0800 Subject: [PATCH 12/14] Cleanup --- Makefile | 19 +++++++++++++------ README.md | 37 +++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index d51e847..6368157 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,19 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") .PHONY: build build: - @versions_json=$$(./scripts/parse-versions-with-hash.sh firecracker_versions.txt); \ - echo "$$versions_json" | jq -r '.[] | "\(.version)|\(.hash)|\(.version_name)"' | \ - while IFS='|' read -r version hash version_name; do \ - echo "Building $$version_name..."; \ - ./build.sh "$$version" "$$hash" "$$version_name"; \ - done + @if ! git diff-index --quiet HEAD --; then \ + echo "Error: Uncommitted changes detected. Please commit or stash changes before building." >&2; \ + exit 1; \ + fi + @hash=$$(git rev-parse HEAD); \ + tag=$$(git describe --tags --abbrev=0 HEAD 2>/dev/null || echo ""); \ + if [ -z "$$tag" ]; then \ + echo "Error: No tag found for current commit" >&2; \ + exit 1; \ + fi; \ + short_hash=$$(git rev-parse --short HEAD); \ + version_name="$${tag}_$${short_hash}"; \ + ./build.sh "$$tag" "$$hash" "$$version_name" .PHONY: upload upload: diff --git a/README.md b/README.md index 11d0b6d..3bc29ca 100644 --- a/README.md +++ b/README.md @@ -2,28 +2,45 @@ ## Overview -This project automates the building of custom Firecracker. It supports building specific firecracker versions and uploading the resulting binaries to a Google Cloud Storage (GCS) bucket. +This project automates the building of custom Firecracker versions. It supports building specific firecracker versions and uploading the resulting binaries to a Google Cloud Storage (GCS) bucket. ## Prerequisites - Linux environment (for building firecracker) +- Git repository with tags ## Building Firecrackers -1. **Configure firecracker versions:** - - Edit `firecracker_versions.txt` to specify which firecracker versions to build (one per line, e.g., `-`). +### Local Build -2. **Build:** +Build the current git version (latest tag + commit hash): - ```sh - make build - ``` +```sh +make build +``` - The built firecrackers will be placed in `builds//firecracker`. +**Requirements:** +- The repository must be in a clean state (no uncommitted changes) +- The current commit must have an associated tag +- The built firecracker will be placed in `builds/_/firecracker` -## Development Workflow +### CI/CD Build -- On every push, GitHub Actions will automatically build the firecrackers and save it as an artifact. +The `firecracker_versions.txt` file specifies which versions to build in CI/CD: + +- Edit `firecracker_versions.txt` to specify firecracker versions (one per line) +- Versions can be tags (e.g., `v1.10.1`) or tag with shorthash (e.g., `v1.12.1_abcdef12`) +- On every push, GitHub Actions will automatically: + 1. Parse versions from `firecracker_versions.txt` and resolve commit hashes + 2. Build each version in parallel + 3. Check CI status for each version + 4. Upload successful builds to GCS and create GitHub releases (on main branch) + +## Scripts + +- `build.sh ` - Builds a single Firecracker version +- `scripts/parse-versions-with-hash.sh` - Parses versions and resolves commit hashes +- `scripts/check-fc-ci.sh ` - Checks CI status for parsed versions ## License From 7fb60bf57fd20cd5016b6e3b782a3819abec1b41 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:52:33 -0800 Subject: [PATCH 13/14] Cleanup --- Makefile | 4 ++-- README.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6368157..c6080b7 100644 --- a/Makefile +++ b/Makefile @@ -8,9 +8,9 @@ build: exit 1; \ fi @hash=$$(git rev-parse HEAD); \ - tag=$$(git describe --tags --abbrev=0 HEAD 2>/dev/null || echo ""); \ + tag=$$(git tag --sort=-version:refname | head -1); \ if [ -z "$$tag" ]; then \ - echo "Error: No tag found for current commit" >&2; \ + echo "Error: No tags found in repository" >&2; \ exit 1; \ fi; \ short_hash=$$(git rev-parse --short HEAD); \ diff --git a/README.md b/README.md index 3bc29ca..11726fc 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This project automates the building of custom Firecracker versions. It supports ### Local Build -Build the current git version (latest tag + commit hash): +Build the current git version (latest tag in repository + current commit hash): ```sh make build @@ -21,8 +21,8 @@ make build **Requirements:** - The repository must be in a clean state (no uncommitted changes) -- The current commit must have an associated tag -- The built firecracker will be placed in `builds/_/firecracker` +- The repository must have at least one tag +- The built firecracker will be placed in `builds/_/firecracker` ### CI/CD Build From d7b3db0bf972d201f2d2ccbb41201d27bca04e41 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 11 Dec 2025 12:55:42 -0800 Subject: [PATCH 14/14] Remove local build as that should be done via the custom FC repo during development --- .gitignore | 1 + Makefile | 25 ------------------------- README.md | 18 +----------------- 3 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 4d3d53d..3610007 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ linux/ .env .terraform .tfplan +.env diff --git a/Makefile b/Makefile deleted file mode 100644 index c6080b7..0000000 --- a/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -ENV := $(shell cat ../../.last_used_env || echo "not-set") --include ../../.env.${ENV} - -.PHONY: build -build: - @if ! git diff-index --quiet HEAD --; then \ - echo "Error: Uncommitted changes detected. Please commit or stash changes before building." >&2; \ - exit 1; \ - fi - @hash=$$(git rev-parse HEAD); \ - tag=$$(git tag --sort=-version:refname | head -1); \ - if [ -z "$$tag" ]; then \ - echo "Error: No tags found in repository" >&2; \ - exit 1; \ - fi; \ - short_hash=$$(git rev-parse --short HEAD); \ - version_name="$${tag}_$${short_hash}"; \ - ./build.sh "$$tag" "$$hash" "$$version_name" - -.PHONY: upload -upload: - ./upload.sh $(GCP_PROJECT_ID) - -.PHONY: build-and-upload -make build-and-upload: build upload \ No newline at end of file diff --git a/README.md b/README.md index 11726fc..eefe422 100644 --- a/README.md +++ b/README.md @@ -7,26 +7,10 @@ This project automates the building of custom Firecracker versions. It supports ## Prerequisites - Linux environment (for building firecracker) -- Git repository with tags ## Building Firecrackers -### Local Build - -Build the current git version (latest tag in repository + current commit hash): - -```sh -make build -``` - -**Requirements:** -- The repository must be in a clean state (no uncommitted changes) -- The repository must have at least one tag -- The built firecracker will be placed in `builds/_/firecracker` - -### CI/CD Build - -The `firecracker_versions.txt` file specifies which versions to build in CI/CD: +The `firecracker_versions.txt` file specifies which versions to build: - Edit `firecracker_versions.txt` to specify firecracker versions (one per line) - Versions can be tags (e.g., `v1.10.1`) or tag with shorthash (e.g., `v1.12.1_abcdef12`)