Switch to custom fc repo; Improve build system

.github/workflows/build.yml

@@ -0,0 +1,58 @@
+name: Build Firecracker Versions
+
+on:
+  workflow_call:
+    outputs:
+      versions:
+        description: "JSON array of versions that were built"
+        value: ${{ jobs.prepare.outputs.versions }}
+  workflow_dispatch:
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      versions: ${{ steps.set-versions.outputs.versions }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Parse versions from file
+        id: set-versions
+        run: |
+          # Read versions, skip comments and empty lines, output as JSON array
+          versions=$(grep -v '^ *#' firecracker_versions.txt | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "versions=$versions" >> $GITHUB_OUTPUT
+          echo "Building versions: $versions"
+
+  build:
+    needs: prepare
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        version: ${{ fromJson(needs.prepare.outputs.versions) }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Firecracker ${{ matrix.version }}
+        id: build
+        run: |
+          ./build.sh "${{ matrix.version }}"
+          # Read the computed version name (with hash) - format is "version_name:commit_hash"
+          version_info=$(cat built_versions.txt | tail -1)
+          version_name=$(echo "$version_info" | cut -d: -f1)
+          echo "version_name=$version_name" >> $GITHUB_OUTPUT
+
+      - name: Upload build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: firecracker-${{ steps.build.outputs.version_name }}
+          path: builds/
+          retention-days: 7
+
+      - name: Upload version info
+        uses: actions/upload-artifact@v4
+        with:
+          name: version-info-${{ steps.build.outputs.version_name }}
+          path: built_versions.txt
+          retention-days: 7

.github/workflows/fc-versions.yml

@@ -2,105 +2,154 @@ name: FC Versions
 
 on:
   push:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
 
 permissions:
   id-token: write
   contents: write
 
 jobs:
+  # Run parallel builds via reusable workflow
+  build:
+    uses: ./.github/workflows/build.yml
+
   publish:
-    name: Build Firecracker and upload
+    name: Collect and upload builds
+    needs: build
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-      - uses: actions/create-github-app-token@v1
-        id: app-token
+      # Download all build artifacts
+      - name: Download all build artifacts
+        uses: actions/download-artifact@v4
         with:
-          app-id: ${{ vars.VERSION_BUMPER_APPID }}
-          private-key: ${{ secrets.VERSION_BUMPER_SECRET }}
+          path: builds
+          pattern: firecracker-*
+          merge-multiple: true
 
-      - name: Get the last release
-        id: last_release
-        continue-on-error: true
-        uses: cardinalby/git-get-release-action@v1
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+      - name: List downloaded builds
+        run: find builds -type f | head -20
+
+      # Download version info to get commit hashes
+      - name: Download version info
+        uses: actions/download-artifact@v4
         with:
-          latest: true
-          prerelease: false
-          draft: false
+          path: version-info
+          pattern: version-info-*
+          merge-multiple: true
 
-      - name: Get next version
-        id: get-version
+      # Check CI status for all commits (runs on all branches for visibility)
+      - name: Check CI status for all versions
+        id: ci-check
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          if [ "${{ steps.last_release.outputs.tag_name }}" == "" ]; then
-            echo "No previous release found, starting with v0.0.1"
-            echo "version=v0.0.1" >> $GITHUB_OUTPUT
+          FIRECRACKER_REPO="e2b-dev/firecracker"
+          all_passed=true
+          failed_versions=""
+
+          # Read all version:hash pairs
+          for line in $(cat version-info/built_versions.txt 2>/dev/null || echo ""); do
+            version_name=$(echo "$line" | cut -d: -f1)
+            commit_hash=$(echo "$line" | cut -d: -f2)
+
+            echo "Checking CI for $version_name (commit: $commit_hash)..."
+
+            # Check combined commit status
+            status=$(gh api "/repos/${FIRECRACKER_REPO}/commits/${commit_hash}/status" --jq '.state' 2>/dev/null || echo "unknown")
+
+            # Check check-runs (GitHub Actions)
+            check_conclusion=$(gh api "/repos/${FIRECRACKER_REPO}/commits/${commit_hash}/check-runs" --jq '
+              if .total_count == 0 then "no_checks"
+              elif ([.check_runs[].conclusion] | all(. == "success" or . == "skipped" or . == null)) then "success"
+              elif ([.check_runs[].status] | any(. == "in_progress" or . == "queued")) then "pending"
+              else "failure"
+              end
+            ' 2>/dev/null || echo "unknown")
+
+            echo "  Status: $status, Check runs: $check_conclusion"
+
+            if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then
+              echo "  ❌ CI failed for $version_name"
+              all_passed=false
+              failed_versions="${failed_versions}${version_name} "
+            elif [[ "$status" == "pending" ]] || [[ "$check_conclusion" == "pending" ]]; then
+              echo "  ⏳ CI still running for $version_name"
+              all_passed=false
+              failed_versions="${failed_versions}${version_name}(pending) "
+            else
+              echo "  ✅ CI passed for $version_name"
+            fi
+          done
+
+          if [[ "$all_passed" == "true" ]]; then
+            echo "All CI checks passed!"
+            echo "ci_passed=true" >> $GITHUB_OUTPUT
           else
-            version=${{ steps.last_release.outputs.tag_name }}
-            result=$(echo ${version} | awk -F. -v OFS=. '{$NF += 1 ; print}')
-            echo "version=$result" >> $GITHUB_OUTPUT
+            echo "CI checks failed or pending for: $failed_versions"
+            echo "ci_passed=false" >> $GITHUB_OUTPUT
+            echo "Skipping GCS upload and release creation."
           fi
 
-      - name: Test next version
-        run: echo "Next version is ${{ steps.get-version.outputs.version }}"
-
       - name: Setup Service Account
+        if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true'
         uses: google-github-actions/auth@v2
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
 
-      - name: Build firecrackers
-        run: sudo make build
-
-      - name: Upload firecrackers as artifact
-        if: github.ref_name != 'main'
-        uses: actions/upload-artifact@v4
-        with:
-          name: firecracker-${{ github.run_id }}
-          path: ./builds
-          retention-days: 7
-
       - name: Upload firecrackers to GCS
-        if: github.ref_name == 'main'
+        if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true'
         uses: "google-github-actions/upload-cloud-storage@v1"
         with:
           path: "./builds"
           destination: ${{ vars.GCP_BUCKET_NAME }}/firecrackers
           gzip: false
           parent: false
 
-      - name: Create Git tag
-        if: github.ref_name == 'main'
+      - name: Create releases for each Firecracker version
+        if: github.ref_name == 'main' && steps.ci-check.outputs.ci_passed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           git config user.name "github-actions"
           git config user.email "[email protected]"
-          git tag ${{ steps.get-version.outputs.version }}
-          git push origin ${{ steps.get-version.outputs.version }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-
-      - name: Prepare release assets
-        if: github.ref_name == 'main'
-        run: |
-          mkdir -p release-assets
           for dir in ./builds/*/; do
-            name=$(basename "$dir")
-            cp "$dir/vmlinux.bin" "release-assets/${name}.bin"
-          done
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            version_name=$(basename "$dir")
+            binary_path="$dir/firecracker"
 
-      - name: Upload Release Asset
-        if: github.ref_name == 'main'
-        uses: softprops/action-gh-release@v2
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          name: Firecrackers ${{ steps.get-version.outputs.version }}
-          tag_name: ${{ steps.get-version.outputs.version }}
-          files: "./release-assets/*"
+            echo "Processing $version_name..."
+
+            # Check if tag already exists
+            if git rev-parse "refs/tags/$version_name" >/dev/null 2>&1; then
+              echo "Tag $version_name already exists, skipping..."
+              continue
+            fi
+
+            # Check if release already exists
+            if gh release view "$version_name" >/dev/null 2>&1; then
+              echo "Release $version_name already exists, skipping..."
+              continue
+            fi
+
+            echo "Creating tag and release for $version_name..."
+
+            # Create and push tag
+            git tag "$version_name"
+            git push origin "$version_name"
+
+            # Create release with the binary
+            gh release create "$version_name" \
+              --title "Firecracker $version_name" \
+              --notes "Firecracker build: $version_name" \
+              "$binary_path#${version_name}"
+          done

README.md

@@ -1,4 +1,4 @@
-# fc-kernels
+# fc-versions
 
 ## Overview
 
@@ -8,22 +8,25 @@ This project automates the building of custom Firecracker. It supports building
 
 - Linux environment (for building firecracker)
 
-## Building Kernels
+## Building Firecrackers
 
 1. **Configure firecracker versions:**
-    - Edit `firecracker_versions.txt` to specify which kernel versions to build (one per line, e.g., `<last_tag-prelease>-<first-8-letters-of-the-specific-commit>`).
+    - Edit `firecracker_versions.txt` to specify which firecracker versions to build (one per line, e.g., `<last_tag-prelease>-<first-8-letters-of-the-specific-commit>`).
 
 2. **Build:**
+
    ```sh
    make build
    # or directly
    ./build.sh
    ```
-   The built kernels will be placed in `builds/vmlinux-<version>/vmlinux.bin`.
+
+   The built firecrackers will be placed in `builds/firecracker-<version>/firecracker`.
 
 ## Development Workflow
-- On every push, GitHub Actions will automatically build the kernels and save it as an artifact.
+
+- On every push, GitHub Actions will automatically build the firecrackers and save it as an artifact.
 
 ## License
 
-This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details. 
+This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details.

build.sh

@@ -2,32 +2,75 @@
 
 set -euo pipefail
 
+FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git"
+
 function build_version {
   local version=$1
-  echo "Starting build for Firecracker commit: $version"
 
-  echo "Checking out repo for Firecracker at commit: $version"
-  git checkout "${version}"
+  # Detect if the version is of the form tag_shorthash (e.g., v1.12.1_abcdef12)
+  if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then
+    local tag="${BASH_REMATCH[1]}"
+    local shorthash="${BASH_REMATCH[2]}"
+    echo "Starting build for Firecracker tag: $tag and shorthash: $shorthash"
+
+    echo "Checking out repo at tag: $tag"
+    git checkout "$tag"
+
+    # Find full hash from shorthash
+    fullhash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || true)
+    if [[ -z "$fullhash" ]]; then
+      echo "Error: Could not resolve hash $shorthash"
+      exit 1
+    fi
+
+    # Ensure that $fullhash is a descendant of $tag
+    if git merge-base --is-ancestor "$tag" "$fullhash"; then
+      echo "Shorthash $shorthash is AFTER tag $tag -- proceeding"
+      git checkout "$fullhash"
+    else
+      echo "Error: shorthash $shorthash is not a descendant of tag $tag"
+      exit 1
+    fi
+    version_name="${tag}_$shorthash"
+  else
+    echo "Starting build for Firecracker at commit: $version"
+    echo "Checking out repo for Firecracker at commit: $version"
+    git checkout "${version}"
+
+    fullhash=$(git rev-parse HEAD)
+    # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311
+    version_name=$(git describe --tags --abbrev=0 "$fullhash")_$(git rev-parse --short HEAD)
+  fi
 
-  # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311
-  version_name=$(git describe --tags --abbrev=0 $(git rev-parse HEAD))_$(git rev-parse --short HEAD)
   echo "Version name: $version_name"
 
   echo "Building Firecracker version: $version_name"
-  tools/devtool -y build --release
+  # Build only the firecracker binary, skip jailer and snapshot-editor for faster builds
+  tools/devtool -y build --release -- --bin firecracker
 
   echo "Copying finished build to builds directory"
   mkdir -p "../builds/${version_name}"
   cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker"
+
+  # Write version name and commit hash to file for CI to use
+  echo "${version_name}:${fullhash}" >> ../built_versions.txt
 }
 
+# If a version is passed as argument, build only that version
+# Otherwise, build all versions from firecracker_versions.txt
+if [[ $# -ge 1 ]]; then
+  versions=("$@")
+else
+  mapfile -t versions < <(grep -v '^ *#' firecracker_versions.txt | grep -v '^$')
+fi
+
 echo "Cloning the Firecracker repository"
-git clone https://github.com/firecracker-microvm/firecracker.git firecracker
+git clone $FIRECRACKER_REPO_URL firecracker
 cd firecracker
 
-grep -v '^ *#' <../firecracker_versions.txt | while IFS= read -r version; do
+for version in "${versions[@]}"; do
   build_version "$version"
 done
 
 cd ..
-rm -rf firecracker
+rm -rf firecracker