Switch to custom fc repo; Improve build system

.github/workflows/build.yml

@@ -0,0 +1,28 @@
+name: Build Firecracker Versions
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      hash:
+        required: true
+        type: string
+      version_name:
+        required: true
+        type: string
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Firecracker ${{ inputs.version_name }}
+        run: ./build.sh "${{ inputs.version }}" "${{ inputs.hash }}" "${{ inputs.version_name }}"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: firecracker-${{ inputs.version_name }}
+          path: builds/
+          retention-days: 7

.github/workflows/fc-versions.yml

@@ -2,105 +2,109 @@ name: FC Versions
 
 on:
   push:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
 
 permissions:
   id-token: write
   contents: write
 
 jobs:
-  publish:
-    name: Build Firecracker and upload
-    runs-on: ubuntu-22.04
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+      - name: Parse versions and resolve hashes
+        id: set-matrix
+        run: |
+          versions_json=$(./scripts/parse-versions-with-hash.sh firecracker_versions.txt)
+          echo "matrix=$versions_json" >> $GITHUB_OUTPUT
 
-      - uses: actions/create-github-app-token@v1
-        id: app-token
-        with:
-          app-id: ${{ vars.VERSION_BUMPER_APPID }}
-          private-key: ${{ secrets.VERSION_BUMPER_SECRET }}
+  build:
+    needs: prepare
+    uses: ./.github/workflows/build.yml
+    strategy:
+      fail-fast: true
+      matrix:
+        include: ${{ fromJson(needs.prepare.outputs.matrix) }}
+    with:
+      version: ${{ matrix.version }}
+      hash: ${{ matrix.hash }}
+      version_name: ${{ matrix.version_name }}
 
-      - name: Get the last release
-        id: last_release
-        continue-on-error: true
-        uses: cardinalby/git-get-release-action@v1
+  check-ci:
+    needs: prepare
+    runs-on: ubuntu-latest
+    outputs:
+      ci_passed: ${{ steps.ci-check.outputs.ci_passed }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check CI status
+        id: ci-check
         env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          latest: true
-          prerelease: false
-          draft: false
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          output=$(./scripts/check-fc-ci.sh '${{ needs.prepare.outputs.matrix }}')
+          echo "$output"
+          ci_passed=$(echo "$output" | grep "^ci_passed=" | cut -d= -f2)
+          echo "ci_passed=$ci_passed" >> $GITHUB_OUTPUT
 
-      - name: Get next version
-        id: get-version
+  publish:
+    name: Collect and upload builds
+    needs: [build, check-ci]
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v4
+        with:
+          path: builds
+          pattern: firecracker-*
+          merge-multiple: true
+      - name: CI check result
         run: |
-          if [ "${{ steps.last_release.outputs.tag_name }}" == "" ]; then
-            echo "No previous release found, starting with v0.0.1"
-            echo "version=v0.0.1" >> $GITHUB_OUTPUT
-          else
-            version=${{ steps.last_release.outputs.tag_name }}
-            result=$(echo ${version} | awk -F. -v OFS=. '{$NF += 1 ; print}')
-            echo "version=$result" >> $GITHUB_OUTPUT
+          if [[ "${{ needs.check-ci.outputs.ci_passed }}" != "true" ]]; then
+            echo "⚠️ CI checks did not pass - skipping GCS upload and release"
           fi
 
-      - name: Test next version
-        run: echo "Next version is ${{ steps.get-version.outputs.version }}"
-
       - name: Setup Service Account
+        if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true'
         uses: google-github-actions/auth@v2
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
 
-      - name: Build firecrackers
-        run: sudo make build
-
-      - name: Upload firecrackers as artifact
-        if: github.ref_name != 'main'
-        uses: actions/upload-artifact@v4
-        with:
-          name: firecracker-${{ github.run_id }}
-          path: ./builds
-          retention-days: 7
-
       - name: Upload firecrackers to GCS
-        if: github.ref_name == 'main'
+        if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true'
         uses: "google-github-actions/upload-cloud-storage@v1"
         with:
           path: "./builds"
           destination: ${{ vars.GCP_BUCKET_NAME }}/firecrackers
           gzip: false
           parent: false
 
-      - name: Create Git tag
-        if: github.ref_name == 'main'
-        run: |
-          git config user.name "github-actions"
-          git config user.email "[email protected]"
-          git tag ${{ steps.get-version.outputs.version }}
-          git push origin ${{ steps.get-version.outputs.version }}
+      - name: Create releases
+        if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-
-      - name: Prepare release assets
-        if: github.ref_name == 'main'
         run: |
-          mkdir -p release-assets
+          git config user.name "github-actions"
+          git config user.email "[email protected]"
           for dir in ./builds/*/; do
-            name=$(basename "$dir")
-            cp "$dir/vmlinux.bin" "release-assets/${name}.bin"
+            version_name=$(basename "$dir")
+            if git rev-parse "refs/tags/$version_name" >/dev/null 2>&1 || gh release view "$version_name" >/dev/null 2>&1; then
+              continue
+            fi
+            git tag "$version_name"
+            git push origin "$version_name"
+            gh release create "$version_name" \
+              --title "Firecracker $version_name" \
+              --notes "Firecracker build: $version_name" \
+              "$dir/firecracker#${version_name}"
           done
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload Release Asset
-        if: github.ref_name == 'main'
-        uses: softprops/action-gh-release@v2
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          name: Firecrackers ${{ steps.get-version.outputs.version }}
-          tag_name: ${{ steps.get-version.outputs.version }}
-          files: "./release-assets/*"

.gitignore

@@ -3,3 +3,4 @@ linux/
 .env
 .terraform
 .tfplan
+.env

README.md

@@ -1,29 +1,31 @@
-# fc-kernels
+# fc-versions
 
 ## Overview
 
-This project automates the building of custom Firecracker. It supports building specific firecracker versions and uploading the resulting binaries to a Google Cloud Storage (GCS) bucket.
+This project automates the building of custom Firecracker versions. It supports building specific firecracker versions and uploading the resulting binaries to a Google Cloud Storage (GCS) bucket.
 
 ## Prerequisites
 
 - Linux environment (for building firecracker)
 
-## Building Kernels
+## Building Firecrackers
 
-1. **Configure firecracker versions:**
-    - Edit `firecracker_versions.txt` to specify which kernel versions to build (one per line, e.g., `<last_tag-prelease>-<first-8-letters-of-the-specific-commit>`).
+The `firecracker_versions.txt` file specifies which versions to build:
 
-2. **Build:**
-   ```sh
-   make build
-   # or directly
-   ./build.sh
-   ```
-   The built kernels will be placed in `builds/vmlinux-<version>/vmlinux.bin`.
+- Edit `firecracker_versions.txt` to specify firecracker versions (one per line)
+- Versions can be tags (e.g., `v1.10.1`) or tag with shorthash (e.g., `v1.12.1_abcdef12`)
+- On every push, GitHub Actions will automatically:
+  1. Parse versions from `firecracker_versions.txt` and resolve commit hashes
+  2. Build each version in parallel
+  3. Check CI status for each version
+  4. Upload successful builds to GCS and create GitHub releases (on main branch)
 
-## Development Workflow
-- On every push, GitHub Actions will automatically build the kernels and save it as an artifact.
+## Scripts
+
+- `build.sh <version> <hash> <version_name>` - Builds a single Firecracker version
+- `scripts/parse-versions-with-hash.sh` - Parses versions and resolves commit hashes
+- `scripts/check-fc-ci.sh <versions_json>` - Checks CI status for parsed versions
 
 ## License
 
-This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details. 
+This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details.

build.sh

@@ -2,32 +2,36 @@
 
 set -euo pipefail
 
-function build_version {
-  local version=$1
-  echo "Starting build for Firecracker commit: $version"
+FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git"
 
-  echo "Checking out repo for Firecracker at commit: $version"
-  git checkout "${version}"
+if [[ $# -lt 3 ]]; then
+  echo "Usage: $0 <version> <hash> <version_name>" >&2
+  exit 1
+fi
 
-  # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311
-  version_name=$(git describe --tags --abbrev=0 $(git rev-parse HEAD))_$(git rev-parse --short HEAD)
-  echo "Version name: $version_name"
+version="$1"
+fullhash="$2"
+version_name="$3"
 
-  echo "Building Firecracker version: $version_name"
-  tools/devtool -y build --release
+git clone $FIRECRACKER_REPO_URL firecracker
+cd firecracker
 
-  echo "Copying finished build to builds directory"
-  mkdir -p "../builds/${version_name}"
-  cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker"
-}
+if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then
+  tag="${BASH_REMATCH[1]}"
+  git checkout "$tag"
+  if ! git merge-base --is-ancestor "$tag" "$fullhash"; then
+    echo "Error: shorthash is not a descendant of tag $tag" >&2
+    exit 1
+  fi
+  git checkout "$fullhash"
+else
+  git checkout "$fullhash"
+fi
 
-echo "Cloning the Firecracker repository"
-git clone https://github.com/firecracker-microvm/firecracker.git firecracker
-cd firecracker
+tools/devtool -y build --release -- --bin firecracker
 
-grep -v '^ *#' <../firecracker_versions.txt | while IFS= read -r version; do
-  build_version "$version"
-done
+mkdir -p "../builds/${version_name}"
+cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker"
 
 cd ..
-rm -rf firecracker
+rm -rf firecracker

scripts/check-fc-ci.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -euo pipefail
+
+FIRECRACKER_REPO_API="e2b-dev/firecracker"
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <versions_json>" >&2
+  exit 1
+fi
+
+versions_json="$1"
+
+all_passed=true
+failed_versions=""
+
+while IFS='|' read -r version commit_hash version_name; do
+  status_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/status" 2>/dev/null || echo '{"state":"unknown","total_count":0}')
+  status=$(echo "$status_response" | jq -r '.state')
+  status_count=$(echo "$status_response" | jq -r '.total_count')
+
+  check_response=$(gh api "/repos/${FIRECRACKER_REPO_API}/commits/${commit_hash}/check-runs" 2>/dev/null || echo '{"total_count":0}')
+  check_count=$(echo "$check_response" | jq -r '.total_count')
+  check_conclusion=$(echo "$check_response" | jq -r '
+    if .total_count == 0 then "no_checks"
+    elif ([.check_runs[].status] | any(. == "in_progress" or . == "queued")) then "pending"
+    elif ([.check_runs[].conclusion] | any(. == "failure" or . == "cancelled" or . == "timed_out")) then "failure"
+    elif ([.check_runs[].conclusion] | all(. == "success" or . == "skipped" or . == "neutral")) then "success"
+    else "unknown"
+    end
+  ')
+
+  if [[ "$status" == "failure" ]] || [[ "$check_conclusion" == "failure" ]]; then
+    echo "  ❌ CI failed for $version_name"
+    all_passed=false
+    failed_versions="${failed_versions}${version_name} "
+  elif [[ "$check_conclusion" == "pending" ]] || ([[ "$status" == "pending" ]] && [[ "$status_count" -gt 0 ]]); then
+    echo "  ⏳ CI still running for $version_name"
+    all_passed=false
+    failed_versions="${failed_versions}${version_name}(pending) "
+  elif [[ "$status" == "unknown" ]] && [[ "$check_conclusion" == "unknown" ]]; then
+    echo "  ⚠️ Could not verify CI status for $version_name (API error)"
+    all_passed=false
+    failed_versions="${failed_versions}${version_name}(unknown) "
+  elif [[ "$status" == "success" ]] || [[ "$check_conclusion" == "success" ]]; then
+    echo "  ✅ CI passed for $version_name"
+  elif [[ "$status_count" -eq 0 ]] && [[ "$check_count" -eq 0 ]]; then
+    echo "  ℹ️ No CI checks found for $version_name (assuming OK)"
+  elif [[ "$status" == "pending" ]] && [[ "$status_count" -eq 0 ]] && [[ "$check_conclusion" == "no_checks" ]]; then
+    echo "  ℹ️ No CI checks found for $version_name (assuming OK)"
+  else
+    echo "  ⚠️ Unexpected CI state for $version_name: status=$status, check_conclusion=$check_conclusion"
+    all_passed=false
+    failed_versions="${failed_versions}${version_name}(unexpected) "
+  fi
+done < <(echo "$versions_json" | jq -r '.[] | "\(.version)|\(.hash)|\(.version_name)"')
+
+echo ""
+[[ "$all_passed" == "true" ]] && echo "ci_passed=true" || echo "ci_passed=false"