Switch to custom fc repo; Improve build system

.github/workflows/build.yml

@@ -0,0 +1,50 @@
+name: Build Firecracker Versions
+
+on:
+  workflow_call:
+    outputs:
+      versions:
+        description: "JSON array of versions that were built"
+        value: ${{ jobs.prepare.outputs.versions }}
+  workflow_dispatch:
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      versions: ${{ steps.set-versions.outputs.versions }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Parse versions from file
+        id: set-versions
+        run: |
+          versions=$(./scripts/parse-versions.sh firecracker_versions.txt)
+          echo "versions=$versions" >> $GITHUB_OUTPUT
+          echo "Building versions: $versions"
+
+  build:
+    needs: prepare
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        version: ${{ fromJson(needs.prepare.outputs.versions) }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Firecracker ${{ matrix.version }}
+        id: build
+        run: |
+          ./build.sh "${{ matrix.version }}"
+          # Read the computed version name (with hash) - format is "version_name:commit_hash"
+          version_info=$(cat built_versions.txt | tail -1)
+          version_name=$(echo "$version_info" | cut -d: -f1)
+          echo "version_name=$version_name" >> $GITHUB_OUTPUT
+
+      - name: Upload build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: firecracker-${{ steps.build.outputs.version_name }}
+          path: builds/
+          retention-days: 7

.github/workflows/fc-versions.yml

@@ -2,105 +2,121 @@ name: FC Versions
 
 on:
   push:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
 
 permissions:
   id-token: write
   contents: write
 
 jobs:
+  # Run parallel builds via reusable workflow
+  build:
+    uses: ./.github/workflows/build.yml
+
+  # Check CI status in parallel with builds (for faster feedback)
+  check-ci:
+    name: Check FC repo CI status
+    runs-on: ubuntu-latest
+    outputs:
+      ci_passed: ${{ steps.ci-check.outputs.ci_passed }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check CI status for all versions
+        id: ci-check
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          output=$(./scripts/check-fc-ci.sh firecracker_versions.txt)
+          echo "$output"
+          # Extract ci_passed from last line
+          ci_passed=$(echo "$output" | grep "^ci_passed=" | cut -d= -f2)
+          echo "ci_passed=$ci_passed" >> $GITHUB_OUTPUT
+
+  # Collect artifacts and publish (waits for both build and CI check)
   publish:
-    name: Build Firecracker and upload
+    name: Collect and upload builds
+    needs: [build, check-ci]
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-
-      - uses: actions/create-github-app-token@v1
-        id: app-token
         with:
-          app-id: ${{ vars.VERSION_BUMPER_APPID }}
-          private-key: ${{ secrets.VERSION_BUMPER_SECRET }}
+          fetch-depth: 0
 
-      - name: Get the last release
-        id: last_release
-        continue-on-error: true
-        uses: cardinalby/git-get-release-action@v1
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+      # Download all build artifacts
+      - name: Download all build artifacts
+        uses: actions/download-artifact@v4
         with:
-          latest: true
-          prerelease: false
-          draft: false
+          path: builds
+          pattern: firecracker-*
+          merge-multiple: true
+
+      - name: List downloaded builds
+        run: find builds -type f | head -20
 
-      - name: Get next version
-        id: get-version
+      - name: CI check result
         run: |
-          if [ "${{ steps.last_release.outputs.tag_name }}" == "" ]; then
-            echo "No previous release found, starting with v0.0.1"
-            echo "version=v0.0.1" >> $GITHUB_OUTPUT
-          else
-            version=${{ steps.last_release.outputs.tag_name }}
-            result=$(echo ${version} | awk -F. -v OFS=. '{$NF += 1 ; print}')
-            echo "version=$result" >> $GITHUB_OUTPUT
+          echo "CI check passed: ${{ needs.check-ci.outputs.ci_passed }}"
+          if [[ "${{ needs.check-ci.outputs.ci_passed }}" != "true" ]]; then
+            echo "⚠️ CI checks did not pass - skipping GCS upload and release"
           fi
 
-      - name: Test next version
-        run: echo "Next version is ${{ steps.get-version.outputs.version }}"
-
       - name: Setup Service Account
+        if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true'
         uses: google-github-actions/auth@v2
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
 
-      - name: Build firecrackers
-        run: sudo make build
-
-      - name: Upload firecrackers as artifact
-        if: github.ref_name != 'main'
-        uses: actions/upload-artifact@v4
-        with:
-          name: firecracker-${{ github.run_id }}
-          path: ./builds
-          retention-days: 7
-
       - name: Upload firecrackers to GCS
-        if: github.ref_name == 'main'
+        if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true'
         uses: "google-github-actions/upload-cloud-storage@v1"
         with:
           path: "./builds"
           destination: ${{ vars.GCP_BUCKET_NAME }}/firecrackers
           gzip: false
           parent: false
 
-      - name: Create Git tag
-        if: github.ref_name == 'main'
+      - name: Create releases for each Firecracker version
+        if: github.ref_name == 'main' && needs.check-ci.outputs.ci_passed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           git config user.name "github-actions"
           git config user.email "[email protected]"
-          git tag ${{ steps.get-version.outputs.version }}
-          git push origin ${{ steps.get-version.outputs.version }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-
-      - name: Prepare release assets
-        if: github.ref_name == 'main'
-        run: |
-          mkdir -p release-assets
           for dir in ./builds/*/; do
-            name=$(basename "$dir")
-            cp "$dir/vmlinux.bin" "release-assets/${name}.bin"
-          done
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            version_name=$(basename "$dir")
+            binary_path="$dir/firecracker"
 
-      - name: Upload Release Asset
-        if: github.ref_name == 'main'
-        uses: softprops/action-gh-release@v2
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          name: Firecrackers ${{ steps.get-version.outputs.version }}
-          tag_name: ${{ steps.get-version.outputs.version }}
-          files: "./release-assets/*"
+            echo "Processing $version_name..."
+
+            # Check if tag already exists
+            if git rev-parse "refs/tags/$version_name" >/dev/null 2>&1; then
+              echo "Tag $version_name already exists, skipping..."
+              continue
+            fi
+
+            # Check if release already exists
+            if gh release view "$version_name" >/dev/null 2>&1; then
+              echo "Release $version_name already exists, skipping..."
+              continue
+            fi
+
+            echo "Creating tag and release for $version_name..."
+
+            # Create and push tag
+            git tag "$version_name"
+            git push origin "$version_name"
+
+            # Create release with the binary
+            gh release create "$version_name" \
+              --title "Firecracker $version_name" \
+              --notes "Firecracker build: $version_name" \
+              "$binary_path#${version_name}"
+          done

README.md

@@ -1,4 +1,4 @@
-# fc-kernels
+# fc-versions
 
 ## Overview
 
@@ -8,22 +8,25 @@ This project automates the building of custom Firecracker. It supports building
 
 - Linux environment (for building firecracker)
 
-## Building Kernels
+## Building Firecrackers
 
 1. **Configure firecracker versions:**
-    - Edit `firecracker_versions.txt` to specify which kernel versions to build (one per line, e.g., `<last_tag-prelease>-<first-8-letters-of-the-specific-commit>`).
+    - Edit `firecracker_versions.txt` to specify which firecracker versions to build (one per line, e.g., `<last_tag-prelease>-<first-8-letters-of-the-specific-commit>`).
 
 2. **Build:**
+
    ```sh
    make build
    # or directly
    ./build.sh
    ```
-   The built kernels will be placed in `builds/vmlinux-<version>/vmlinux.bin`.
+
+   The built firecrackers will be placed in `builds/firecracker-<version>/firecracker`.
 
 ## Development Workflow
-- On every push, GitHub Actions will automatically build the kernels and save it as an artifact.
+
+- On every push, GitHub Actions will automatically build the firecrackers and save it as an artifact.
 
 ## License
 
-This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details. 
+This project is licensed under the Apache License 2.0. See [LICENSE](LICENSE) for details.

build.sh

@@ -2,32 +2,75 @@
 
 set -euo pipefail
 
+FIRECRACKER_REPO_URL="https://github.com/e2b-dev/firecracker.git"
+
 function build_version {
   local version=$1
-  echo "Starting build for Firecracker commit: $version"
 
-  echo "Checking out repo for Firecracker at commit: $version"
-  git checkout "${version}"
+  # Detect if the version is of the form tag_shorthash (e.g., v1.12.1_abcdef12)
+  if [[ "$version" =~ ^([^_]+)_([0-9a-fA-F]+)$ ]]; then
+    local tag="${BASH_REMATCH[1]}"
+    local shorthash="${BASH_REMATCH[2]}"
+    echo "Starting build for Firecracker tag: $tag and shorthash: $shorthash"
+
+    echo "Checking out repo at tag: $tag"
+    git checkout "$tag"
+
+    # Find full hash from shorthash
+    fullhash=$(git rev-parse --verify "$shorthash^{commit}" 2>/dev/null || true)
+    if [[ -z "$fullhash" ]]; then
+      echo "Error: Could not resolve hash $shorthash"
+      exit 1
+    fi
+
+    # Ensure that $fullhash is a descendant of $tag
+    if git merge-base --is-ancestor "$tag" "$fullhash"; then
+      echo "Shorthash $shorthash is AFTER tag $tag -- proceeding"
+      git checkout "$fullhash"
+    else
+      echo "Error: shorthash $shorthash is not a descendant of tag $tag"
+      exit 1
+    fi
+    version_name="${tag}_$shorthash"
+  else
+    echo "Starting build for Firecracker at commit: $version"
+    echo "Checking out repo for Firecracker at commit: $version"
+    git checkout "${version}"
+
+    fullhash=$(git rev-parse HEAD)
+    # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311
+    version_name=$(git describe --tags --abbrev=0 "$fullhash")_$(git rev-parse --short HEAD)
+  fi
 
-  # The format will be: latest_tag_latest_commit_hash — v1.7.0-dev_g8bb88311
-  version_name=$(git describe --tags --abbrev=0 $(git rev-parse HEAD))_$(git rev-parse --short HEAD)
   echo "Version name: $version_name"
 
   echo "Building Firecracker version: $version_name"
-  tools/devtool -y build --release
+  # Build only the firecracker binary, skip jailer and snapshot-editor for faster builds
+  tools/devtool -y build --release -- --bin firecracker
 
   echo "Copying finished build to builds directory"
   mkdir -p "../builds/${version_name}"
   cp build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "../builds/${version_name}/firecracker"
+
+  # Write version name and commit hash to file for CI to use
+  echo "${version_name}:${fullhash}" >> ../built_versions.txt
 }
 
+# If a version is passed as argument, build only that version
+# Otherwise, build all versions from firecracker_versions.txt
+if [[ $# -ge 1 ]]; then
+  versions=("$@")
+else
+  mapfile -t versions < <(grep -v '^ *#' firecracker_versions.txt | grep -v '^$')
+fi
+
 echo "Cloning the Firecracker repository"
-git clone https://github.com/firecracker-microvm/firecracker.git firecracker
+git clone $FIRECRACKER_REPO_URL firecracker
 cd firecracker
 
-grep -v '^ *#' <../firecracker_versions.txt | while IFS= read -r version; do
+for version in "${versions[@]}"; do
   build_version "$version"
 done
 
 cd ..
-rm -rf firecracker
+rm -rf firecracker