Install pre-compiled files into source tree (#242)

* Copy newly compiled files over when using USE_PRECOMPILED=FALSE

* Add action to check precompiled binaries

* Fix build with precompiled

* Update precompiled binaries

* Remove `if: runner.os == 'Linux'` where not needed

Keep the step name "Install dependencies (Linux)" in case dependencies change later, to make changing all of them simpler

* Change workflow to just check for file updates, rather than identical files

* Run checks in separate script

* Anchor start/end of greps

* Review fixups

Simplify CMakeLists.txt and BUILD.bazel skips

Install all ELFs as files, so they aren't marked as executable

* Tidy up file checks

* Add add_embedded_data_project function

This fixes the BUILD_ALWAYS issues as it's now only set when `USE_PRECOMPILED=false`, and also removes the need for `${CMAKE_COMMAND} --install .`

* Add back BUILD_ALWAYS

It's needed so if you update the binaries (eg `git pull`) they get re-copied and re-installed

Author: will-v-pi

.github/workflows/check_help_text.yml

@@ -13,7 +13,6 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install dependencies (Linux)
-        if: runner.os == 'Linux'
         run: sudo apt install cmake ninja-build python3 build-essential libusb-1.0-0-dev
 
       - name: Checkout Pico SDK

.github/workflows/check_precompiled.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Pass updated files in $updated_files environment variable
+
+# enc_bootloader
+if ! echo "$updated_files" | grep -q "^enc_bootloader/.*\.elf$"; then
+    echo "Checking enc_bootloader files for modifications as ELFs have not been updated"
+    for file in enc_bootloader/*; do
+        if [[ "$file" == "enc_bootloader/CMakeLists.txt" || "$file" == "enc_bootloader/BUILD.bazel" ]]; then
+            continue
+        fi
+        if [[ "$updated_files" == *"$file"* ]]; then
+            echo "File $file is in the PR but enc_bootloader ELFs have not been updated"
+            exit 1
+        fi
+    done
+fi
+
+# picoboot_flash_id
+if ! echo "$updated_files" | grep -q "^picoboot_flash_id/.*\.bin$"; then
+    echo "Checking picoboot_flash_id files for modifications as BINs have not been updated"
+    for file in picoboot_flash_id/*; do
+        if [[ "$file" == "picoboot_flash_id/CMakeLists.txt" || "$file" == "picoboot_flash_id/BUILD.bazel" ]]; then
+            continue
+        fi
+        if [[ "$updated_files" == *"$file"* ]]; then
+            echo "File $file is in the PR but flash_id BINs have not been updated"
+            exit 1
+        fi
+    done
+fi
+
+# xip_ram_perms
+if ! echo "$updated_files" | grep -q "^xip_ram_perms/.*\.elf$"; then
+    echo "Checking xip_ram_perms files for modifications as ELFs have not been updated"
+    for file in xip_ram_perms/*; do
+        if [[ "$file" == "xip_ram_perms/CMakeLists.txt" || "$file" == "xip_ram_perms/BUILD.bazel" ]]; then
+            continue
+        fi
+        if [[ "$updated_files" == *"$file"* ]]; then
+            echo "File $file is in the PR but xip_ram_perms ELFs have not been updated"
+            exit 1
+        fi
+    done
+fi

.github/workflows/check_precompiled.yml

@@ -0,0 +1,52 @@
+name: Check Precompiled Binaries
+
+on:
+  pull_request:
+    paths:
+      - 'enc_bootloader/**'
+      - 'picoboot_flash_id/**'
+      - 'xip_ram_perms/**'
+
+jobs:
+  check-precompiled:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies (Linux)
+        run: sudo apt install cmake ninja-build python3 build-essential gcc-arm-none-eabi libnewlib-arm-none-eabi libstdc++-arm-none-eabi-newlib libusb-1.0-0-dev
+
+      - name: Checkout Pico SDK
+        uses: actions/checkout@v4
+        with:
+          repository: raspberrypi/pico-sdk
+          ref: develop
+          path: pico-sdk
+          submodules: 'true'
+
+      - name: Build and Install
+        run: |
+          cmake -S . -B build -G "Ninja" -D PICO_SDK_PATH="${{ github.workspace }}/pico-sdk" -D USE_PRECOMPILED=FALSE
+          cmake --build build
+          sudo cmake --install build
+
+      - name: Check precompiled binaries have been updated
+        run: |
+          updated_files=$(curl -s -u "${{ github.repository_owner }}":"${{ github.token }}" -H "Accept: application/vnd.github.v3+json" "${{ github.event.pull_request._links.self.href }}/files")
+          updated_files=$(jq -r '.[] | .filename' <<<"$updated_files")
+
+          export updated_files
+          echo "Updated files: $updated_files"
+          ./.github/workflows/check_precompiled.sh
+
+      - name: Upload new precompiled binaries
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: precompiled-binaries
+          path: |
+            enc_bootloader/enc_bootloader.elf
+            enc_bootloader/enc_bootloader_mbedtls.elf
+            picoboot_flash_id/flash_id.bin
+            xip_ram_perms/xip_ram_perms.elf

CMakeLists.txt

@@ -62,8 +62,33 @@ if (NOT DEFINED USE_PRECOMPILED)
     set(USE_PRECOMPILED true)
 endif()
 
+
+function(add_embedded_data_project TARGET)
+    cmake_parse_arguments(PARSE_ARGV 1 OPTS "" "PREFIX;SOURCE_DIR;BINARY_DIR" "CMAKE_ARGS")
+
+    if (USE_PRECOMPILED)
+        ExternalProject_Add(${TARGET}
+            PREFIX ${OPTS_PREFIX}
+            SOURCE_DIR ${OPTS_SOURCE_DIR}
+            BINARY_DIR ${OPTS_BINARY_DIR}
+            CMAKE_ARGS ${OPTS_CMAKE_ARGS}
+            INSTALL_COMMAND ""
+            BUILD_ALWAYS 1
+        )
+    else()
+        ExternalProject_Add(${TARGET}
+            PREFIX ${OPTS_PREFIX}
+            SOURCE_DIR ${OPTS_SOURCE_DIR}
+            BINARY_DIR ${OPTS_BINARY_DIR}
+            CMAKE_ARGS ${OPTS_CMAKE_ARGS}
+            BUILD_ALWAYS 1
+        )
+    endif()
+endfunction()
+
+
 # compile enc_bootloader.elf
-ExternalProject_Add(enc_bootloader
+add_embedded_data_project(enc_bootloader
         PREFIX enc_bootloader
         SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/enc_bootloader
         BINARY_DIR ${CMAKE_BINARY_DIR}/enc_bootloader
@@ -73,14 +98,12 @@ ExternalProject_Add(enc_bootloader
             "-DUSE_PRECOMPILED:BOOL=${USE_PRECOMPILED}"
             "-DUSE_MBEDTLS=0"
             "-DPICO_DEBUG_INFO_IN_RELEASE=OFF"
-        BUILD_ALWAYS 1 # todo remove this
-        INSTALL_COMMAND ""
         )
 
 set(ENC_BOOTLOADER_ELF ${CMAKE_BINARY_DIR}/enc_bootloader/enc_bootloader.elf)
 
 if (TARGET mbedtls)
-    ExternalProject_Add(enc_bootloader_mbedtls
+    add_embedded_data_project(enc_bootloader_mbedtls
             PREFIX enc_bootloader_mbedtls
             SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/enc_bootloader
             BINARY_DIR ${CMAKE_BINARY_DIR}/enc_bootloader_mbedtls
@@ -90,16 +113,14 @@ if (TARGET mbedtls)
                 "-DUSE_PRECOMPILED:BOOL=${USE_PRECOMPILED}"
                 "-DUSE_MBEDTLS=1"
                 "-DPICO_DEBUG_INFO_IN_RELEASE=OFF"
-            BUILD_ALWAYS 1 # todo remove this
-            INSTALL_COMMAND ""
             )
 
     set(ENC_BOOTLOADER_MBEDTLS_ELF ${CMAKE_BINARY_DIR}/enc_bootloader_mbedtls/enc_bootloader.elf)
 endif()
 
 if (NOT PICOTOOL_NO_LIBUSB)
     # compile xip_ram_perms.elf
-    ExternalProject_Add(xip_ram_perms
+    add_embedded_data_project(xip_ram_perms
             PREFIX xip_ram_perms
             SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/xip_ram_perms
             BINARY_DIR ${CMAKE_BINARY_DIR}/xip_ram_perms
@@ -108,14 +129,12 @@ if (NOT PICOTOOL_NO_LIBUSB)
                 "-DPICO_SDK_PATH:FILEPATH=${PICO_SDK_PATH}"
                 "-DUSE_PRECOMPILED:BOOL=${USE_PRECOMPILED}"
                 "-DPICO_DEBUG_INFO_IN_RELEASE=OFF"
-            BUILD_ALWAYS 1 # todo remove this
-            INSTALL_COMMAND ""
             )
 
     set(XIP_RAM_PERMS_ELF ${CMAKE_BINARY_DIR}/xip_ram_perms/xip_ram_perms.elf)
 
     # compile flash_id
-    ExternalProject_Add(flash_id
+    add_embedded_data_project(flash_id
             PREFIX picoboot_flash_id
             SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/picoboot_flash_id
             BINARY_DIR ${CMAKE_BINARY_DIR}/picoboot_flash_id
@@ -124,8 +143,6 @@ if (NOT PICOTOOL_NO_LIBUSB)
                 "-DPICO_SDK_PATH:FILEPATH=${PICO_SDK_PATH}"
                 "-DUSE_PRECOMPILED:BOOL=${USE_PRECOMPILED}"
                 "-DPICO_DEBUG_INFO_IN_RELEASE=OFF"
-            BUILD_ALWAYS 1 # todo remove this
-            INSTALL_COMMAND ""
             )
 
     set(FLASH_ID_BIN ${CMAKE_BINARY_DIR}/picoboot_flash_id/flash_id.bin)

enc_bootloader/CMakeLists.txt

@@ -112,6 +112,12 @@ if (NOT USE_PRECOMPILED)
 
     pico_set_binary_type(enc_bootloader no_flash)
     pico_add_dis_output(enc_bootloader)
+
+    if (USE_MBEDTLS)
+        install(FILES ${CMAKE_CURRENT_BINARY_DIR}/enc_bootloader.elf DESTINATION ${CMAKE_CURRENT_LIST_DIR} RENAME enc_bootloader_mbedtls.elf)
+    else()
+        install(FILES ${CMAKE_CURRENT_BINARY_DIR}/enc_bootloader.elf DESTINATION ${CMAKE_CURRENT_LIST_DIR})
+    endif()
 else()
     project(enc_bootloader C CXX ASM)
     message("Using precompiled enc_bootloader.elf")

enc_bootloader/enc_bootloader.elf

@@ -25,6 +25,8 @@ if (NOT USE_PRECOMPILED)
     target_link_options(flash_id PRIVATE -nostartfiles -nodefaultlibs -Ttext=0)
     pico_add_bin_output(flash_id)
     pico_add_dis_output(flash_id)
+
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/flash_id.bin DESTINATION ${CMAKE_CURRENT_LIST_DIR})
 else()
     project(flash_id C CXX ASM)
     message("Using precompiled flash_id.bin")

enc_bootloader/enc_bootloader_mbedtls.elf

@@ -42,6 +42,8 @@ if (NOT USE_PRECOMPILED)
     string(REPLACE "RAM(rwx) : ORIGIN =  0x20000000, LENGTH = 512k" "RAM(rwx) : ORIGIN =  0x13ffc000, LENGTH = 16k" LINKER_SCRIPT "${LINKER_SCRIPT}")
     file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/memmap_xip_ram.ld "${LINKER_SCRIPT}")
     pico_set_linker_script(xip_ram_perms ${CMAKE_CURRENT_BINARY_DIR}/memmap_xip_ram.ld)
+
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/xip_ram_perms.elf DESTINATION ${CMAKE_CURRENT_LIST_DIR})
 else()
     project(xip_ram_perms C CXX ASM)
     message("Using precompiled xip_ram_perms.elf")