Poc split images for hup app engine

meta-balena-common/classes/image-balena.bbclass

@@ -454,24 +454,20 @@ def get_dir_size_kb(path):
 python do_image_size_check() {
     imgfile = d.getVar("BALENA_DOCKER_IMG")
     ext4file = d.getVar("BALENA_ROOTB_FS")
-    rfs_alignment = d.getVar("IMAGE_ROOTFS_ALIGNMENT")
-    rfs_size = int(get_rootfs_size(d))
     image_size_aligned = int(disk_aligned(d, os.stat(imgfile).st_size / 1024))
     available = int(disk_aligned(d, available_space(ext4file, d)))
 
-    # Calculate /boot directory size - copied to volume during HUP
-    boot_dir = os.path.join(d.getVar("IMAGE_ROOTFS"), "boot")
-    boot_size_kb = get_dir_size_kb(boot_dir)
-    boot_size_aligned = int(disk_aligned(d, boot_size_kb))
+    boot_imgfile = d.getVar("BALENA_BOOT_DOCKER_IMG")
+    boot_size_aligned = int(disk_aligned(d, os.stat(boot_imgfile).st_size / 1024))
 
-    # Total space required = docker image + boot volume
+    # Total space required = main hostapp image + boot image staged on root slot
     total_required = image_size_aligned + boot_size_aligned
 
     if total_required > available:
-        bb.fatal("HUP size check failed: docker image (%d KiB) + /boot volume (%d KiB) = %d KiB exceeds available space %d KiB"
+        bb.fatal("HUP size check failed: hostapp image (%d KiB) + boot image (%d KiB) = %d KiB exceeds available space %d KiB"
                  % (image_size_aligned, boot_size_aligned, total_required, available))
 
-    bb.debug(1, 'HUP size check: docker image %d KiB, /boot volume %d KiB, total %d KiB, available %d KiB'
+    bb.debug(1, 'HUP size check: hostapp image %d KiB, boot image %d KiB, total %d KiB, available %d KiB'
              % (image_size_aligned, boot_size_aligned, total_required, available))
 }
 

meta-balena-common/classes/image_types_balena.bbclass

@@ -72,12 +72,14 @@ python() {
         d.setVar('BALENA_RAW_IMG', '${IMGDEPLOYDIR}/${IMAGE_NAME}.balenaos-img')
         d.setVar('BALENA_RAW_BMAP', '${IMGDEPLOYDIR}/${IMAGE_NAME}.bmap')
         d.setVar('BALENA_DOCKER_IMG', '${IMGDEPLOYDIR}/${IMAGE_NAME}.docker')
+        d.setVar('BALENA_BOOT_DOCKER_IMG', '${IMGDEPLOYDIR}/${IMAGE_NAME}-boot.docker')
         d.setVar('BALENA_HOSTAPP_IMG', '${IMGDEPLOYDIR}/${IMAGE_NAME}.${BALENA_ROOT_FSTYPE}')
     else:
         d.setVar('BALENA_ROOT_FS', '${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.${BALENA_ROOT_FSTYPE}')
         d.setVar('BALENA_RAW_IMG', '${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.balenaos-img')
         d.setVar('BALENA_RAW_BMAP', '${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.bmap')
         d.setVar('BALENA_DOCKER_IMG', '${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.docker')
+        d.setVar('BALENA_BOOT_DOCKER_IMG', '${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}-boot.docker')
         d.setVar('BALENA_HOSTAPP_IMG', '${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.${BALENA_ROOT_FSTYPE}')
 
     d.setVar('BALENA_IMAGE_BOOTLOADER_DEPLOY_TASK', ' '.join(bootloader + ':do_populate_sysroot' for bootloader in d.getVar("BALENA_IMAGE_BOOTLOADER", True).split()))
@@ -386,19 +388,31 @@ do_rootfs[vardeps] += "BALENA_BOOT_PARTITION_FILES"
 
 # XXX(petrosagg): This should be eventually implemented using a docker-native daemon
 IMAGE_CMD:docker () {
-    DOCKER_IMAGE=$(${IMAGE_CMD_TAR} -cv -C ${IMAGE_ROOTFS} . | DOCKER_API_VERSION=${BALENA_API_VERSION} docker import -)
+    DOCKER_IMAGE=$(${IMAGE_CMD_TAR} --exclude=./boot -cv -C ${IMAGE_ROOTFS} . | DOCKER_API_VERSION=${BALENA_API_VERSION} docker import -)
     DOCKER_API_VERSION=${BALENA_API_VERSION} docker save ${DOCKER_IMAGE} > ${BALENA_DOCKER_IMG}
 }
 
-IMAGE_TYPEDEP:hostapp-ext4 = "docker"
+IMAGE_CMD:boot-docker () {
+    if [ ! -d "${IMAGE_ROOTFS}/boot" ] || [ -z "$(ls -A "${IMAGE_ROOTFS}/boot" 2>/dev/null)" ]; then
+        bbfatal "Boot image /boot is empty"
+    fi
+    BOOT_STAGE="${WORKDIR}/boot-image-root"
+    rm -rf "${BOOT_STAGE}"
+    mkdir -p "${BOOT_STAGE}/boot"
+    cp -a "${IMAGE_ROOTFS}/boot/." "${BOOT_STAGE}/boot/"
+    DOCKER_IMAGE=$(${IMAGE_CMD_TAR} -cv -C "${BOOT_STAGE}" . | DOCKER_API_VERSION=${BALENA_API_VERSION} docker import -)
+    DOCKER_API_VERSION=${BALENA_API_VERSION} docker save ${DOCKER_IMAGE} > ${BALENA_BOOT_DOCKER_IMG}
+}
+
+IMAGE_TYPEDEP:hostapp-ext4 = "docker boot-docker"
 
 do_image_hostapp_ext4[depends] = " \
     mkfs-hostapp-native:do_populate_sysroot \
     "
 
 IMAGE_CMD:hostapp-ext4 () {
     truncate -s "$(expr ${ROOTFS_SIZE} \* 1024)" "${BALENA_HOSTAPP_IMG}"
-    mkfs.hostapp -t "${TMPDIR}" -s "${STAGING_DIR_NATIVE}" -i ${BALENA_DOCKER_IMG} -o ${BALENA_HOSTAPP_IMG}
+    mkfs.hostapp -t "${TMPDIR}" -s "${STAGING_DIR_NATIVE}" -i ${BALENA_DOCKER_IMG} -b ${BALENA_BOOT_DOCKER_IMG} -o ${BALENA_HOSTAPP_IMG}
 }
 
 IMAGE_TYPEDEP:balenaos-img.sig = "balenaos-img"

meta-balena-common/recipes-containers/hostapp-update/files/hostapp-update

@@ -38,13 +38,15 @@ run_current_hooks_and_recover () {
 }
 
 local_image=""
+boot_image=""
 remote_image=""
 reboot=0
 hooks=1
 hooks_rollback=1
 
-while getopts 'f:i:rnx' flag; do
+while getopts 'b:f:i:rnx' flag; do
 	case "${flag}" in
+	b) boot_image="${OPTARG}" ;;
 	f) local_image=$(realpath "${OPTARG}") ;;
 	i) remote_image="${OPTARG}" ;;
 	r) reboot=1 ;;
@@ -54,14 +56,21 @@ while getopts 'f:i:rnx' flag; do
 	esac
 done
 
-if [ "$local_image" =  "" ] && [ "$remote_image" = "" ]; then
+if [ "$local_image" = "" ] && [ "$remote_image" = "" ]; then
 	ERROR "At least one of -f or -i is required"
 	exit 1
 fi
 
+if [ "$boot_image" != "" ] && [ "$local_image" != "" ] && [ "$remote_image" != "" ]; then
+	ERROR "Cannot use -b with both -f and -i"
+	exit 1
+fi
+
 INFO "Running hostapp update..."
 
-export DOCKER_HOST="unix:///var/run/balena-host.sock"
+HOST_DOCKER_HOST="unix:///var/run/balena-host.sock"
+APP_DOCKER_HOST="unix:///var/run/balena.sock"
+export DOCKER_HOST="$HOST_DOCKER_HOST"
 SYSROOT="/mnt/sysroot/inactive"
 LOADTMP="/mnt/data/resin-data/tmp"
 
@@ -105,6 +114,54 @@ for hostapp in "$SYSROOT/hostapps/"*; do
 done
 sync -f "$SYSROOT"
 
+rm -rf "$SYSROOT/hostapps/.new"
+mkdir -p "$SYSROOT/hostapps/.new"
+
+# Dual-image path (-b): stage boot on sysroot before main image load/pull.
+# Boot image load/pull uses the app engine; main hostapp uses the host engine.
+# -b is a local tarball if the path exists, otherwise a remote image reference.
+if [ "$boot_image" != "" ]; then
+	export DOCKER_HOST="$APP_DOCKER_HOST"
+	if [ -f "$boot_image" ]; then
+		INFO "Loading boot image from local: $boot_image"
+		BOOT_IMAGE=$(balena load --quiet -i "$(realpath "$boot_image")" | cut -d: -f1 --complement | tr -d ' ')
+	else
+		INFO "Pulling boot image from remote: $boot_image"
+		balena rmi --force "$boot_image" >/dev/null 2>&1 || true
+		_out="$(balena pull "$boot_image" 2>&1)"
+		INFO "${_out}"
+		BOOT_IMAGE="$boot_image"
+	fi
+	if [ -z "$BOOT_IMAGE" ]; then
+		ERROR "Failed to resolve boot image reference after load/pull"
+		exit 1
+	fi
+	INFO "Creating boot container from image: $BOOT_IMAGE"
+	BOOT_CONTAINER_ID=$(balena create --runtime="bare" "$BOOT_IMAGE" /bin/sh)
+	INFO "Extracting boot files from container: $BOOT_CONTAINER_ID"
+	mkdir -p "$SYSROOT/hostapps/.new/boot"
+	if ! _out=$(balena cp "$BOOT_CONTAINER_ID:/boot/." "$SYSROOT/hostapps/.new/boot/" 2>&1); then
+		ERROR "Failed to copy boot files from boot image: ${_out}"
+		exit 1
+	fi
+	INFO "${_out}"
+	if [ -z "$(ls -A "$SYSROOT/hostapps/.new/boot" 2>/dev/null)" ]; then
+		ERROR "Boot image /boot is empty"
+		exit 1
+	fi
+	if ! _out=$(balena rm --force "$BOOT_CONTAINER_ID" 2>&1); then
+		ERROR "Failed to remove boot container: ${_out}"
+		exit 1
+	fi
+	INFO "${_out}"
+	if ! _out=$(balena rmi --force "$BOOT_IMAGE" 2>&1); then
+		ERROR "Failed to remove boot image: ${_out}"
+		exit 1
+	fi
+	INFO "${_out}"
+	export DOCKER_HOST="$HOST_DOCKER_HOST"
+fi
+
 # Load new hostapp
 if [ "$local_image" != "" ]; then
 	# bind mount the data partition for temporary extract/load files
@@ -124,13 +181,16 @@ elif [ "$remote_image" != "" ]; then
 	_out="$(balena pull "$HOSTAPP_IMAGE" 2>&1)"
 	INFO "${_out}"
 fi
-CONTAINER_ID=$(balena create --runtime="bare" --volume=/boot "$HOSTAPP_IMAGE" /bin/sh)
-BOOTSTRAP=$(balena inspect -f "{{range .Mounts}}{{.Destination}} {{.Source}}{{end}}" "$CONTAINER_ID" | awk '$1 == "/boot" { print $2 }' | head -n1)
 
-# Create boot entry
-rm -rf "$SYSROOT/hostapps/.new"
-mkdir -p "$SYSROOT/hostapps/.new"
-ln -sr "$BOOTSTRAP" "$SYSROOT/hostapps/.new/boot"
+if [ "$boot_image" != "" ]; then
+	CONTAINER_ID=$(balena create --runtime="bare" "$HOSTAPP_IMAGE" /bin/sh)
+else
+	CONTAINER_ID=$(balena create --runtime="bare" --volume=/boot "$HOSTAPP_IMAGE" /bin/sh)
+	BOOTSTRAP=$(balena inspect -f "{{range .Mounts}}{{.Destination}} {{.Source}}{{end}}" "$CONTAINER_ID" | awk '$1 == "/boot" { print $2 }' | head -n1)
+
+	# Wire the anonymous /boot volume into the root-slot layout.
+	ln -sr "$BOOTSTRAP" "$SYSROOT/hostapps/.new/boot"
+fi
 sync -f "$SYSROOT"
 mv -T "$SYSROOT/hostapps/.new" "$SYSROOT/hostapps/$CONTAINER_ID"
 sync -f "$SYSROOT"

meta-balena-common/recipes-containers/mkfs-hostapp-native/files/create.ext4

@@ -3,14 +3,63 @@
 set -ex
 
 SYSROOT="/mnt/sysroot/inactive"
+HOST_DOCKER_HOST="unix:///var/run/balena-host.sock"
+APP_DOCKER_HOST="unix:///var/run/balena.sock"
+HOST_DATA_ROOT="$SYSROOT/balena"
+APP_DATA_ROOT="/var/lib/docker"
 
-balenad -s=@BALENA_STORAGE@ --data-root="$SYSROOT/balena" -H unix:///var/run/balena-host.sock --iptables=false &
-pid=$!
-sleep 5
+mkdir -p /var/run "$APP_DATA_ROOT" "$HOST_DATA_ROOT"
 
-hostapp-update -f /input -n
+wait_for_engine() {
+	_socket="$1"
+	_i=0
+	while [ "$_i" -lt 30 ]; do
+		if DOCKER_HOST="$_socket" balena info >/dev/null 2>&1; then
+			return 0
+		fi
+		_i=$((_i + 1))
+		sleep 1
+	done
+	return 1
+}
 
-kill $pid
-wait $pid
+# App engine (boot image). 
+# a second overlay2 daemon would fail inside the mkfs build container (docker-in-docker)
+# vfs is fixing this issue by using a different storage driver.
+balenad -s=@BALENA_STORAGE@ \
+	--storage-driver=vfs \
+	--data-root="$APP_DATA_ROOT" \
+	--exec-root=/var/run/balena-app \
+	--pidfile=/var/run/balena-app.pid \
+	-H "$APP_DOCKER_HOST" \
+	--iptables=false &
+app_pid=$!
+
+# Host engine (main hostapp)
+balenad -s=@BALENA_STORAGE@ \
+	--data-root="$HOST_DATA_ROOT" \
+	--exec-root=/var/run/balena-host \
+	--pidfile=/var/run/balena-host.pid \
+	-H "$HOST_DOCKER_HOST" \
+	--iptables=false &
+host_pid=$!
+
+cleanup_engines() {
+	kill "$host_pid" "$app_pid" 2>/dev/null || true
+	wait "$host_pid" "$app_pid" 2>/dev/null || true
+}
+
+if ! wait_for_engine "$APP_DOCKER_HOST"; then
+	echo "App engine failed to start at $APP_DOCKER_HOST" >&2
+	exit 1
+fi
+if ! wait_for_engine "$HOST_DOCKER_HOST"; then
+	echo "Host engine failed to start at $HOST_DOCKER_HOST" >&2
+	exit 1
+fi
+
+hostapp-update -f /input -b /input-boot -n
+
+cleanup_engines
 
 mkfs.ext4 -F -E lazy_itable_init=0,lazy_journal_init=0 -i 8192 -d "$SYSROOT" /output

meta-balena-common/recipes-containers/mkfs-hostapp-native/files/mkfs.hostapp

@@ -12,8 +12,9 @@ sysroot="/"
 tmpdir=""
 fstype="ext4"
 
-while getopts 'i:o:s:t:f:' flag; do
+while getopts 'b:i:o:s:t:f:' flag; do
 	case "${flag}" in
+        b) boot_input=$(realpath "${OPTARG}") ;;
         i) input=$(realpath "${OPTARG}") ;;
         o) output=$(realpath "${OPTARG}") ;;
         s) sysroot=$(realpath "${OPTARG}") ;;
@@ -33,6 +34,11 @@ if ! [ -f "$input" ]; then
     exit 1
 fi
 
+if ! [ -f "$boot_input" ]; then
+    echo "File does not exist: $boot_input"
+    exit 1
+fi
+
 if ! [ -f "$output" ]; then
     echo "File does not exist: $output"
     exit 1
@@ -45,4 +51,10 @@ cleanup_docker() {
 $DOCKER load -i "$sysroot/usr/share/mkfs-hostapp-image.tar"
 trap cleanup_docker EXIT
 
-$DOCKER run --privileged --rm -v "$input:/input:ro" -v "$tmpdir:$tmpdir:ro" -v "$output:/output" -e "PATH=$PATH:$HOST_PATH" @IMAGE@ create.${fstype}
+$DOCKER run --privileged --rm \
+    -v "$input:/input:ro" \
+    -v "$boot_input:/input-boot:ro" \
+    -v "$tmpdir:$tmpdir:ro" \
+    -v "$output:/output" \
+    -e "PATH=$PATH:$HOST_PATH" \
+    @IMAGE@ create.${fstype}