buildextend-metal: Add options for rhcos secex build pipeline

Allow customizing qemu arguments and add the option to execute
genprotimg outside of cosa.
Add the option to run a post script after disk creation.

Background: To build RHCOS Secure Execution Images, we need to use a special
hostkey, which we do not have direct access to. Therefore we need to execute
extra steps in this case.

Signed-off-by: Jan Schintag <[email protected]>

Author: jschintag

Makefile

@@ -104,6 +104,8 @@ install:
 	cp -df -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/ci $$(find ci/ -maxdepth 1 -type f)
 	install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib
 	install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib $$(find src/cosalib/ -maxdepth 1 -type f)
+	install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/secex-genprotimgvm-scripts
+	install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/secex-genprotimgvm-scripts $$(find src/secex-genprotimgvm-scripts/ -maxdepth 1 -type f)
 	install -d $(DESTDIR)$(PREFIX)/bin
 	install bin/coreos-assembler $(DESTDIR)$(PREFIX)/bin/
 	ln -sf ../lib/coreos-assembler/cp-reflink $(DESTDIR)$(PREFIX)/bin/

src/cmd-buildextend-metal

@@ -35,10 +35,11 @@ EOF
 
 # Parse options
 hostkey=src/config/secex-hostkey
+genprotimgvm=
 rc=0
 build=
 force=
-options=$(getopt --options h --longoptions help,force,build:,hostkey: -- "$@") || rc=$?
+options=$(getopt --options h --longoptions help,force,build:,hostkey:,genprotimgvm: -- "$@") || rc=$?
 [ $rc -eq 0 ] || {
     print_help
     exit 1
@@ -58,8 +59,13 @@ while true; do
             shift
             ;;
         --hostkey)
-            hostkey="$2"
-            shift;;
+            hostkey=$(realpath "$2")
+            shift
+            ;;
+        --genprotimgvm)
+            genprotimgvm="$2"
+            shift
+            ;;
         --)
             shift
             break
@@ -88,15 +94,6 @@ esac
 if [[ "$basearch" != "s390x" && $image_type == dasd ]]; then
     fatal "$basearch is not supported for building dasd images"
 fi
-disk_args=()
-# SecureExecution extra stuff
-secex_hostkey_drive=()
-if [[ $secure_execution -eq "1" ]]; then
-    hostkey=$(realpath "$hostkey")
-    disk_args+=("--with-secure-execution")
-    secex_hostkey_drive=("-drive" "if=none,id=hostkey,format=raw,file=$hostkey,readonly=on" \
-                         "-device" "virtio-blk,serial=hostkey,drive=hostkey")
-fi
 
 # shellcheck disable=SC2031
 export LIBGUESTFS_BACKEND=direct
@@ -185,6 +182,23 @@ if [ "${rootfs_type}" = "ext4verity" ]; then
     BLKSIZE="$(getconf PAGE_SIZE)"
 fi
 
+disk_args=()
+qemu_args=()
+# SecureExecution extra stuff
+if [[ $secure_execution -eq "1" ]]; then
+    disk_args+=("--with-secure-execution")
+    if [ -z "${genprotimgvm}" ]; then
+        qemu_args+=("-drive" "if=none,id=hostkey,format=raw,file=$hostkey,readonly=on" \
+                            "-device" "virtio-blk,serial=hostkey,drive=hostkey")
+    else
+        genprotimg_img="${PWD}/secex-genprotimg.img"
+        qemu-img create -f raw "${genprotimg_img}" 512M
+        mkfs.ext4 "${genprotimg_img}"
+        qemu_args+=("-drive" "if=none,id=genprotimg,format=raw,file=${genprotimg_img}" \
+                            "-device" "virtio-blk,serial=genprotimg,drive=genprotimg")
+    fi
+fi
+
 echo "Estimating disk size..."
 # The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem,
 # and doing so has apparently negative performance implications.
@@ -223,7 +237,7 @@ extra_target_device_opts=""
 if [[ $image_type == dasd || $image_type == metal4k ]]; then
   extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096"
 fi
-target_drive=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \
+qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \
               "-device" "virtio-blk,serial=target,drive=target${extra_target_device_opts}")
 
 # Generate the JSON describing the disk we want to build
@@ -246,13 +260,19 @@ if [ -e "${configdir}/platforms.yaml" ]; then
     platforms_json="${workdir}/tmp/platforms.json"
     yaml2json "${configdir}/platforms.yaml" "${platforms_json}"
 fi
-runvm "${target_drive[@]}" "${secex_hostkey_drive[@]}" -- \
+runvm "${qemu_args[@]}" -- \
         /usr/lib/coreos-assembler/create_disk.sh \
             --config "$(pwd)"/image-for-disk.json \
             --kargs "\"${kargs}\"" \
             --platform "${ignition_platform_id}" \
             ${platforms_json:+--platforms-json "${platforms_json}"} \
             "${disk_args[@]}"
+
+if [[ $secure_execution -eq "1" && -n "${genprotimgvm}" ]]; then
+    /usr/lib/coreos-assembler/secex-genprotimgvm-scripts/runvm.sh \
+        --genprotimgvm "${genprotimgvm}" -- "${qemu_args[@]}"
+fi
+
 /usr/lib/coreos-assembler/finalize-artifact "${path}.tmp" "${path}"
 
 sha256=$(sha256sum_str < "${img}")

src/create_disk.sh

@@ -544,16 +544,61 @@ create_dmverity() {
     mount -o ro "/dev/mapper/${partlabel}" "${mountpoint}"
 }
 
+# Save genprotimg input for later and don't run zipl here
+rdcore_replacement() {
+    local se_kargs_append se_initrd se_kernel se_parmfile
+    local blsfile kernel initrd
+    local se_script_dir se_tmp_disk se_tmp_mount se_tmp_boot
+
+    se_kargs_append=("ignition.firstboot")
+    while [ $# -gt 0 ]; do
+        se_kargs_append+=("$1")
+        shift
+    done
+
+    se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts"
+    se_tmp_disk=$(realpath /dev/disk/by-id/virtio-genprotimg)
+    se_tmp_mount=$(mktemp -d /tmp/genprotimg-XXXXXX)
+    se_tmp_boot="${se_tmp_mount}/genprotimg"
+    mount "${se_tmp_disk}" "${se_tmp_mount}"
+    mkdir "${se_tmp_boot}"
+
+    se_initrd="${se_tmp_boot}/initrd.img"
+    se_kernel="${se_tmp_boot}/vmlinuz"
+    se_parmfile="${se_tmp_boot}/parmfile"
+
+    blsfile=$(find "${rootfs}"/boot/loader/entries/*.conf)
+    echo "$(grep options "${blsfile}" | cut -d' ' -f2-)" "${se_kargs_append[@]}" > "${se_parmfile}"
+    kernel="${rootfs}/boot/$(grep linux "${blsfile}" | cut -d' ' -f2)"
+    initrd="${rootfs}/boot/$(grep initrd "${blsfile}" | cut -d' ' -f2)"
+    cp "${kernel}" "${se_kernel}"
+    cp "${initrd}" "${se_initrd}"
+
+    # genprotimg and zipl will be done outside this script
+    # copy scripts for that step to the tmp disk
+    cp "${se_script_dir}/genprotimg-script.sh" "${se_script_dir}/post-script.sh" "${se_tmp_mount}/"
+
+    umount "${se_tmp_mount}"
+    rmdir "${se_tmp_mount}"
+}
+
 if [[ ${secure_execution} -eq 1 ]]; then
     # set up dm-verity for the rootfs and bootfs
     create_dmverity root $rootfs
     create_dmverity boot $rootfs/boot
 
-    # run zipl with root hashes as kargs
-    rdcore_zipl_args+=("--secex-mode=enforce" "--hostkey=/dev/disk/by-id/virtio-hostkey")
-    rdcore_zipl_args+=("--append-karg=rootfs.roothash=$(cat /tmp/root-roothash)")
-    rdcore_zipl_args+=("--append-karg=bootfs.roothash=$(cat /tmp/boot-roothash)")
-    chroot_run /usr/lib/dracut/modules.d/50rdcore/rdcore zipl "${rdcore_zipl_args[@]}"
+    # We need to run the genprotimg step in a separate step for rhcos release images
+    if [ ! -e /dev/disk/by-id/virtio-genprotimg ]; then
+        echo "Building local Secure Execution Image, running zipl and genprotimg"
+        # run zipl with root hashes as kargs
+        rdcore_zipl_args+=("--secex-mode=enforce" "--hostkey=/dev/disk/by-id/virtio-hostkey")
+        rdcore_zipl_args+=("--append-karg=rootfs.roothash=$(cat /tmp/root-roothash)")
+        rdcore_zipl_args+=("--append-karg=bootfs.roothash=$(cat /tmp/boot-roothash)")
+        chroot_run /usr/lib/dracut/modules.d/50rdcore/rdcore zipl "${rdcore_zipl_args[@]}"
+    else
+        echo "Building release Secure Execution Image, zipl and genprotimg will be run later"
+        rdcore_replacement "rootfs.roothash=$(cat /tmp/root-roothash)" "bootfs.roothash=$(cat /tmp/boot-roothash)"
+    fi
 
     # unmount and close everything
     umount -R $rootfs

src/secex-genprotimgvm-scripts/genprotimg-script.sh

@@ -0,0 +1,25 @@
+#!bin/bash
+
+set -exuo pipefail
+
+echo "Preparing for genprotimg-daemon"
+
+source="/build/genprotimg"
+destination="/genprotimg"
+
+# Files need to be named correctly
+# genprotimg daemon can only see /genprotimg folder
+cp "${source}/vmlinuz" "${source}/initrd.img" "${source}/parmfile" "${destination}/"
+
+# Signal daemon that it can run genprotimg
+touch "${destination}/signal.file"
+
+# Wait for genprotimg execution
+while [ -e "$destination/signal.file" ] && [ ! -e "$destination/error" ]; do
+    sleep 5
+done
+if [ -e "$destination/error" ] || [ ! -e "${destination}/se.img" ]; then
+    ls -lha $destination
+    echo "Failed to run genprotimg"
+    exit 1
+fi

src/secex-genprotimgvm-scripts/post-script.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+echo "Moving sdboot and executing zipl"
+
+workdir="/build"
+sdboot="/genprotimg/se.img"
+genprotimg_dir="${workdir}/genprotimg"
+se_boot=$(mktemp -d /tmp/se-XXXXXX)
+
+disk=$(realpath /dev/disk/by-id/virtio-target)
+disk_se="${disk}1"
+
+mount "${disk_se}" "${se_boot}"
+cp "${sdboot}" "${se_boot}/sdboot"
+zipl -V -i ${se_boot}/sdboot -t ${se_boot}
+
+# Disable debug output, the last message should be success
+set +x
+echo "Success, added sdboot to image and executed zipl"
+
+umount "${se_boot}"
+rm -rf "${se_boot}"

src/secex-genprotimgvm-scripts/runvm.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euo pipefail
+
+workdir="/srv"
+vmrundir="${workdir}/tmp/build.secex"
+memory_default=2048
+runvm_console="${vmrundir}/runvm-console.txt"
+genprotimgvm=
+qemu_args=()
+while true; do
+    case "$1" in
+    --genprotimgvm)
+        genprotimgvm="$2"
+        shift
+        ;;
+    --)
+        shift
+        break
+        ;;
+    -*)
+        fatal "$0: unrecognized option: $1"
+        exit 1
+        ;;
+    *)
+        break
+        ;;
+    esac
+    shift
+done
+if [ -z "${genprotimgvm}" ]; then
+    echo "Missing option --genprotimgvm"
+fi
+while [ $# -gt 0 ]; do
+    qemu_args+=("$1")
+    shift
+done
+
+set -x
+
+[[ -d "${vmrundir}" ]] && rm -rf "${vmrundir}"
+mkdir "${vmrundir}"
+touch "${runvm_console}"
+
+kola_args=(kola qemuexec -m "${COSA_SUPERMIN_MEMORY:-${memory_default}}" --auto-cpus -U --workdir none \
+       --console-to-file "${runvm_console}")
+
+base_qemu_args=(-drive "if=none,id=buildvm,format=qcow2,snapshot=on,file=${genprotimgvm},index=1" -device virtio-blk-ccw,drive=buildvm,bootindex=1 \
+        -no-reboot -nodefaults -device virtio-serial \
+        -device virtserialport,chardev=virtioserial0,name=cosa-cmdout -chardev stdio,id=virtioserial0
+           )
+
+if [ -z "${GENPROTIMGVM_SE_OFF:-}" ]; then
+    base_qemu_args+=(-object s390-pv-guest,id=pv0 -machine confidential-guest-support=pv0)
+else
+    echo "No secure execution enabled for build-VM, happy debugging"
+fi
+
+if ! "${kola_args[@]}" -- "${base_qemu_args[@]}" \
+    "${qemu_args[@]}" <&-; then # the <&- here closes stdin otherwise qemu waits forever
+    cat "${runvm_console}"
+    echo "Failed to run 'kola qemuexec'"
+    exit 1
+fi
+
+cat "${runvm_console}"
+
+if ! grep -q "Success, added sdboot to image and executed zipl" "${runvm_console}"; then
+   echo "Could not find success message, genprotimg failed."
+   exit 1
+fi
+
+exit 0