diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91cc1d42..003c77b7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,14 +9,167 @@ on: jobs: snap: runs-on: ubuntu-latest + outputs: + runtime: ${{ steps.upload-runtime.outputs.artifact-id }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 + - name: Free disk space run: | sudo rm -rf /usr/local/lib/android - - uses: snapcore/action-build@v1 + + - name: Build SDK + uses: snapcore/action-build@v1 id: snapcraft - - uses: actions/upload-artifact@v4 + + - uses: actions/upload-artifact@v6 with: name: 'snap' path: ${{ steps.snapcraft.outputs.snap}} + + - name: Prepare runtime + run: | + set -eu + git clone --depth 1 --branch gnome-46-2404 https://github.com/ubuntu/gnome-sdk gnome-runtime + mkdir gnome-runtime/base-gnome-sdk + mv ${{ steps.snapcraft.outputs.snap}} gnome-runtime/base-gnome-sdk/ + cd gnome-runtime + ./local-build.py --prepare-only + + - name: Build runtime + uses: snapcore/action-build@v1 + id: snapcraft-runtime + with: + path: gnome-runtime + + - name: Delete old snap + run: | + rm -f gnome-runtime/base-gnome-sdk/*.snap + + - uses: actions/upload-artifact@v6 + id: upload-runtime + with: + name: 'snap-runtime' + path: ${{ steps.snapcraft-runtime.outputs.snap}} + + testing: + strategy: + fail-fast: false + matrix: + # define in 'snapname' the name of each snap to test + snapname: + - cheese + - chromium + - darktable + - evince + - epiphany + - firefox + - gimp + - gnome-calculator + - gnome-system-monitor + - gnome-text-editor + - kicad + - mattermost-desktop + - shotwell + - telegram-desktop + - thunderbird + # here, in waittext, define the text to find for each snap + # more variables are: + # - commandname: the command to run instead of 'snap run $snapname' + # - timeout: the timeout to wait for the text to appear. Default is 20 seconds. + # - connections: an space-separated list of plugs that must be connected + # - enablex11: if true, will enable Xwayland + # - separated: if true, the words will be split and must be found each anywhere in the screen + include: + - snapname: "cheese" + waittext: "Take a Photo" + - snapname: "chromium" + waittext: "New Tab" + enablex11: "true" + - snapname: "darktable" + waittext: "darktable lighttable darkroom" + separated: "true" + - snapname: "evince" + waittext: "Document Viewer Open" + - snapname: "epiphany" + waittext: "Set as Default Browser" + enablex11: "true" + - snapname: "firefox" + waittext: "Welcome to Firefox" + - snapname: "gimp" + waittext: "Welcome to GIMP" + - snapname: "gnome-calculator" + waittext: "Basic mod" + separated: "true" + - snapname: "gnome-system-monitor" + waittext: "Network Resources Processes Disk Memory Swap CPU" + separated: "true" + - snapname: "gnome-text-editor" + waittext: "New Document" + - snapname: "kicad" + waittext: "Configure KiCad" + enablex11: "true" + - snapname: "mattermost-desktop" + waittext: "Mattermost" + separated: "true" + commandname: "snap run mattermost-desktop" + enablex11: "true" + connections: "login-session-observe" + - snapname: "shotwell" + waittext: "Shotwell" + - snapname: "telegram-desktop" + waittext: "Welcome to the official Telegram Desktop app" + - snapname: "thunderbird" + waittext: "Thunderbird will automatically search" + runs-on: ubuntu-latest + needs: snap + steps: + - name: Get data + uses: actions/checkout@v6 + - name: Download artifact + id: download-snap + uses: actions/download-artifact@v8 + with: + artifact-ids: ${{ needs.snap.outputs.runtime }} + - name: Install tools + run: | + set -eu + sudo snap refresh + sudo apt update + # dbus-x11 is required because it contains dbus-launch!! + # x11-apps are required to wait for XWayland + sudo apt install -y \ + dbus-daemon \ + dconf-gsettings-backend \ + xwayland \ + gnome-keyring \ + wayland-utils \ + xdg-desktop-portal-gnome \ + gnome-session-common \ + gnome-settings-daemon \ + x11-apps + # to be able to use Xwayland, --devmode is required + sudo snap install --devmode mir-test-tools + sudo snap install --beta yarf + sudo snap connect yarf:process-control + sudo snap install --dangerous ${{ steps.download-snap.outputs.download-path}}/*.snap + +# test application with the new runtime + - name: Test application + run: | + export TEST_SNAPNAME=${{ matrix.snapname }} + export TEST_WAITTEXT="${{ matrix.waittext }}" + export TEST_COMMAND_NAME="${{ matrix.commandname }}" + export TEST_DO_ENABLE_X11=${{ matrix.enablex11 }} + export TEST_DO_SEPARATED=${{ matrix.separated }} + export TEST_TIMEOUT=${{ matrix.timeout }} + export TEST_IFACE_CONNECTIONS=${{ matrix.connections }} + + ./.github/workflows/test_script.sh + + - uses: actions/upload-artifact@v6 + id: upload-yarf-log + if: always() + with: + path: ~/snap/yarf/common/yarf-outdir + name: yarf-logs-${{ matrix.snapname }} diff --git a/.github/workflows/test_script.sh b/.github/workflows/test_script.sh new file mode 100755 index 00000000..7265a764 --- /dev/null +++ b/.github/workflows/test_script.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +set -eux -o pipefail + +export TEST_COMMAND_NAME="${TEST_COMMAND_NAME:-snap run ${TEST_SNAPNAME}}" +# default value for TEST_TIMEOUT: 20 seconds +export TEST_TIMEOUT=${TEST_TIMEOUT:-20} + +echo Testing ${TEST_SNAPNAME} with $TEST_COMMAND_NAME and waiting for ${TEST_WAITTEXT} +rm -rf ./testRobot +mkdir ./testRobot + +TEST_ROBOT_FILE=./testRobot/app.robot +cat > $TEST_ROBOT_FILE <> ${TEST_ROBOT_FILE} + export TEST_TIMEOUT="" + done + echo " Sleep 5" >> ${TEST_ROBOT_FILE} + for entry in ${TEST_WAITTEXT}; do + echo " Match Text ${entry}" >> ${TEST_ROBOT_FILE} + done +else + cat >> $TEST_ROBOT_FILE <> $TEST_ROBOT_FILE < /dev/null || (( ${counter} > 30 )); do + echo waiting for mir server... + sleep 1 + counter=$((counter+1)) +done + +if [[ "${TEST_DO_ENABLE_X11}" == "true" ]]; then + export DISPLAY=:0 + counter=0 + until (( ${counter} > 30 )); do + timeout 2 xclock || EXIT_CODE=$? + # 124 is "timeout" + if (( "EXIT_CODE" == "124" )); then + # end loop if timed out + echo XWayland available + counter=50 + else + echo Waiting for XWayland to be available + sleep 1 + fi + counter=$((counter+1)) + done + unset WAYLAND_DISPLAY + export XDG_SESSION_TYPE=x11 +fi + +echo Launching ${TEST_COMMAND_NAME} +${TEST_COMMAND_NAME} & command_pid=$! + +echo Launching YARF +YARF_RETERR=0 +APP_RETERR=0 +MIR_RETERR=0 +yarf --platform Mir ./testRobot || YARF_RETERR=$? + +echo Journal +journalctl --user --no-pager + +echo Waiting for application to die +# wait up to 10 seconds for the snap to finish. +timeout 10 tail --pid=${command_pid} -f /dev/null || /bin/true + +echo killing MIR server +# kill DBus and MIR and wait for it to fully finish. +kill $MIR_PID +wait $MIR_PID + +echo Cleaning up +# remove user data, just in case +rm -rf ~/snap/${TEST_SNAPNAME} +sudo snap remove ${TEST_SNAPNAME} || echo Failed to remove the snap +rm -rf ./testRobot +# only fail after cleanup +exit ${YARF_RETERR} diff --git a/README.md b/README.md index 0a1edd16..90f02c01 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,21 @@ Core22/Gnome-42-2204: Core24/Gnome-46-2404: git clone -b gnome-46-2404-sdk https://github.com/ubuntu/gnome-sdk.git gnome-46-2404-sdk + +## CI tests + +The `build.yml` CI test builds the SDK, then builds the RUNTIME from it, and finally +tests it against a curated list of snaps. It installs the needed dependencies, connects +the RUNTIME being tested to each snap, launches the snap itself and waits for some specific +text to appear on the screen. When it happens, it waits five seconds and checks that the +text is still there, to ensure that the application hasn't crashed. + +When adding new tests, the `local-test.py` script can be useful. It uses multipass to +create a VM and run the test inside. To use it, first initialize the VM with + + local-test.py init + +and then you can run the test for a specific snap by calling it with the snap name. +For example, if you want to test `chromium`, just run + + local-test.py chromium diff --git a/local-test.py b/local-test.py new file mode 100755 index 00000000..afafb896 --- /dev/null +++ b/local-test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import yaml + +vm_name = 'local-test' +# Ubuntu version name +vm_version = 'noble' +# Memory in GB +vm_memory = 16 +# number of cores assigned to the VM +vm_cpu_cores = 2 +# disk space for the VM in GB +vm_disk_space = 10 + +def run_command_inside(command): + print(f'Running {command}') + os.system(f'multipass exec {vm_name} -- {command}') + +def create_vm(): + print('Creating VM') + # first, delete any old VM + os.system(f'multipass delete {vm_name}') + os.system(f'multipass purge') + time.sleep(2) + os.system(f'multipass launch {vm_version} -n {vm_name} -m {vm_memory}G -c {vm_cpu_cores} -d {vm_disk_space}G') + run_command_inside('sudo apt update') + run_command_inside('sudo apt dist-upgrade -y') + run_command_inside('mkdir -p /home/ubuntu/basedir') + os.system(f'multipass stop {vm_name}') + os.system(f'multipass mount -t native . {vm_name}:/home/ubuntu/basedir') + os.system(f'multipass start {vm_name}') + + +if (len(sys.argv) != 2): + print('Usage:') + print(' local-test init Initializes the VM and exits') + print(' local-text SNAP-NAME Tests the specified snap') + sys.exit(-1) + +if (sys.argv[1] == 'init'): + create_vm() + print('System created') + sys.exit(0) + +# load YAML test file + +snapName = sys.argv[1] + +with open('./.github/workflows/build.yml', 'r') as testFile: + testData = yaml.load(testFile, yaml.CLoader) + +job = testData['jobs']['testing'] +snapList = job['strategy']['matrix']['snapname'] +snapParams = job['strategy']['matrix']['include'] + +if snapName not in snapList: + print(f'Error: the snap "{snapName}" is not in the test list') + sys.exit(-1) + +params = {} +for param in snapParams: + if param['snapname'] == snapName: + params = param + break + +correspondences = { + 'snapname': 'TEST_SNAPNAME', + 'waittext': 'TEST_WAITTEXT', + 'commandname': 'TEST_COMMAND_NAME', + 'enablex11': 'TEST_DO_ENABLE_X11', + 'separated': 'TEST_DO_SEPARATED', + 'timeout': 'TEST_TIMEOUT', + 'connections': 'TEST_IFACE_CONNECTIONS' +} + +test_script_path = './test.sh' + +with open(test_script_path, "w") as test_script: + test_script.write('#!/bin/bash\n\n') + + for step in job['steps']: + if 'run' not in step: + continue + if step['name'] == 'Test application': + continue + for line in step['run'].split('\n'): + if line.find('steps.download-snap.outputs.download-path') != -1: + continue + test_script.write(line + '\n') + test_script.write('\n') + for param in correspondences: + if param in params: + test_script.write(f'export {correspondences[param]}="{params[param]}"\n') + else: + test_script.write(f'export {correspondences[param]}=""\n') + test_script.write(f'./.github/workflows/test_script.sh\n') + test_script.write(f'cp -a ~/snap/yarf/common/yarf-outdir .') + +os.chmod(test_script_path, 0o755) +run_command_inside(test_script_path)