From ca59eedabd381579823efbd8493dc514bdad9583 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Mon, 28 Mar 2022 11:22:31 +0530 Subject: [PATCH 01/40] Added PLATFORM environment variable for starting up containers using different architecture before running tests. Added 10 second delay when platform is specified to give containers time to start when running under emulation. Experimented with geckodriver armv7 binary, which results in Firefox tests passing using qemu-user-static --- Hub/Dockerfile | 4 +-- NodeBase/Dockerfile | 4 +-- NodeChromium/Dockerfile | 4 +-- NodeFirefox/Dockerfile | 68 +++++++++++++++++++++-------------- StandaloneChromium/Dockerfile | 4 +-- StandaloneFirefox/Dockerfile | 4 +-- tests/test.py | 23 +++++++++--- 7 files changed, 70 insertions(+), 41 deletions(-) diff --git a/Hub/Dockerfile b/Hub/Dockerfile index 4996e9e206..0e3ea1fe21 100644 --- a/Hub/Dockerfile +++ b/Hub/Dockerfile @@ -2,8 +2,8 @@ # NOTE: DO *NOT* EDIT THIS FILE. IT IS GENERATED. # PLEASE UPDATE Dockerfile.txt INSTEAD OF THIS FILE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -FROM seleniarm/base:4.1.2-20220327 -LABEL authors=SeleniumHQ,sj26,jamesmortensen +FROM selenium/base:multi-arch-tests-20220327 +LABEL authors=SeleniumHQ USER 1200 diff --git a/NodeBase/Dockerfile b/NodeBase/Dockerfile index d1f7667da3..9fbf42d779 100644 --- a/NodeBase/Dockerfile +++ b/NodeBase/Dockerfile @@ -2,8 +2,8 @@ # NOTE: DO *NOT* EDIT THIS FILE. IT IS GENERATED. # PLEASE UPDATE Dockerfile.txt INSTEAD OF THIS FILE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -FROM seleniarm/base:4.1.2-20220327 -LABEL authors=SeleniumHQ,sj26,jamesmortensen +FROM selenium/base:multi-arch-tests-20220327 +LABEL authors=SeleniumHQ USER root diff --git a/NodeChromium/Dockerfile b/NodeChromium/Dockerfile index 9cad9bfd21..e1aff5fce1 100644 --- a/NodeChromium/Dockerfile +++ b/NodeChromium/Dockerfile @@ -2,8 +2,8 @@ # NOTE: DO *NOT* EDIT THIS FILE. IT IS GENERATED. # PLEASE UPDATE Dockerfile.txt INSTEAD OF THIS FILE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -FROM seleniarm/node-base:4.1.2-20220327 -LABEL authors=SeleniumHQ,sj26,jamesmortensen +FROM selenium/node-base:multi-arch-tests-20220327 +LABEL authors=SeleniumHQ USER root diff --git a/NodeFirefox/Dockerfile b/NodeFirefox/Dockerfile index 08c628ceaf..7d0044f6e0 100644 --- a/NodeFirefox/Dockerfile +++ b/NodeFirefox/Dockerfile @@ -2,40 +2,54 @@ # NOTE: DO *NOT* EDIT THIS FILE. IT IS GENERATED. # PLEASE UPDATE Dockerfile.txt INSTEAD OF THIS FILE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -FROM seleniarm/node-base:4.1.2-20220327 -LABEL authors=SeleniumHQ,sj26,jamesmortensen +FROM selenium/node-base:multi-arch-tests-20220327 +LABEL authors=SeleniumHQ + +#FROM local-seleniarm/node-base:4.0.0-20211018 +#LABEL authors=SeleniumHQ,sj26,jamesmortensen + +ARG TARGETARCH +ARG TARGETVARIANT USER root #========= # Firefox #========= -ARG FIREFOX_VERSION=latest -RUN FIREFOX_DOWNLOAD_URL=$(if [ $FIREFOX_VERSION = "latest" ] || [ $FIREFOX_VERSION = "nightly-latest" ] || [ $FIREFOX_VERSION = "devedition-latest" ] || [ $FIREFOX_VERSION = "esr-latest" ]; then echo "https://download.mozilla.org/?product=firefox-$FIREFOX_VERSION-ssl&os=linux64&lang=en-US"; else echo "https://download-installer.cdn.mozilla.net/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2"; fi) \ +RUN echo "deb http://deb.debian.org/debian/ sid main" >> /etc/apt/sources.list \ && apt-get update -qqy \ - && apt-get -qqy --no-install-recommends install firefox libavcodec-extra \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/* \ - && wget --no-verbose -O /tmp/firefox.tar.bz2 $FIREFOX_DOWNLOAD_URL \ - && apt-get -y purge firefox \ - && rm -rf /opt/firefox \ - && tar -C /opt -xjf /tmp/firefox.tar.bz2 \ - && rm /tmp/firefox.tar.bz2 \ - && mv /opt/firefox /opt/firefox-$FIREFOX_VERSION \ - && ln -fs /opt/firefox-$FIREFOX_VERSION/firefox /usr/bin/firefox - -#============ -# GeckoDriver -#============ -ARG GECKODRIVER_VERSION=latest -RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.30.0"; else echo $GECKODRIVER_VERSION; fi) \ - && echo "Using GeckoDriver version: "$GK_VERSION \ - && wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \ - && rm -rf /opt/geckodriver \ - && tar -C /opt -zxf /tmp/geckodriver.tar.gz \ - && rm /tmp/geckodriver.tar.gz \ - && mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \ - && chmod 755 /opt/geckodriver-$GK_VERSION \ - && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver + && apt-get -qqy install firefox-esr libavcodec-extra \ + # && apt-get -qqy install firefox libavcodec-extra \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +#============= +# geckodriver +#============= +RUN if [ $TARGETARCH = "arm" ] && [ $TARGETVARIANT = "v7" ]; then \ + export ARCH=armhf ; \ + else \ + export ARCH=$TARGETARCH ; \ + fi ; \ + if [ -z "$ARCH" ]; then \ + echo "*** BUILD ERROR: \$TARGETARCH must be arm64, amd64, or arm with \$TARGETVARIANT set to v7... exiting..." ; \ + exit 1 ; \ + fi ; \ + if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "armhf" ]; then \ + wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/jamesmortensen/geckodriver-arm-binaries/releases/download/v0.30.0/geckodriver-v0.30.0-linux-aarch64.tar.gz ; \ + else \ + wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz ; \ + fi ; \ + tar -C /tmp -zxf /tmp/geckodriver.tar.gz ; \ + rm /tmp/geckodriver.tar.gz ; \ + mkdir -p /opt/geckodriver-bin ; \ + mv /tmp/geckodriver /opt/geckodriver-bin/geckodriver ; \ + echo "Symlinking geckodriver to /usr/local/bin/geckodriver" ; \ + ln -s /opt/geckodriver-bin/geckodriver /usr/local/bin/geckodriver-orig ; \ + chmod 755 /usr/local/bin/geckodriver-orig + +COPY geckodriver-armhf /usr/local/bin/geckodriver + +RUN chmod 755 /usr/local/bin/geckodriver && apt-get update -y && apt-get install file -y USER 1200 diff --git a/StandaloneChromium/Dockerfile b/StandaloneChromium/Dockerfile index 394026e156..12dce7087c 100644 --- a/StandaloneChromium/Dockerfile +++ b/StandaloneChromium/Dockerfile @@ -2,8 +2,8 @@ # NOTE: DO *NOT* EDIT THIS FILE. IT IS GENERATED. # PLEASE UPDATE Dockerfile.txt INSTEAD OF THIS FILE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -FROM seleniarm/node-chromium:4.1.2-20220327 -LABEL authors=SeleniumHQ,sj26,jamesmortensen +FROM selenium/node-chromium:multi-arch-tests-20220327 +LABEL authors=SeleniumHQ USER 1200 diff --git a/StandaloneFirefox/Dockerfile b/StandaloneFirefox/Dockerfile index 1455f78172..11e03164ca 100644 --- a/StandaloneFirefox/Dockerfile +++ b/StandaloneFirefox/Dockerfile @@ -2,8 +2,8 @@ # NOTE: DO *NOT* EDIT THIS FILE. IT IS GENERATED. # PLEASE UPDATE Dockerfile.txt INSTEAD OF THIS FILE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -FROM seleniarm/node-firefox:4.1.2-20220327 -LABEL authors=SeleniumHQ,sj26,jamesmortensen +FROM selenium/node-firefox:multi-arch-tests-20220327 +LABEL authors=SeleniumHQ USER 1200 diff --git a/tests/test.py b/tests/test.py index c3f7e0e777..0c6ac3b652 100644 --- a/tests/test.py +++ b/tests/test.py @@ -3,6 +3,8 @@ import random import sys import unittest +import requests +import time import docker from docker.errors import NotFound @@ -22,6 +24,7 @@ https_proxy = os.environ.get('https_proxy', '') no_proxy = os.environ.get('no_proxy', '') SKIP_BUILD = os.environ.get('SKIP_BUILD', False) +PLATFORM = os.environ.get('PLATFORM', '') IMAGE_NAME_MAP = { # Hub @@ -106,6 +109,12 @@ def prune_networks(): client.networks.prune() +def wait_for_grid(platform): + if platform != '': + logger.info('Starting container on platform %s so we will wait 10 more seconds since emulation is slower...' % platform) + time.sleep(10) + + def launch_container(container, **kwargs): """ Launch a specific container @@ -150,6 +159,7 @@ def launch_container(container, **kwargs): use_random_user_id = USE_RANDOM_USER_ID == 'true' run_in_docker_compose = RUN_IN_DOCKER_COMPOSE == 'true' random_user_id = random.randint(100000, 2147483647) + platform = PLATFORM if use_random_user_id: logger.info("Running tests with a random user ID -> %s" % random_user_id) @@ -171,9 +181,12 @@ def launch_container(container, **kwargs): """ ports = {'4444': 4444} if use_random_user_id: - test_container_id = launch_container(image, ports=ports, user=random_user_id) + test_container_id = launch_container(image, ports=ports, user=random_user_id, platform=platform) else: - test_container_id = launch_container(image, ports=ports) + test_container_id = launch_container(image, ports=ports,platform=platform) + + wait_for_grid(platform) + else: """ Hub / Node Configuration @@ -183,11 +196,13 @@ def launch_container(container, **kwargs): hub_id = launch_hub("grid") ports = {'5555': 5555, '7900': 7900} if use_random_user_id: - test_container_id = launch_container(image, network='grid', ports=ports, user=random_user_id) + test_container_id = launch_container(image, network='grid', ports=ports, user=random_user_id, platform=platform) else: - test_container_id = launch_container(image, network='grid', ports=ports) + test_container_id = launch_container(image, network='grid', ports=ports, platform=platform) prune_networks() + wait_for_grid(platform) + logger.info('========== / Containers ready to go ==========') try: From 06bf70b2799deefaffd59dbe85995002977023af Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 00:41:59 +0530 Subject: [PATCH 02/40] Added a third job for building and running tests for armv7 variant using an armv7 VM --- .circleci/config.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index de3339b878..4e24ec4994 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,27 @@ jobs: BUILD_DATE: today PLATFORMS: << parameters.platforms >> steps: + - run: + name: Start VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + echo "Download armv7l VM..." + curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ + -o debian-11.3-armhf-ci-v1.0.tar.xz + echo "Extract VM..." + tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz + cd debian-11.3-armhf-ci + echo "Install QEMU binaries" + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ + && source ~/.bashrc \ + && qemu-system-arm --version + bash start-vm-and-tunnels.sh + cd .. + echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + source $BASH_ENV + fi - run: name: "Prepare workflow environment variables" command: | @@ -79,6 +100,17 @@ jobs: - restore_cache: keys: - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> + - run: + name: Start VM if armv7l and connect to Docker socket with SSH port forwarding + command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz -o debian-11.3-armhf-ci-v1.0.tar.xz + tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz + cd debian-11.3-armhf-ci + bash start-vm-and-tunnels.sh + ssh -p 8022 debianhf@127.0.0.1 -N -f -L4444:127.0.0.1:4444 -L5555:127.0.0.1:5555 + cd .. + fi - run: uname -a - run: docker info - run: @@ -187,6 +219,10 @@ workflows: name: build-multi-arch-amd64 platforms: linux/amd64 machine-type: ubuntu2004amd64 + - build-multi-arch: + name: build-multi-arch-armv7l + platforms: linux/arm/v7 + machine-type: ubuntu2004arm64 - test-multi-arch: name: test-multi-arch-arm64 requires: [build-multi-arch-arm64] @@ -197,6 +233,11 @@ workflows: requires: [build-multi-arch-amd64] platforms: linux/amd64 machine-type: ubuntu2004amd64 + - test-multi-arch: + name: test-multi-arch-armv7l + requires: [build-multi-arch-armv7l] + platforms: linux/arm/v7 + machine-type: ubuntu2004arm64 deploy-multi-arch: jobs: From 44167d913fbc774608c305ecdb92896ce746fc62 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 01:11:46 +0530 Subject: [PATCH 03/40] Run start-vm in background so it won't terminate when moving onto the next step. --- .circleci/config.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e24ec4994..ce43155156 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,22 +25,37 @@ jobs: PLATFORMS: << parameters.platforms >> steps: - run: - name: Start VM if armv7l + name: Download VM if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then echo "Download armv7l VM..." + cd ~ curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ -o debian-11.3-armhf-ci-v1.0.tar.xz echo "Extract VM..." tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz - cd debian-11.3-armhf-ci + rm debian-11.3-armhf-ci-v1.0.tar.xz + fi + - run: + name: Start VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sh start-vm.sh + fi + background: true + - run: + name: Wait for VM if armv7l + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci echo "Install QEMU binaries" curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ && source ~/.bashrc \ && qemu-system-arm --version - bash start-vm-and-tunnels.sh + sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh + bash wait-for-vm-boot.sh cd .. echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV source $BASH_ENV From 8a2a4a7d758a3b57bff67b7af1aa0757c618eb46 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 01:14:03 +0530 Subject: [PATCH 04/40] Updated config.yml --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce43155156..92466a23a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,7 @@ jobs: background: true - run: name: Wait for VM if armv7l + command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci echo "Install QEMU binaries" From 99e9600a580e8f550cc5089e55ab1d23489c6e38 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 01:24:14 +0530 Subject: [PATCH 05/40] Fixed some config issues and installing qemu before attempting to start the VM --- .circleci/config.yml | 49 +++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce43155156..04bbb484ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: PLATFORMS: << parameters.platforms >> steps: - run: - name: Download VM if armv7l + name: Download VM and install qemu, if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then echo "Download armv7l VM..." @@ -35,6 +35,12 @@ jobs: echo "Extract VM..." tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz rm debian-11.3-armhf-ci-v1.0.tar.xz + echo "Install QEMU binaries" + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ + && source ~/.bashrc \ + && qemu-system-arm --version fi - run: name: Start VM if armv7l @@ -48,12 +54,6 @@ jobs: name: Wait for VM if armv7l if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci - echo "Install QEMU binaries" - curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ - && source ~/.bashrc \ - && qemu-system-arm --version sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh bash wait-for-vm-boot.sh cd .. @@ -116,15 +116,40 @@ jobs: keys: - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> - run: - name: Start VM if armv7l and connect to Docker socket with SSH port forwarding + name: Download VM and install qemu, if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz -o debian-11.3-armhf-ci-v1.0.tar.xz + echo "Download armv7l VM..." + cd ~ + curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ + -o debian-11.3-armhf-ci-v1.0.tar.xz + echo "Extract VM..." tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz - cd debian-11.3-armhf-ci - bash start-vm-and-tunnels.sh - ssh -p 8022 debianhf@127.0.0.1 -N -f -L4444:127.0.0.1:4444 -L5555:127.0.0.1:5555 + rm debian-11.3-armhf-ci-v1.0.tar.xz + echo "Install QEMU binaries" + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ + && source ~/.bashrc \ + && qemu-system-arm --version + fi + - run: + name: Start VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sh start-vm.sh + fi + background: true + - run: + name: Wait for VM if armv7l + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh + bash wait-for-vm-boot.sh cd .. + echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + source $BASH_ENV fi - run: uname -a - run: docker info From 0757be5d0177896ea29dfee276fa7a178310271c Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 01:27:13 +0530 Subject: [PATCH 06/40] Updated config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index adb13f6e8e..f740861afb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: PLATFORMS: << parameters.platforms >> steps: - run: - name: Download VM and install qemu, if armv7l + name: Download VM and install qemu if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then echo "Download armv7l VM..." From dd5e754d9db3ecb23f652240beb812a2cf110953 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 01:28:03 +0530 Subject: [PATCH 07/40] Updated config.yml --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f740861afb..de7f32bc4f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -144,6 +144,7 @@ jobs: background: true - run: name: Wait for VM if armv7l + command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh From 82c3b4a240d77f6a69044b83dcef7722e69216af Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 08:52:05 +0530 Subject: [PATCH 08/40] Putting DOCKER_HOST in with the other environment variables --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index de7f32bc4f..9db8eb4ea1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,12 +58,13 @@ jobs: sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh bash wait-for-vm-boot.sh cd .. - echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV - source $BASH_ENV fi - run: name: "Prepare workflow environment variables" command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + fi echo 'export BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV cat $BASH_ENV source $BASH_ENV From a2bdffd1c5aada3e83fb1bca90de20f3ee839169 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 09:06:04 +0530 Subject: [PATCH 09/40] Ordering of setup --- .circleci/config.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9db8eb4ea1..e4ef1217dd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,18 @@ jobs: BUILD_DATE: today PLATFORMS: << parameters.platforms >> steps: + - run: + name: "Prepare workflow environment variables" + command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + fi + echo 'export BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV + cat $BASH_ENV + source $BASH_ENV + echo "Workflow environment variables:" + echo $BRANCH + - run: uname -a - run: name: Download VM and install qemu if armv7l command: | @@ -54,23 +66,12 @@ jobs: name: Wait for VM if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + pgrep qemu cd ~/debian-11.3-armhf-ci sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh bash wait-for-vm-boot.sh cd .. fi - - run: - name: "Prepare workflow environment variables" - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV - fi - echo 'export BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV - cat $BASH_ENV - source $BASH_ENV - echo "Workflow environment variables:" - echo $BRANCH - - run: uname -a - run: docker info - checkout - run: From cfd42648f1c897a93b870736587aa5572d1d4fc2 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 09:14:23 +0530 Subject: [PATCH 10/40] fixes --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4ef1217dd..8b914e1878 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,12 +28,14 @@ jobs: name: "Prepare workflow environment variables" command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + echo "Set DOCKER_HOST to unix:///tmp/docker-on-debianhf" echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV fi echo 'export BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV cat $BASH_ENV source $BASH_ENV echo "Workflow environment variables:" + echo $DOCKER_HOST echo $BRANCH - run: uname -a - run: @@ -60,12 +62,15 @@ jobs: if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci sh start-vm.sh + sleep 4 + pgrep qemu fi background: true - run: name: Wait for VM if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + echo "Wait for VM to boot..." pgrep qemu cd ~/debian-11.3-armhf-ci sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh From a27e6ef4ba8c2f7e2c77664c10569acb9a0948aa Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 09:30:38 +0530 Subject: [PATCH 11/40] fixes --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b914e1878..542e4b411d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,10 +60,11 @@ jobs: name: Start VM if armv7l command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + echo "Start the armv7l VM with qemu-system-arm..." cd ~/debian-11.3-armhf-ci + sed -i 's/&//g' start-vm.sh sh start-vm.sh - sleep 4 - pgrep qemu + echo "Process blocking since running in the background..." fi background: true - run: From e3223552fee069261103870af8325e88a2c8f76a Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 09:46:53 +0530 Subject: [PATCH 12/40] Env variables are still not propogating to other run blocks, so we'll set the DOCKER_HOST when running the build. --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 542e4b411d..340941279c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,10 @@ jobs: - run: name: "Build Docker images" command: | + export BRANCH=$CIRCLE_BRANCH + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock echo "Branch is $BRANCH" + echo "DOCKER_HOST is $DOCKER_HOST" NAME=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} PLATFORMS=${PLATFORMS} BUILD_ARGS=${BUILD_ARGS} make build_multi - run: name: "Save Docker Images in Cache" From 6d6dc0cc1f84b7209f0a606988736d8ad61a1233 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 10:03:25 +0530 Subject: [PATCH 13/40] More stuff to help in debugging the docker socket --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 340941279c..09e1842fe7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,6 +76,7 @@ jobs: cd ~/debian-11.3-armhf-ci sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh bash wait-for-vm-boot.sh + ls /tmp/docker-on-debianhf.sock cd .. fi - run: docker info @@ -84,9 +85,13 @@ jobs: name: "Build Docker images" command: | export BRANCH=$CIRCLE_BRANCH - export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + fi echo "Branch is $BRANCH" echo "DOCKER_HOST is $DOCKER_HOST" + ls /tmp/docker-on-debianhf.sock + docker info NAME=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} PLATFORMS=${PLATFORMS} BUILD_ARGS=${BUILD_ARGS} make build_multi - run: name: "Save Docker Images in Cache" From 1ece3ae190c1e7602a559b1b0a081f15bbf3d18c Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 13:10:15 +0530 Subject: [PATCH 14/40] trying other qemu settings to speed up the VM --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09e1842fe7..c8e9803e97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,10 @@ jobs: echo "Start the armv7l VM with qemu-system-arm..." cd ~/debian-11.3-armhf-ci sed -i 's/&//g' start-vm.sh + sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm' start-vm.sh + sed -i 's/-m 1024/-m 6144/' start-vm.sh + sed -i 's/cpus=4,sockets=1,cores=4/cpus=2,sockets=1,cores=4/g' start-vm.sh + sed -i 's/-cpu cortex-a15/-cpu host/g' start-vm.sh sh start-vm.sh echo "Process blocking since running in the background..." fi @@ -87,10 +91,10 @@ jobs: export BRANCH=$CIRCLE_BRANCH if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + ls /tmp/docker-on-debianhf.sock fi echo "Branch is $BRANCH" echo "DOCKER_HOST is $DOCKER_HOST" - ls /tmp/docker-on-debianhf.sock docker info NAME=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} PLATFORMS=${PLATFORMS} BUILD_ARGS=${BUILD_ARGS} make build_multi - run: From 4e7476913341ef4c009619331c51ade3c7839889 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 13:46:50 +0530 Subject: [PATCH 15/40] trying other qemu settings to speed up the VM --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8e9803e97..f275ea8df3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ jobs: echo "Start the armv7l VM with qemu-system-arm..." cd ~/debian-11.3-armhf-ci sed -i 's/&//g' start-vm.sh - sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm' start-vm.sh + sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm/' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh sed -i 's/cpus=4,sockets=1,cores=4/cpus=2,sockets=1,cores=4/g' start-vm.sh sed -i 's/-cpu cortex-a15/-cpu host/g' start-vm.sh From 630c2a89de6513733cf90c3b7015eba1fa295a9d Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 13:51:51 +0530 Subject: [PATCH 16/40] trying other qemu settings to speed up the VM --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f275ea8df3..a637421671 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm/' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh - sed -i 's/cpus=4,sockets=1,cores=4/cpus=2,sockets=1,cores=4/g' start-vm.sh + #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2,sockets=1,cores=4/g' start-vm.sh sed -i 's/-cpu cortex-a15/-cpu host/g' start-vm.sh sh start-vm.sh echo "Process blocking since running in the background..." From f03f8aa52cf4731d0d0fa8572ab2248ee8824695 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 14:14:50 +0530 Subject: [PATCH 17/40] trying other qemu settings to speed up the VM --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a637421671..82024062cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,8 +65,8 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm/' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh - #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2,sockets=1,cores=4/g' start-vm.sh - sed -i 's/-cpu cortex-a15/-cpu host/g' start-vm.sh + #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2/g' start-vm.sh + sed -i 's/-cpu cortex-a15/-cpu host,aarch64=off/g' start-vm.sh sh start-vm.sh echo "Process blocking since running in the background..." fi From dbc14686b7011ff180e77fa6881237dd9955e10a Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 19:05:35 +0530 Subject: [PATCH 18/40] Show CPU features as well as try out some different qemu args --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 82024062cb..13ad67eee6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,11 +62,16 @@ jobs: if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then echo "Start the armv7l VM with qemu-system-arm..." cd ~/debian-11.3-armhf-ci + echo "does it support kvm? " + cat /proc/cpuinfo + grep -E 'svm|vmx' /proc/cpuinfo + egrep -c '(vmx|svm)' /proc/cpuinfo sed -i 's/&//g' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm/' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2/g' start-vm.sh sed -i 's/-cpu cortex-a15/-cpu host,aarch64=off/g' start-vm.sh + cat start-vm.sh sh start-vm.sh echo "Process blocking since running in the background..." fi From 90b0177ae803b6eb3d5697df396b15a2afcac851 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 20:20:49 +0530 Subject: [PATCH 19/40] try building without the VM, just use it for testing --- .circleci/config.yml | 121 ++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 58 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13ad67eee6..0ad5d3d5e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,10 +27,10 @@ jobs: - run: name: "Prepare workflow environment variables" command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - echo "Set DOCKER_HOST to unix:///tmp/docker-on-debianhf" - echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV - fi + # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + # echo "Set DOCKER_HOST to unix:///tmp/docker-on-debianhf" + # echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + # fi echo 'export BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV cat $BASH_ENV source $BASH_ENV @@ -38,66 +38,62 @@ jobs: echo $DOCKER_HOST echo $BRANCH - run: uname -a - - run: - name: Download VM and install qemu if armv7l - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - echo "Download armv7l VM..." - cd ~ - curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ - -o debian-11.3-armhf-ci-v1.0.tar.xz - echo "Extract VM..." - tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz - rm debian-11.3-armhf-ci-v1.0.tar.xz - echo "Install QEMU binaries" - curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ - && source ~/.bashrc \ - && qemu-system-arm --version - fi - - run: - name: Start VM if armv7l - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - echo "Start the armv7l VM with qemu-system-arm..." - cd ~/debian-11.3-armhf-ci - echo "does it support kvm? " - cat /proc/cpuinfo - grep -E 'svm|vmx' /proc/cpuinfo - egrep -c '(vmx|svm)' /proc/cpuinfo - sed -i 's/&//g' start-vm.sh - sed -i 's/qemu-system-arm/qemu-system-aarch64 -enable-kvm/' start-vm.sh - sed -i 's/-m 1024/-m 6144/' start-vm.sh - #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2/g' start-vm.sh - sed -i 's/-cpu cortex-a15/-cpu host,aarch64=off/g' start-vm.sh - cat start-vm.sh - sh start-vm.sh - echo "Process blocking since running in the background..." - fi - background: true - - run: - name: Wait for VM if armv7l - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - echo "Wait for VM to boot..." - pgrep qemu - cd ~/debian-11.3-armhf-ci - sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh - bash wait-for-vm-boot.sh - ls /tmp/docker-on-debianhf.sock - cd .. - fi + # - run: + # name: Download VM and install qemu if armv7l + # command: | + # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + # echo "Download armv7l VM..." + # cd ~ + # curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ + # -o debian-11.3-armhf-ci-v1.0.tar.xz + # echo "Extract VM..." + # tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz + # rm debian-11.3-armhf-ci-v1.0.tar.xz + # echo "Install QEMU binaries" + # curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + # && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + # && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ + # && source ~/.bashrc \ + # && qemu-system-arm --version + # fi + # - run: + # name: Start VM if armv7l + # command: | + # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + # echo "Start the armv7l VM with qemu-system-arm..." + # cd ~/debian-11.3-armhf-ci + # sed -i 's/&//g' start-vm.sh + # sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh + # sed -i 's/-m 1024/-m 6144/' start-vm.sh + # #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2/g' start-vm.sh + # #sed -i 's/-cpu cortex-a15/-cpu host,aarch64=off/g' start-vm.sh + # cat start-vm.sh + # sh start-vm.sh + # echo "Process blocking since running in the background..." + # fi + # background: true + # - run: + # name: Wait for VM if armv7l + # command: | + # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + # echo "Wait for VM to boot..." + # pgrep qemu + # cd ~/debian-11.3-armhf-ci + # sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh + # bash wait-for-vm-boot.sh + # ls /tmp/docker-on-debianhf.sock + # cd .. + # fi - run: docker info - checkout - run: name: "Build Docker images" command: | export BRANCH=$CIRCLE_BRANCH - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock - ls /tmp/docker-on-debianhf.sock - fi + # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + # export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + # ls /tmp/docker-on-debianhf.sock + # fi echo "Branch is $BRANCH" echo "DOCKER_HOST is $DOCKER_HOST" docker info @@ -164,6 +160,7 @@ jobs: command: | if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci + sed -i 's/&//g' start-vm.sh sh start-vm.sh fi background: true @@ -183,6 +180,9 @@ jobs: - run: name: "Load built images from cache into Docker" command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID docker load -i multi-arch-images.tar - run: @@ -196,6 +196,11 @@ jobs: name: "Test Docker images" no_output_timeout: 2m command: | + if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + export GRID_STATUS_MAX_ATTEMPTS=15 + export PLATFORM=linux/arm/v7 + fi export USE_RANDOM_USER=false export BRANCH=$CIRCLE_BRANCH USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch From c392708c2bd1a552ce4f73d767a1561b61adbb52 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 21:36:40 +0530 Subject: [PATCH 20/40] Run the tests again with the cached built images. --- .circleci/config.yml | 49 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ad5d3d5e3..a3675b1d5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,7 @@ jobs: - checkout - restore_cache: keys: - - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> + - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >> - run: name: Download VM and install qemu, if armv7l command: | @@ -201,6 +201,7 @@ jobs: export GRID_STATUS_MAX_ATTEMPTS=15 export PLATFORM=linux/arm/v7 fi + docker run --rm seleniarm/standalone-firefox:armv7-tests-today arch export USE_RANDOM_USER=false export BRANCH=$CIRCLE_BRANCH USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch @@ -283,31 +284,31 @@ jobs: workflows: build-and-test-multi-arch: jobs: - - build-multi-arch: - name: build-multi-arch-arm64 - platforms: linux/arm64 - machine-type: ubuntu2004arm64 - - build-multi-arch: - name: build-multi-arch-amd64 - platforms: linux/amd64 - machine-type: ubuntu2004amd64 - - build-multi-arch: - name: build-multi-arch-armv7l - platforms: linux/arm/v7 - machine-type: ubuntu2004arm64 - - test-multi-arch: - name: test-multi-arch-arm64 - requires: [build-multi-arch-arm64] - platforms: linux/arm64 - machine-type: ubuntu2004arm64 - - test-multi-arch: - name: test-multi-arch-amd64 - requires: [build-multi-arch-amd64] - platforms: linux/amd64 - machine-type: ubuntu2004amd64 + # - build-multi-arch: + # name: build-multi-arch-arm64 + # platforms: linux/arm64 + # machine-type: ubuntu2004arm64 + # - build-multi-arch: + # name: build-multi-arch-amd64 + # platforms: linux/amd64 + # machine-type: ubuntu2004amd64 + # - build-multi-arch: + # name: build-multi-arch-armv7l + # platforms: linux/arm/v7 + # machine-type: ubuntu2004arm64 + # - test-multi-arch: + # name: test-multi-arch-arm64 + # requires: [build-multi-arch-arm64] + # platforms: linux/arm64 + # machine-type: ubuntu2004arm64 + # - test-multi-arch: + # name: test-multi-arch-amd64 + # requires: [build-multi-arch-amd64] + # platforms: linux/amd64 + # machine-type: ubuntu2004amd64 - test-multi-arch: name: test-multi-arch-armv7l - requires: [build-multi-arch-armv7l] + # requires: [build-multi-arch-armv7l] platforms: linux/arm/v7 machine-type: ubuntu2004arm64 From bb201a175560fd4e7156406260f7da71090b989a Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 21:40:21 +0530 Subject: [PATCH 21/40] Checking architecture of cached built images to confirm it's armv7l not aarch64 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a3675b1d5d..04fde42cdc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,10 +201,10 @@ jobs: export GRID_STATUS_MAX_ATTEMPTS=15 export PLATFORM=linux/arm/v7 fi - docker run --rm seleniarm/standalone-firefox:armv7-tests-today arch export USE_RANDOM_USER=false export BRANCH=$CIRCLE_BRANCH USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch + docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch deploy-multi-arch: parameters: From 9087a35e1370ef1f92d8a766881cbcc955c7e5b0 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 21:57:22 +0530 Subject: [PATCH 22/40] Checking arch of cached armv7 container build and the armv7 latest release. --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04fde42cdc..056096a378 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -196,15 +196,18 @@ jobs: name: "Test Docker images" no_output_timeout: 2m command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + # export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock export GRID_STATUS_MAX_ATTEMPTS=15 export PLATFORM=linux/arm/v7 fi export USE_RANDOM_USER=false export BRANCH=$CIRCLE_BRANCH USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch + docker run --rm --privileged aptman/qus -- -r + docker run --rm --privileged aptman/qus -s -- -p docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch + docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:latest arch deploy-multi-arch: parameters: From b2a3bbc78b2cd11412764142f28604798cac03a5 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 22:53:10 +0530 Subject: [PATCH 23/40] removed VM setup code --- .circleci/config.yml | 106 ++----------------------------------------- 1 file changed, 4 insertions(+), 102 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 056096a378..cb70969b46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,76 +27,19 @@ jobs: - run: name: "Prepare workflow environment variables" command: | - # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - # echo "Set DOCKER_HOST to unix:///tmp/docker-on-debianhf" - # echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV - # fi echo 'export BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV cat $BASH_ENV source $BASH_ENV echo "Workflow environment variables:" - echo $DOCKER_HOST - echo $BRANCH + echo "BRANCH=$BRANCH" - run: uname -a - # - run: - # name: Download VM and install qemu if armv7l - # command: | - # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - # echo "Download armv7l VM..." - # cd ~ - # curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ - # -o debian-11.3-armhf-ci-v1.0.tar.xz - # echo "Extract VM..." - # tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz - # rm debian-11.3-armhf-ci-v1.0.tar.xz - # echo "Install QEMU binaries" - # curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - # && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - # && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ - # && source ~/.bashrc \ - # && qemu-system-arm --version - # fi - # - run: - # name: Start VM if armv7l - # command: | - # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - # echo "Start the armv7l VM with qemu-system-arm..." - # cd ~/debian-11.3-armhf-ci - # sed -i 's/&//g' start-vm.sh - # sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh - # sed -i 's/-m 1024/-m 6144/' start-vm.sh - # #sed -i 's/cpus=4,sockets=1,cores=4/cpus=2/g' start-vm.sh - # #sed -i 's/-cpu cortex-a15/-cpu host,aarch64=off/g' start-vm.sh - # cat start-vm.sh - # sh start-vm.sh - # echo "Process blocking since running in the background..." - # fi - # background: true - # - run: - # name: Wait for VM if armv7l - # command: | - # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - # echo "Wait for VM to boot..." - # pgrep qemu - # cd ~/debian-11.3-armhf-ci - # sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh - # bash wait-for-vm-boot.sh - # ls /tmp/docker-on-debianhf.sock - # cd .. - # fi - run: docker info - checkout - run: name: "Build Docker images" command: | export BRANCH=$CIRCLE_BRANCH - # if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - # export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock - # ls /tmp/docker-on-debianhf.sock - # fi echo "Branch is $BRANCH" - echo "DOCKER_HOST is $DOCKER_HOST" - docker info NAME=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} PLATFORMS=${PLATFORMS} BUILD_ARGS=${BUILD_ARGS} make build_multi - run: name: "Save Docker Images in Cache" @@ -136,53 +79,13 @@ jobs: - checkout - restore_cache: keys: + # - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >> - - run: - name: Download VM and install qemu, if armv7l - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - echo "Download armv7l VM..." - cd ~ - curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ - -o debian-11.3-armhf-ci-v1.0.tar.xz - echo "Extract VM..." - tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz - rm debian-11.3-armhf-ci-v1.0.tar.xz - echo "Install QEMU binaries" - curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ - && source ~/.bashrc \ - && qemu-system-arm --version - fi - - run: - name: Start VM if armv7l - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - cd ~/debian-11.3-armhf-ci - sed -i 's/&//g' start-vm.sh - sh start-vm.sh - fi - background: true - - run: - name: Wait for VM if armv7l - command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - cd ~/debian-11.3-armhf-ci - sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh - bash wait-for-vm-boot.sh - cd .. - echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV - source $BASH_ENV - fi - run: uname -a - run: docker info - run: name: "Load built images from cache into Docker" command: | - if [ "$CIRCLE_JOB" = "build-multi-arch-armv7l" ]; then - export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock - fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID docker load -i multi-arch-images.tar - run: @@ -197,15 +100,14 @@ jobs: no_output_timeout: 2m command: | if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then - # export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock export GRID_STATUS_MAX_ATTEMPTS=15 export PLATFORM=linux/arm/v7 fi export USE_RANDOM_USER=false export BRANCH=$CIRCLE_BRANCH USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch - docker run --rm --privileged aptman/qus -- -r - docker run --rm --privileged aptman/qus -s -- -p + # docker run --rm --privileged aptman/qus -- -r + # docker run --rm --privileged aptman/qus -s -- -p docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:latest arch From 9e33b3d30e104e28a58e3fadb9e8feff92ed1961 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 23:04:15 +0530 Subject: [PATCH 24/40] Registering other platforms in Makefile for tests --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 23189c0168..31364ec3a8 100644 --- a/Makefile +++ b/Makefile @@ -438,7 +438,8 @@ test_firefox_standalone: # Test multi-arch container images -test_multi_arch: test_chromium_multi \ +test_multi_arch: qemu_user_static \ + test_chromium_multi \ test_firefox_multi \ test_chromium_standalone_multi \ test_firefox_standalone_multi From ba1f808e8ddeaa9cb9b4096a86120265a8c789a4 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 23:21:08 +0530 Subject: [PATCH 25/40] Running tests with the armv7 VM. They don't pass with qemu-user-static and that's what was being used in the Makefile. --- .circleci/config.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb70969b46..ee13f5574b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,11 +81,53 @@ jobs: keys: # - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >> + - run: + name: Download VM and install qemu, if armv7l + command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + echo "Download armv7l VM..." + cd ~ + curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ + -o debian-11.3-armhf-ci-v1.0.tar.xz + echo "Extract VM..." + tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz + rm debian-11.3-armhf-ci-v1.0.tar.xz + echo "Install QEMU binaries" + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ + && source ~/.bashrc \ + && qemu-system-arm --version + fi + - run: + name: Start VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sed -i 's/&//g' start-vm.sh + sed -i 's/-m 1024/-m 6144/' start-vm.sh + sh start-vm.sh + fi + background: true + - run: + name: Wait for VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh + bash wait-for-vm-boot.sh + cd .. + echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + source $BASH_ENV + fi - run: uname -a - run: docker info - run: name: "Load built images from cache into Docker" command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID docker load -i multi-arch-images.tar - run: @@ -100,6 +142,7 @@ jobs: no_output_timeout: 2m command: | if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock export GRID_STATUS_MAX_ATTEMPTS=15 export PLATFORM=linux/arm/v7 fi @@ -110,6 +153,9 @@ jobs: # docker run --rm --privileged aptman/qus -s -- -p docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:latest arch + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + sh stop-vm.sh + fi deploy-multi-arch: parameters: From 3677ae638fb709ce7eb9ba18266078975cb9988a Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 21 May 2022 23:38:50 +0530 Subject: [PATCH 26/40] Opening ports for running the tests against Docker engine in the virtual machine. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ee13f5574b..e693a79252 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -119,6 +119,7 @@ jobs: cd .. echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV source $BASH_ENV + ssh -p 8022 debianhf@127.0.0.1 -N -f -L4444:127.0.0.1:4444 -L5555:127.0.0.1:5555 -L7900:127.0.0.1:7900 fi - run: uname -a - run: docker info From 82c6df464e650267e9c48f6f30ffeff288eb25e1 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 15:14:31 +0530 Subject: [PATCH 27/40] Now that CircleCI released arm-large on the free plan, we'll try testing armv7l in a QEMU VM again... --- .circleci/config.yml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e693a79252..30bf77d5df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,11 @@ executors: machine: image: ubuntu-2004:current resource_class: medium + ubuntu2004arm64-large: + machine: + image: ubuntu-2004:current + resource_class: arm-large + ubuntu2004amd64: jobs: build-multi-arch: @@ -77,10 +82,10 @@ jobs: BUILD_DATE: today steps: - checkout - - restore_cache: - keys: + #- restore_cache: + # keys: # - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> - - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >> + # - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >> - run: name: Download VM and install qemu, if armv7l command: | @@ -93,9 +98,9 @@ jobs: tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz rm debian-11.3-armhf-ci-v1.0.tar.xz echo "Install QEMU binaries" - curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v6.2.0-2/xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && tar xvfz xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ - && echo "export PATH=$PWD/xpack-qemu-arm-6.2.0-2/bin:$PATH" > ~/.bashrc \ + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.0.0-1/xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && tar xvfz xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && echo "export PATH=$PWD/xpack-qemu-arm-7.0.0-1/bin:$PATH" > ~/.bashrc \ && source ~/.bashrc \ && qemu-system-arm --version fi @@ -124,13 +129,19 @@ jobs: - run: uname -a - run: docker info - run: - name: "Load built images from cache into Docker" + name: "Load built images into Docker" command: | if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID - docker load -i multi-arch-images.tar + # docker load -i multi-arch-images.tar + docker pull seleniarm/hub:latest + docker pull seleniarm/node-chromium:latest + docker pull seleniarm/node-firefox:latest + docker pull seleniarm/standalone-chromium:latest + docker pull seleniarm/standalone-firefox:latest + docker info - run: name: "Use Python3 and pip3 instead of Python2.7" command: | @@ -262,7 +273,7 @@ workflows: name: test-multi-arch-armv7l # requires: [build-multi-arch-armv7l] platforms: linux/arm/v7 - machine-type: ubuntu2004arm64 + machine-type: ubuntu2004arm64-large deploy-multi-arch: jobs: @@ -278,4 +289,4 @@ workflows: - \ No newline at end of file + From d7476007750e4a1810f8f8d5993b693bd5a94d18 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 15:16:16 +0530 Subject: [PATCH 28/40] fix typo --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30bf77d5df..f5be99ee8c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,6 @@ executors: machine: image: ubuntu-2004:current resource_class: arm-large - ubuntu2004amd64: jobs: build-multi-arch: From 1171b72940b0634e7c55e7de1e0757af0103ff7b Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 15:28:07 +0530 Subject: [PATCH 29/40] Fixed qemu version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5be99ee8c..3c68c9965f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,7 +97,7 @@ jobs: tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz rm debian-11.3-armhf-ci-v1.0.tar.xz echo "Install QEMU binaries" - curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.0.0-1/xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-6.2.0-2-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.0.0-1/xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ && tar xvfz xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ && echo "export PATH=$PWD/xpack-qemu-arm-7.0.0-1/bin:$PATH" > ~/.bashrc \ && source ~/.bashrc \ From f0768330853ab667015c63f7aca3c62801185219 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:09:55 +0530 Subject: [PATCH 30/40] Removing highmem=off since we're using QEMU 7.0.0 --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c68c9965f..e0ba0072ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,6 +110,7 @@ jobs: cd ~/debian-11.3-armhf-ci sed -i 's/&//g' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh + sed -i 's/,highmem=off//' start-vm.sh sh start-vm.sh fi background: true From 1b8b41578bce602d99716276144a45ce78ad8349 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:14:55 +0530 Subject: [PATCH 31/40] debug view of start-vm.sh --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e0ba0072ba..12151fdf0c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,6 +111,7 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh + cat start-vm.sh sh start-vm.sh fi background: true From 9da0a9ab8e0b99d0411b770cdb7787bc892ab1c0 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:19:40 +0530 Subject: [PATCH 32/40] reduce number of cores --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12151fdf0c..6a335e4532 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,6 +111,7 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh + sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh cat start-vm.sh sh start-vm.sh fi From 0917c9a50f710b4a13efb0e7ad098a2d78250c83 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:26:24 +0530 Subject: [PATCH 33/40] Change cpu cortex-a15 to cpu host --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a335e4532..a270bba820 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,6 +111,7 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/-m 1024/-m 6144/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh + sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh cat start-vm.sh sh start-vm.sh From 11c68b8df7285b020581415d3cf0f7fc15f2db83 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:33:56 +0530 Subject: [PATCH 34/40] Change to qemu-system-aarch64 --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a270bba820..3a88a72188 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,6 +102,7 @@ jobs: && echo "export PATH=$PWD/xpack-qemu-arm-7.0.0-1/bin:$PATH" > ~/.bashrc \ && source ~/.bashrc \ && qemu-system-arm --version + && qemu-system-aarch64 --version fi - run: name: Start VM if armv7l @@ -112,6 +113,7 @@ jobs: sed -i 's/-m 1024/-m 6144/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh + sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh cat start-vm.sh sh start-vm.sh From 42473e60e21294019930138ea2c7848946233bb5 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:35:44 +0530 Subject: [PATCH 35/40] typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a88a72188..721c9f0e6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,7 +101,7 @@ jobs: && tar xvfz xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ && echo "export PATH=$PWD/xpack-qemu-arm-7.0.0-1/bin:$PATH" > ~/.bashrc \ && source ~/.bashrc \ - && qemu-system-arm --version + && qemu-system-arm --version \ && qemu-system-aarch64 --version fi - run: From f8f054c9f21d0799a811fecd1f2453b752169abe Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:39:43 +0530 Subject: [PATCH 36/40] Try 4gb of memory --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 721c9f0e6e..ed1595e49e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,7 +110,7 @@ jobs: if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci sed -i 's/&//g' start-vm.sh - sed -i 's/-m 1024/-m 6144/' start-vm.sh + sed -i 's/-m 1024/-m 4096/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh From e3989b3954c13600f88325210c636f9388619321 Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:41:43 +0530 Subject: [PATCH 37/40] Try 3gb of memory --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed1595e49e..568d84d653 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,7 +110,7 @@ jobs: if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then cd ~/debian-11.3-armhf-ci sed -i 's/&//g' start-vm.sh - sed -i 's/-m 1024/-m 4096/' start-vm.sh + sed -i 's/-m 1024/-m 3072/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh From 85ca350d5b3c509afa62119aa2ae4b84f84dcd9e Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 16:46:04 +0530 Subject: [PATCH 38/40] Go back to cortex-a15 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 568d84d653..4e7a03d3e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,7 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/-m 1024/-m 3072/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh - sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh + #sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh cat start-vm.sh From 70cf534f3fdcda23e434eee81506d3ef7483278d Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 25 Jun 2022 18:12:50 +0530 Subject: [PATCH 39/40] Need a version and build date, latest won't work. Also, increased tcg size --- .circleci/config.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e7a03d3e0..c8abbeb2a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,6 +112,7 @@ jobs: sed -i 's/&//g' start-vm.sh sed -i 's/-m 1024/-m 3072/' start-vm.sh sed -i 's/,highmem=off//' start-vm.sh + sed -i 's/tb-size=256/tb-size=1024/' start-vm.sh #sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh @@ -141,11 +142,11 @@ jobs: fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID # docker load -i multi-arch-images.tar - docker pull seleniarm/hub:latest - docker pull seleniarm/node-chromium:latest - docker pull seleniarm/node-firefox:latest - docker pull seleniarm/standalone-chromium:latest - docker pull seleniarm/standalone-firefox:latest + docker pull seleniarm/hub:4.3.0-20220624 + docker pull seleniarm/node-chromium:4.3.0-20220624 + docker pull seleniarm/node-firefox:4.3.0-20220624 + docker pull seleniarm/standalone-chromium:4.3.0-20220624 + docker pull seleniarm/standalone-firefox:4.3.0-20220624 docker info - run: name: "Use Python3 and pip3 instead of Python2.7" @@ -164,12 +165,14 @@ jobs: export PLATFORM=linux/arm/v7 fi export USE_RANDOM_USER=false - export BRANCH=$CIRCLE_BRANCH + #export BRANCH=$CIRCLE_BRANCH + export BRANCH=4.3.0 + export BUILD_DATE=20220624 USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch # docker run --rm --privileged aptman/qus -- -r # docker run --rm --privileged aptman/qus -s -- -p - docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch - docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:latest arch + #docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch + docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:4.3.0-20220624 arch if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then sh stop-vm.sh fi From 7b813f9bf047955c2f47457bef9cf3752585143c Mon Sep 17 00:00:00 2001 From: James Mortensen Date: Sat, 3 Sep 2022 18:25:01 +0530 Subject: [PATCH 40/40] Pulling latest published images and attempt to test armv7l on them --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c837c19e53..8b24a1c457 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -142,11 +142,11 @@ jobs: fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID # docker load -i multi-arch-images.tar - docker pull seleniarm/hub:4.3.0-20220624 - docker pull seleniarm/node-chromium:4.3.0-20220624 - docker pull seleniarm/node-firefox:4.3.0-20220624 - docker pull seleniarm/standalone-chromium:4.3.0-20220624 - docker pull seleniarm/standalone-firefox:4.3.0-20220624 + docker pull seleniarm/hub:4.4.0-20220814 + docker pull seleniarm/node-chromium:4.4.0-20220814 + docker pull seleniarm/node-firefox:4.4.0-20220814 + docker pull seleniarm/standalone-chromium:4.4.0-20220814 + docker pull seleniarm/standalone-firefox:4.4.0-20220814 docker info - run: name: "Use Python3 and pip3 instead of Python2.7" @@ -172,7 +172,7 @@ jobs: # docker run --rm --privileged aptman/qus -- -r # docker run --rm --privileged aptman/qus -s -- -p #docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch - docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:4.3.0-20220624 arch + docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:4.4.0-20220814 arch if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then sh stop-vm.sh fi