diff --git a/.circleci/config.yml b/.circleci/config.yml index 436314171f..d7d460f655 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,10 @@ executors: machine: image: ubuntu-2004:current resource_class: medium + ubuntu2004arm64-large: + machine: + image: ubuntu-2004:current + resource_class: arm-large jobs: build-multi-arch: @@ -31,13 +35,14 @@ jobs: cat $BASH_ENV source $BASH_ENV echo "Workflow environment variables:" - echo $BRANCH + echo "BRANCH=$BRANCH" - run: uname -a - run: docker info - checkout - run: name: "Build Docker images" command: | + export BRANCH=$CIRCLE_BRANCH echo "Branch is $BRANCH" NAME=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} PLATFORMS=${PLATFORMS} BUILD_ARGS=${BUILD_ARGS} make build_multi - run: @@ -76,16 +81,73 @@ jobs: BUILD_DATE: today steps: - checkout - - restore_cache: - keys: - - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> + #- restore_cache: + # keys: + # - multi-arch-images-{{ .Branch }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}-<< parameters.platforms >> + # - multi-arch-images-{{ .Branch }}-d9897deb-4482-4d3a-b3cf-b9b36b69df31-<< parameters.platforms >> + - run: + name: Download VM and install qemu, if armv7l + command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + echo "Download armv7l VM..." + cd ~ + curl -L https://github.com/jamesmortensen/virtual-machine-releases/releases/download/debian-11.3-armhf-ci-v1.0/debian-11.3-armhf-ci-v1.0.tar.xz \ + -o debian-11.3-armhf-ci-v1.0.tar.xz + echo "Extract VM..." + tar xvfJ debian-11.3-armhf-ci-v1.0.tar.xz + rm debian-11.3-armhf-ci-v1.0.tar.xz + echo "Install QEMU binaries" + curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.0.0-1/xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz -o xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && tar xvfz xpack-qemu-arm-7.0.0-1-linux-$(uname -m | sed 's/x86_64/x64/' | sed 's/aarch64/arm64/').tar.gz \ + && echo "export PATH=$PWD/xpack-qemu-arm-7.0.0-1/bin:$PATH" > ~/.bashrc \ + && source ~/.bashrc \ + && qemu-system-arm --version \ + && qemu-system-aarch64 --version + fi + - run: + name: Start VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sed -i 's/&//g' start-vm.sh + sed -i 's/-m 1024/-m 3072/' start-vm.sh + sed -i 's/,highmem=off//' start-vm.sh + sed -i 's/tb-size=256/tb-size=1024/' start-vm.sh + #sed -i 's/cpu cortex-a15/cpu host/' start-vm.sh + sed -i 's/qemu-system-arm/qemu-system-aarch64/' start-vm.sh + sed -i 's/cpus=4,sockets=1,cores=4,threads=1/cpus=2,sockets=1,cores=2,threads=1/' start-vm.sh + cat start-vm.sh + sh start-vm.sh + fi + background: true + - run: + name: Wait for VM if armv7l + command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + cd ~/debian-11.3-armhf-ci + sed -i 's/sh start-vm.sh//g' start-vm-and-tunnels.sh && mv start-vm-and-tunnels.sh wait-for-vm-boot.sh + bash wait-for-vm-boot.sh + cd .. + echo 'export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock' >> $BASH_ENV + source $BASH_ENV + ssh -p 8022 debianhf@127.0.0.1 -N -f -L4444:127.0.0.1:4444 -L5555:127.0.0.1:5555 -L7900:127.0.0.1:7900 + fi - run: uname -a - run: docker info - run: - name: "Load built images from cache into Docker" + name: "Load built images into Docker" command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + fi echo "CIRCLE_WORKFLOW_ID = " $CIRCLE_WORKFLOW_ID - docker load -i multi-arch-images.tar + # docker load -i multi-arch-images.tar + docker pull seleniarm/hub:4.4.0-20220814 + docker pull seleniarm/node-chromium:4.4.0-20220814 + docker pull seleniarm/node-firefox:4.4.0-20220814 + docker pull seleniarm/standalone-chromium:4.4.0-20220814 + docker pull seleniarm/standalone-firefox:4.4.0-20220814 + docker info - run: name: "Use Python3 and pip3 instead of Python2.7" command: | @@ -97,9 +159,23 @@ jobs: name: "Test Docker images" no_output_timeout: 2m command: | + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + export DOCKER_HOST=unix:///tmp/docker-on-debianhf.sock + export GRID_STATUS_MAX_ATTEMPTS=15 + export PLATFORM=linux/arm/v7 + fi export USE_RANDOM_USER=false - export BRANCH=${CIRCLE_BRANCH//\//-} + #export BRANCH=${CIRCLE_BRANCH//\//-} + export BRANCH=4.3.0 + export BUILD_DATE=20220624 USE_RANDOM_USER_ID=${USE_RANDOM_USER} NAMESPACE=${NAMESPACE} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SKIP_BUILD=true make test_multi_arch + # docker run --rm --privileged aptman/qus -- -r + # docker run --rm --privileged aptman/qus -s -- -p + #docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:armv7-tests-today arch + docker run --platform linux/arm/v7 --rm seleniarm/standalone-firefox:4.4.0-20220814 arch + if [ "$CIRCLE_JOB" = "test-multi-arch-armv7l" ]; then + sh stop-vm.sh + fi deploy-multi-arch: parameters: @@ -184,24 +260,33 @@ jobs: workflows: build-and-test-multi-arch: jobs: - - build-multi-arch: - name: build-multi-arch-arm64 - platforms: linux/arm64 - machine-type: ubuntu2004arm64 - - build-multi-arch: - name: build-multi-arch-amd64 - platforms: linux/amd64 - machine-type: ubuntu2004amd64 - - test-multi-arch: - name: test-multi-arch-arm64 - requires: [build-multi-arch-arm64] - platforms: linux/arm64 - machine-type: ubuntu2004arm64 + # - build-multi-arch: + # name: build-multi-arch-arm64 + # platforms: linux/arm64 + # machine-type: ubuntu2004arm64 + # - build-multi-arch: + # name: build-multi-arch-amd64 + # platforms: linux/amd64 + # machine-type: ubuntu2004amd64 + # - build-multi-arch: + # name: build-multi-arch-armv7l + # platforms: linux/arm/v7 + # machine-type: ubuntu2004arm64 + # - test-multi-arch: + # name: test-multi-arch-arm64 + # requires: [build-multi-arch-arm64] + # platforms: linux/arm64 + # machine-type: ubuntu2004arm64 + # - test-multi-arch: + # name: test-multi-arch-amd64 + # requires: [build-multi-arch-amd64] + # platforms: linux/amd64 + # machine-type: ubuntu2004amd64 - test-multi-arch: - name: test-multi-arch-amd64 - requires: [build-multi-arch-amd64] - platforms: linux/amd64 - machine-type: ubuntu2004amd64 + name: test-multi-arch-armv7l + # requires: [build-multi-arch-armv7l] + platforms: linux/arm/v7 + machine-type: ubuntu2004arm64-large deploy-multi-arch: jobs: @@ -214,4 +299,3 @@ workflows: branches: only: - trunk - diff --git a/Makefile b/Makefile index 0a6f9a30f9..ff70081fed 100644 --- a/Makefile +++ b/Makefile @@ -393,7 +393,8 @@ test_firefox_standalone: # Test multi-arch container images -test_multi_arch: test_chromium_multi \ +test_multi_arch: qemu_user_static \ + test_chromium_multi \ test_firefox_multi \ test_chromium_standalone_multi \ test_firefox_standalone_multi diff --git a/tests/test.py b/tests/test.py index 3268dc9d07..0885c42936 100644 --- a/tests/test.py +++ b/tests/test.py @@ -3,6 +3,7 @@ import random import sys import unittest +import time import re import docker @@ -23,6 +24,7 @@ https_proxy = os.environ.get('https_proxy', '') no_proxy = os.environ.get('no_proxy', '') SKIP_BUILD = os.environ.get('SKIP_BUILD', False) +PLATFORM = os.environ.get('PLATFORM', '') IMAGE_NAME_MAP = { # Hub @@ -111,6 +113,12 @@ def prune_networks(): client.networks.prune() +def wait_for_grid(platform): + if platform != '': + logger.info('Starting container on platform %s so we will wait 10 more seconds since emulation is slower...' % platform) + time.sleep(10) + + def launch_container(container, **kwargs): """ Launch a specific container @@ -176,6 +184,7 @@ def standalone_browser_container_matches(container): use_random_user_id = USE_RANDOM_USER_ID == 'true' run_in_docker_compose = RUN_IN_DOCKER_COMPOSE == 'true' random_user_id = random.randint(100000, 2147483647) + platform = PLATFORM if use_random_user_id: logger.info("Running tests with a random user ID -> %s" % random_user_id) @@ -197,9 +206,12 @@ def standalone_browser_container_matches(container): """ ports = {'4444': 4444} if use_random_user_id: - test_container_id = launch_container(image, ports=ports, user=random_user_id) + test_container_id = launch_container(image, ports=ports, user=random_user_id, platform=platform) else: - test_container_id = launch_container(image, ports=ports) + test_container_id = launch_container(image, ports=ports,platform=platform) + + wait_for_grid(platform) + else: """ Hub / Node Configuration @@ -209,11 +221,13 @@ def standalone_browser_container_matches(container): hub_id = launch_hub("grid") ports = {'5555': 5555, '7900': 7900} if use_random_user_id: - test_container_id = launch_container(image, network='grid', ports=ports, user=random_user_id) + test_container_id = launch_container(image, network='grid', ports=ports, user=random_user_id, platform=platform) else: - test_container_id = launch_container(image, network='grid', ports=ports) + test_container_id = launch_container(image, network='grid', ports=ports, platform=platform) prune_networks() + wait_for_grid(platform) + logger.info('========== / Containers ready to go ==========') try: