diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 9714e371c228..57fa8b89d640 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -149,3 +149,186 @@ jobs: build/workspace/*/err build/workspace/*.ledger/* if-no-files-found: ignore + + # All e2e tests without sanitizers in debug mode; needed because: + # - Sanitizer builds may slightly differ. + # - Test durations may also differ, which is important for -L "suite" because they have fixed timeouts. + long-e2e-debug: + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}} + name: Long E2E - all (Debug) + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Install deps" + run: | + sudo apt-get -y update + sudo apt install ansible -y + cd getting_started/setup_vm + ansible-playbook ccf-extended-testing.yml + + - name: "Build" + run: | + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON .. + ninja + + - name: "Test" + run: | + set +x + cd build + ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark" + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: logs-e2e-debug + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + build/workspace/*.ledger/* + if-no-files-found: ignore + + # All e2e tests in release mode (same as release build). + long-e2e-release: + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}} + name: Long E2E - all (Release) + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Build" + run: | + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Release .. + ninja + + - name: "Test" + run: | + set +x + cd build + ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark" + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: logs-e2e-release + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + build/workspace/*.ledger/* + if-no-files-found: ignore + + # End-to-end test suites with shuffling enabled. + e2e-suite-shuffled: + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}} + name: Long E2E suite (Shuffled) + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Build" + run: | + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DSHUFFLE_SUITE=ON .. + ninja + + - name: "Test" + run: | + set +x + cd build + ./tests.sh --output-on-failure --timeout 1600 -R "suite" + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: logs-e2e-shuffled + path: | + build/workspace/*/*.config.json + build/workspace/*/out + build/workspace/*/err + build/workspace/*.ledger/* + if-no-files-found: ignore + + # All end-to-end tests in debug mode on Azure Linux (except benchmarks and tests which are not supported yet). + # The main purpose of these is to keep the Azure Linux build/run up-to-date to facilitate an easy migration in the future. + long-azure-linux: + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}} + name: "Azure Linux Long Test" + runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub] + container: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro + + steps: + - name: "Install dependencies" + run: | + set -ex + gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY + tdnf -y update + # Source control + tdnf -y install ca-certificates git + # Build tools + tdnf -y install build-essential clang cmake ninja-build which + # libc++ + tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static + # Dependencies + tdnf -y install openssl-devel libuv-devel nghttp2-devel + # Test dependencies + tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect + # Install CDDL via rubygems + tdnf -y install rubygems + gem install cddl + shell: bash + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Build Debug (Long Test)" + run: | + set -ex + git config --global --add safe.directory /__w/CCF/CCF + mkdir build + cd build + CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TEST=ON .. + ninja + shell: bash + + - name: "Test virtual (Long Test)" + run: | + set -ex + cd build + rm -rf /github/home/.cache + mkdir -p /github/home/.cache + # Unit tests + ./tests.sh --output-on-failure -L unit -j$(nproc --all) + # All e2e tests, which are now supported on Mariner. + ./tests.sh --timeout 1600 --output-on-failure -LE "benchmark|lts_compatibility" + shell: bash diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ef7454928f0..1fc4cae12ef7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -987,7 +987,7 @@ if(BUILD_TESTS) LABEL suite ADDITIONAL_ARGS --test-duration - 150 + 300 --test-suite rekey_recovery --test-suite @@ -1001,7 +1001,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py LABEL suite ADDITIONAL_ARGS - --test-duration 200 --test-suite reconfiguration --jinja-templates-path + --test-duration 300 --test-suite reconfiguration --jinja-templates-path ${CMAKE_SOURCE_DIR}/samples/templates ) @@ -1010,7 +1010,7 @@ if(BUILD_TESTS) NAME regression_test_suite PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py LABEL suite - ADDITIONAL_ARGS --test-duration 200 --test-suite regression_5236 + ADDITIONAL_ARGS --test-duration 300 --test-suite regression_5236 ) endif() @@ -1024,7 +1024,7 @@ if(BUILD_TESTS) --ledger-recovery-timeout 20 --test-duration - 200 + 300 --test-suite all --jinja-templates-path diff --git a/tests/recovery.py b/tests/recovery.py index 2de2936eb1a4..48ec2b01312f 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -220,6 +220,7 @@ def test_recover_service(network, args, from_snapshot=True, no_ledger=False): @reqs.description("Recover a service with wrong service identity") @reqs.recover(number_txs=2) +@reqs.sufficient_network_recovery_count(required_count=1) def test_recover_service_with_wrong_identity(network, args): old_primary, _ = network.find_primary() diff --git a/tests/suite/test_requirements.py b/tests/suite/test_requirements.py index 01c24262d233..b7c3f21391d9 100644 --- a/tests/suite/test_requirements.py +++ b/tests/suite/test_requirements.py @@ -111,6 +111,16 @@ def check(network, args, recovery_member=True, *nargs, **kwargs): return ensure_reqs(check) +def sufficient_network_recovery_count(required_count): + def check(network, args, *nargs, **kwargs): + if network.recovery_count < required_count: + raise TestRequirementsNotMet( + f"Test requires {required_count} network recoveries, but only {network.recovery_count} have been performed" + ) + + return ensure_reqs(check) + + def can_kill_n_nodes(nodes_to_kill_count): def check(network, args, *nargs, **kwargs): running_nodes_count = len(network.get_joined_nodes())