More tests in Long Test job

.github/workflows/long-test.yml

@@ -149,3 +149,186 @@ jobs:
             build/workspace/*/err
             build/workspace/*.ledger/*
           if-no-files-found: ignore
+
+  # All e2e tests without sanitizers in debug mode; needed because:
+  # - Sanitizer builds may slightly differ.
+  # - Test durations may also differ, which is important for -L "suite" because they have fixed timeouts.
+  long-e2e-debug:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: Long E2E - all (Debug)
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Install deps"
+        run: |
+          sudo apt-get -y update
+          sudo apt install ansible -y
+          cd getting_started/setup_vm
+          ansible-playbook ccf-extended-testing.yml
+
+      - name: "Build"
+        run: |
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-e2e-debug
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+  # All e2e tests in release mode (same as release build).
+  long-e2e-release:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: Long E2E - all (Release)
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build"
+        run: |
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Release ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-e2e-release
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+  # End-to-end test suites with shuffling enabled.
+  e2e-suite-shuffled:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: Long E2E suite (Shuffled)
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build"
+        run: |
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DSHUFFLE_SUITE=ON ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -R "suite"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-e2e-shuffled
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+  # All end-to-end tests in debug mode on Azure Linux (except benchmarks and tests which are not supported yet).
+  # The main purpose of these is to keep the Azure Linux build/run up-to-date to facilitate an easy migration in the future.
+  long-azure-linux:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: "Azure Linux Long Test"
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: mcr.microsoft.com/azurelinux/base/core:3.0
+      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
+
+    steps:
+      - name: "Install dependencies"
+        run: |
+          set -ex
+          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+          tdnf -y update
+          # Source control
+          tdnf -y install ca-certificates git
+          # Build tools
+          tdnf -y install build-essential clang cmake ninja-build which
+          # libc++
+          tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static
+          # Dependencies
+          tdnf -y install openssl-devel libuv-devel nghttp2-devel
+          # Test dependencies
+          tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect
+          # Install CDDL via rubygems
+          tdnf -y install rubygems
+          gem install cddl
+        shell: bash
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build Debug (Long Test)"
+        run: |
+          set -ex
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TEST=ON ..
+          ninja
+        shell: bash
+
+      - name: "Test virtual (Long Test)"
+        run: |
+          set -ex
+          cd build
+          rm -rf /github/home/.cache
+          mkdir -p /github/home/.cache
+          # Unit tests
+          ./tests.sh --output-on-failure -L unit -j$(nproc --all)
+          # All e2e tests, which are now supported on Mariner.
+          ./tests.sh --timeout 1600 --output-on-failure -LE "benchmark|lts_compatibility"
+        shell: bash

CMakeLists.txt

@@ -987,7 +987,7 @@ if(BUILD_TESTS)
       LABEL suite
       ADDITIONAL_ARGS
         --test-duration
-        150
+        300
         --test-suite
         rekey_recovery
         --test-suite
@@ -1001,7 +1001,7 @@ if(BUILD_TESTS)
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py
       LABEL suite
       ADDITIONAL_ARGS
-        --test-duration 200 --test-suite reconfiguration --jinja-templates-path
+        --test-duration 300 --test-suite reconfiguration --jinja-templates-path
         ${CMAKE_SOURCE_DIR}/samples/templates
     )
 
@@ -1010,7 +1010,7 @@ if(BUILD_TESTS)
         NAME regression_test_suite
         PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py
         LABEL suite
-        ADDITIONAL_ARGS --test-duration 200 --test-suite regression_5236
+        ADDITIONAL_ARGS --test-duration 300 --test-suite regression_5236
       )
     endif()
 
@@ -1024,7 +1024,7 @@ if(BUILD_TESTS)
         --ledger-recovery-timeout
         20
         --test-duration
-        200
+        300
         --test-suite
         all
         --jinja-templates-path

tests/recovery.py

@@ -220,6 +220,7 @@ def test_recover_service(network, args, from_snapshot=True, no_ledger=False):
 
 @reqs.description("Recover a service with wrong service identity")
 @reqs.recover(number_txs=2)
+@reqs.sufficient_network_recovery_count(required_count=1)
 def test_recover_service_with_wrong_identity(network, args):
     old_primary, _ = network.find_primary()
 

tests/suite/test_requirements.py

@@ -111,6 +111,16 @@ def check(network, args, recovery_member=True, *nargs, **kwargs):
     return ensure_reqs(check)
 
 
+def sufficient_network_recovery_count(required_count):
+    def check(network, args, *nargs, **kwargs):
+        if network.recovery_count < required_count:
+            raise TestRequirementsNotMet(
+                f"Test requires {required_count} network recoveries, but only {network.recovery_count} have been performed"
+            )
+
+    return ensure_reqs(check)
+
+
 def can_kill_n_nodes(nodes_to_kill_count):
     def check(network, args, *nargs, **kwargs):
         running_nodes_count = len(network.get_joined_nodes())