From 415b9d07e6ad8fa51b0b1af6f17da6809e7c9b50 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 8 Jan 2025 14:32:51 +0000
Subject: [PATCH 01/20] Add Azure Long and all e2e to long test

---
 .github/workflows/long-test.yml | 93 +++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index d216aa08c50..711a4f6ecbc 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -149,3 +149,96 @@ jobs:
             build/workspace/*/err
             build/workspace/*.ledger/*
           if-no-files-found: ignore
+
+  long-e2e:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: Long E2E
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build"
+        run: |
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DWORKER_THREADS=2 ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-lts
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+  long-azure-linux:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: "Azure Linux Long Test"
+    needs: checks
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: mcr.microsoft.com/azurelinux/base/core:3.0
+      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
+
+    steps:
+      - name: "Install dependencies"
+        run: |
+          set -ex
+          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+          tdnf -y update
+          # Source control
+          tdnf -y install ca-certificates git
+          # Build tools
+          tdnf -y install build-essential clang cmake ninja-build which
+          # Dependencies
+          tdnf -y install openssl-devel libuv-devel curl-devel
+          # Test dependencies
+          tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect
+          # Install CDDL via rubygems
+          tdnf -y install rubygems
+          gem install cddl
+        shell: bash
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build Debug (Long Test)"
+        run: |
+          set -ex
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DUSE_LIBCXX=OFF -DLONG_TEST=ON ..
+          ninja
+        shell: bash
+
+      - name: "Test virtual (Long Test)"
+        run: |
+          set -ex
+          cd build
+          rm -rf /github/home/.cache
+          mkdir -p /github/home/.cache
+          export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
+          # Unit tests
+          ./tests.sh --output-on-failure -L unit -j$(nproc --all)
+          # All e2e tests, which are now supported on Mariner.
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|lts_compatibility|raft_scenario"
+        shell: bash
\ No newline at end of file

From d832eec742f978761cd1d4e328befc6b798df504 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 8 Jan 2025 14:36:09 +0000
Subject: [PATCH 02/20] remove irrelevant needs

---
 .github/workflows/long-test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 711a4f6ecbc..2d3a36b724a 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -191,7 +191,6 @@ jobs:
   long-azure-linux:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: "Azure Linux Long Test"
-    needs: checks
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
       image: mcr.microsoft.com/azurelinux/base/core:3.0

From 0884bc231733b90f3591dc88ba919d7bdd253b07 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 8 Jan 2025 15:21:49 +0000
Subject: [PATCH 03/20] Format and fix upload

---
 .github/workflows/long-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 2d3a36b724a..8eb892c4132 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -180,7 +180,7 @@ jobs:
         if: success() || failure()
         uses: actions/upload-artifact@v4
         with:
-          name: logs-lts
+          name: logs-e2e
           path: |
             build/workspace/*/*.config.json
             build/workspace/*/out
@@ -240,4 +240,4 @@ jobs:
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
           # All e2e tests, which are now supported on Mariner.
           ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|lts_compatibility|raft_scenario"
-        shell: bash
\ No newline at end of file
+        shell: bash

From 3ca0c6241ce5d46f4f43d2f709d9d2922d7b1560 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 8 Jan 2025 16:39:51 +0000
Subject: [PATCH 04/20] Try if long_test=on hides failures

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 8eb892c4132..a1d9b2380d7 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -167,7 +167,7 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DWORKER_THREADS=2 ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TEST=ON -DLVI_MITIGATIONS=OFF -DWORKER_THREADS=2 ..
           ninja
 
       - name: "Test"

From ade7b5b4dff946a16ff9908a5656af9a306bab51 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 8 Jan 2025 21:10:07 +0000
Subject: [PATCH 05/20] Revert "Try if long_test=on hides failures"

This reverts commit 70f4f8e6321b6f3cad28ede43dd7ed9a55ef8d6b.
---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index a1d9b2380d7..8eb892c4132 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -167,7 +167,7 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TEST=ON -DLVI_MITIGATIONS=OFF -DWORKER_THREADS=2 ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DWORKER_THREADS=2 ..
           ninja
 
       - name: "Test"

From 6cd3344c44e81a44630c0f9bf50509c1303fe458 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 11:12:22 +0000
Subject: [PATCH 06/20] Try if worker threads are a problem

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 8eb892c4132..b0c6e52d4b0 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -167,7 +167,7 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DWORKER_THREADS=2 ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF ..
           ninja
 
       - name: "Test"

From 5c38717f46624b72ab2fc9d85104130e3b9c9759 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 11:29:23 +0000
Subject: [PATCH 07/20] New image ref

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index b0c6e52d4b0..ae3d3ae44f5 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -155,7 +155,7 @@ jobs:
     name: Long E2E
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ghcr.io/microsoft/ccf/ci/default:build-05-12-2024
+      image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
 
     steps:
       - uses: actions/checkout@v4

From a1fce46a19bc277ff5b472cbf5b315dfcfc4d30b Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 12:47:44 +0000
Subject: [PATCH 08/20] LONG_TEST back

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index ae3d3ae44f5..28c777b27c7 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -167,7 +167,7 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DLVI_MITIGATIONS=OFF ..
           ninja
 
       - name: "Test"

From f460ca352824567733c7c90a396c6d32d3d5c1ab Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 12:59:25 +0000
Subject: [PATCH 09/20] Debug + Release

---
 .github/workflows/long-test.yml | 42 +++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 28c777b27c7..f5b251c57af 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -150,7 +150,7 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
-  long-e2e:
+  long-e2e-debug:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: Long E2E
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
@@ -180,7 +180,45 @@ jobs:
         if: success() || failure()
         uses: actions/upload-artifact@v4
         with:
-          name: logs-e2e
+          name: logs-e2e-debug
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+  long-e2e-release:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: Long E2E
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build"
+        run: |
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Release ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-e2e-release
           path: |
             build/workspace/*/*.config.json
             build/workspace/*/out

From fa852c7972ee26d0b0ca69cab0a7f3e1d82d0553 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 13:02:30 +0000
Subject: [PATCH 10/20] Shuffled tests

---
 .github/workflows/long-test.yml | 39 +++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index f5b251c57af..00892329d3e 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -226,6 +226,45 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
+  e2e-suite-shuffled:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
+    name: Long E2E
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: "Build"
+        run: |
+          git config --global --add safe.directory /__w/CCF/CCF
+          mkdir build
+          cd build
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DSHUFFLE_SUITE=ON ..
+          ninja
+
+      - name: "Test"
+        run: |
+          set +x
+          cd build
+          ./tests.sh --output-on-failure --timeout 1600 -R "suite"
+
+      - name: "Upload logs"
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-e2e-shuffled
+          path: |
+            build/workspace/*/*.config.json
+            build/workspace/*/out
+            build/workspace/*/err
+            build/workspace/*.ledger/*
+          if-no-files-found: ignore
+
+
   long-azure-linux:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: "Azure Linux Long Test"

From 9f789b011f7b08b3c57b4c0a7263e5d6bd3f000d Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 13:07:17 +0000
Subject: [PATCH 11/20] Name

---
 .github/workflows/long-test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 00892329d3e..4697e8abff3 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -152,7 +152,7 @@ jobs:
 
   long-e2e-debug:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
-    name: Long E2E
+    name: Long E2E - all (Debug)
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
       image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
@@ -190,7 +190,7 @@ jobs:
 
   long-e2e-release:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
-    name: Long E2E
+    name: Long E2E - all (Release)
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
       image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2
@@ -228,7 +228,7 @@ jobs:
 
   e2e-suite-shuffled:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
-    name: Long E2E
+    name: Long E2E suite (Shuffled)
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
       image: ghcr.io/microsoft/ccf/ci/default:build-08-01-2025-2

From 86d44290c484bd94ea71b0dbb7bab39cc229c26d Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 14:16:58 +0000
Subject: [PATCH 12/20] Format

---
 .github/workflows/long-test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 4697e8abff3..f5b513f53f5 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -264,7 +264,6 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
-
   long-azure-linux:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: "Azure Linux Long Test"

From b59f6159937777fd5209a8b9796999f3f2e9af99 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Thu, 9 Jan 2025 14:26:20 +0000
Subject: [PATCH 13/20] Raft scenario back after 6748

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index f5b513f53f5..a8f7fb60e08 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -315,5 +315,5 @@ jobs:
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
           # All e2e tests, which are now supported on Mariner.
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|lts_compatibility|raft_scenario"
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|lts_compatibility"
         shell: bash

From e92ef0d707df366efc6932396eb75389d6c00eec Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Mon, 13 Jan 2025 11:13:59 +0000
Subject: [PATCH 14/20] Fix dep

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index a8f7fb60e08..87c048b982b 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -283,7 +283,7 @@ jobs:
           # Build tools
           tdnf -y install build-essential clang cmake ninja-build which
           # Dependencies
-          tdnf -y install openssl-devel libuv-devel curl-devel
+          tdnf -y install openssl-devel libuv-devel nghttp2-devel
           # Test dependencies
           tdnf -y install libarrow-devel parquet-libs-devel lldb npm jq expect
           # Install CDDL via rubygems

From 3585a5ac977b81c2d9cfd7b0bde78ff8a831c1ef Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Tue, 14 Jan 2025 13:13:08 +0000
Subject: [PATCH 15/20] Cleanup + pebble

---
 .github/workflows/long-test.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 0ade40d8b85..a5b692ab86f 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -162,12 +162,19 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: "Install deps"
+        run: |
+          sudo apt-get -y update
+          sudo apt install ansible -y
+          cd getting_started/setup_vm
+          ansible-playbook ccf-extended-testing.yml
+
       - name: "Build"
         run: |
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON -DLVI_MITIGATIONS=OFF ..
+          cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON ..
           ninja
 
       - name: "Test"
@@ -301,7 +308,7 @@ jobs:
           git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
-          CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLVI_MITIGATIONS=OFF -DUSE_LIBCXX=OFF -DLONG_TEST=ON ..
+          CC=`which clang` CXX=`which clang++` cmake -GNinja -DCOMPILE_TARGET=virtual -DCMAKE_BUILD_TYPE=Debug -DLONG_TEST=ON ..
           ninja
         shell: bash
 
@@ -311,7 +318,6 @@ jobs:
           cd build
           rm -rf /github/home/.cache
           mkdir -p /github/home/.cache
-          export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
           # All e2e tests, which are now supported on Mariner.

From c54810ea3dab6dd9468312e46f4270c29d50be01 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Tue, 14 Jan 2025 13:29:54 +0000
Subject: [PATCH 16/20] More timeout for suites

---
 .github/workflows/long-test.yml | 7 +++++++
 CMakeLists.txt                  | 8 ++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index a5b692ab86f..3ed0d4faa84 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -150,6 +150,9 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
+  # All e2e tests without sanitizers in debug mode; needed because:
+  # - Sanitizer builds may slightly differ.
+  # - Test durations may also differ, which is important for -L "suite" because they have fixed timeouts.
   long-e2e-debug:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: Long E2E - all (Debug)
@@ -195,6 +198,7 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
+  # All e2e tests in release mode (same as release build).
   long-e2e-release:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: Long E2E - all (Release)
@@ -233,6 +237,7 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
+  # End-to-end test suites with shuffling enabled.
   e2e-suite-shuffled:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: Long E2E suite (Shuffled)
@@ -271,6 +276,8 @@ jobs:
             build/workspace/*.ledger/*
           if-no-files-found: ignore
 
+  # All end-to-end tests in debug mode on Azure Linux (except benchmarks and tests which are not supported yet).
+  # The main purpose of these is to keep the Azure Linux build/run up-to-date to facilitate an easy migration in the future.
   long-azure-linux:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'}}
     name: "Azure Linux Long Test"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6ef7454928f..1fc4cae12ef 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -987,7 +987,7 @@ if(BUILD_TESTS)
       LABEL suite
       ADDITIONAL_ARGS
         --test-duration
-        150
+        300
         --test-suite
         rekey_recovery
         --test-suite
@@ -1001,7 +1001,7 @@ if(BUILD_TESTS)
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py
       LABEL suite
       ADDITIONAL_ARGS
-        --test-duration 200 --test-suite reconfiguration --jinja-templates-path
+        --test-duration 300 --test-suite reconfiguration --jinja-templates-path
         ${CMAKE_SOURCE_DIR}/samples/templates
     )
 
@@ -1010,7 +1010,7 @@ if(BUILD_TESTS)
         NAME regression_test_suite
         PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py
         LABEL suite
-        ADDITIONAL_ARGS --test-duration 200 --test-suite regression_5236
+        ADDITIONAL_ARGS --test-duration 300 --test-suite regression_5236
       )
     endif()
 
@@ -1024,7 +1024,7 @@ if(BUILD_TESTS)
         --ledger-recovery-timeout
         20
         --test-duration
-        200
+        300
         --test-suite
         all
         --jinja-templates-path

From 5cef14bc59788246bb6ea35e832d101cb3b0f6fe Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Tue, 14 Jan 2025 13:31:43 +0000
Subject: [PATCH 17/20] Azure linux build fix libc++

---
 .github/workflows/long-test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index 3ed0d4faa84..d15ed9388eb 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -296,6 +296,8 @@ jobs:
           tdnf -y install ca-certificates git
           # Build tools
           tdnf -y install build-essential clang cmake ninja-build which
+          # libc++
+          tdnf -y install libcxx-devel llvm-libunwind-devel llvm-libunwind-static
           # Dependencies
           tdnf -y install openssl-devel libuv-devel nghttp2-devel
           # Test dependencies

From c99744ad4558606c6cde57d79e3c3d0d78b4530e Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Tue, 14 Jan 2025 15:27:27 +0000
Subject: [PATCH 18/20] Bigger timeout for long test

---
 .github/workflows/long-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml
index d15ed9388eb..38999d52704 100644
--- a/.github/workflows/long-test.yml
+++ b/.github/workflows/long-test.yml
@@ -330,5 +330,5 @@ jobs:
           # Unit tests
           ./tests.sh --output-on-failure -L unit -j$(nproc --all)
           # All e2e tests, which are now supported on Mariner.
-          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|lts_compatibility"
+          ./tests.sh --timeout 1600 --output-on-failure -LE "benchmark|lts_compatibility"
         shell: bash

From e538eb65eda98188efdec53dd30695ba3d7728fc Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 15 Jan 2025 12:56:54 +0000
Subject: [PATCH 19/20] Fix recovery condition

---
 tests/recovery.py                |  1 +
 tests/suite/test_requirements.py | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/tests/recovery.py b/tests/recovery.py
index 2de2936eb1a..48ec2b01312 100644
--- a/tests/recovery.py
+++ b/tests/recovery.py
@@ -220,6 +220,7 @@ def test_recover_service(network, args, from_snapshot=True, no_ledger=False):
 
 @reqs.description("Recover a service with wrong service identity")
 @reqs.recover(number_txs=2)
+@reqs.sufficient_network_recovery_count(required_count=1)
 def test_recover_service_with_wrong_identity(network, args):
     old_primary, _ = network.find_primary()
 
diff --git a/tests/suite/test_requirements.py b/tests/suite/test_requirements.py
index 01c24262d23..fa25341a848 100644
--- a/tests/suite/test_requirements.py
+++ b/tests/suite/test_requirements.py
@@ -111,6 +111,16 @@ def check(network, args, recovery_member=True, *nargs, **kwargs):
     return ensure_reqs(check)
 
 
+def sufficient_network_recovery_count(required_count):
+    def check(network, args,  *nargs, **kwargs):
+        if network.recovery_count < required_count:
+            raise TestRequirementsNotMet(
+                f"Test requires {required_count} network recoveries, but only {network.recovery_count} have been performed"
+            )
+
+    return ensure_reqs(check)
+
+
 def can_kill_n_nodes(nodes_to_kill_count):
     def check(network, args, *nargs, **kwargs):
         running_nodes_count = len(network.get_joined_nodes())

From 8e2a46759e660b46203cd5877a958d9c31cc01a8 Mon Sep 17 00:00:00 2001
From: Max Tropets <maxtropets@gmail.com>
Date: Wed, 15 Jan 2025 12:58:09 +0000
Subject: [PATCH 20/20] Format

---
 tests/suite/test_requirements.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/suite/test_requirements.py b/tests/suite/test_requirements.py
index fa25341a848..b7c3f21391d 100644
--- a/tests/suite/test_requirements.py
+++ b/tests/suite/test_requirements.py
@@ -112,7 +112,7 @@ def check(network, args, recovery_member=True, *nargs, **kwargs):
 
 
 def sufficient_network_recovery_count(required_count):
-    def check(network, args,  *nargs, **kwargs):
+    def check(network, args, *nargs, **kwargs):
         if network.recovery_count < required_count:
             raise TestRequirementsNotMet(
                 f"Test requires {required_count} network recoveries, but only {network.recovery_count} have been performed"