ci: capture native abort messages across Linux runners

Author: abetlen

.github/scripts/diagnose-recurrent.py

@@ -0,0 +1,28 @@
+import ctypes
+import os
+import subprocess
+import sys
+
+if len(sys.argv) == 1:
+    for iteration in range(5):
+        print(f"Captured test iteration {iteration + 1}", flush=True)
+        result = subprocess.run([sys.executable, __file__, "once"])
+        if result.returncode:
+            raise SystemExit(result.returncode)
+    raise SystemExit(0)
+
+import llama_cpp.llama_cpp as lc
+import pytest
+
+stderr = os.dup(2)
+
+
+@ctypes.CFUNCTYPE(None, ctypes.c_char_p)
+def report_abort(message):
+    os.write(stderr, b"NATIVE ABORT: " + message + b"\n")
+
+
+lc._lib.ggml_set_abort_callback.argtypes = [type(report_abort)]
+lc._lib.ggml_set_abort_callback.restype = ctypes.c_void_p
+lc._lib.ggml_set_abort_callback(report_abort)
+raise SystemExit(pytest.main(["-q", "-x"]))

.github/workflows/debug-recurrent.yaml

@@ -12,9 +12,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        native: ["ON", "OFF"]
+        trial: [1, 2, 3, 4]
     env:
-      CMAKE_ARGS: -DGGML_NATIVE=${{ matrix.native }}
+      CMAKE_ARGS: -DGGML_NATIVE=ON
     steps:
       - uses: actions/checkout@v6
         with:
@@ -32,5 +32,5 @@ jobs:
         run: |
           python -m pip install uv
           python -m uv pip install --system -e '.[all]' --verbose
-      - name: Test without native stderr capture
-        run: python -m pytest -s -v -x
+      - name: Test with abort diagnostics
+        run: python .github/scripts/diagnose-recurrent.py