From 1924e18f6f5cd3afec91e8b5e14cb55806c23848 Mon Sep 17 00:00:00 2001
From: Dextroid17 <217507631+Dextroid17@users.noreply.github.com>
Date: Tue, 6 Oct 2026 21:53:02 -0400
Subject: [PATCH 1/2] ci: fail when a test skips that the job's device is not
 expected to skip

The Linux job reports "100% tests passed out of 371" with 20 agc_driver execution tests skipped
(llvmpipe reports subgroup size 8, so the tests that need a wider wave exit 77), and the Windows
job reports "100% tests passed, 0 tests failed out of 368" with 143 skipped - the runner has no
Vulkan driver and ANYPS5_REQUIRE_VULKAN is not set there, so 143 of the 147 GPU execution tests
silently do not run. ctest counts a skip as a pass, so neither job notices when the set grows.

Add tools/check_skips.py. It reads the file ctest writes with --output-junit and fails when a test
skips for a reason that is not expected:

- linux: a skip is expected only when the test's own output reports that the device's subgroups
  are too narrow for it, so adding a wave32 test does not need the rule edited;
- windows: any agc_driver test, since the runner has no driver for them, while a skip of anything
  else fails.

Both jobs now pass --output-junit and run the check after ctest. The JUnit file needs no
redirection, so the console output stays visible and no pipe can hide a failing ctest.

Verified here with the CI command and the CI device (lavapipe, subgroup 8): the real run reports
20 skipped, all expected; a 21st wave32 test skipping for the same reason passes; a skip whose
reason is something else exits 1; a file with 143 agc_driver skips passes with --job windows and
one non-agc skip in it exits 1.
---
 .github/workflows/build.yml |  6 +++--
 tools/check_skips.py        | 45 +++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 tools/check_skips.py

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c067651f3..7d0cfab4e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -166,10 +166,11 @@ jobs:
         run: python3 tools/check_test_registration.py build
       - run: cmake --build build
       - run: cmake --build build --target libs
-      - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log
+      - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
         env:
           ANYPS5_REQUIRE_VULKAN: 1
           GALLIUM_OVERRIDE_CPU_CAPS: avx
+      - run: python3 tools/check_skips.py build/ctest.xml
       - name: Report tests slower than 30 s
         if: ${{ !cancelled() }}
         shell: bash
@@ -221,7 +222,8 @@ jobs:
       - shell: cmd
         run: cmake --build build --target libs
       - shell: cmd
-        run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log
+        run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
+      - run: python tools/check_skips.py build/ctest.xml --job windows
       - name: Report tests slower than 30 s
         if: ${{ !cancelled() }}
         shell: bash
diff --git a/tools/check_skips.py b/tools/check_skips.py
new file mode 100644
index 000000000..0644b271f
--- /dev/null
+++ b/tools/check_skips.py
@@ -0,0 +1,45 @@
+import argparse
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+NARROW_SUBGROUPS = re.compile(r"skipped,.*subgroup", re.IGNORECASE)
+
+WINDOWS_PREFIX = "agc_driver_"
+
+
+def skips(path):
+    for case in ET.parse(path).getroot().iter("testcase"):
+        if case.find("skipped") is None:
+            continue
+        name = case.get("name")
+        if not name:
+            continue
+        output = case.find("system-out")
+        yield name, (output.text or "") if output is not None else ""
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="fail when a test is skipped that the job's device is not expected to skip")
+    parser.add_argument("junit", type=Path, help="the file written by ctest --output-junit")
+    parser.add_argument("--job", choices=("linux", "windows"), default="linux")
+    parser.add_argument("--list", action="store_true", help="print the skipped tests and why the file says they skipped")
+    args = parser.parse_args()
+    found = sorted(skips(args.junit))
+    if args.list:
+        for name, output in found:
+            reason = next((line.strip() for line in output.splitlines() if line.startswith("skipped")), "")
+            print(f"{name}: {reason}")
+        raise SystemExit
+    if args.job == "windows":
+        unexpected = [name for name, _ in found if not name.startswith(WINDOWS_PREFIX)]
+        expected = f"any {WINDOWS_PREFIX}* test, which the runner has no Vulkan driver for"
+    else:
+        unexpected = [name for name, output in found if not NARROW_SUBGROUPS.search(output)]
+        expected = "any test that reports the device's subgroups are too narrow for it"
+    for name in unexpected:
+        print(f"error: {name} skipped")
+    if unexpected:
+        print(f"{len(unexpected)} test(s) skipped for a reason that is not expected: the job lost that coverage and ctest still reports every test as passed")
+        raise SystemExit(1)
+    print(f"{len(found)} test(s) skipped, all expected ({expected})")

From ad1c3924daed9d8810a3bb26d3235e8df01ef108 Mon Sep 17 00:00:00 2001
From: Dextroid17 <217507631+Dextroid17@users.noreply.github.com>
Date: Thu, 8 Oct 2026 14:14:59 -0400
Subject: [PATCH 2/2] ci: accept the device limitations the CI jobs actually
 report

---
 tools/check_skips.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/check_skips.py b/tools/check_skips.py
index 0644b271f..5b2098c83 100644
--- a/tools/check_skips.py
+++ b/tools/check_skips.py
@@ -3,9 +3,9 @@ import re
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-NARROW_SUBGROUPS = re.compile(r"skipped,.*subgroup", re.IGNORECASE)
+DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device has no|the device reports)", re.IGNORECASE)
 
-WINDOWS_PREFIX = "agc_driver_"
+WINDOWS_PREFIX = "agc"
 
 
 def skips(path):
@@ -35,8 +35,8 @@ if __name__ == "__main__":
         unexpected = [name for name, _ in found if not name.startswith(WINDOWS_PREFIX)]
         expected = f"any {WINDOWS_PREFIX}* test, which the runner has no Vulkan driver for"
     else:
-        unexpected = [name for name, output in found if not NARROW_SUBGROUPS.search(output)]
-        expected = "any test that reports the device's subgroups are too narrow for it"
+        unexpected = [name for name, output in found if not DEVICE_LIMITATION.search(output)]
+        expected = "any test that reports the device limitation it needs, such as narrow subgroups or a missing extension"
     for name in unexpected:
         print(f"error: {name} skipped")
     if unexpected: