53 lines
2.6 KiB
Python
53 lines
2.6 KiB
Python
import argparse
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device (has no|reports|lacks|does not|imports no|flushes|binds fewer)|no display|no usable Vulkan device|host imports are unavailable)", re.IGNORECASE)
|
|
|
|
NO_DISPLAY = re.compile(r"skipped,.*no display", re.IGNORECASE)
|
|
|
|
WINDOWS_PREFIX = "agc"
|
|
|
|
|
|
def skips(path):
|
|
for case in ET.parse(path).getroot().iter("testcase"):
|
|
skipped = case.find("skipped")
|
|
if skipped is None:
|
|
continue
|
|
name = case.get("name")
|
|
if not name:
|
|
raise ValueError(f"{path}: skipped testcase has no name")
|
|
output = ["skipped, " + reason for reason in (skipped.get("message", ""), skipped.text or "") if reason]
|
|
output += [case.findtext(channel) or "" for channel in ("system-out", "system-err")]
|
|
yield name, "\n".join(output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="fail when a test is skipped that the job's device is not expected to skip")
|
|
parser.add_argument("junit", type=Path, help="the file written by ctest --output-junit")
|
|
parser.add_argument("--job", choices=("linux", "windows"), default="linux")
|
|
parser.add_argument("--list", action="store_true", help="print the skipped tests and why the file says they skipped")
|
|
args = parser.parse_args()
|
|
try:
|
|
found = sorted(skips(args.junit))
|
|
except (OSError, ET.ParseError, ValueError) as error:
|
|
parser.error(str(error))
|
|
if args.list:
|
|
for name, output in found:
|
|
reason = next((line.strip() for line in output.splitlines() if line.startswith("skipped")), "")
|
|
print(f"{name}: {reason}")
|
|
raise SystemExit
|
|
if args.job == "windows":
|
|
unexpected = [name for name, output in found if not name.startswith(WINDOWS_PREFIX) and not NO_DISPLAY.search(output)]
|
|
expected = f"any {WINDOWS_PREFIX}* test, which the runner has no Vulkan driver for, or a test that reports no display or Vulkan device"
|
|
else:
|
|
unexpected = [name for name, output in found if not DEVICE_LIMITATION.search(output)]
|
|
expected = "any test that reports the device limitation it needs, such as narrow subgroups, a missing extension or no display for a Vulkan window"
|
|
for name in unexpected:
|
|
print(f"error: {name} skipped")
|
|
if unexpected:
|
|
print(f"{len(unexpected)} test(s) skipped for a reason that is not expected: the job lost that coverage and ctest still reports every test as passed")
|
|
raise SystemExit(1)
|
|
print(f"{len(found)} test(s) skipped, all expected ({expected})")
|