diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e2215f19..0bae125d7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,8 @@ jobs: - name: Install lavapipe for the GPU execution tests run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON + - name: Check that every test executable is registered with ctest + run: python3 tools/check_test_registration.py build - run: cmake --build build - run: cmake --build build --target libs - run: ctest --test-dir build --output-on-failure --timeout 120 --repeat until-fail:3 @@ -73,6 +75,9 @@ jobs: evict-old-files: job - shell: cmd run: cmake -S . -B build %CONFIGURE% + - name: Check that every test executable is registered with ctest + shell: cmd + run: python tools/check_test_registration.py build - shell: cmd run: cmake --build build - shell: cmd diff --git a/tools/check_test_registration.py b/tools/check_test_registration.py new file mode 100644 index 000000000..4931f8836 --- /dev/null +++ b/tools/check_test_registration.py @@ -0,0 +1,68 @@ +import argparse +import re +import subprocess +from pathlib import Path + +TEST_OUTPUT = "tests/" + +KNOWN_UNRUN = { + "agc_driver_bda_device_tests", + "agc_driver_mesh_tests", + "agc_driver_recorder_tests", + "guest_formatting_tests", + "video_out_flip_tests", +} + + +def built(build): + listed = subprocess.run(["ninja", "-C", str(build), "-t", "targets", "all"], + capture_output=True, text=True, check=True).stdout + names = set() + for line in listed.splitlines(): + found = re.match(rf"^{TEST_OUTPUT}([A-Za-z0-9_]+)(?::|\s|$)", line) + if found: + names.add(found.group(1)) + return names + + +def registered(build): + names = set() + for file in Path(build).rglob("CTestTestfile.cmake"): + for line in file.read_text(errors="replace").splitlines(): + found = re.match(r"\s*add_test\s*\((.*)\)\s*$", line) + if not found: + continue + for token in re.findall(r'"([^"]*)"|(\S+)', found.group(1)): + argument = token[0] or token[1] + base = re.split(r"[\\/]", argument)[-1] + if base.endswith(".exe"): + base = base[:-4] + if base: + names.add(base) + return names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="fail when a test executable is built that no ctest test runs") + parser.add_argument("build", type=Path, nargs="?", default=Path("build"), help="configured build directory") + parser.add_argument("--list", action="store_true", help="print every test executable and whether ctest runs it") + args = parser.parse_args() + every = sorted(built(args.build)) + run = registered(args.build) + if args.list: + for name in every: + print(f"{'run' if name in run else 'NOT RUN'} {name}") + raise SystemExit + unrun = sorted(set(every) - run) + if not every: + print(f"no test executables found under {args.build}; is it configured with -DBUILD_TESTING=ON?") + raise SystemExit(1) + for name in sorted(set(unrun) & KNOWN_UNRUN): + print(f"note: {name} is built and not run, as recorded") + fresh = sorted(set(unrun) - KNOWN_UNRUN) + for name in fresh: + print(f"error: {name} is built but no ctest test runs it") + if fresh: + print(f"{len(fresh)} test executable(s) would never run: ctest reports a full pass over what it knows, so a test nobody registered is invisible in CI") + raise SystemExit(1) + print(f"{len(every)} test executable(s), {len(unrun)} built and not run as recorded, none new")