AnyPS5/tools/check_test_registration.py

import argparse
import re
import subprocess
from pathlib import Path

TEST_OUTPUT = r"tests[\\/]"
COMMAND = re.compile(r"^\s*(add_test|set_tests_properties|subdirs)\s*\(", re.M | re.I)
ARGUMENT = re.compile(r'"(?:\\.|[^"\\])*"|\[(?P<equals>=*)\[.*?\](?P=equals)\]|[^\s()]+', re.S)

KNOWN_UNRUN = set()


def built(build):
    program, generator = None, None
    for line in (Path(build) / "CMakeCache.txt").read_text(encoding="utf-8").splitlines():
        if line.startswith("CMAKE_MAKE_PROGRAM:"):
            program = line.partition("=")[2]
        elif line.startswith("CMAKE_GENERATOR:"):
            generator = line.partition("=")[2]
    if generator not in ("Ninja", "Ninja Multi-Config"):
        raise ValueError(f"test registration audit requires a Ninja generator, got {generator!r}")
    if not program or program.endswith("-NOTFOUND"):
        raise FileNotFoundError(f"CMAKE_MAKE_PROGRAM is not set in {Path(build) / 'CMakeCache.txt'}")
    listed = subprocess.run([program, "-C", str(build), "-t", "targets", "all"],
                            capture_output=True, text=True, check=True).stdout
    names = set()
    for line in listed.splitlines():
        path, _, rule = line.partition(": ")
        if "EXECUTABLE" not in rule:
            continue
        found = re.match(rf"^{TEST_OUTPUT}(?:.*[\\/])?([^\\/]+)$", path)
        if not found:
            continue
        name = found.group(1)
        if name.endswith(".exe"):
            name = name[:-4]
        if name:
            names.add(name)
    return names


def commands(content):
    parsed_until = 0
    for command in COMMAND.finditer(content):
        if command.start() < parsed_until:
            continue
        position, arguments = command.end(), []
        while position < len(content):
            if content[position].isspace():
                position += 1
                continue
            if content[position] == ")":
                parsed_until = position + 1
                yield command.group(1).lower(), arguments
                break
            if content[position] == "#":
                position = content.find("\n", position)
                if position < 0:
                    break
                continue
            token = ARGUMENT.match(content, position)
            if token is None:
                raise ValueError(f"invalid CTest argument at position {position}")
            argument = token.group(0)
            if argument.startswith('"'):
                argument = re.sub(r"\\(.)", lambda match: {"n": "\n", "r": "\r", "t": "\t"}.get(match[1], match[1]), argument[1:-1], flags=re.S)
            elif token.group("equals") is not None:
                width = len(token.group("equals")) + 2
                argument = argument[width:-width]
            arguments.append(argument)
            position = token.end()


def registered(build):
    names = set()
    pending = [Path(build).resolve()]
    visited = set()
    while pending:
        directory = pending.pop().resolve()
        file = directory / "CTestTestfile.cmake"
        if directory in visited or not file.is_file():
            continue
        visited.add(directory)
        tests, disabled = {}, {}
        for command, arguments in commands(file.read_text(encoding="utf-8", errors="replace")):
            if command == "subdirs":
                pending.extend(directory / argument for argument in arguments)
            elif command == "add_test" and arguments:
                tests[arguments[0]] = arguments[1:]
            elif command == "set_tests_properties" and "PROPERTIES" in arguments:
                properties = arguments.index("PROPERTIES")
                values = arguments[properties + 1:]
                for key, value in zip(values[::2], values[1::2]):
                    if key == "DISABLED":
                        for name in arguments[:properties]:
                            disabled[name] = value.upper() in ("1", "ON", "YES", "TRUE", "Y")
        for test, arguments in tests.items():
            if disabled.get(test, False):
                continue
            for index, argument in enumerate(arguments):
                if index and (argument.startswith("-") or re.match(r"^[A-Za-z_]\w*=", argument)):
                    continue
                base = re.split(r"[\\/]", argument)[-1]
                if base.endswith(".exe"):
                    base = base[:-4]
                if base:
                    names.add(base)
    return names


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="fail when a test executable is built that no ctest test runs")
    parser.add_argument("build", type=Path, nargs="?", default=Path("build"), help="configured build directory")
    parser.add_argument("--list", action="store_true", help="print every test executable and whether ctest runs it")
    args = parser.parse_args()
    every = sorted(built(args.build))
    run = registered(args.build)
    if args.list:
        for name in every:
            print(f"{'run' if name in run else 'NOT RUN'}  {name}")
        raise SystemExit
    unrun = sorted(set(every) - run)
    if not every:
        print(f"no test executables found under {args.build}; is it configured with -DBUILD_TESTING=ON?")
        raise SystemExit(1)
    for name in sorted(set(unrun) & KNOWN_UNRUN):
        print(f"note: {name} is built and not run, as recorded")
    stale = sorted(KNOWN_UNRUN & run)
    for name in stale:
        print(f"error: {name} is run by ctest now; remove it from KNOWN_UNRUN")
    fresh = sorted(set(unrun) - KNOWN_UNRUN)
    for name in fresh:
        print(f"error: {name} is built but no ctest test runs it")
    if fresh:
        print(f"{len(fresh)} test executable(s) would never run: ctest reports a full pass over what it knows, so a test nobody registered is invisible in CI")
    if stale or fresh:
        raise SystemExit(1)
    print(f"{len(every)} test executable(s), {len(unrun)} built and not run as recorded, none new")