From 08b1b7808aae9fdfc961df0cb3071de073a1fad5 Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:22:21 +0400 Subject: [PATCH 1/2] fix(nid-names): ignore comments and literals in symbol inventories --- tools/nid_names.py | 88 ++++++++++++++++++++---------- tools/tests/test_nid_names.py | 100 ++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 28 deletions(-) diff --git a/tools/nid_names.py b/tools/nid_names.py index 35ee96582..edb87dc69 100644 --- a/tools/nid_names.py +++ b/tools/nid_names.py @@ -18,7 +18,12 @@ DEFAULT_CACHE = Path(tempfile.gettempdir()) / "anyps5-nid-db" / "aerolib.csv" CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-" SUFFIX = bytes([0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30]) -ALIAS = re.compile(r'APS5_EXPORT\("([A-Za-z0-9+\-]{11})",\s*(\w*[Uu]nknown\w*)\)') +CPP_TOKEN = re.compile( + r'(?P//[^\n]*|/\*[\s\S]*?\*/)' + r'|(?P(?:u8|u|U|L)?R"(?P[^\s()\\]{0,16})\([\s\S]*?\)(?P=delimiter)")' + r'|(?P(?:u8|u|U|L)?"(?:\\[\s\S]|[^"\\])*")' + r"|(?P(?:u8|u|U|L)?'(?:\\[\s\S]|[^'\\])*')" + r"|(?P(?:[0-9]|\.[0-9])[A-Za-z0-9_.']*|[A-Za-z_][A-Za-z0-9_]*|\S)") DEFINITION = re.compile(r"\bAPS5_VABI\s+(\w+)\s*\([^;{]*\)\s*(?:noexcept\s*)?(?:try\s*)?\{") NID_POSTFIX = "_nid_postfix" @@ -49,35 +54,58 @@ def load_db(path): finally: temporary.unlink(missing_ok=True) db = {} - with open(path, errors="replace") as handle: + with open(path, encoding="utf-8-sig", errors="replace") as handle: for line in handle: parts = line.strip().split() - if len(parts) >= 2: - db.setdefault(parts[0], parts[1]) + if len(parts) >= 2 and not parts[0].startswith("#"): + nid, name = (part.split("#", 1)[0] for part in parts[:2]) + if nid and name: + db.setdefault(nid, name) return db +def source_files(root): + return (path for path in sorted(root.rglob("*.cpp")) + if not any(part.lower() in {"test", "tests"} + for part in path.relative_to(root).parts[:-1])) + + +def source_aliases(text): + tokens = [match.group() for match in CPP_TOKEN.finditer(text.replace("\\\n", "").replace("\\\r\n", "")) + if match.group("comment") is None] + for index, token in enumerate(tokens): + if token != "APS5_EXPORT" or index + 5 >= len(tokens): + continue + opening, literal, comma, name, closing = tokens[index + 1:index + 6] + if opening != "(" or comma != "," or closing != ")": + continue + if re.fullmatch(r'"[A-Za-z0-9+\-]{11}"', literal) is None: + continue + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) and "unknown" in name.lower(): + yield literal[1:-1], name + + +def source_definitions(text): + def mask(match): + return match.group() if match.group("code") is not None else re.sub(r"[^\n]", " ", match.group()) + + return DEFINITION.finditer(CPP_TOKEN.sub(mask, text.replace("\\\n", "").replace("\\\r\n", ""))) + + def collect_unknowns(): found = {} - for path in sorted(PRX.rglob("*.cpp")): - try: - text = path.read_text(errors="replace") - except OSError: - continue - for match in ALIAS.finditer(text): - found.setdefault(match.group(1), "%s:%s" % ( - path.relative_to(ROOT), match.group(2))) + for path in source_files(PRX): + text = path.read_text(encoding="utf-8") + for nid, name in source_aliases(text): + found.setdefault(nid, "%s:%s" % (path.relative_to(ROOT), name)) return found def collect_real_names(): names = set() - for path in sorted((ROOT / "core" / "libs").rglob("*.cpp")): - try: - text = path.read_text(errors="replace") - except OSError: - continue - for match in DEFINITION.finditer(text): + for path in source_files(ROOT / "core" / "libs"): + text = path.read_text(encoding="utf-8") + for match in source_definitions(text): name = match.group(1) names.add(name) if name.endswith(NID_POSTFIX): @@ -118,26 +146,30 @@ def nid_arguments(arguments, options): return normalized -def main(): +def main(argv=None): parser = argparse.ArgumentParser( description="Suggest real names for Unknown NID stubs") parser.add_argument("--db", default=os.environ.get("ANYPS5_NID_DB"), help="NID database CSV (downloaded to a cache dir by default)") - parser.add_argument("--nid", action="append", default=[], + parser.add_argument("--nid", action="append", default=None, help="resolve only these NIDs instead of scanning the tree") parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") options = [option for action in parser._actions for option in action.option_strings] - args = parser.parse_args(nid_arguments(sys.argv[1:], options)) + arguments = sys.argv[1:] if argv is None else argv + args = parser.parse_args(nid_arguments(arguments, options)) + requested_nids = any(option_name(argument, options) == "--nid" for argument in arguments) db_path = Path(args.db) if args.db else DEFAULT_CACHE - db = load_db(db_path) - - if args.nid: - unknowns = {nid: "" for nid in args.nid} - else: - unknowns = collect_unknowns() - real_names = collect_real_names() + try: + db = load_db(db_path) + if requested_nids: + unknowns = {nid: "" for nid in (args.nid or [])} + else: + unknowns = collect_unknowns() + real_names = collect_real_names() if unknowns else set() + except (OSError, UnicodeError) as error: + parser.error(str(error)) rows = [] failures = 0 diff --git a/tools/tests/test_nid_names.py b/tools/tests/test_nid_names.py index 1c7453e93..39711f949 100644 --- a/tools/tests/test_nid_names.py +++ b/tools/tests/test_nid_names.py @@ -1,3 +1,5 @@ +import contextlib +import io import json import tempfile import sys @@ -68,6 +70,27 @@ class DatabaseCacheTests(unittest.TestCase): self.assertEqual(nid_names.load_db(cache), {"nid1": "sceExisting"}) download.assert_not_called() + def test_utf8_bom_does_not_change_the_first_nid(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "aerolib.csv" + nid = nid_names.compute_nid("sceExample") + cache.write_text(f"{nid} sceExample\n", encoding="utf-8-sig") + self.assertEqual(nid_names.load_db(cache), {nid: "sceExample"}) + + def test_comment_rows_do_not_become_database_entries(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "aerolib.csv" + nid = nid_names.compute_nid("sceExample") + cache.write_text(f"# Database names\n{nid} sceExample\n", encoding="utf-8") + self.assertEqual(nid_names.load_db(cache), {nid: "sceExample"}) + + def test_annotations_do_not_change_hash_inputs(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "aerolib.csv" + nid = nid_names.compute_nid("sceExample") + cache.write_text(f"{nid}#A#B sceExample#verified\n", encoding="utf-8") + self.assertEqual(nid_names.load_db(cache), {nid: "sceExample"}) + class NidArgumentTests(unittest.TestCase): options = ["-h", "--help", "--db", "--nid", "--json"] @@ -102,5 +125,82 @@ class NidArgumentTests(unittest.TestCase): self.assertEqual({row["nid"]: row["suggestion"] for row in rows}, database) +class SourceCollectionTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.prx = self.root / "core" / "libs" / "prx" + self.module = self.prx / "libSceExample" + self.module.mkdir(parents=True) + self.root_patch = patch.object(nid_names, "ROOT", self.root) + self.prx_patch = patch.object(nid_names, "PRX", self.prx) + self.root_patch.start() + self.prx_patch.start() + self.addCleanup(self.root_patch.stop) + self.addCleanup(self.prx_patch.stop) + + def write_source(self, text, relative="Export.cpp"): + path = self.module / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + def test_comments_and_literals_do_not_create_export_records(self): + self.write_source('''// APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00); +/* int APS5_VABI sceCommented() { return 0; } */ +auto example = R"text(APS5_EXPORT("BBBBBBBBBBB", exampleUnknown01); +int APS5_VABI sceQuoted() { return 0; })text"; +APS5_EXPORT("CCCCCCCCCCC", exampleUnknown02); +int APS5_VABI sceReal() { return 0; } +''') + self.assertEqual(nid_names.collect_unknowns(), {"CCCCCCCCCCC": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown02"}) + self.assertEqual(nid_names.collect_real_names(), {"sceReal"}) + + def test_formatted_alias_macro_is_detected(self): + self.write_source('APS5_EXPORT (\n "AAAAAAAAAAA" ,\n exampleUnknown00\n );\n') + self.assertEqual(nid_names.collect_unknowns(), {"AAAAAAAAAAA": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown00"}) + + def test_test_fixtures_are_not_runtime_implementations(self): + self.write_source('APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\nint APS5_VABI sceFixture() { return 0; }\n', "tests/Fixture.cpp") + self.assertEqual(nid_names.collect_unknowns(), {}) + self.assertEqual(nid_names.collect_real_names(), set()) + + def test_unreadable_sources_fail_instead_of_returning_incomplete_names(self): + self.write_source('int APS5_VABI sceReal() { return 0; }\n') + with patch.object(Path, "read_text", side_effect=OSError("read denied")): + with self.assertRaisesRegex(OSError, "read denied"): + nid_names.collect_unknowns() + with self.assertRaisesRegex(OSError, "read denied"): + nid_names.collect_real_names() + + def test_explicit_empty_nid_filter_does_not_scan_the_tree(self): + cache = self.root / "aerolib.csv" + cache.write_text("AAAAAAAAAAA example\n", encoding="utf-8") + output = io.StringIO() + with patch.object(sys, "argv", ["nid_names.py", "--db", str(cache), "--nid", "--json"]), patch.object(nid_names, "collect_unknowns", side_effect=AssertionError("unexpected tree scan")), contextlib.redirect_stdout(output): + self.assertEqual(nid_names.main(), 0) + self.assertEqual(json.loads(output.getvalue())["rows"], []) + + def test_empty_abbreviated_filter_skips_all_source_scans(self): + cache = self.root / "aerolib.csv" + cache.write_text("AAAAAAAAAAA example\n", encoding="utf-8") + arguments = ["nid_names.py", "--db", str(cache), "--ni", "--json"] + with patch.object(sys, "argv", arguments), patch.object(nid_names, "collect_unknowns", side_effect=AssertionError("unexpected aliases scan")), patch.object(nid_names, "collect_real_names", side_effect=AssertionError("unexpected definitions scan")), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(nid_names.main(), 0) + + def test_literals_and_digit_separators_keep_following_definitions(self): + self.write_source('''auto count = 1'000; +auto quote = '\\''; +auto string = "\\\" APS5_VABI sceQuoted() { }"; +int APS5_VABI sceReal() { return 0; } +''') + self.assertEqual(nid_names.collect_real_names(), {"sceReal"}) + + def test_spliced_line_comment_and_macro_tokens(self): + self.write_source('// continued \\\nAPS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\nAPS5_EX\\\nPORT("BBBBBBBBBBB", exampleUnknown01);\n') + self.assertEqual(nid_names.collect_unknowns(), {"BBBBBBBBBBB": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown01"}) + + if __name__ == "__main__": unittest.main() From 2f6db82da7f4fb25e2cc7bbb872665ce7b1c812d Mon Sep 17 00:00:00 2001 From: MrMha <114929652+MrMha@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:30:28 +0400 Subject: [PATCH 2/2] fix(nid-names): normalize Windows inventory paths --- tools/nid_names.py | 2 +- tools/tests/test_nid_names.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/nid_names.py b/tools/nid_names.py index edb87dc69..8dd9382e7 100644 --- a/tools/nid_names.py +++ b/tools/nid_names.py @@ -97,7 +97,7 @@ def collect_unknowns(): for path in source_files(PRX): text = path.read_text(encoding="utf-8") for nid, name in source_aliases(text): - found.setdefault(nid, "%s:%s" % (path.relative_to(ROOT), name)) + found.setdefault(nid, "%s:%s" % (path.relative_to(ROOT).as_posix(), name)) return found diff --git a/tools/tests/test_nid_names.py b/tools/tests/test_nid_names.py index 39711f949..0ac509da5 100644 --- a/tools/tests/test_nid_names.py +++ b/tools/tests/test_nid_names.py @@ -4,7 +4,7 @@ import json import tempfile import sys import unittest -from pathlib import Path +from pathlib import Path, PureWindowsPath from unittest.mock import patch from urllib.error import URLError @@ -161,6 +161,12 @@ int APS5_VABI sceReal() { return 0; } self.write_source('APS5_EXPORT (\n "AAAAAAAAAAA" ,\n exampleUnknown00\n );\n') self.assertEqual(nid_names.collect_unknowns(), {"AAAAAAAAAAA": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown00"}) + def test_windows_source_paths_use_portable_separators(self): + self.write_source('APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\n') + relative = PureWindowsPath("core/libs/prx/libSceExample/Export.cpp") + with patch.object(Path, "relative_to", return_value=relative): + self.assertEqual(nid_names.collect_unknowns(), {"AAAAAAAAAAA": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown00"}) + def test_test_fixtures_are_not_runtime_implementations(self): self.write_source('APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\nint APS5_VABI sceFixture() { return 0; }\n', "tests/Fixture.cpp") self.assertEqual(nid_names.collect_unknowns(), {})