Merge pull request #3672 from MrMha/fix/nid-inventory-current
fix(nid-names): ignore comments and literals in symbol inventories
This commit is contained in:
Adrià
GitHub
commit
45682f090d
|
|
@ -18,7 +18,12 @@ DEFAULT_CACHE = Path(tempfile.gettempdir()) / "anyps5-nid-db" / "aerolib.csv"
|
|||
CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"
|
||||
SUFFIX = bytes([0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1,
|
||||
0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30])
|
||||
ALIAS = re.compile(r'APS5_EXPORT\("([A-Za-z0-9+\-]{11})",\s*(\w*[Uu]nknown\w*)\)')
|
||||
CPP_TOKEN = re.compile(
|
||||
r'(?P<comment>//[^\n]*|/\*[\s\S]*?\*/)'
|
||||
r'|(?P<raw>(?:u8|u|U|L)?R"(?P<delimiter>[^\s()\\]{0,16})\([\s\S]*?\)(?P=delimiter)")'
|
||||
r'|(?P<string>(?:u8|u|U|L)?"(?:\\[\s\S]|[^"\\])*")'
|
||||
r"|(?P<char>(?:u8|u|U|L)?'(?:\\[\s\S]|[^'\\])*')"
|
||||
r"|(?P<code>(?:[0-9]|\.[0-9])[A-Za-z0-9_.']*|[A-Za-z_][A-Za-z0-9_]*|\S)")
|
||||
DEFINITION = re.compile(r"\bAPS5_VABI\s+(\w+)\s*\([^;{]*\)\s*(?:noexcept\s*)?(?:try\s*)?\{")
|
||||
NID_POSTFIX = "_nid_postfix"
|
||||
|
||||
|
|
@ -49,35 +54,58 @@ def load_db(path):
|
|||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
db = {}
|
||||
with open(path, errors="replace") as handle:
|
||||
with open(path, encoding="utf-8-sig", errors="replace") as handle:
|
||||
for line in handle:
|
||||
parts = line.strip().split()
|
||||
if len(parts) >= 2:
|
||||
db.setdefault(parts[0], parts[1])
|
||||
if len(parts) >= 2 and not parts[0].startswith("#"):
|
||||
nid, name = (part.split("#", 1)[0] for part in parts[:2])
|
||||
if nid and name:
|
||||
db.setdefault(nid, name)
|
||||
return db
|
||||
|
||||
|
||||
def source_files(root):
|
||||
return (path for path in sorted(root.rglob("*.cpp"))
|
||||
if not any(part.lower() in {"test", "tests"}
|
||||
for part in path.relative_to(root).parts[:-1]))
|
||||
|
||||
|
||||
def source_aliases(text):
|
||||
tokens = [match.group() for match in CPP_TOKEN.finditer(text.replace("\\\n", "").replace("\\\r\n", ""))
|
||||
if match.group("comment") is None]
|
||||
for index, token in enumerate(tokens):
|
||||
if token != "APS5_EXPORT" or index + 5 >= len(tokens):
|
||||
continue
|
||||
opening, literal, comma, name, closing = tokens[index + 1:index + 6]
|
||||
if opening != "(" or comma != "," or closing != ")":
|
||||
continue
|
||||
if re.fullmatch(r'"[A-Za-z0-9+\-]{11}"', literal) is None:
|
||||
continue
|
||||
if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) and "unknown" in name.lower():
|
||||
yield literal[1:-1], name
|
||||
|
||||
|
||||
def source_definitions(text):
|
||||
def mask(match):
|
||||
return match.group() if match.group("code") is not None else re.sub(r"[^\n]", " ", match.group())
|
||||
|
||||
return DEFINITION.finditer(CPP_TOKEN.sub(mask, text.replace("\\\n", "").replace("\\\r\n", "")))
|
||||
|
||||
|
||||
def collect_unknowns():
|
||||
found = {}
|
||||
for path in sorted(PRX.rglob("*.cpp")):
|
||||
try:
|
||||
text = path.read_text(errors="replace")
|
||||
except OSError:
|
||||
continue
|
||||
for match in ALIAS.finditer(text):
|
||||
found.setdefault(match.group(1), "%s:%s" % (
|
||||
path.relative_to(ROOT), match.group(2)))
|
||||
for path in source_files(PRX):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for nid, name in source_aliases(text):
|
||||
found.setdefault(nid, "%s:%s" % (path.relative_to(ROOT).as_posix(), name))
|
||||
return found
|
||||
|
||||
|
||||
def collect_real_names():
|
||||
names = set()
|
||||
for path in sorted((ROOT / "core" / "libs").rglob("*.cpp")):
|
||||
try:
|
||||
text = path.read_text(errors="replace")
|
||||
except OSError:
|
||||
continue
|
||||
for match in DEFINITION.finditer(text):
|
||||
for path in source_files(ROOT / "core" / "libs"):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for match in source_definitions(text):
|
||||
name = match.group(1)
|
||||
names.add(name)
|
||||
if name.endswith(NID_POSTFIX):
|
||||
|
|
@ -118,26 +146,30 @@ def nid_arguments(arguments, options):
|
|||
return normalized
|
||||
|
||||
|
||||
def main():
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Suggest real names for Unknown NID stubs")
|
||||
parser.add_argument("--db", default=os.environ.get("ANYPS5_NID_DB"),
|
||||
help="NID database CSV (downloaded to a cache dir by default)")
|
||||
parser.add_argument("--nid", action="append", default=[],
|
||||
parser.add_argument("--nid", action="append", default=None,
|
||||
help="resolve only these NIDs instead of scanning the tree")
|
||||
parser.add_argument("--json", action="store_true",
|
||||
help="emit machine-readable JSON")
|
||||
options = [option for action in parser._actions for option in action.option_strings]
|
||||
args = parser.parse_args(nid_arguments(sys.argv[1:], options))
|
||||
arguments = sys.argv[1:] if argv is None else argv
|
||||
args = parser.parse_args(nid_arguments(arguments, options))
|
||||
requested_nids = any(option_name(argument, options) == "--nid" for argument in arguments)
|
||||
|
||||
db_path = Path(args.db) if args.db else DEFAULT_CACHE
|
||||
db = load_db(db_path)
|
||||
|
||||
if args.nid:
|
||||
unknowns = {nid: "<cli>" for nid in args.nid}
|
||||
else:
|
||||
unknowns = collect_unknowns()
|
||||
real_names = collect_real_names()
|
||||
try:
|
||||
db = load_db(db_path)
|
||||
if requested_nids:
|
||||
unknowns = {nid: "<cli>" for nid in (args.nid or [])}
|
||||
else:
|
||||
unknowns = collect_unknowns()
|
||||
real_names = collect_real_names() if unknowns else set()
|
||||
except (OSError, UnicodeError) as error:
|
||||
parser.error(str(error))
|
||||
|
||||
rows = []
|
||||
failures = 0
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from unittest.mock import patch
|
||||
from urllib.error import URLError
|
||||
|
||||
|
|
@ -68,6 +70,27 @@ class DatabaseCacheTests(unittest.TestCase):
|
|||
self.assertEqual(nid_names.load_db(cache), {"nid1": "sceExisting"})
|
||||
download.assert_not_called()
|
||||
|
||||
def test_utf8_bom_does_not_change_the_first_nid(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
cache = Path(directory) / "aerolib.csv"
|
||||
nid = nid_names.compute_nid("sceExample")
|
||||
cache.write_text(f"{nid} sceExample\n", encoding="utf-8-sig")
|
||||
self.assertEqual(nid_names.load_db(cache), {nid: "sceExample"})
|
||||
|
||||
def test_comment_rows_do_not_become_database_entries(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
cache = Path(directory) / "aerolib.csv"
|
||||
nid = nid_names.compute_nid("sceExample")
|
||||
cache.write_text(f"# Database names\n{nid} sceExample\n", encoding="utf-8")
|
||||
self.assertEqual(nid_names.load_db(cache), {nid: "sceExample"})
|
||||
|
||||
def test_annotations_do_not_change_hash_inputs(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
cache = Path(directory) / "aerolib.csv"
|
||||
nid = nid_names.compute_nid("sceExample")
|
||||
cache.write_text(f"{nid}#A#B sceExample#verified\n", encoding="utf-8")
|
||||
self.assertEqual(nid_names.load_db(cache), {nid: "sceExample"})
|
||||
|
||||
class NidArgumentTests(unittest.TestCase):
|
||||
options = ["-h", "--help", "--db", "--nid", "--json"]
|
||||
|
||||
|
|
@ -102,5 +125,88 @@ class NidArgumentTests(unittest.TestCase):
|
|||
self.assertEqual({row["nid"]: row["suggestion"] for row in rows}, database)
|
||||
|
||||
|
||||
class SourceCollectionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.root = Path(self.directory.name)
|
||||
self.prx = self.root / "core" / "libs" / "prx"
|
||||
self.module = self.prx / "libSceExample"
|
||||
self.module.mkdir(parents=True)
|
||||
self.root_patch = patch.object(nid_names, "ROOT", self.root)
|
||||
self.prx_patch = patch.object(nid_names, "PRX", self.prx)
|
||||
self.root_patch.start()
|
||||
self.prx_patch.start()
|
||||
self.addCleanup(self.root_patch.stop)
|
||||
self.addCleanup(self.prx_patch.stop)
|
||||
|
||||
def write_source(self, text, relative="Export.cpp"):
|
||||
path = self.module / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def test_comments_and_literals_do_not_create_export_records(self):
|
||||
self.write_source('''// APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);
|
||||
/* int APS5_VABI sceCommented() { return 0; } */
|
||||
auto example = R"text(APS5_EXPORT("BBBBBBBBBBB", exampleUnknown01);
|
||||
int APS5_VABI sceQuoted() { return 0; })text";
|
||||
APS5_EXPORT("CCCCCCCCCCC", exampleUnknown02);
|
||||
int APS5_VABI sceReal() { return 0; }
|
||||
''')
|
||||
self.assertEqual(nid_names.collect_unknowns(), {"CCCCCCCCCCC": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown02"})
|
||||
self.assertEqual(nid_names.collect_real_names(), {"sceReal"})
|
||||
|
||||
def test_formatted_alias_macro_is_detected(self):
|
||||
self.write_source('APS5_EXPORT (\n "AAAAAAAAAAA" ,\n exampleUnknown00\n );\n')
|
||||
self.assertEqual(nid_names.collect_unknowns(), {"AAAAAAAAAAA": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown00"})
|
||||
|
||||
def test_windows_source_paths_use_portable_separators(self):
|
||||
self.write_source('APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\n')
|
||||
relative = PureWindowsPath("core/libs/prx/libSceExample/Export.cpp")
|
||||
with patch.object(Path, "relative_to", return_value=relative):
|
||||
self.assertEqual(nid_names.collect_unknowns(), {"AAAAAAAAAAA": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown00"})
|
||||
|
||||
def test_test_fixtures_are_not_runtime_implementations(self):
|
||||
self.write_source('APS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\nint APS5_VABI sceFixture() { return 0; }\n', "tests/Fixture.cpp")
|
||||
self.assertEqual(nid_names.collect_unknowns(), {})
|
||||
self.assertEqual(nid_names.collect_real_names(), set())
|
||||
|
||||
def test_unreadable_sources_fail_instead_of_returning_incomplete_names(self):
|
||||
self.write_source('int APS5_VABI sceReal() { return 0; }\n')
|
||||
with patch.object(Path, "read_text", side_effect=OSError("read denied")):
|
||||
with self.assertRaisesRegex(OSError, "read denied"):
|
||||
nid_names.collect_unknowns()
|
||||
with self.assertRaisesRegex(OSError, "read denied"):
|
||||
nid_names.collect_real_names()
|
||||
|
||||
def test_explicit_empty_nid_filter_does_not_scan_the_tree(self):
|
||||
cache = self.root / "aerolib.csv"
|
||||
cache.write_text("AAAAAAAAAAA example\n", encoding="utf-8")
|
||||
output = io.StringIO()
|
||||
with patch.object(sys, "argv", ["nid_names.py", "--db", str(cache), "--nid", "--json"]), patch.object(nid_names, "collect_unknowns", side_effect=AssertionError("unexpected tree scan")), contextlib.redirect_stdout(output):
|
||||
self.assertEqual(nid_names.main(), 0)
|
||||
self.assertEqual(json.loads(output.getvalue())["rows"], [])
|
||||
|
||||
def test_empty_abbreviated_filter_skips_all_source_scans(self):
|
||||
cache = self.root / "aerolib.csv"
|
||||
cache.write_text("AAAAAAAAAAA example\n", encoding="utf-8")
|
||||
arguments = ["nid_names.py", "--db", str(cache), "--ni", "--json"]
|
||||
with patch.object(sys, "argv", arguments), patch.object(nid_names, "collect_unknowns", side_effect=AssertionError("unexpected aliases scan")), patch.object(nid_names, "collect_real_names", side_effect=AssertionError("unexpected definitions scan")), contextlib.redirect_stdout(io.StringIO()):
|
||||
self.assertEqual(nid_names.main(), 0)
|
||||
|
||||
def test_literals_and_digit_separators_keep_following_definitions(self):
|
||||
self.write_source('''auto count = 1'000;
|
||||
auto quote = '\\'';
|
||||
auto string = "\\\" APS5_VABI sceQuoted() { }";
|
||||
int APS5_VABI sceReal() { return 0; }
|
||||
''')
|
||||
self.assertEqual(nid_names.collect_real_names(), {"sceReal"})
|
||||
|
||||
def test_spliced_line_comment_and_macro_tokens(self):
|
||||
self.write_source('// continued \\\nAPS5_EXPORT("AAAAAAAAAAA", exampleUnknown00);\nAPS5_EX\\\nPORT("BBBBBBBBBBB", exampleUnknown01);\n')
|
||||
self.assertEqual(nid_names.collect_unknowns(), {"BBBBBBBBBBB": "core/libs/prx/libSceExample/Export.cpp:exampleUnknown01"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue