fix(tools): audit the imports of --modules files
(cherry picked from commit 2d0d707802c190f4d8ed93d01427576accbc08b1)
This commit is contained in:
Leandro Soares
MarcZX100
parent
1acfa7f788
commit
6054923cd5
|
|
@ -168,7 +168,9 @@ python3 tools/import_audit.py app.registry.json app.libc.prx.guest.prx.registry.
|
|||
| `implemented` | A built library exports the NID and its function is not a throwing stub. |
|
||||
| `stub` | Exported, but the whole function calls `NotImplemented_nid_no_patch`: the game loads and throws when it calls it. |
|
||||
| `absent` | No built library exports the NID: the loader fails. |
|
||||
| `module` | The import names a file found in `--modules` (the title's own library); its exports are not checked. |
|
||||
| `module` | The import names a file found in `--modules` (the title's own library), which is trusted to provide it. |
|
||||
|
||||
Each `--modules` file is also audited itself: its undefined symbols are classified against the built libraries and the other modules' exports, with the module file named in the report, and the libraries it needs must have a file in `--libs` or `--modules`.
|
||||
|
||||
The report also lists needed libraries that have no file in `--libs` or `--modules`, and imports exported only by a library other than the one they name; those resolve on Linux and can fail on Windows.
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ ROOT = Path(__file__).resolve().parent.parent
|
|||
PRX = ROOT / "core" / "libs" / "prx"
|
||||
NID = re.compile(r"[A-Za-z0-9+\-]{11}")
|
||||
NID_POSTFIX = "_nid_postfix"
|
||||
MODULE_SUFFIX = "#"
|
||||
CLASSES = ("implemented", "stub", "absent", "module")
|
||||
PE_DIRECTORY_OFFSET = {0x20b: 112, 0x10b: 96}
|
||||
|
||||
|
|
@ -43,17 +44,26 @@ def cstring(data, offset, limit, path, what, encoding="utf-8", errors="surrogate
|
|||
raise AuditError(f"{path}: {what} at 0x{offset:x} is not {encoding}: {error}") from error
|
||||
|
||||
|
||||
def elf_exports(path, data):
|
||||
def elf_sections(path, data):
|
||||
header = unpack("<16sHHIQQQIHHHHHH", data, 0, path, "ELF header")
|
||||
ident, section_offset, section_size, section_count = header[0], header[6], header[11], header[12]
|
||||
if ident[4] != 2 or ident[5] != 1 or section_size != 64:
|
||||
raise AuditError(f"{path}: only little-endian ELF64 with 64-byte section headers is supported (class {ident[4]}, data {ident[5]}, shentsize {section_size})")
|
||||
if not section_offset:
|
||||
raise AuditError(f"{path}: ELF exports without a section header table are not supported")
|
||||
raise AuditError(f"{path}: ELF file without a section header table is not supported")
|
||||
if not section_count:
|
||||
section_count = unpack("<IIQQQQIIQQ", data, section_offset, path, "extended section count")[5]
|
||||
take(data, section_offset, section_count * 64, path, "section header table")
|
||||
sections = [unpack("<IIQQQQIIQQ", data, section_offset + index * 64, path, f"section header {index}") for index in range(section_count)]
|
||||
return [unpack("<IIQQQQIIQQ", data, section_offset + index * 64, path, f"section header {index}") for index in range(section_count)]
|
||||
|
||||
|
||||
def elf_exports(path, data):
|
||||
try:
|
||||
sections = elf_sections(path, data)
|
||||
except AuditError as error:
|
||||
if "without a section header table" in str(error):
|
||||
return set()
|
||||
raise
|
||||
names = set()
|
||||
for section_index, section in enumerate(sections):
|
||||
kind, offset, size, link, entry = section[1], section[4], section[5], section[6], section[9]
|
||||
|
|
@ -83,6 +93,39 @@ def elf_exports(path, data):
|
|||
return names
|
||||
|
||||
|
||||
def elf_imports(path, data):
|
||||
try:
|
||||
sections = elf_sections(path, data)
|
||||
except AuditError as error:
|
||||
if "without a section header table" in str(error):
|
||||
return None
|
||||
raise
|
||||
symbols = set()
|
||||
needed = set()
|
||||
for section in sections:
|
||||
kind, offset, size, link, entry = section[1], section[4], section[5], section[6], section[9]
|
||||
if kind == 11:
|
||||
if entry != 24 or size % 24 or link >= len(sections) or sections[link][1] != 3:
|
||||
raise AuditError(f"{path}: invalid dynamic symbol table (entry size {entry}, size {size}, string table section {link})")
|
||||
strings_offset, strings_size = sections[link][4], sections[link][5]
|
||||
take(data, strings_offset, strings_size, path, "dynamic string table")
|
||||
for position in range(0, size, 24):
|
||||
name, _, _, index, _, _ = unpack("<IBBHQQ", data, offset + position, path, "dynamic symbol")
|
||||
if name and index == 0:
|
||||
symbol = cstring(data, strings_offset + name, strings_offset + strings_size, path, "import name")
|
||||
symbols.add(symbol.split(MODULE_SUFFIX, 1)[0])
|
||||
elif kind == 6:
|
||||
if entry != 16 or size % 16 or link >= len(sections) or sections[link][1] != 3:
|
||||
raise AuditError(f"{path}: invalid dynamic section (entry size {entry}, size {size}, string table section {link})")
|
||||
strings_offset, strings_size = sections[link][4], sections[link][5]
|
||||
take(data, strings_offset, strings_size, path, "dynamic string table")
|
||||
for position in range(0, size, 16):
|
||||
tag, value = unpack("<qQ", data, offset + position, path, "dynamic entry")
|
||||
if tag == 1:
|
||||
needed.add(cstring(data, strings_offset + value, strings_offset + strings_size, path, "needed library name"))
|
||||
return symbols, needed
|
||||
|
||||
|
||||
def pe_exports(path, data):
|
||||
header = unpack("<I", data, 0x3c, path, "PE header offset")[0]
|
||||
if unpack("<I", data, header, path, "PE signature")[0] != 0x4550:
|
||||
|
|
@ -190,8 +233,19 @@ def built_libraries(directories):
|
|||
return {name: sorted(files) for name, files in exports.items()}, libraries
|
||||
|
||||
|
||||
def module_files(directories):
|
||||
return {file.name for directory in directories for file in directory_files(directory, "*")}
|
||||
def read_modules(directories):
|
||||
imports = {}
|
||||
exports = defaultdict(set)
|
||||
for directory in directories:
|
||||
for file in directory_files(directory, "*"):
|
||||
data = file.read_bytes()
|
||||
own = elf_imports(file, data)
|
||||
if own is not None:
|
||||
symbols, needed = own
|
||||
imports[file.name] = (sorted(symbols), sorted(needed))
|
||||
for name in elf_exports(file, data):
|
||||
exports[name].add(file.name)
|
||||
return imports, {name: sorted(files) for name, files in exports.items()}
|
||||
|
||||
|
||||
def nid_of(name, aliases):
|
||||
|
|
@ -243,33 +297,46 @@ def check_conservation(references, records):
|
|||
raise AuditError(f"conservation failed: {references} references in, {classified} classified, unknown classes {unknown}")
|
||||
|
||||
|
||||
def audit(imports, exports, stubs, libraries, modules):
|
||||
def classify(nid, library, providers, stubs, modules):
|
||||
if library and library in modules:
|
||||
return "module"
|
||||
if not providers:
|
||||
return "absent"
|
||||
if all((provider, nid) in stubs for provider in providers):
|
||||
return "stub"
|
||||
return "implemented"
|
||||
|
||||
|
||||
def audit(imports, exports, stubs, libraries, modules, module_imports=None, module_exports=None):
|
||||
records = {}
|
||||
for nid, library in imports:
|
||||
record = records.setdefault((nid, library), {"nid": nid, "library": library, "references": 0})
|
||||
record["references"] += 1
|
||||
for module, (symbols, needed) in (module_imports or {}).items():
|
||||
for nid in symbols:
|
||||
record = records.setdefault((nid, None, module), {"nid": nid, "library": None, "module": module, "references": 0})
|
||||
record["references"] += 1
|
||||
for record in records.values():
|
||||
nid, library = record["nid"], record["library"]
|
||||
providers = exports.get(nid, [])
|
||||
if library and library in modules:
|
||||
kind = "module"
|
||||
elif not providers:
|
||||
kind = "absent"
|
||||
elif all((provider, nid) in stubs for provider in providers):
|
||||
kind = "stub"
|
||||
else:
|
||||
kind = "implemented"
|
||||
if library is None:
|
||||
providers = sorted(set(providers) | set((module_exports or {}).get(nid, [])))
|
||||
kind = classify(nid, library, providers, stubs, modules)
|
||||
record["class"] = kind
|
||||
record["providers"] = providers
|
||||
record["library_mismatch"] = kind in ("implemented", "stub") and bool(library) and library not in providers and "libc.prx" not in providers
|
||||
record["name"] = next((stubs[(provider, nid)] for provider in providers if (provider, nid) in stubs), None) if kind == "stub" else None
|
||||
ordered = sorted(records.values(), key=lambda record: (record["class"], record["library"], record["nid"]))
|
||||
check_conservation(len(imports), ordered)
|
||||
ordered = sorted(records.values(), key=lambda record: (record["class"], record["library"] or "", record.get("module") or "", record["nid"]))
|
||||
check_conservation(len(imports) + sum(len(symbols) for symbols, _ in (module_imports or {}).values()), ordered)
|
||||
missing = defaultdict(int)
|
||||
for record in ordered:
|
||||
library = record["library"]
|
||||
if library and library not in libraries and library not in modules:
|
||||
missing[library] += 1
|
||||
for _, needed in (module_imports or {}).values():
|
||||
for library in needed:
|
||||
if library not in libraries and library not in modules:
|
||||
missing[library] += 1
|
||||
return ordered, dict(sorted(missing.items()))
|
||||
|
||||
|
||||
|
|
@ -311,9 +378,11 @@ def summarize(records, references, missing):
|
|||
|
||||
def label(record):
|
||||
if record["name"] is None:
|
||||
return record["nid"]
|
||||
mark = "" if record.get("name_verified", True) else " (name does not hash to this NID)"
|
||||
return f"{record['nid']} {record['name']}{mark}"
|
||||
text = record["nid"]
|
||||
else:
|
||||
mark = "" if record.get("name_verified", True) else " (name does not hash to this NID)"
|
||||
text = f"{record['nid']} {record['name']}{mark}"
|
||||
return f"{text} (imported by {record['module']})" if record.get("module") else text
|
||||
|
||||
|
||||
def render(summary, records, missing):
|
||||
|
|
@ -325,7 +394,7 @@ def render(summary, records, missing):
|
|||
if selected:
|
||||
lines.append("")
|
||||
lines.append(f"{title} ({len(selected)})")
|
||||
current = None
|
||||
current = False
|
||||
for record in selected:
|
||||
if record["library"] != current:
|
||||
current = record["library"]
|
||||
|
|
@ -345,7 +414,7 @@ def main(argv=None):
|
|||
parser = argparse.ArgumentParser(description="classify the imports of a converted game against the built system libraries")
|
||||
parser.add_argument("registries", nargs="+", type=Path, help="JSON written by 'relinker --registry'")
|
||||
parser.add_argument("--libs", action="append", required=True, help="directory of built .prx files; repeat for several")
|
||||
parser.add_argument("--modules", action="append", default=[], help="directory of the title's own modules; libraries named there are not checked")
|
||||
parser.add_argument("--modules", action="append", default=[], help="directory of the title's own modules; each module's own imports are audited")
|
||||
parser.add_argument("--source", default=str(PRX), help="core/libs/prx tree used to find throwing stubs (default: this repository's)")
|
||||
parser.add_argument("--names", help="aerolib.csv style file (NID, name per line) used to name absent imports")
|
||||
parser.add_argument("--json", type=Path, help="write the full result to this file")
|
||||
|
|
@ -361,11 +430,11 @@ def main(argv=None):
|
|||
raise AuditError(f"{args.json}: JSON output would overwrite an input file")
|
||||
imports = [entry for registry in args.registries for entry in read_registry(registry)]
|
||||
exports, libraries = built_libraries(args.libs)
|
||||
modules = module_files(args.modules)
|
||||
records, missing = audit(imports, exports, source_stubs(args.source), libraries, modules)
|
||||
module_imports, module_exports = read_modules(args.modules)
|
||||
records, missing = audit(imports, exports, source_stubs(args.source), libraries, set(module_imports), module_imports, module_exports)
|
||||
if args.names:
|
||||
attach_names(records, load_names(args.names))
|
||||
summary = summarize(records, len(imports), missing)
|
||||
summary = summarize(records, sum(record["references"] for record in records), missing)
|
||||
if args.json:
|
||||
commit, dirty = git_stamp(Path(args.source))
|
||||
result = {"source_commit": commit, "source_dirty": dirty, "registries": [str(path) for path in args.registries], **summary, "imports": records}
|
||||
|
|
|
|||
|
|
@ -20,23 +20,36 @@ MESSAGE_NID = "wTpfglkmv34"
|
|||
MESSAGE_NAME = "sceMsgDialogProgressBarSetValue"
|
||||
|
||||
|
||||
def build_elf(symbols):
|
||||
def build_elf(symbols, needed=()):
|
||||
strings = b"\0"
|
||||
entries = [struct.pack("<IBBHQQ", 0, 0, 0, 0, 0, 0)]
|
||||
for name, index, info in symbols:
|
||||
entries.append(struct.pack("<IBBHQQ", len(strings), info, 0, index, 0, 0))
|
||||
strings += name.encode() + b"\0"
|
||||
libraries = []
|
||||
for library in needed:
|
||||
libraries.append(len(strings))
|
||||
strings += library.encode() + b"\0"
|
||||
table = b"".join(entries)
|
||||
table_offset = 64 + len(strings)
|
||||
table_offset += -table_offset % 8
|
||||
sections_offset = table_offset + len(table)
|
||||
dynamic_offset = table_offset + len(table)
|
||||
dynamic_offset += -dynamic_offset % 8
|
||||
pairs = [(1, offset) for offset in libraries]
|
||||
if pairs:
|
||||
pairs.append((0, 0))
|
||||
dynamic = b"".join(struct.pack("<qQ", tag, value) for tag, value in pairs)
|
||||
sections_offset = dynamic_offset + len(dynamic)
|
||||
sections_offset += -sections_offset % 8
|
||||
image = bytearray(sections_offset + 3 * 64)
|
||||
image = bytearray(sections_offset + (4 if dynamic else 3) * 64)
|
||||
image[64:64 + len(strings)] = strings
|
||||
image[table_offset:table_offset + len(table)] = table
|
||||
if dynamic:
|
||||
image[dynamic_offset:dynamic_offset + len(dynamic)] = dynamic
|
||||
image[sections_offset + 192:sections_offset + 256] = struct.pack("<IIQQQQIIQQ", 0, 6, 2, 0, dynamic_offset, len(dynamic), 2, 0, 8, 16)
|
||||
image[sections_offset + 64:sections_offset + 128] = struct.pack("<IIQQQQIIQQ", 0, 11, 2, 0, table_offset, len(table), 2, 1, 8, 24)
|
||||
image[sections_offset + 128:sections_offset + 192] = struct.pack("<IIQQQQIIQQ", 0, 3, 2, 0, 64, len(strings), 0, 0, 1, 0)
|
||||
image[:64] = struct.pack("<16sHHIQQQIHHHHHH", b"\x7fELF\x02\x01\x01" + bytes(9), 3, 0x3e, 1, 0, 0, sections_offset, 0, 64, 0, 0, 64, 3, 0)
|
||||
image[:64] = struct.pack("<16sHHIQQQIHHHHHH", b"\x7fELF\x02\x01\x01" + bytes(9), 3, 0x3e, 1, 0, 0, sections_offset, 0, 64, 0, 0, 64, 4 if dynamic else 3, 0)
|
||||
return bytes(image)
|
||||
|
||||
|
||||
|
|
@ -111,7 +124,7 @@ class ImportAuditTests(unittest.TestCase):
|
|||
(MESSAGE_NID, 1, GLOBAL_FUNCTION), ("AAAAAAAAAAA", 1, GLOBAL_FUNCTION), ("BBBBBBBBBBB", 1, GLOBAL_FUNCTION),
|
||||
("UNDEFINED11", 0, GLOBAL_FUNCTION), ("LOCALLOCAL1", 1, LOCAL_FUNCTION)]))
|
||||
(work.libs / "libc.prx").write_bytes(exporting("CCCCCCCCCCC"))
|
||||
(work.modules / "libSceNpCppWebApi.prx").write_bytes(b"\x7fELF")
|
||||
(work.modules / "libSceNpCppWebApi.prx").write_bytes(exporting())
|
||||
write_source(work.source, "libSceMsgDialog", STUB_SOURCE.format(name=MESSAGE_NAME) + 'APS5_EXPORT("BBBBBBBBBBB", sceFooUnknown_B);\n' + STUB_SOURCE.format(name="sceFooUnknown_B"))
|
||||
registry = work.root / "game.registry.json"
|
||||
write_registry(registry, [
|
||||
|
|
@ -139,6 +152,94 @@ class ImportAuditTests(unittest.TestCase):
|
|||
self.assertIn(f"{MESSAGE_NID} {MESSAGE_NAME}", out)
|
||||
self.assertIn("DDDDDDDDDDD", out)
|
||||
|
||||
def test_module_imports_are_audited(self):
|
||||
work = self.work
|
||||
stub = nid_names.compute_nid("sceModuleStub")
|
||||
(work.libs / "libSceA.prx").write_bytes(exporting("AAAAAAAAAAA"))
|
||||
(work.libs / "libSceB.prx").write_bytes(exporting(stub))
|
||||
write_source(work.source, "libSceB", STUB_SOURCE.format(name="sceModuleStub"))
|
||||
(work.modules / "sibling.prx").write_bytes(exporting("BBBBBBBBBBB"))
|
||||
(work.modules / "main.prx").write_bytes(build_elf([
|
||||
("AAAAAAAAAAA", 0, GLOBAL_FUNCTION), ("BBBBBBBBBBB", 0, GLOBAL_FUNCTION),
|
||||
(stub, 0, GLOBAL_FUNCTION), ("EEEEEEEEEEE", 0, GLOBAL_FUNCTION)], ["libSceA.prx", "sibling.prx"]))
|
||||
registry = work.root / "game.json"
|
||||
write_registry(registry, [])
|
||||
result = work.root / "audit.json"
|
||||
code, out, err = work.run(registry, "--json", str(result))
|
||||
self.assertEqual((code, err), (1, ""))
|
||||
data = json.loads(result.read_text())
|
||||
self.assertEqual(data["references"], 4)
|
||||
self.assertEqual(data["unique_by_class"], {"implemented": 2, "stub": 1, "absent": 1, "module": 0})
|
||||
self.assertEqual(data["missing_libraries"], {})
|
||||
records = {(record["nid"], record["module"]): record for record in data["imports"]}
|
||||
self.assertEqual(records[("AAAAAAAAAAA", "main.prx")]["class"], "implemented")
|
||||
self.assertEqual(records[("AAAAAAAAAAA", "main.prx")]["providers"], ["libSceA.prx"])
|
||||
self.assertEqual(records[("BBBBBBBBBBB", "main.prx")]["class"], "implemented")
|
||||
self.assertEqual(records[("BBBBBBBBBBB", "main.prx")]["providers"], ["sibling.prx"])
|
||||
self.assertEqual(records[(stub, "main.prx")]["class"], "stub")
|
||||
self.assertEqual(records[(stub, "main.prx")]["name"], "sceModuleStub")
|
||||
self.assertEqual(records[("EEEEEEEEEEE", "main.prx")]["class"], "absent")
|
||||
self.assertIn("[library not named]", out)
|
||||
self.assertIn("EEEEEEEEEEE (imported by main.prx)", out)
|
||||
|
||||
def test_module_needed_library_without_a_file_is_missing(self):
|
||||
work = self.work
|
||||
(work.libs / "libc.prx").write_bytes(exporting())
|
||||
(work.modules / "main.prx").write_bytes(build_elf([], ["libSceMissing.prx"]))
|
||||
registry = work.root / "game.json"
|
||||
write_registry(registry, [])
|
||||
result = work.root / "audit.json"
|
||||
code, out, err = work.run(registry, "--json", str(result))
|
||||
self.assertEqual((code, err), (1, ""))
|
||||
data = json.loads(result.read_text())
|
||||
self.assertEqual((data["references"], data["imports"]), (0, []))
|
||||
self.assertEqual(data["missing_libraries"], {"libSceMissing.prx": 1})
|
||||
self.assertIn("libSceMissing.prx", out)
|
||||
|
||||
def test_module_files_must_be_elf(self):
|
||||
work = self.work
|
||||
(work.libs / "libc.prx").write_bytes(exporting())
|
||||
(work.modules / "guest.prx").write_bytes(build_pe(["alpha"]))
|
||||
registry = work.root / "game.json"
|
||||
write_registry(registry, [])
|
||||
code, out, err = work.run(registry)
|
||||
self.assertEqual((code, out), (2, ""))
|
||||
self.assertIn("guest.prx", err)
|
||||
self.assertIn("ELF", err)
|
||||
|
||||
def test_module_without_section_headers_is_skipped(self):
|
||||
work = self.work
|
||||
(work.libs / "libSceA.prx").write_bytes(exporting("AAAAAAAAAAA"))
|
||||
raw = bytearray(build_elf([("AAAAAAAAAAA", 0, GLOBAL_FUNCTION)], []))
|
||||
struct.pack_into("<Q", raw, 40, 0)
|
||||
struct.pack_into("<H", raw, 60, 0)
|
||||
(work.modules / "stripped.prx").write_bytes(bytes(raw))
|
||||
(work.modules / "main.prx").write_bytes(build_elf([("BBBBBBBBBBB", 0, GLOBAL_FUNCTION)], []))
|
||||
registry = work.root / "game.json"
|
||||
write_registry(registry, [])
|
||||
result = work.root / "audit.json"
|
||||
code, out, err = work.run(registry, "--json", str(result))
|
||||
self.assertEqual((code, err), (1, ""))
|
||||
data = json.loads(result.read_text())
|
||||
stripped = [record for record in data["imports"] if record["module"] == "stripped.prx"]
|
||||
self.assertEqual(stripped, [])
|
||||
self.assertIn("BBBBBBBBBBB (imported by main.prx)", out)
|
||||
|
||||
def test_module_symbol_suffix_is_stripped(self):
|
||||
work = self.work
|
||||
exported = nid_names.compute_nid("sceRealName")
|
||||
(work.libs / "libSceA.prx").write_bytes(exporting(exported))
|
||||
(work.modules / "main.prx").write_bytes(build_elf([
|
||||
(exported + "#lib#mod", 0, GLOBAL_FUNCTION)], []))
|
||||
registry = work.root / "game.json"
|
||||
write_registry(registry, [])
|
||||
result = work.root / "audit.json"
|
||||
code, out, err = work.run(registry, "--json", str(result))
|
||||
self.assertEqual((code, err), (0, ""))
|
||||
data = json.loads(result.read_text())
|
||||
self.assertEqual(data["unique_by_class"], {"implemented": 1, "stub": 0, "absent": 0, "module": 0})
|
||||
self.assertEqual(data["references"], 1)
|
||||
|
||||
def test_clean_audit_exits_zero_and_empty_registry_is_empty(self):
|
||||
work = self.work
|
||||
(work.libs / "libSceA.prx").write_bytes(exporting("AAAAAAAAAAA"))
|
||||
|
|
@ -261,6 +362,15 @@ class ImportAuditTests(unittest.TestCase):
|
|||
with self.assertRaisesRegex(import_audit.AuditError, "lib.prx"):
|
||||
import_audit.elf_exports(path, image[:cut])
|
||||
|
||||
def test_elf_imports_reader(self):
|
||||
image = build_elf([("one", 1, GLOBAL_FUNCTION), ("two", 0, GLOBAL_FUNCTION), ("three", 0, 0x22)], ["libc.prx", "libSceZip.prx"])
|
||||
path = Path("mod.prx")
|
||||
self.assertEqual(import_audit.elf_imports(path, image), ({"two", "three"}, {"libc.prx", "libSceZip.prx"}))
|
||||
self.assertEqual(import_audit.elf_imports(path, exporting()), (set(), set()))
|
||||
for cut in (10, 64, len(image) - 5):
|
||||
with self.assertRaisesRegex(import_audit.AuditError, "mod.prx"):
|
||||
import_audit.elf_imports(path, image[:cut])
|
||||
|
||||
def test_pe_reader(self):
|
||||
path = Path("lib.prx")
|
||||
self.assertEqual(import_audit.pe_exports(path, build_pe(["alpha", "beta"])), {"alpha", "beta"})
|
||||
|
|
|
|||
Loading…
Reference in New Issue