import argparse import hashlib import json import re import struct import subprocess import sys from collections import defaultdict from pathlib import Path import nid_names import progress ROOT = Path(__file__).resolve().parent.parent PRX = ROOT / "core" / "libs" / "prx" NID = re.compile(r"[A-Za-z0-9+\-]{11}") NID_POSTFIX = "_nid_postfix" MODULE_SUFFIX = "#" CLASSES = ("implemented", "stub", "absent", "module") PE_DIRECTORY_OFFSET = {0x20b: 112, 0x10b: 96} class AuditError(Exception): pass def take(data, offset, size, path, what): if offset < 0 or size < 0 or offset + size > len(data): raise AuditError(f"{path}: {what} at 0x{offset:x} with size {size} lies outside the file ({len(data)} bytes)") return data[offset:offset + size] def unpack(layout, data, offset, path, what): return struct.unpack(layout, take(data, offset, struct.calcsize(layout), path, what)) def cstring(data, offset, limit, path, what, encoding="utf-8", errors="surrogateescape"): end = data.find(b"\0", offset, limit) if offset < 0 or offset >= limit or end < 0: raise AuditError(f"{path}: {what} at 0x{offset:x} is not a NUL-terminated string inside 0x{limit:x} bytes") try: return data[offset:end].decode(encoding, errors=errors) except UnicodeDecodeError as error: raise AuditError(f"{path}: {what} at 0x{offset:x} is not {encoding}: {error}") from error def elf_sections(path, data): header = unpack("<16sHHIQQQIHHHHHH", data, 0, path, "ELF header") ident, section_offset, section_size, section_count = header[0], header[6], header[11], header[12] if ident[4] != 2 or ident[5] != 1 or section_size != 64: raise AuditError(f"{path}: only little-endian ELF64 with 64-byte section headers is supported (class {ident[4]}, data {ident[5]}, shentsize {section_size})") if not section_offset: raise AuditError(f"{path}: ELF file without a section header table is not supported") if not section_count: section_count = unpack("= virtual and address - virtual <= file_size and size <= file_size - (address - virtual): result = offset + address - virtual take(data, result, size, path, what) return result raise AuditError(f"{path}: {what} address 0x{address:x} is outside file-backed load segments") dynamic_segments = [segment for segment in segments if segment[0] == 2] if len(dynamic_segments) != 1: raise AuditError(f"{path}: expected one dynamic segment, found {len(dynamic_segments)}") _, _, dynamic_offset, _, _, dynamic_size, _, _ = dynamic_segments[0] if dynamic_size % 16: raise AuditError(f"{path}: dynamic segment size {dynamic_size} is not a multiple of 16") dynamic = {} needed_offsets = [] for position in range(0, dynamic_size, 16): tag, value = unpack("> 4 in (1, 2, 10) and other & 3 in (0, 3): exports.add(symbol) needed = {cstring(data, string_offset + value, string_offset + string_size, path, "needed library name") for value in needed_offsets} return imports, exports, needed def elf_exports(path, data): header = unpack("<16sHHIQQQIHHHHHH", data, 0, path, "ELF header") if not header[6]: return elf_program_dynamic_symbols(path, data)[1] sections = elf_sections(path, data) names = set() for section_index, section in enumerate(sections): kind, offset, size, link, entry = section[1], section[4], section[5], section[6], section[9] if kind != 11: continue if entry != 24 or size % 24 or link >= len(sections) or sections[link][1] != 3: raise AuditError(f"{path}: invalid dynamic symbol table (entry size {entry}, size {size}, string table section {link})") strings_offset, strings_size = sections[link][4], sections[link][5] take(data, strings_offset, strings_size, path, "dynamic string table") indexes = [candidate for candidate in sections if candidate[1] == 18 and candidate[6] == section_index] if len(indexes) > 1: raise AuditError(f"{path}: multiple extended section index tables for section {section_index}") for position in range(0, size, 24): name, info, other, index, _, _ = unpack("= len(sections): raise AuditError(f"{path}: extended symbol section index {index} lies outside the section table") elif 0 < index < 0xff00 and index >= len(sections): raise AuditError(f"{path}: symbol section index {index} lies outside the section table") binding, visibility = info >> 4, other & 3 if name == 0 or index == 0 or binding not in (1, 2, 10) or visibility not in (0, 3): continue names.add(cstring(data, strings_offset + name, strings_offset + strings_size, path, "export name")) return names def elf_imports(path, data): header = unpack("<16sHHIQQQIHHHHHH", data, 0, path, "ELF header") if not header[6]: imports, _, needed = elf_program_dynamic_symbols(path, data) return imports, needed sections = elf_sections(path, data) symbols = set() needed = set() for section in sections: kind, offset, size, link, entry = section[1], section[4], section[5], section[6], section[9] if kind == 11: if entry != 24 or size % 24 or link >= len(sections) or sections[link][1] != 3: raise AuditError(f"{path}: invalid dynamic symbol table (entry size {entry}, size {size}, string table section {link})") strings_offset, strings_size = sections[link][4], sections[link][5] take(data, strings_offset, strings_size, path, "dynamic string table") for position in range(0, size, 24): name, _, _, index, _, _ = unpack("= len(sections) or sections[link][1] != 3: raise AuditError(f"{path}: invalid dynamic section (entry size {entry}, size {size}, string table section {link})") strings_offset, strings_size = sections[link][4], sections[link][5] take(data, strings_offset, strings_size, path, "dynamic string table") for position in range(0, size, 16): tag, value = unpack("= raw_size or length > raw_size - relative: continue take(data, raw_offset + relative, length, path, f"data at RVA 0x{address:x}") return raw_offset + relative, min(raw_offset + raw_size, len(data)) raise AuditError(f"{path}: RVA 0x{address:x} is not inside any section") table = unpack("= function_count: raise AuditError(f"{path}: export ordinal {ordinal} lies outside the address table ({function_count} entries)") function_rva = unpack("6} imports {summary['references_by_class'][kind]:>6} references") for title, kind in (("Absent: no built library exports these, so the loader fails", "absent"), ("Stub: exported but throws when called", "stub")): selected = [record for record in records if record["class"] == kind] if selected: lines.append("") lines.append(f"{title} ({len(selected)})") current = False for record in selected: if record["library"] != current: current = record["library"] lines.append(f" [{current or 'library not named'}]") lines.append(f" {label(record)}") if stops: lines.append("") lines.append("Current stop (first unresolved import, in registry order)") for entry in stops: stop = entry["stop"] lines.append(f" {entry['registry']}: none" if stop is None else f" {entry['registry']}: {label(stop)} [{stop['library']}]") if missing: lines.append("") lines.append(f"Needed libraries with no file in --libs or --modules ({len(missing)})") lines.extend(f" {library}: {count} imports" for library, count in missing.items()) if summary["library_mismatch"]: lines.append("") lines.append(f"{summary['library_mismatch']} imports are exported only by a library other than the one they name: they resolve on Linux and can fail on Windows") return "\n".join(lines) def main(argv=None): parser = argparse.ArgumentParser(description="classify the imports of a converted game against the built system libraries") parser.add_argument("registries", nargs="+", type=Path, help="JSON written by 'relinker --registry'") parser.add_argument("--libs", action="append", required=True, help="directory of built .prx files; repeat for several") parser.add_argument("--modules", action="append", default=[], help="directory of the title's own modules; each module's own imports are audited") parser.add_argument("--source", default=str(PRX), help="core/libs/prx tree used to find throwing stubs (default: this repository's)") parser.add_argument("--names", help="aerolib.csv style file (NID, name per line) used to name absent imports") parser.add_argument("--json", type=Path, help="write the full result to this file") args = parser.parse_args(argv) try: if args.json: inputs = args.registries + ([Path(args.names)] if args.names else []) inputs += [file for directory in args.libs for file in directory_files(directory, "*.prx")] inputs += [file for directory in args.modules for file in directory_files(directory, "*")] inputs += list(Path(args.source).rglob("*.cpp")) output = args.json.resolve() if any(output == path.resolve() or (args.json.exists() and args.json.samefile(path)) for path in inputs): raise AuditError(f"{args.json}: JSON output would overwrite an input file") registries = [(registry, read_registry(registry)) for registry in args.registries] imports = [entry for _, entries in registries for entry in entries] exports, libraries = built_libraries(args.libs) module_imports, module_exports = read_modules(args.modules) records, missing = audit(imports, exports, source_stubs(args.source), libraries, set(module_imports), module_imports, module_exports) if args.names: attach_names(records, load_names(args.names)) summary = summarize(records, sum(record["references"] for record in records), missing) stops = [{"registry": str(path), "stop": current_stop(entries, records)} for path, entries in registries] if args.json: commit, dirty = git_stamp(Path(args.source)) json_stops = [{"registry": entry["registry"], "nid": entry["stop"]["nid"] if entry["stop"] else None, "library": entry["stop"]["library"] if entry["stop"] else None, "name": entry["stop"]["name"] if entry["stop"] else None} for entry in stops] result = {"source_commit": commit, "source_dirty": dirty, "registries": [str(path) for path in args.registries], **summary, "stops": json_stops, "imports": records} args.json.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") except AuditError as error: print(f"FAIL: {error}", file=sys.stderr) return 2 except OSError as error: print(f"FAIL: {error.strerror}: {error.filename}" if error.filename else f"FAIL: {error}", file=sys.stderr) return 2 print(render(summary, records, missing, stops)) return 1 if summary["unique_by_class"]["absent"] or missing else 0 if __name__ == "__main__": if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") raise SystemExit(main())