diff --git a/core/relinker/cli/include/ConsoleProgress.hpp b/core/relinker/cli/include/ConsoleProgress.hpp index 63a485e51..9030cceff 100644 --- a/core/relinker/cli/include/ConsoleProgress.hpp +++ b/core/relinker/cli/include/ConsoleProgress.hpp @@ -19,6 +19,7 @@ public: case Domain::ProgressStage::Modules: label = "PRX conversion"; break; case Domain::ProgressStage::Syscalls: label = "Syscall scan"; break; case Domain::ProgressStage::Decode: label = "Instruction scan"; break; + case Domain::ProgressStage::Lowering: label = "Instruction conversion"; break; case Domain::ProgressStage::Imports: label = "Import analysis"; break; case Domain::ProgressStage::Image: label = "Output image"; break; case Domain::ProgressStage::Write: label = "Write output"; break; diff --git a/core/relinker/codegen/include/codegen/IAmd64OnlyConverter.hpp b/core/relinker/codegen/include/codegen/IAmd64OnlyConverter.hpp index e133c533f..ef1718927 100644 --- a/core/relinker/codegen/include/codegen/IAmd64OnlyConverter.hpp +++ b/core/relinker/codegen/include/codegen/IAmd64OnlyConverter.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace Codegen { @@ -12,13 +13,10 @@ class IAmd64OnlyConverter { public: virtual ~IAmd64OnlyConverter() = default; - [[nodiscard]] virtual ConvertResult Convert( - std::vector fileBytes, - const std::vector& segments - ) const = 0; + [[nodiscard]] virtual ConvertResult Convert(std::vector fileBytes, const std::vector& segments) const = 0; }; -std::unique_ptr MakeAmd64OnlyConverter(Amd64OnlyTarget target = Amd64OnlyTarget::Intel); +std::unique_ptr MakeAmd64OnlyConverter(Amd64OnlyTarget target = Amd64OnlyTarget::Intel, std::function progress = {}); } diff --git a/core/relinker/codegen/include/codegen/IInstructionScanner.hpp b/core/relinker/codegen/include/codegen/IInstructionScanner.hpp index 515ff800c..969ea443b 100644 --- a/core/relinker/codegen/include/codegen/IInstructionScanner.hpp +++ b/core/relinker/codegen/include/codegen/IInstructionScanner.hpp @@ -13,11 +13,7 @@ class IInstructionScanner { public: virtual ~IInstructionScanner() = default; - [[nodiscard]] virtual std::vector ScanCodeSection( - const std::vector& codeSection, - Domain::FileByteOffset codeSectionOffset, - Domain::FileByteOffset codeSectionSize - ) const = 0; + [[nodiscard]] virtual std::vector ScanCodeSection(const std::vector& codeSection, Domain::FileByteOffset codeSectionOffset, Domain::FileByteOffset codeSectionSize) const = 0; }; std::unique_ptr MakeInstructionScanner(std::function progress = {}); diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 6990cf40f..7e1966132 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -209,7 +209,7 @@ private: class Amd64OnlyConverter : public IAmd64OnlyConverter { public: - explicit Amd64OnlyConverter(const Amd64OnlyTarget target) : _matcher(MakeAmd64OnlyInstructionMatcher(target)) {} + Amd64OnlyConverter(const Amd64OnlyTarget target, std::function progress) : _matcher(MakeAmd64OnlyInstructionMatcher(target)), progress(std::move(progress)) {} [[nodiscard]] ConvertResult Convert( std::vector fileBytes, @@ -224,13 +224,15 @@ private: }; std::unique_ptr _matcher; - std::unique_ptr _scanner = MakeInstructionScanner(); + std::function progress; void _convertSegment( std::vector& fileBytes, const Domain::ProgramHeader& ph, const std::vector& segments, - ConvertResult& result + ConvertResult& result, + std::uint64_t completed, + std::uint64_t total ) const; [[nodiscard]] static std::set _collectBranchTargets( @@ -290,7 +292,9 @@ void Amd64OnlyConverter::_convertSegment( std::vector& fileBytes, const Domain::ProgramHeader& ph, const std::vector& segments, - ConvertResult& result + ConvertResult& result, + std::uint64_t completed, + std::uint64_t total ) const { const auto segOffset = static_cast(ph.Offset); const auto segSize = static_cast(ph.FileSize); @@ -302,17 +306,23 @@ void Amd64OnlyConverter::_convertSegment( fileBytes.begin() + static_cast(segOffset + segSize) ); - const auto matches = _atFileOffset(ph.Offset, [&] { return _scanner->ScanCodeSection(seg, 0, seg.size()); }); + const auto scanner = MakeInstructionScanner([&](auto offset, auto) { if (progress) progress(completed + offset, total); }); + const auto matches = _atFileOffset(ph.Offset, [&] { return scanner->ScanCodeSection(seg, 0, seg.size()); }); + if (progress) progress(completed + segSize, total); std::vector pending; for (std::size_t index = 0; index < matches.size(); ++index) { + if (progress && index != 0 && index % 4096 == 0) progress(completed + segSize + matches[index].Offset, total); const auto& match = matches[index]; auto substitution = _atFileOffset(ph.Offset + match.Offset, [&] { return _matcher->Match(seg.data() + match.Offset, match.Length); }); if (substitution.has_value()) pending.push_back({index, match, std::move(*substitution)}); } - if (pending.empty()) + if (progress) progress(completed + 2 * segSize, total); + if (pending.empty()) { + if (progress) progress(completed + 3 * segSize, total); return; + } const bool needsBranchTargets = std::any_of(pending.begin(), pending.end(), [](const Pending& item) { return item.Substitution.Lowering == Amd64OnlyLowering::Trampoline; @@ -324,7 +334,11 @@ void Amd64OnlyConverter::_convertSegment( pendingIndices.insert(item.Index); std::set claimed; std::set consumed; + + std::size_t processed = 0; for (const auto& item : pending) { + if (progress && processed != 0 && processed % 256 == 0) progress(completed + 2 * segSize + item.Instruction.Offset, total); + ++processed; if (consumed.contains(item.Index)) continue; const auto& match = item.Instruction; @@ -425,6 +439,7 @@ void Amd64OnlyConverter::_convertSegment( if (seg.size() != segSize) throw CodegenException("Code segment size changed during Intel conversion", ph.Offset); std::copy(seg.begin(), seg.end(), fileBytes.begin() + static_cast(segOffset)); + if (progress) progress(completed + 3 * segSize, total); } ConvertResult Amd64OnlyConverter::Convert( @@ -432,18 +447,28 @@ ConvertResult Amd64OnlyConverter::Convert( const std::vector& segments ) const { ConvertResult result{}; + std::uint64_t total = 0; for (const auto& ph : segments) { - if (ph.Type == kLoadSegment && (ph.Flags & kExecutableSegment) != 0) - _convertSegment(fileBytes, ph, segments, result); + if (ph.Type != kLoadSegment || (ph.Flags & kExecutableSegment) == 0) continue; + if (ph.FileSize > (std::numeric_limits::max() - total) / 3) throw CodegenException("Instruction conversion progress size overflow"); + total += 3 * ph.FileSize; } + if (progress) progress(0, total); + std::uint64_t completed = 0; + for (const auto& ph : segments) { + if (ph.Type != kLoadSegment || (ph.Flags & kExecutableSegment) == 0) continue; + _convertSegment(fileBytes, ph, segments, result, completed, total); + completed += 3 * ph.FileSize; + } + if (progress) progress(total, total); result.Bytes = std::move(fileBytes); return result; } } -std::unique_ptr MakeAmd64OnlyConverter(const Amd64OnlyTarget target) { - return std::make_unique(target); +std::unique_ptr MakeAmd64OnlyConverter(const Amd64OnlyTarget target, std::function progress) { + return std::make_unique(target, std::move(progress)); } } diff --git a/core/relinker/codegen/src/InstructionScanner.cpp b/core/relinker/codegen/src/InstructionScanner.cpp index 37aa11e06..3797a2b09 100644 --- a/core/relinker/codegen/src/InstructionScanner.cpp +++ b/core/relinker/codegen/src/InstructionScanner.cpp @@ -10,21 +10,13 @@ namespace Codegen { class InstructionScanner : public IInstructionScanner { public: explicit InstructionScanner(std::function progress) : progress(std::move(progress)) {} - [[nodiscard]] std::vector ScanCodeSection( - const std::vector& codeSection, - Domain::FileByteOffset codeSectionOffset, - Domain::FileByteOffset codeSectionSize - ) const override; + [[nodiscard]] std::vector ScanCodeSection(const std::vector& codeSection, Domain::FileByteOffset codeSectionOffset, Domain::FileByteOffset codeSectionSize) const override; private: std::function progress; }; -std::vector InstructionScanner::ScanCodeSection( - const std::vector& codeSection, - const Domain::FileByteOffset codeSectionOffset, - const Domain::FileByteOffset codeSectionSize -) const { +std::vector InstructionScanner::ScanCodeSection(const std::vector& codeSection, const Domain::FileByteOffset codeSectionOffset, const Domain::FileByteOffset codeSectionSize) const { const std::size_t limit = std::min(codeSection.size(), static_cast(codeSectionSize)); const X64InstructionDecoder decoder; diff --git a/core/relinker/domain/include/domain/IProgress.hpp b/core/relinker/domain/include/domain/IProgress.hpp index b48ad7f6c..27f40568d 100644 --- a/core/relinker/domain/include/domain/IProgress.hpp +++ b/core/relinker/domain/include/domain/IProgress.hpp @@ -7,7 +7,7 @@ namespace Domain { -enum class ProgressStage { Executable, Modules, Syscalls, Decode, Imports, Image, Write }; +enum class ProgressStage { Executable, Modules, Syscalls, Decode, Lowering, Imports, Image, Write }; struct ProgressEvent { ProgressStage Stage; diff --git a/core/relinker/relinker/src/analysis/SyscallScanner.cpp b/core/relinker/relinker/src/analysis/SyscallScanner.cpp index d8a880fef..dccd9816a 100644 --- a/core/relinker/relinker/src/analysis/SyscallScanner.cpp +++ b/core/relinker/relinker/src/analysis/SyscallScanner.cpp @@ -71,9 +71,7 @@ static constexpr std::array SDK_MMAP_EXEC_INLINE{ }; template -static bool MatchesSdkInlineStub(const std::vector& code, std::size_t syscallOffset, std::size_t limit, - const std::array& pattern, std::size_t beforeSyscall, - const std::vector& matches, FileByteOffset codeOffset) { +static bool MatchesSdkInlineStub(const std::vector& code, std::size_t syscallOffset, std::size_t limit, const std::array& pattern, std::size_t beforeSyscall, const std::vector& matches, FileByteOffset codeOffset) { if (syscallOffset < beforeSyscall || syscallOffset > limit) return false; const auto start = syscallOffset - beforeSyscall; if (pattern.size() > limit - start || !std::equal(pattern.begin(), pattern.end(), code.begin() + static_cast(start))) return false; @@ -82,17 +80,11 @@ static bool MatchesSdkInlineStub(const std::vector& code, std::siz return instruction != matches.end() && instruction->Offset == codeOffset + start; } -static bool IsSdkMmanSyscallStub(const std::vector& code, std::size_t syscallOffset, std::size_t limit, - const std::vector& matches, FileByteOffset codeOffset) { - return MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MPROTECT_INLINE, 42, matches, codeOffset) - || MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_INLINE, 17, matches, codeOffset) - || MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_EXEC_INLINE, 23, matches, codeOffset); +static bool IsSdkMmanSyscallStub(const std::vector& code, std::size_t syscallOffset, std::size_t limit, const std::vector& matches, FileByteOffset codeOffset) { + return MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MPROTECT_INLINE, 42, matches, codeOffset) || MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_INLINE, 17, matches, codeOffset) || MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_EXEC_INLINE, 23, matches, codeOffset); } -static ForbiddenInstruction ClassifyForbiddenInstruction( - const std::vector& codeSection, - const std::size_t opcodeOffset, - const std::size_t limit) { +static ForbiddenInstruction ClassifyForbiddenInstruction(const std::vector& codeSection, const std::size_t opcodeOffset, const std::size_t limit) { if (opcodeOffset + 1 >= limit) return ForbiddenInstruction::None; @@ -133,11 +125,7 @@ static std::optional ConstantRaxLoad(const std::uint8_t* bytes, c return std::nullopt; } -static std::optional ConstantSyscallNumber( - const std::vector& codeSection, - const FileByteOffset codeSectionOffset, - const std::vector& matches, - const std::size_t syscallIndex) { +static std::optional ConstantSyscallNumber(const std::vector& codeSection, const FileByteOffset codeSectionOffset, const std::vector& matches, const std::size_t syscallIndex) { for (std::size_t step = 1; step <= SYSCALL_NUMBER_LOOKBACK && step <= syscallIndex; ++step) { const auto& match = matches[syscallIndex - step]; const std::size_t start = static_cast(match.Offset - codeSectionOffset); @@ -166,20 +154,14 @@ class SyscallScanner : public ISyscallScanner { public: SyscallScanner(Domain::IProgress& progress, const bool trapSdkStubs) : progress(progress), trapSdkStubs(trapSdkStubs) {} - std::vector ScanCodeSectionForSyscalls( - const std::vector& codeSection, - FileByteOffset codeSectionOffset, - FileByteOffset codeSectionSize, std::string_view input) override; + std::vector ScanCodeSectionForSyscalls(const std::vector& codeSection, FileByteOffset codeSectionOffset, FileByteOffset codeSectionSize, std::string_view input) override; private: Domain::IProgress& progress; bool trapSdkStubs; }; -std::vector SyscallScanner::ScanCodeSectionForSyscalls( - const std::vector& codeSection, - const FileByteOffset codeSectionOffset, - const FileByteOffset codeSectionSize, std::string_view input) { +std::vector SyscallScanner::ScanCodeSectionForSyscalls(const std::vector& codeSection, const FileByteOffset codeSectionOffset, const FileByteOffset codeSectionSize, std::string_view input) { const std::size_t limit = std::min(codeSection.size(), static_cast(codeSectionSize)); const auto scanner = Codegen::MakeInstructionScanner([&](auto completed, auto total) { progress.Report({Domain::ProgressStage::Decode, std::string(input), completed, total}); }); @@ -228,12 +210,7 @@ std::unique_ptr MakeSyscallScanner(Domain::IProgress& progress, class NullSyscallScanner : public ISyscallScanner { public: - std::vector ScanCodeSectionForSyscalls( - const std::vector&, - FileByteOffset, - FileByteOffset, - std::string_view - ) override { return {}; } + std::vector ScanCodeSectionForSyscalls(const std::vector&, FileByteOffset, FileByteOffset, std::string_view) override { return {}; } }; std::unique_ptr MakeNullSyscallScanner() { diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp index c391a5306..de2662f6a 100644 --- a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -294,7 +294,7 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path std::vector codeHeaders; for (const auto& header : image.Headers) if (header.Type == 1 && (header.Flags & 1) != 0) codeHeaders.push_back(header); if (toIntel) { - auto converted = Codegen::MakeAmd64OnlyConverter(toRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel)->Convert(std::move(image.Bytes), image.Headers); + auto converted = Codegen::MakeAmd64OnlyConverter(toRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel, [&](auto completed, auto total) { progress.Report({Domain::ProgressStage::Lowering, Io::Utf8Path(image.SourcePath), completed, total}); })->Convert(std::move(image.Bytes), image.Headers); image.Trampolines = std::move(converted.Trampolines); image.Bytes = std::move(converted.Bytes); } diff --git a/core/relinker/relinker/src/pipeline/Conversion.cpp b/core/relinker/relinker/src/pipeline/Conversion.cpp index 1ca72e6dd..99fa3a340 100644 --- a/core/relinker/relinker/src/pipeline/Conversion.cpp +++ b/core/relinker/relinker/src/pipeline/Conversion.cpp @@ -51,7 +51,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti std::vector trampolines; if (options.ToIntel) { const auto segments = Relinker::ElfReader(sourceBytes).ReadProgramHeaders(); - auto converted = Codegen::MakeAmd64OnlyConverter(options.ToRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel)->Convert(std::move(sourceBytes), segments); + auto converted = Codegen::MakeAmd64OnlyConverter(options.ToRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel, [&](auto completed, auto total) { inputProgress.Report({Domain::ProgressStage::Lowering, {}, completed, total}); })->Convert(std::move(sourceBytes), segments); sourceBytes = std::move(converted.Bytes); trampolines = std::move(converted.Trampolines); std::map stubsByName; diff --git a/core/relinker/relinker/tests/ConversionTests.cpp b/core/relinker/relinker/tests/ConversionTests.cpp index 21240cbc7..d80f012c5 100644 --- a/core/relinker/relinker/tests/ConversionTests.cpp +++ b/core/relinker/relinker/tests/ConversionTests.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -73,6 +74,7 @@ void convert() { for (const auto target : {Relinker::TargetSystem::Windows, Relinker::TargetSystem::Linux}) { Relinker::ConversionOptions options; options.Target = target; + options.ToIntel = true; auto source = input(); source.Modules.push_back({"virtual/title/sce_module/plugin.prx", fixture()}); const auto output = converter.Convert(std::move(source), options); @@ -109,6 +111,20 @@ void scanning() { require(scanner->ScanCodeSectionForSyscalls(code, 0, code.size(), "sample.prx").empty(), "Unexpected syscall trap"); require(std::any_of(progress.Events.begin(), progress.Events.end(), [](const auto& event) { return event.Completed > 0 && event.Completed < event.Total; }), "Only start and finish were reported"); for (const auto& event : progress.Events) require(event.Input == "sample.prx", "Lost scanner input identity"); + progress.Events.clear(); + Domain::ProgramHeader header{}; + header.Type = 1; + header.Flags = 1; + header.FileSize = code.size(); + header.MemorySize = code.size(); + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Intel, [&](auto completed, auto total) { progress.Report({Domain::ProgressStage::Lowering, "sample.prx", completed, total}); }); + require(converter->Convert(code, {header}).Bytes == code, "Progress changed instruction conversion"); + std::uint64_t previous = 0; + for (const auto& event : progress.Events) { + require(event.Completed >= previous && event.Completed <= event.Total, "Instruction conversion progress moved backwards"); + previous = event.Completed; + } + require(previous == progress.Events.back().Total, "Instruction conversion progress did not finish"); code[70000] = 0x0f; code[70001] = 0x05; progress.Events.clear();