From a534bd67caed622ce8e8035903478e422238041d Mon Sep 17 00:00:00 2001 From: boykopovar Date: Tue, 13 Oct 2026 12:47:30 +0300 Subject: [PATCH] fix(relinker): report intermediate output image progress --- .../elfpatcher/windows/WindowsElfPatcher.hpp | 4 +- .../elfpatcher/windows/WindowsTlsBuilder.hpp | 3 +- .../src/windows/WindowsPePatcher.cpp | 17 +++++-- .../src/windows/WindowsTlsBuilder.cpp | 13 ++++- .../analysis/CodeInstructionCollector.hpp | 3 +- .../UnusedNidFilter/IControlFlowGraph.hpp | 11 +---- .../src/analysis/CodeInstructionCollector.cpp | 33 +++++++++++-- .../UnusedNidFilter/ControlFlowGraph.cpp | 26 ++-------- .../relinker/src/pipeline/Conversion.cpp | 6 +-- .../relinker/tests/ConversionTests.cpp | 49 ++++++++++++++++--- 10 files changed, 114 insertions(+), 51 deletions(-) diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp index a9b348fc4..ac4be9dfe 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsElfPatcher.hpp @@ -3,18 +3,20 @@ #include #include +#include #include namespace Elfpatcher::Windows { class WindowsPePatcher : public IElfPatcher { public: - explicit WindowsPePatcher(bool windowsGui = false, std::optional> icon = std::nullopt, std::filesystem::path iconName = "icon0.png"); + explicit WindowsPePatcher(bool windowsGui = false, std::optional> icon = std::nullopt, std::filesystem::path iconName = "icon0.png", std::function progress = {}); const std::string& GetDiagnostics() const; std::vector Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, std::uint64_t originalPltGotVaddr, const std::string& runPath, bool lazyBinding, bool dependencyDiagnostics, const std::vector& trampolines) override; private: + std::function progress; bool windowsGui; std::optional> icon; std::string diagnostics; diff --git a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp index e7dc1ed7d..207ef639b 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/windows/WindowsTlsBuilder.hpp @@ -2,12 +2,13 @@ #define ELFPATCHER_WINDOWS_WINDOWSTLSBUILDER_HPP #include +#include namespace Elfpatcher::Windows { class WindowsTlsBuilder { public: - PeDirectory Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva, std::uint32_t* tlsIndexRva = nullptr) const; + PeDirectory Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva, std::uint32_t* tlsIndexRva = nullptr, const std::function& progress = {}) const; }; } diff --git a/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp b/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp index 5d1082bcb..0f91dbccc 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsPePatcher.cpp @@ -28,19 +28,24 @@ void writeGotStub(std::vector& sections, const std::uint32_t targetRv } -WindowsPePatcher::WindowsPePatcher(const bool windowsGui, std::optional> icon, std::filesystem::path iconName) : windowsGui(windowsGui), icon(std::move(icon)), iconName(std::move(iconName)) { +WindowsPePatcher::WindowsPePatcher(const bool windowsGui, std::optional> icon, std::filesystem::path iconName, std::function progress) : progress(std::move(progress)), windowsGui(windowsGui), icon(std::move(icon)), iconName(std::move(iconName)) { } const std::string& WindowsPePatcher::GetDiagnostics() const { return diagnostics; } std::vector WindowsPePatcher::Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, const std::uint64_t originalPltGotVaddr, const std::string& runPath, const bool lazyBinding, const bool dependencyDiagnostics, const std::vector& trampolines) { + const auto report = [&](std::uint64_t completed) { if (progress) progress(completed, 10000); }; + report(0); diagnostics.clear(); WindowsLoadImage image(sourceElf, originalHeaders); if (originalPltGotVaddr != 0) image.GetRva(originalPltGotVaddr, 8); + report(500); const WindowsRelocationBuilder relocationBuilder; auto relocations = relocationBuilder.Apply(image, dynamicSection); + report(1000); auto sections = image.BuildSections(); + report(1500); std::array directories{}; auto nextRva = image.GetEndRva(); bool hasProcessParameters = false; @@ -61,7 +66,8 @@ std::vector WindowsPePatcher::Patch(const std::vector WindowsPePatcher::Patch(const std::vector WindowsPePatcher::Patch(const std::vector& sections, const TlsAccess& access, cons } -PeDirectory WindowsTlsBuilder::Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva, std::uint32_t* tlsIndexRva) const { +PeDirectory WindowsTlsBuilder::Build(const std::vector& source, const std::vector& headers, const WindowsLoadImage& image, std::vector& sections, std::vector& relocations, std::uint32_t& nextRva, std::uint32_t* tlsIndexRva, const std::function& progress) const { + const auto report = [&](std::uint64_t completed) { if (progress) progress(completed, 10000); }; const Domain::ProgramHeader* tls = nullptr; const Codegen::X64InstructionDecoder decoder; std::vector accesses; std::set branchTargets; - const auto instructions = Relinker::CodeInstructionCollector().Collect(source, headers); + const auto instructions = Relinker::CodeInstructionCollector().Collect(source, headers, [&](auto completed, auto total) { report(7000 * completed / total); }); + std::uint64_t scanned = 0; for (const auto& header : headers) { if (header.Type == 7) { @@ -110,6 +112,8 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co continue; std::uint64_t decodedEnd = 0; for (auto instruction = instructions.lower_bound(header.MappedAddress); instruction != instructions.end() && *instruction - header.MappedAddress < header.FileSize; ++instruction) { + if (scanned % 4096 == 0) report(7000 + 2000 * scanned / std::max(instructions.size(), 1)); + ++scanned; const auto offset = *instruction - header.MappedAddress; const auto* bytes = source.data() + header.Offset + offset; const auto info = decoder.DecodeInstruction(bytes, header.FileSize - offset); @@ -187,6 +191,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co } } + report(9000); if (tls == nullptr) { if (!accesses.empty()) throw Domain::RelinkerException("Guest TLS access without PT_TLS"); @@ -238,7 +243,10 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co code.PatchBranch(skipCallback, code.GetRva()); code.Emit({0xc3}); + std::uint64_t patched = 0; for (const auto& access : accesses) { + if (patched % 256 == 0) report(9000 + 900 * patched / accesses.size()); + ++patched; const auto target = branchTargets.upper_bound(access.Rva); if (target != branchTargets.end() && *target < access.Rva + access.Length) throw Domain::RelinkerException("Branch enters a guest TLS instruction", *target); @@ -390,6 +398,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co code.Rip({0xe9}, CheckedRva(access.Rva + access.Length)); } + report(9900); const auto templateBytes = WindowsTlsTemplateBuilder().Build(*tls, image, sections, templateRva, relocations); std::copy(templateBytes.begin(), templateBytes.end(), data.Data.begin() + templateOffset); const auto writeAddress = [&](const std::size_t offset, const std::uint32_t rva) { diff --git a/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp b/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp index 2146ab9cd..9169186da 100644 --- a/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp +++ b/core/relinker/relinker/include/relinker/analysis/CodeInstructionCollector.hpp @@ -3,12 +3,13 @@ #include #include +#include namespace Relinker { class CodeInstructionCollector { public: - std::set Collect(const std::vector& bytes, const std::vector& headers) const; + std::set Collect(const std::vector& bytes, const std::vector& headers, const std::function& progress = {}) const; }; } diff --git a/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp b/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp index 7b1add380..77e6466fc 100644 --- a/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp +++ b/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace Relinker::UnusedNidFilter { @@ -27,15 +28,7 @@ std::unique_ptr BuildControlFlowGraph( bool followCodeAddresses = false ); -std::unique_ptr BuildControlFlowGraph( - std::span text, - VirtualAddress textVaddr, - VirtualAddress entryVaddr, - const std::vector& extraEntries, - const IRelativeRelocationIndex& relativeRelocations, - bool followCodeAddresses, - std::unordered_set& reachable -); +std::unique_ptr BuildControlFlowGraph(std::span text, VirtualAddress textVaddr, VirtualAddress entryVaddr, const std::vector& extraEntries, const IRelativeRelocationIndex& relativeRelocations, bool followCodeAddresses, std::unordered_set& reachable, const std::function& progress = {}); std::unique_ptr BuildControlFlowGraph( const std::vector& text, diff --git a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp index 876fa53ea..2fd1a9a2e 100644 --- a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp +++ b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp @@ -34,7 +34,13 @@ public: } -std::set CodeInstructionCollector::Collect(const std::vector& bytes, const std::vector& headers) const { +std::set CodeInstructionCollector::Collect(const std::vector& bytes, const std::vector& headers, const std::function& progress) const { + std::uint64_t lastProgress = 0; + const auto report = [&](std::uint64_t completed) { + if (completed <= lastProgress) return; + lastProgress = completed; + if (progress) progress(completed, 10000); + }; const auto range = [&](std::uint64_t offset, std::uint64_t size) { if (offset > bytes.size() || size > bytes.size() - offset) throw Domain::RelinkerException("Code analysis: file range exceeds image", offset); }; @@ -223,6 +229,10 @@ std::set CodeInstructionCollector::Collect(const std::ve const Codegen::X64InstructionDecoder decoder; std::vector> skipped; std::set staticTargets; + std::uint64_t functionBytes = 0; + for (const auto& [begin, end] : functions) functionBytes += end - begin; + std::uint64_t decodedBytes = 0; + report(1000); for (const auto& [begin, end] : functions) { const auto offset = fileOffset(begin, end - begin); for (auto address = begin; address < end;) { @@ -253,8 +263,14 @@ std::set CodeInstructionCollector::Collect(const std::ve addRoot(target); } address += info.Length; + decodedBytes += info.Length; + if (progress && (decodedBytes & 65535) < info.Length) report(1000 + 2500 * decodedBytes / functionBytes); } } + report(3500); + std::uint64_t scanBytes = 0; + for (const auto& header : headers) if (header.Type == 1 && (header.Flags & 1) != 0) scanBytes += header.FileSize; + std::uint64_t scannedBytes = 0; for (const auto& header : headers) { if (header.Type != 1 || (header.Flags & 1) == 0 || header.FileSize < 16) continue; range(header.Offset, header.FileSize); @@ -263,11 +279,15 @@ std::set CodeInstructionCollector::Collect(const std::ve const auto paddedStart = [&](std::size_t address) { return address % 16 == 0 && address >= 2 && text[address - 1] == 0xcc && text[address - 2] == 0xcc; }; const auto nextPaddedStart = [&](std::size_t address) { address = (address + 16) & ~std::size_t{15}; - while (address < size && !paddedStart(address)) address += 16; + while (address < size && !paddedStart(address)) { + if (progress && address % 65536 == 0) report(3500 + 2500 * (scannedBytes + address) / scanBytes); + address += 16; + } return address; }; bool afterTransfer = false; for (std::size_t position = nextPaddedStart(0); position < size;) { + if (progress && position % 65536 < 16) report(3500 + 2500 * (scannedBytes + position) / scanBytes); const auto aligned = (position + 15) & ~std::size_t{15}; const bool zeroPadding = afterTransfer && aligned != position && aligned <= size && std::all_of(text + position, text + aligned, [](const std::uint8_t value) { return value == 0; }); afterTransfer = false; @@ -301,7 +321,9 @@ std::set CodeInstructionCollector::Collect(const std::ve afterTransfer = info.FlowKind != Codegen::ControlFlowKind::Sequential && info.FlowKind != Codegen::ControlFlowKind::ConditionalBranch; position = next; } + scannedBytes += header.FileSize; } + report(6000); const auto decodesInSegment = [&](std::uint64_t address) { for (const auto& header : headers) { if (header.Type != 1 || (header.Flags & 1) == 0 || address < header.MappedAddress || address - header.MappedAddress >= header.FileSize) continue; @@ -333,7 +355,7 @@ std::set CodeInstructionCollector::Collect(const std::ve for (const auto root : roots) if (root >= header.MappedAddress && root - header.MappedAddress < header.FileSize) entries.push_back(root); if (entries.empty()) continue; const std::span text(bytes.data() + header.Offset, header.FileSize); - const auto graph = UnusedNidFilter::BuildControlFlowGraph(text, header.MappedAddress, entries.front(), entries, pointers, true, sectionReachable[i]); + const auto graph = UnusedNidFilter::BuildControlFlowGraph(text, header.MappedAddress, entries.front(), entries, pointers, true, sectionReachable[i], [&](std::uint64_t count) { report(6000 + 2500 * count / std::max(scanBytes, 1)); }); for (const auto address : graph->ReachableVaddrs()) { if (!decodedInstructions.insert(address).second) continue; instructions.insert(address); @@ -344,8 +366,12 @@ std::set CodeInstructionCollector::Collect(const std::ve } } } while (roots.size() != previousRoots); + report(8500); + std::uint64_t checked = 0; std::uint64_t previousEnd = 0; for (const auto address : instructions) { + if (progress && checked % 4096 == 0) report(8500 + 1499 * checked / instructions.size()); + ++checked; if (address < previousEnd) { std::uint64_t spanEnd = previousEnd; try { @@ -394,6 +420,7 @@ std::set CodeInstructionCollector::Collect(const std::ve break; } } + report(10000); return instructions; } diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp index 18842fd8c..bf4c1f76d 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp @@ -29,16 +29,7 @@ private: namespace { -void buildGraph( - std::span text, - VirtualAddress textVaddr, - VirtualAddress entryVaddr, - const std::vector& extraEntries, - const IRelativeRelocationIndex& relativeRelocations, - const std::vector* data, - bool followCodeAddresses, - std::unordered_set& reachable -) { +void buildGraph(std::span text, VirtualAddress textVaddr, VirtualAddress entryVaddr, const std::vector& extraEntries, const IRelativeRelocationIndex& relativeRelocations, const std::vector* data, bool followCodeAddresses, std::unordered_set& reachable, const std::function& progress) { const Codegen::X64InstructionDecoder decoder; std::unordered_set visited; std::queue worklist; @@ -90,6 +81,7 @@ void buildGraph( info = decoder.DecodeInstruction(text.data() + bufOff, available); } reachable.insert(va); + if (progress && reachable.size() % 4096 == 0) progress(reachable.size()); visited.erase(va); VirtualAddress nextVaddr = va + static_cast(info.Length); @@ -162,16 +154,8 @@ void buildGraph( } -std::unique_ptr BuildControlFlowGraph( - std::span text, - VirtualAddress textVaddr, - VirtualAddress entryVaddr, - const std::vector& extraEntries, - const IRelativeRelocationIndex& relativeRelocations, - bool followCodeAddresses, - std::unordered_set& reachable -) { - buildGraph(text, textVaddr, entryVaddr, extraEntries, relativeRelocations, nullptr, followCodeAddresses, reachable); +std::unique_ptr BuildControlFlowGraph(std::span text, VirtualAddress textVaddr, VirtualAddress entryVaddr, const std::vector& extraEntries, const IRelativeRelocationIndex& relativeRelocations, bool followCodeAddresses, std::unordered_set& reachable, const std::function& progress) { + buildGraph(text, textVaddr, entryVaddr, extraEntries, relativeRelocations, nullptr, followCodeAddresses, reachable, progress); return std::make_unique(&reachable); } @@ -208,7 +192,7 @@ std::unique_ptr BuildAddressTakenControlFlowGraph( const std::vector& data ) { std::unordered_set reachable; - buildGraph(std::span(text.data(), text.size()), textVaddr, entryVaddr, extraEntries, relativeRelocations, &data, false, reachable); + buildGraph(std::span(text.data(), text.size()), textVaddr, entryVaddr, extraEntries, relativeRelocations, &data, false, reachable, {}); return std::make_unique(std::move(reachable)); } diff --git a/core/relinker/relinker/src/pipeline/Conversion.cpp b/core/relinker/relinker/src/pipeline/Conversion.cpp index 99fa3a340..5ff4f4d23 100644 --- a/core/relinker/relinker/src/pipeline/Conversion.cpp +++ b/core/relinker/relinker/src/pipeline/Conversion.cpp @@ -102,7 +102,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti if (macos) { patcher = std::make_shared(); } else if (windows) { - patcher = std::make_shared(options.WindowsGui, std::move(input.Icon), input.InputPath.parent_path() / "sce_sys" / "icon0.png"); + patcher = std::make_shared(options.WindowsGui, std::move(input.Icon), input.InputPath.parent_path() / "sce_sys" / "icon0.png", [&](auto completed, auto total) { inputProgress.Report({Domain::ProgressStage::Image, {}, completed, total}); }); } else { patcher = std::make_shared( std::make_shared(), @@ -115,7 +115,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti ); } - inputProgress.Report({Domain::ProgressStage::Image, Io::Utf8Path(input.InputPath), 0, 1}); + if (!windows) inputProgress.Report({Domain::ProgressStage::Image, Io::Utf8Path(input.InputPath), 0, 1}); std::vector executableBytes; try { executableBytes = patcher->Patch(sourceBytes, result.OriginalHeaders, result.DynamicSection, result.OriginalPltGotVaddr, options.RunPath, options.LazyBinding, options.WindowsDiagnostics, trampolines); @@ -124,7 +124,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti throw; } - inputProgress.Report({Domain::ProgressStage::Image, Io::Utf8Path(input.InputPath), 1, 1}); + if (!windows) inputProgress.Report({Domain::ProgressStage::Image, Io::Utf8Path(input.InputPath), 1, 1}); std::string diagnostics; if (windows) diagnostics = std::static_pointer_cast(patcher)->GetDiagnostics(); inputProgress.Report({Domain::ProgressStage::Executable, Io::Utf8Path(input.InputPath), 1, 1}); diff --git a/core/relinker/relinker/tests/ConversionTests.cpp b/core/relinker/relinker/tests/ConversionTests.cpp index a1d3cd4b9..a2ff847e1 100644 --- a/core/relinker/relinker/tests/ConversionTests.cpp +++ b/core/relinker/relinker/tests/ConversionTests.cpp @@ -1,6 +1,7 @@ #include "TestProgress.hpp" #include #include +#include #include #include #include @@ -85,6 +86,19 @@ void convert() { require(output.Modules.front().Bytes.front() == output.Executable.front(), "Wrong module format"); require(progress.Events.back().Stage == Domain::ProgressStage::Executable && progress.Events.back().Completed == 1, "Conversion did not finish"); for (const auto& event : progress.Events) require(event.Completed <= event.Total, "Invalid counters"); + if (target == Relinker::TargetSystem::Windows) { + std::uint64_t previous = 0; + bool intermediate = false; + for (const auto& event : progress.Events) { + if (event.Stage != Domain::ProgressStage::Image) continue; + require(event.Input == "virtual/title/custom-name.bin", "Lost image input identity"); + require(event.Total == 10000 && event.Completed >= previous, "Image progress moved backwards"); + intermediate |= event.Completed > 0 && event.Completed < event.Total; + previous = event.Completed; + } + require(intermediate && previous == 10000, "Missing image progress"); + } + progress.Events.clear(); } auto source = input(); source.Modules.push_back({"virtual/escaped.prx", fixture()}); @@ -126,6 +140,21 @@ void scanning() { } require(previous == progress.Events.back().Total, "Instruction conversion progress did not finish"); reject([&] { scanner->ScanCodeSectionForSyscalls(code, 0, (std::uint64_t{1} << 32) + code.size()); }); + auto image = code; + image.resize(code.size() + 64, 0x90); + image.back() = 0xc3; + Io::WriteU64(image, 24, 64); + header.Offset = 64; + header.MappedAddress = 64; + previous = 0; + std::size_t updates = 0; + const auto instructions = Relinker::CodeInstructionCollector().Collect(image, {header}, [&](auto completed, auto total) { + require(completed >= previous && completed <= total, "Image analysis progress moved backwards"); + previous = completed; + ++updates; + }); + require(instructions.size() == code.size(), "Progress changed image code analysis"); + require(updates > 10 && previous == 10000, "Missing intermediate image analysis progress"); code[70000] = 0x0f; code[70001] = 0x05; progress.Events.clear(); @@ -135,20 +164,26 @@ void scanning() { class CancellingProgress final : public Domain::IProgress { public: + Domain::ProgressStage Stage = Domain::ProgressStage::Decode; void Report(const Domain::ProgressEvent& event) override { - if (event.Stage == Domain::ProgressStage::Decode && event.Completed > 0) throw std::runtime_error("cancelled"); + if (event.Stage == Stage && event.Completed > 0) throw std::runtime_error("cancelled"); } void Message(std::string_view) override {} }; void cancellation() { - CancellingProgress progress; - try { Relinker::Conversion(progress).Convert(input(), {}); } - catch (const std::runtime_error& error) { - require(std::string(error.what()) == "cancelled", "Callback exception was replaced"); - return; + for (const auto stage : {Domain::ProgressStage::Decode, Domain::ProgressStage::Image}) { + CancellingProgress progress; + progress.Stage = stage; + Relinker::ConversionOptions options; + options.Target = Relinker::TargetSystem::Windows; + try { Relinker::Conversion(progress).Convert(input(), options); } + catch (const std::runtime_error& error) { + require(std::string(error.what()) == "cancelled", "Callback exception was replaced"); + continue; + } + throw std::runtime_error("Callback exception was swallowed"); } - throw std::runtime_error("Callback exception was swallowed"); } }