shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(relinker): report instruction conversion progress for executables and modules

This commit is contained in:
boykopovar 2026-10-13 10:58:29 +03:00
parent b42aa8c540
commit 6224febfb0
10 changed files with 69 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/relinker/cli/include/ConsoleProgress.hpp
View File

@ -19,6 +19,7 @@ public:
case Domain::ProgressStage::Modules: label = "PRX conversion"; break;
case Domain::ProgressStage::Syscalls: label = "Syscall scan"; break;
case Domain::ProgressStage::Decode: label = "Instruction scan"; break;
case Domain::ProgressStage::Lowering: label = "Instruction conversion"; break;
case Domain::ProgressStage::Imports: label = "Import analysis"; break;
case Domain::ProgressStage::Image: label = "Output image"; break;
case Domain::ProgressStage::Write: label = "Write output"; break;

8
core/relinker/codegen/include/codegen/IAmd64OnlyConverter.hpp
View File

@ -4,6 +4,7 @@
#include <codegen/CodegenTypes.hpp>
#include <domain/Types.hpp>
#include <memory>
#include <functional>
#include <vector>
namespace Codegen {
@ -12,13 +13,10 @@ class IAmd64OnlyConverter {
public:
virtual ~IAmd64OnlyConverter() = default;
[[nodiscard]] virtual ConvertResult Convert(
std::vector<std::uint8_t> fileBytes,
const std::vector<Domain::ProgramHeader>& segments
) const = 0;
[[nodiscard]] virtual ConvertResult Convert(std::vector<std::uint8_t> fileBytes, const std::vector<Domain::ProgramHeader>& segments) const = 0;
};
std::unique_ptr<IAmd64OnlyConverter> MakeAmd64OnlyConverter(Amd64OnlyTarget target = Amd64OnlyTarget::Intel);
std::unique_ptr<IAmd64OnlyConverter> MakeAmd64OnlyConverter(Amd64OnlyTarget target = Amd64OnlyTarget::Intel, std::function<void(std::uint64_t, std::uint64_t)> progress = {});
}

6
core/relinker/codegen/include/codegen/IInstructionScanner.hpp
View File

@ -13,11 +13,7 @@ class IInstructionScanner {
public:
virtual ~IInstructionScanner() = default;
[[nodiscard]] virtual std::vector<InstructionMatch> ScanCodeSection(
const std::vector<std::uint8_t>& codeSection,
Domain::FileByteOffset codeSectionOffset,
Domain::FileByteOffset codeSectionSize
) const = 0;
[[nodiscard]] virtual std::vector<InstructionMatch> ScanCodeSection(const std::vector<std::uint8_t>& codeSection, Domain::FileByteOffset codeSectionOffset, Domain::FileByteOffset codeSectionSize) const = 0;
};
std::unique_ptr<IInstructionScanner> MakeInstructionScanner(std::function<void(std::uint64_t, std::uint64_t)> progress = {});

45
core/relinker/codegen/src/Amd64OnlyConverter.cpp
View File

@ -209,7 +209,7 @@ private:
class Amd64OnlyConverter : public IAmd64OnlyConverter {
public:
explicit Amd64OnlyConverter(const Amd64OnlyTarget target) : _matcher(MakeAmd64OnlyInstructionMatcher(target)) {}
Amd64OnlyConverter(const Amd64OnlyTarget target, std::function<void(std::uint64_t, std::uint64_t)> progress) : _matcher(MakeAmd64OnlyInstructionMatcher(target)), progress(std::move(progress)) {}
[[nodiscard]] ConvertResult Convert(
std::vector<std::uint8_t> fileBytes,
@ -224,13 +224,15 @@ private:
};
std::unique_ptr<IAmd64OnlyInstructionMatcher> _matcher;
std::unique_ptr<IInstructionScanner> _scanner = MakeInstructionScanner();
std::function<void(std::uint64_t, std::uint64_t)> progress;
void _convertSegment(
std::vector<std::uint8_t>& fileBytes,
const Domain::ProgramHeader& ph,
const std::vector<Domain::ProgramHeader>& segments,
ConvertResult& result
ConvertResult& result,
std::uint64_t completed,
std::uint64_t total
) const;
[[nodiscard]] static std::set<std::uint64_t> _collectBranchTargets(
@ -290,7 +292,9 @@ void Amd64OnlyConverter::_convertSegment(
std::vector<std::uint8_t>& fileBytes,
const Domain::ProgramHeader& ph,
const std::vector<Domain::ProgramHeader>& segments,
ConvertResult& result
ConvertResult& result,
std::uint64_t completed,
std::uint64_t total
) const {
const auto segOffset = static_cast<std::size_t>(ph.Offset);
const auto segSize = static_cast<std::size_t>(ph.FileSize);
@ -302,17 +306,23 @@ void Amd64OnlyConverter::_convertSegment(
fileBytes.begin() + static_cast<std::ptrdiff_t>(segOffset + segSize)
);
const auto matches = _atFileOffset(ph.Offset, [&] { return _scanner->ScanCodeSection(seg, 0, seg.size()); });
const auto scanner = MakeInstructionScanner([&](auto offset, auto) { if (progress) progress(completed + offset, total); });
const auto matches = _atFileOffset(ph.Offset, [&] { return scanner->ScanCodeSection(seg, 0, seg.size()); });
if (progress) progress(completed + segSize, total);
std::vector<Pending> pending;
for (std::size_t index = 0; index < matches.size(); ++index) {
if (progress && index != 0 && index % 4096 == 0) progress(completed + segSize + matches[index].Offset, total);
const auto& match = matches[index];
auto substitution = _atFileOffset(ph.Offset + match.Offset, [&] { return _matcher->Match(seg.data() + match.Offset, match.Length); });
if (substitution.has_value())
pending.push_back({index, match, std::move(*substitution)});
}
if (pending.empty())
if (progress) progress(completed + 2 * segSize, total);
if (pending.empty()) {
if (progress) progress(completed + 3 * segSize, total);
return;
}
const bool needsBranchTargets = std::any_of(pending.begin(), pending.end(), [](const Pending& item) {
return item.Substitution.Lowering == Amd64OnlyLowering::Trampoline;
@ -324,7 +334,11 @@ void Amd64OnlyConverter::_convertSegment(
pendingIndices.insert(item.Index);
std::set<std::size_t> claimed;
std::set<std::size_t> consumed;
std::size_t processed = 0;
for (const auto& item : pending) {
if (progress && processed != 0 && processed % 256 == 0) progress(completed + 2 * segSize + item.Instruction.Offset, total);
++processed;
if (consumed.contains(item.Index))
continue;
const auto& match = item.Instruction;
@ -425,6 +439,7 @@ void Amd64OnlyConverter::_convertSegment(
if (seg.size() != segSize)
throw CodegenException("Code segment size changed during Intel conversion", ph.Offset);
std::copy(seg.begin(), seg.end(), fileBytes.begin() + static_cast<std::ptrdiff_t>(segOffset));
if (progress) progress(completed + 3 * segSize, total);
}
ConvertResult Amd64OnlyConverter::Convert(
@ -432,18 +447,28 @@ ConvertResult Amd64OnlyConverter::Convert(
const std::vector<Domain::ProgramHeader>& segments
) const {
ConvertResult result{};
std::uint64_t total = 0;
for (const auto& ph : segments) {
if (ph.Type == kLoadSegment && (ph.Flags & kExecutableSegment) != 0)
_convertSegment(fileBytes, ph, segments, result);
if (ph.Type != kLoadSegment || (ph.Flags & kExecutableSegment) == 0) continue;
if (ph.FileSize > (std::numeric_limits<std::uint64_t>::max() - total) / 3) throw CodegenException("Instruction conversion progress size overflow");
total += 3 * ph.FileSize;
}
if (progress) progress(0, total);
std::uint64_t completed = 0;
for (const auto& ph : segments) {
if (ph.Type != kLoadSegment || (ph.Flags & kExecutableSegment) == 0) continue;
_convertSegment(fileBytes, ph, segments, result, completed, total);
completed += 3 * ph.FileSize;
}
if (progress) progress(total, total);
result.Bytes = std::move(fileBytes);
return result;
}
}
std::unique_ptr<IAmd64OnlyConverter> MakeAmd64OnlyConverter(const Amd64OnlyTarget target) {
return std::make_unique<Amd64OnlyConverter>(target);
std::unique_ptr<IAmd64OnlyConverter> MakeAmd64OnlyConverter(const Amd64OnlyTarget target, std::function<void(std::uint64_t, std::uint64_t)> progress) {
return std::make_unique<Amd64OnlyConverter>(target, std::move(progress));
}
}

12
core/relinker/codegen/src/InstructionScanner.cpp
View File

@ -10,21 +10,13 @@ namespace Codegen {
class InstructionScanner : public IInstructionScanner {
public:
explicit InstructionScanner(std::function<void(std::uint64_t, std::uint64_t)> progress) : progress(std::move(progress)) {}
[[nodiscard]] std::vector<InstructionMatch> ScanCodeSection(
const std::vector<std::uint8_t>& codeSection,
Domain::FileByteOffset codeSectionOffset,
Domain::FileByteOffset codeSectionSize
) const override;
[[nodiscard]] std::vector<InstructionMatch> ScanCodeSection(const std::vector<std::uint8_t>& codeSection, Domain::FileByteOffset codeSectionOffset, Domain::FileByteOffset codeSectionSize) const override;
private:
std::function<void(std::uint64_t, std::uint64_t)> progress;
};
std::vector<InstructionMatch> InstructionScanner::ScanCodeSection(
const std::vector<std::uint8_t>& codeSection,
const Domain::FileByteOffset codeSectionOffset,
const Domain::FileByteOffset codeSectionSize
) const {
std::vector<InstructionMatch> InstructionScanner::ScanCodeSection(const std::vector<std::uint8_t>& codeSection, const Domain::FileByteOffset codeSectionOffset, const Domain::FileByteOffset codeSectionSize) const {
const std::size_t limit = std::min(codeSection.size(), static_cast<std::size_t>(codeSectionSize));
const X64InstructionDecoder decoder;

2
core/relinker/domain/include/domain/IProgress.hpp
View File

@ -7,7 +7,7 @@
namespace Domain {
enum class ProgressStage { Executable, Modules, Syscalls, Decode, Imports, Image, Write };
enum class ProgressStage { Executable, Modules, Syscalls, Decode, Lowering, Imports, Image, Write };
struct ProgressEvent {
ProgressStage Stage;

39
core/relinker/relinker/src/analysis/SyscallScanner.cpp
View File

@ -71,9 +71,7 @@ static constexpr std::array<std::uint8_t, 43> SDK_MMAP_EXEC_INLINE{
};
template<std::size_t TSize>
static bool MatchesSdkInlineStub(const std::vector<std::uint8_t>& code, std::size_t syscallOffset, std::size_t limit,
const std::array<std::uint8_t, TSize>& pattern, std::size_t beforeSyscall,
const std::vector<Codegen::InstructionMatch>& matches, FileByteOffset codeOffset) {
static bool MatchesSdkInlineStub(const std::vector<std::uint8_t>& code, std::size_t syscallOffset, std::size_t limit, const std::array<std::uint8_t, TSize>& pattern, std::size_t beforeSyscall, const std::vector<Codegen::InstructionMatch>& matches, FileByteOffset codeOffset) {
if (syscallOffset < beforeSyscall || syscallOffset > limit) return false;
const auto start = syscallOffset - beforeSyscall;
if (pattern.size() > limit - start || !std::equal(pattern.begin(), pattern.end(), code.begin() + static_cast<std::ptrdiff_t>(start))) return false;
@ -82,17 +80,11 @@ static bool MatchesSdkInlineStub(const std::vector<std::uint8_t>& code, std::siz
return instruction != matches.end() && instruction->Offset == codeOffset + start;
}
static bool IsSdkMmanSyscallStub(const std::vector<std::uint8_t>& code, std::size_t syscallOffset, std::size_t limit,
const std::vector<Codegen::InstructionMatch>& matches, FileByteOffset codeOffset) {
return MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MPROTECT_INLINE, 42, matches, codeOffset)
|| MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_INLINE, 17, matches, codeOffset)
|| MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_EXEC_INLINE, 23, matches, codeOffset);
static bool IsSdkMmanSyscallStub(const std::vector<std::uint8_t>& code, std::size_t syscallOffset, std::size_t limit, const std::vector<Codegen::InstructionMatch>& matches, FileByteOffset codeOffset) {
return MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MPROTECT_INLINE, 42, matches, codeOffset) || MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_INLINE, 17, matches, codeOffset) || MatchesSdkInlineStub(code, syscallOffset, limit, SDK_MMAP_EXEC_INLINE, 23, matches, codeOffset);
}
static ForbiddenInstruction ClassifyForbiddenInstruction(
const std::vector<std::uint8_t>& codeSection,
const std::size_t opcodeOffset,
const std::size_t limit) {
static ForbiddenInstruction ClassifyForbiddenInstruction(const std::vector<std::uint8_t>& codeSection, const std::size_t opcodeOffset, const std::size_t limit) {
if (opcodeOffset + 1 >= limit)
return ForbiddenInstruction::None;
@ -133,11 +125,7 @@ static std::optional<std::uint32_t> ConstantRaxLoad(const std::uint8_t* bytes, c
return std::nullopt;
}
static std::optional<std::uint32_t> ConstantSyscallNumber(
const std::vector<std::uint8_t>& codeSection,
const FileByteOffset codeSectionOffset,
const std::vector<Codegen::InstructionMatch>& matches,
const std::size_t syscallIndex) {
static std::optional<std::uint32_t> ConstantSyscallNumber(const std::vector<std::uint8_t>& codeSection, const FileByteOffset codeSectionOffset, const std::vector<Codegen::InstructionMatch>& matches, const std::size_t syscallIndex) {
for (std::size_t step = 1; step <= SYSCALL_NUMBER_LOOKBACK && step <= syscallIndex; ++step) {
const auto& match = matches[syscallIndex - step];
const std::size_t start = static_cast<std::size_t>(match.Offset - codeSectionOffset);
@ -166,20 +154,14 @@ class SyscallScanner : public ISyscallScanner {
public:
SyscallScanner(Domain::IProgress& progress, const bool trapSdkStubs) : progress(progress), trapSdkStubs(trapSdkStubs) {}
std::vector<std::size_t> ScanCodeSectionForSyscalls(
const std::vector<std::uint8_t>& codeSection,
FileByteOffset codeSectionOffset,
FileByteOffset codeSectionSize, std::string_view input) override;
std::vector<std::size_t> ScanCodeSectionForSyscalls(const std::vector<std::uint8_t>& codeSection, FileByteOffset codeSectionOffset, FileByteOffset codeSectionSize, std::string_view input) override;
private:
Domain::IProgress& progress;
bool trapSdkStubs;
};
std::vector<std::size_t> SyscallScanner::ScanCodeSectionForSyscalls(
const std::vector<std::uint8_t>& codeSection,
const FileByteOffset codeSectionOffset,
const FileByteOffset codeSectionSize, std::string_view input) {
std::vector<std::size_t> SyscallScanner::ScanCodeSectionForSyscalls(const std::vector<std::uint8_t>& codeSection, const FileByteOffset codeSectionOffset, const FileByteOffset codeSectionSize, std::string_view input) {
const std::size_t limit = std::min(codeSection.size(), static_cast<std::size_t>(codeSectionSize));
const auto scanner = Codegen::MakeInstructionScanner([&](auto completed, auto total) { progress.Report({Domain::ProgressStage::Decode, std::string(input), completed, total}); });
@ -228,12 +210,7 @@ std::unique_ptr<ISyscallScanner> MakeSyscallScanner(Domain::IProgress& progress,
class NullSyscallScanner : public ISyscallScanner {
public:
std::vector<std::size_t> ScanCodeSectionForSyscalls(
const std::vector<std::uint8_t>&,
FileByteOffset,
FileByteOffset,
std::string_view
) override { return {}; }
std::vector<std::size_t> ScanCodeSectionForSyscalls(const std::vector<std::uint8_t>&, FileByteOffset, FileByteOffset, std::string_view) override { return {}; }
};
std::unique_ptr<ISyscallScanner> MakeNullSyscallScanner() {

2
core/relinker/relinker/src/guest/GuestModuleBuilder.cpp
View File

@ -294,7 +294,7 @@ std::vector<GuestArtifact> GuestModuleBuilder::Build(const std::filesystem::path
std::vector<Domain::ProgramHeader> codeHeaders;
for (const auto& header : image.Headers) if (header.Type == 1 && (header.Flags & 1) != 0) codeHeaders.push_back(header);
if (toIntel) {
auto converted = Codegen::MakeAmd64OnlyConverter(toRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel)->Convert(std::move(image.Bytes), image.Headers);
auto converted = Codegen::MakeAmd64OnlyConverter(toRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel, [&](auto completed, auto total) { progress.Report({Domain::ProgressStage::Lowering, Io::Utf8Path(image.SourcePath), completed, total}); })->Convert(std::move(image.Bytes), image.Headers);
image.Trampolines = std::move(converted.Trampolines);
image.Bytes = std::move(converted.Bytes);
}

2
core/relinker/relinker/src/pipeline/Conversion.cpp
View File

@ -51,7 +51,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti
std::vector<Codegen::TrampolineSite> trampolines;
if (options.ToIntel) {
const auto segments = Relinker::ElfReader(sourceBytes).ReadProgramHeaders();
auto converted = Codegen::MakeAmd64OnlyConverter(options.ToRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel)->Convert(std::move(sourceBytes), segments);
auto converted = Codegen::MakeAmd64OnlyConverter(options.ToRosetta ? Codegen::Amd64OnlyTarget::Rosetta : Codegen::Amd64OnlyTarget::Intel, [&](auto completed, auto total) { inputProgress.Report({Domain::ProgressStage::Lowering, {}, completed, total}); })->Convert(std::move(sourceBytes), segments);
sourceBytes = std::move(converted.Bytes);
trampolines = std::move(converted.Trampolines);
std::map<std::string, std::size_t> stubsByName;

16
core/relinker/relinker/tests/ConversionTests.cpp
View File

@ -2,6 +2,7 @@
#include <relinker/pipeline/Conversion.hpp>
#include <relinker/analysis/SyscallScanner.hpp>
#include <io/BufferUtils.hpp>
#include <codegen/IAmd64OnlyConverter.hpp>
#include <algorithm>
#include <array>
#include <iostream>
@ -73,6 +74,7 @@ void convert() {
for (const auto target : {Relinker::TargetSystem::Windows, Relinker::TargetSystem::Linux}) {
Relinker::ConversionOptions options;
options.Target = target;
options.ToIntel = true;
auto source = input();
source.Modules.push_back({"virtual/title/sce_module/plugin.prx", fixture()});
const auto output = converter.Convert(std::move(source), options);
@ -109,6 +111,20 @@ void scanning() {
require(scanner->ScanCodeSectionForSyscalls(code, 0, code.size(), "sample.prx").empty(), "Unexpected syscall trap");
require(std::any_of(progress.Events.begin(), progress.Events.end(), [](const auto& event) { return event.Completed > 0 && event.Completed < event.Total; }), "Only start and finish were reported");
for (const auto& event : progress.Events) require(event.Input == "sample.prx", "Lost scanner input identity");
progress.Events.clear();
Domain::ProgramHeader header{};
header.Type = 1;
header.Flags = 1;
header.FileSize = code.size();
header.MemorySize = code.size();
const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Intel, [&](auto completed, auto total) { progress.Report({Domain::ProgressStage::Lowering, "sample.prx", completed, total}); });
require(converter->Convert(code, {header}).Bytes == code, "Progress changed instruction conversion");
std::uint64_t previous = 0;
for (const auto& event : progress.Events) {
require(event.Completed >= previous && event.Completed <= event.Total, "Instruction conversion progress moved backwards");
previous = event.Completed;
}
require(previous == progress.Events.back().Total, "Instruction conversion progress did not finish");
code[70000] = 0x0f;
code[70001] = 0x05;
progress.Events.clear();