loader: native trampoline for VRSQRTPS under --amd-cpu (#893)
loader: add AMD CPU compatibility patches Co-authored-by: nmzik <Nmzik@mail.ru>
This commit is contained in:
Ewen trividic
GitHub
parent
feeced81d2
commit
b11aea803c
|
|
@ -25,7 +25,7 @@ FetchContent_MakeAvailable(opus)
|
|||
set(BUILD_TESTING "${kyty_build_testing_saved}")
|
||||
install(FILES "${opus_SOURCE_DIR}/COPYING" DESTINATION "licenses/opus")
|
||||
|
||||
# The Windows guest red-zone patcher needs a decoder/encoder and a small
|
||||
# The guest instruction patcher needs a decoder/encoder and a small
|
||||
# trampoline assembler. Pin the same revisions used by shadPS4's implementation.
|
||||
set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
||||
FetchContent_Declare(
|
||||
|
|
@ -48,6 +48,7 @@ FetchContent_Declare(
|
|||
)
|
||||
|
||||
FetchContent_MakeAvailable(xbyak zydis)
|
||||
target_compile_definitions(xbyak INTERFACE XBYAK_NO_EXCEPTION)
|
||||
|
||||
# ZArchive provides random-access reads for compressed game dumps. Build only
|
||||
# the reader/writer library and its zstd dependency; the command-line tool is
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "loader/redZonePatcher.h"
|
||||
#include "loader/guestInstructionPatcher.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
#include "common/virtualMemory.h"
|
||||
#include "loader/x64InstructionEmulator.h"
|
||||
|
||||
#include <Zydis/Zydis.h>
|
||||
#include <algorithm>
|
||||
|
|
@ -22,7 +21,7 @@
|
|||
#include <set>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
#if defined(_WIN32)
|
||||
#if !defined(__APPLE__)
|
||||
#include <xbyak/xbyak.h>
|
||||
#include <xbyak/xbyak_util.h>
|
||||
#endif
|
||||
|
|
@ -34,7 +33,7 @@
|
|||
#undef max
|
||||
#endif
|
||||
|
||||
#if defined(_WIN32)
|
||||
#if !defined(__APPLE__)
|
||||
using namespace Xbyak::util;
|
||||
#endif
|
||||
|
||||
|
|
@ -53,7 +52,7 @@ using s64 = int64_t;
|
|||
|
||||
constexpr size_t NearJumpSize = 5;
|
||||
|
||||
#if defined(_WIN32)
|
||||
#if !defined(__APPLE__)
|
||||
|
||||
struct PatchModule {
|
||||
std::mutex mutex {};
|
||||
|
|
@ -62,7 +61,7 @@ struct PatchModule {
|
|||
std::set<u8*> patched;
|
||||
Xbyak::CodeGenerator patch_gen;
|
||||
Xbyak::CodeGenerator trampoline_gen;
|
||||
bool trampoline_exhausted = false;
|
||||
bool trampoline_exhaustion_reported = false;
|
||||
|
||||
PatchModule(u8* module_ptr, u64 module_size, u8* trampoline_ptr, u64 trampoline_size)
|
||||
: start(module_ptr), end(module_ptr + module_size), patch_gen(module_size, module_ptr),
|
||||
|
|
@ -81,14 +80,14 @@ static PatchModule* GetContainingModule(const void* ptr) {
|
|||
return address >= module->start && address < module->end ? module : nullptr;
|
||||
}
|
||||
|
||||
static bool HandleTrampolineError(PatchModule* module, const Xbyak::Error& error) {
|
||||
if (static_cast<int>(error) != Xbyak::ERR_CODE_IS_TOO_BIG) {
|
||||
static bool HandleTrampolineError(PatchModule* module, int error) {
|
||||
if (error != Xbyak::ERR_CODE_IS_TOO_BIG) {
|
||||
return false;
|
||||
}
|
||||
if (!module->trampoline_exhausted) {
|
||||
LOGF("Windows guest red-zone trampoline space exhausted for module %p\n",
|
||||
if (!module->trampoline_exhaustion_reported) {
|
||||
LOGF("Guest instruction trampoline space exhausted for module %p\n",
|
||||
static_cast<void*>(module->start));
|
||||
module->trampoline_exhausted = true;
|
||||
module->trampoline_exhaustion_reported = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
@ -103,9 +102,6 @@ static ZydisDecoder& GetDecoder() {
|
|||
return decoder;
|
||||
}
|
||||
|
||||
#endif
|
||||
#if defined(_WIN32)
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr size_t GuestRedZoneSize = 128;
|
||||
|
|
@ -139,6 +135,7 @@ struct DecodedFunction {
|
|||
struct InstructionRewrite {
|
||||
bool protect_red_zone {};
|
||||
bool protected_indirect_call {};
|
||||
bool emulate_rsqrt {};
|
||||
};
|
||||
|
||||
bool IsStackPointerRegister(ZydisRegister reg) {
|
||||
|
|
@ -266,7 +263,7 @@ DecodedCodeInstruction DecodeCodeInstruction(uintptr_t address, uintptr_t end) {
|
|||
const s64 access_start = operand.mem.disp.value;
|
||||
const s64 access_size = std::max<s64>(operand.size / 8, 1);
|
||||
const s64 range_start = std::max(access_start, -static_cast<s64>(GuestRedZoneSize));
|
||||
const s64 range_end = std::min(access_start + access_size, 0LL);
|
||||
const s64 range_end = std::min<s64>(access_start + access_size, 0);
|
||||
for (s64 offset = range_start; offset < range_end; ++offset) {
|
||||
const size_t bit = static_cast<size_t>(offset + static_cast<s64>(GuestRedZoneSize));
|
||||
if ((operand.actions & ZYDIS_OPERAND_ACTION_MASK_READ) != 0) {
|
||||
|
|
@ -738,9 +735,9 @@ bool GenerateProtectedIndirectCall(const DecodedCodeInstruction& decoded,
|
|||
return true;
|
||||
}
|
||||
|
||||
void CollectRedZoneMemoryInstructions(const DecodedFunction& function,
|
||||
void CollectRedZoneMemoryInstructions(const DecodedFunction& function,
|
||||
std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||
RedZonePatchResult& result) {
|
||||
GuestInstructionPatchResult& result) {
|
||||
if (!function.uses_red_zone) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -777,98 +774,173 @@ void CollectRedZoneMemoryInstructions(const DecodedFunction& function,
|
|||
}
|
||||
}
|
||||
|
||||
struct ReciprocalSquareRootSite {
|
||||
uintptr_t address;
|
||||
u8 length;
|
||||
bool requires_red_zone_protection;
|
||||
};
|
||||
// Compute 1/sqrt in double precision, rounded to float, with denormals treated as
|
||||
// signed zero. Preserve guest MXCSR, RFLAGS and the scratch register's full YMM.
|
||||
void GenerateReciprocalSquareRoot(const DecodedCodeInstruction& decoded,
|
||||
Xbyak::CodeGenerator& generator) {
|
||||
const int destination = decoded.operands[0].reg.value - ZYDIS_REGISTER_XMM0;
|
||||
const int source = decoded.operands[1].reg.value - ZYDIS_REGISTER_XMM0;
|
||||
int scratch_index = 0;
|
||||
while (scratch_index == destination || scratch_index == source) {
|
||||
++scratch_index;
|
||||
}
|
||||
const Xbyak::Ymm scratch(scratch_index);
|
||||
Xbyak::Label one;
|
||||
Xbyak::Label done;
|
||||
constexpr size_t SpillSize = 48;
|
||||
|
||||
void CollectReciprocalSquareRoots(const DecodedFunction& function,
|
||||
std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||
std::vector<ReciprocalSquareRootSite>& sites) {
|
||||
// Guest code uses the SysV red zone on both hosts; put our spills below it.
|
||||
// LEA/MOV/SIMD leave RFLAGS intact, including when the guest red zone is live.
|
||||
generator.lea(rsp, ptr[rsp - GuestRedZoneSize - SpillSize]);
|
||||
generator.vmovdqu(ptr[rsp], scratch);
|
||||
generator.stmxcsr(ptr[rsp + 32]);
|
||||
generator.mov(dword[rsp + 36], 0x1fc0); // round to nearest, exceptions masked, DAZ
|
||||
generator.ldmxcsr(ptr[rsp + 36]);
|
||||
generator.vcvtps2pd(scratch, Xbyak::Xmm(source));
|
||||
generator.vsqrtpd(scratch, scratch);
|
||||
generator.vbroadcastsd(Xbyak::Ymm(destination), ptr[rip + one]);
|
||||
generator.vdivpd(scratch, Xbyak::Ymm(destination), scratch);
|
||||
generator.vcvtpd2ps(Xbyak::Xmm(destination), scratch); // Clears upper YMM lanes.
|
||||
generator.ldmxcsr(ptr[rsp + 32]);
|
||||
generator.vmovdqu(scratch, ptr[rsp]);
|
||||
generator.lea(rsp, ptr[rsp + GuestRedZoneSize + SpillSize]);
|
||||
generator.jmp(done);
|
||||
// A failed label reference can have an incomplete displacement. Do not bind it.
|
||||
if (Xbyak::GetError() != 0) {
|
||||
return;
|
||||
}
|
||||
generator.L(one);
|
||||
generator.dq(0x3ff0000000000000ULL);
|
||||
generator.L(done);
|
||||
}
|
||||
|
||||
void CollectReciprocalSquareRoots(const DecodedFunction& function,
|
||||
std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||
GuestInstructionPatchResult& result) {
|
||||
for (const auto& [address, decoded]: function.instructions) {
|
||||
if (!X64InstructionEmulator::IsReciprocalSquareRoot(decoded.instruction,
|
||||
decoded.operands.data())) {
|
||||
const auto& instruction = decoded.instruction;
|
||||
if (instruction.mnemonic != ZYDIS_MNEMONIC_VRSQRTPS ||
|
||||
instruction.encoding != ZYDIS_INSTRUCTION_ENCODING_VEX ||
|
||||
instruction.raw.vex.offset != 0 || decoded.operands[0].size != 128 ||
|
||||
decoded.operands[1].type != ZYDIS_OPERAND_TYPE_REGISTER) {
|
||||
continue;
|
||||
}
|
||||
const bool protect_red_zone = decoded.red_zone_live.any();
|
||||
sites.push_back({address, decoded.instruction.length, protect_red_zone});
|
||||
if (protect_red_zone) {
|
||||
rewrite_sites[address].protect_red_zone = true;
|
||||
}
|
||||
++result.reciprocal_sqrt_instruction_count;
|
||||
rewrite_sites[address].emulate_rsqrt = true;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t ApplyReciprocalSquareRootPatches(const PatchModule& module,
|
||||
std::span<const ReciprocalSquareRootSite> sites,
|
||||
uint64_t trampoline_addr, uint64_t trampoline_size) {
|
||||
// Validate every required relocation before introducing any traps.
|
||||
for (const auto& site: sites) {
|
||||
if (site.requires_red_zone_protection &&
|
||||
!module.patched.contains(reinterpret_cast<u8*>(site.address))) {
|
||||
EXIT("Cannot preserve the guest red zone at emulated VRSQRTPS 0x%016" PRIx64 "\n",
|
||||
static_cast<u64>(site.address));
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t patched = 0;
|
||||
for (const auto& site: sites) {
|
||||
if (!module.patched.contains(reinterpret_cast<u8*>(site.address))) {
|
||||
patched += X64InstructionEmulator::PatchReciprocalSquareRoots(site.address, site.length);
|
||||
}
|
||||
}
|
||||
return patched +
|
||||
X64InstructionEmulator::PatchReciprocalSquareRoots(trampoline_addr, trampoline_size);
|
||||
void MarkReciprocalSquareRootTrap(u8* code, const ZydisDecodedInstruction& instruction) {
|
||||
// Clear a reserved VEX.vvvv bit, retaining both register operands for the handler.
|
||||
code[instruction.raw.vex.size - 1] &= ~0x08u;
|
||||
}
|
||||
|
||||
void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& function,
|
||||
const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||
RedZonePatchResult& result) {
|
||||
void GenerateReciprocalSquareRootTrap(const DecodedCodeInstruction& decoded,
|
||||
Xbyak::CodeGenerator& generator) {
|
||||
std::array<u8, ZYDIS_MAX_INSTRUCTION_LENGTH> code {};
|
||||
std::memcpy(code.data(), reinterpret_cast<const void*>(decoded.address),
|
||||
decoded.instruction.length);
|
||||
MarkReciprocalSquareRootTrap(code.data(), decoded.instruction);
|
||||
generator.db(code.data(), decoded.instruction.length);
|
||||
}
|
||||
|
||||
bool NeedsTrapRedZoneProtection(const DecodedCodeInstruction& decoded) {
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
return decoded.red_zone_live.any();
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
void TrapUnrelocatedReciprocalSquareRoots(
|
||||
const PatchModule& module, const DecodedFunction& function,
|
||||
const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||
GuestInstructionPatchResult& result) {
|
||||
for (const auto& [address, rewrite]: rewrite_sites) {
|
||||
if (!rewrite.emulate_rsqrt || module.patched.contains(reinterpret_cast<u8*>(address))) {
|
||||
continue;
|
||||
}
|
||||
const auto& decoded = function.instructions.at(address);
|
||||
if (NeedsTrapRedZoneProtection(decoded)) {
|
||||
continue;
|
||||
}
|
||||
MarkReciprocalSquareRootTrap(reinterpret_cast<u8*>(address), decoded.instruction);
|
||||
++result.trapped_reciprocal_sqrt_instruction_count;
|
||||
}
|
||||
}
|
||||
|
||||
void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& function,
|
||||
const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
|
||||
GuestInstructionPatchResult& result) {
|
||||
struct RelocationSpan {
|
||||
std::vector<const DecodedCodeInstruction*> instructions;
|
||||
uintptr_t patch_start {};
|
||||
uintptr_t continuation {};
|
||||
size_t patch_size {};
|
||||
bool trap_rsqrt {};
|
||||
};
|
||||
|
||||
const auto emit_span = [&](const RelocationSpan& span) -> std::optional<size_t> {
|
||||
const size_t trampoline_offset = module->trampoline_gen.getSize();
|
||||
if (module->trampoline_exhausted) {
|
||||
return std::nullopt;
|
||||
}
|
||||
try {
|
||||
const auto emit_span = [&](RelocationSpan& span) -> std::optional<size_t> {
|
||||
auto& generator = module->trampoline_gen;
|
||||
const size_t trampoline_offset = generator.getSize();
|
||||
const bool has_rsqrt = std::ranges::any_of(span.instructions, [&](const auto* decoded) {
|
||||
const auto rewrite = rewrite_sites.find(decoded->address);
|
||||
return rewrite != rewrite_sites.end() && rewrite->second.emulate_rsqrt;
|
||||
});
|
||||
for (bool trap_rsqrt: {false, true}) {
|
||||
span.trap_rsqrt = trap_rsqrt;
|
||||
Xbyak::ClearError();
|
||||
bool encoded = true;
|
||||
for (const auto* decoded: span.instructions) {
|
||||
const auto rewrite = rewrite_sites.find(decoded->address);
|
||||
const bool emulate_rsqrt =
|
||||
rewrite != rewrite_sites.end() && rewrite->second.emulate_rsqrt;
|
||||
const bool protected_indirect_call =
|
||||
rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call;
|
||||
const bool protect_red_zone = rewrite != rewrite_sites.end() &&
|
||||
rewrite->second.protect_red_zone &&
|
||||
!protected_indirect_call;
|
||||
const bool protect_red_zone =
|
||||
(rewrite != rewrite_sites.end() && rewrite->second.protect_red_zone &&
|
||||
!protected_indirect_call) ||
|
||||
(emulate_rsqrt && trap_rsqrt && NeedsTrapRedZoneProtection(*decoded));
|
||||
if (protect_red_zone) {
|
||||
module->trampoline_gen.lea(rsp, ptr[rsp - GuestRedZoneSize]);
|
||||
generator.lea(rsp, ptr[rsp - GuestRedZoneSize]);
|
||||
}
|
||||
if (protected_indirect_call) {
|
||||
if (!GenerateProtectedIndirectCall(*decoded, module->trampoline_gen)) {
|
||||
module->trampoline_gen.setSize(trampoline_offset);
|
||||
return std::nullopt;
|
||||
if (emulate_rsqrt) {
|
||||
if (trap_rsqrt) {
|
||||
GenerateReciprocalSquareRootTrap(*decoded, generator);
|
||||
} else {
|
||||
GenerateReciprocalSquareRoot(*decoded, generator);
|
||||
}
|
||||
} else if (!EncodeRelocatedInstruction(*decoded, module->trampoline_gen)) {
|
||||
module->trampoline_gen.setSize(trampoline_offset);
|
||||
return std::nullopt;
|
||||
} else if (protected_indirect_call) {
|
||||
encoded = GenerateProtectedIndirectCall(*decoded, generator);
|
||||
} else {
|
||||
encoded = EncodeRelocatedInstruction(*decoded, generator);
|
||||
}
|
||||
if (protect_red_zone && !decoded->replaces_stack_pointer) {
|
||||
module->trampoline_gen.lea(rsp, ptr[rsp + GuestRedZoneSize]);
|
||||
generator.lea(rsp, ptr[rsp + GuestRedZoneSize]);
|
||||
}
|
||||
if (!encoded || Xbyak::GetError() != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
module->trampoline_gen.jmp(reinterpret_cast<void*>(span.continuation));
|
||||
} catch (const Xbyak::Error& error) {
|
||||
module->trampoline_gen.setSize(trampoline_offset);
|
||||
if (HandleTrampolineError(module, error)) {
|
||||
return std::nullopt;
|
||||
if (encoded && Xbyak::GetError() == 0) {
|
||||
generator.jmp(reinterpret_cast<void*>(span.continuation));
|
||||
if (Xbyak::GetError() == 0) {
|
||||
return trampoline_offset;
|
||||
}
|
||||
}
|
||||
const int error = Xbyak::GetError();
|
||||
// Roll back before retrying this uncommitted span with smaller trap replacements.
|
||||
generator.reset();
|
||||
generator.setSize(trampoline_offset);
|
||||
if (error != 0 && !HandleTrampolineError(module, error)) {
|
||||
EXIT("Guest instruction trampoline generation failed: %s\n",
|
||||
Xbyak::ConvertErrorToString(error));
|
||||
}
|
||||
if (error != Xbyak::ERR_CODE_IS_TOO_BIG || !has_rsqrt) {
|
||||
break;
|
||||
}
|
||||
throw;
|
||||
}
|
||||
return trampoline_offset;
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
const auto record_rewrites = [&](const RelocationSpan& span) {
|
||||
|
|
@ -881,6 +953,13 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
if (rewrite->second.protect_red_zone && decoded->accesses_memory) {
|
||||
++result.patched_memory_instruction_count;
|
||||
}
|
||||
if (rewrite->second.emulate_rsqrt) {
|
||||
if (span.trap_rsqrt) {
|
||||
++result.trapped_reciprocal_sqrt_instruction_count;
|
||||
} else {
|
||||
++result.patched_reciprocal_sqrt_instruction_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -983,6 +1062,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
patch_gen.setSize(span.patch_start - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
|
||||
patch_gen.jmp(trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
|
||||
patch_gen.nop(span.patch_size - NearJumpSize);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
|
||||
record_rewrites(span);
|
||||
patched_spans.emplace_back(span.patch_start, span.continuation);
|
||||
|
|
@ -1021,7 +1101,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
continue;
|
||||
}
|
||||
|
||||
const RelocationSpan site_span {
|
||||
RelocationSpan site_span {
|
||||
.instructions = {&site_instruction->second},
|
||||
.patch_start = site,
|
||||
.continuation = site + site_instruction->second.instruction.length,
|
||||
|
|
@ -1049,12 +1129,14 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
patch_gen.reset();
|
||||
patch_gen.setSize(*relay_slot - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
|
||||
patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
|
||||
patch_gen.reset();
|
||||
patch_gen.setSize(site - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
|
||||
patch_gen.jmp(reinterpret_cast<void*>(*relay_slot),
|
||||
Xbyak::CodeGenerator::LabelType::T_SHORT);
|
||||
patch_gen.nop(site_span.patch_size - ShortJumpSize);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
|
||||
record_rewrites(site_span);
|
||||
patched_spans.emplace_back(site_span.patch_start, site_span.continuation);
|
||||
|
|
@ -1187,6 +1269,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
patch_gen.reset();
|
||||
patch_gen.setSize(relay_address - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
|
||||
patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
std::erase(relay_slots, relay_address);
|
||||
std::erase(short_relay_slots, relay_address);
|
||||
} else {
|
||||
|
|
@ -1201,6 +1284,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
patch_gen.jmp(host_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
|
||||
patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
|
||||
patch_gen.nop(host_span->patch_size - NearJumpSize * 2);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
}
|
||||
|
||||
uintptr_t jump_target = relay_address;
|
||||
|
|
@ -1210,6 +1294,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
reinterpret_cast<uintptr_t>(patch_gen.getCode()));
|
||||
patch_gen.jmp(reinterpret_cast<void*>(jump_target),
|
||||
Xbyak::CodeGenerator::LabelType::T_SHORT);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
jump_target = final_short_jump;
|
||||
const auto parent = relay_parent.find(final_short_jump);
|
||||
ASSERT(parent != relay_parent.end());
|
||||
|
|
@ -1223,6 +1308,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
patch_gen.jmp(reinterpret_cast<void*>(jump_target),
|
||||
Xbyak::CodeGenerator::LabelType::T_SHORT);
|
||||
patch_gen.nop(site_span.patch_size - ShortJumpSize);
|
||||
EXIT_IF(Xbyak::GetError() != 0);
|
||||
|
||||
if (host_span) {
|
||||
record_rewrites(*host_span);
|
||||
|
|
@ -1240,10 +1326,13 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
|
|||
}
|
||||
} // namespace
|
||||
|
||||
RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
||||
std::span<const uintptr_t> function_starts,
|
||||
bool protect_memory, bool emulate_rsqrt) {
|
||||
RedZonePatchResult result {};
|
||||
GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
||||
std::span<const uintptr_t> function_starts,
|
||||
bool protect_memory, bool emulate_rsqrt) {
|
||||
GuestInstructionPatchResult result {};
|
||||
if (!protect_memory && !emulate_rsqrt) {
|
||||
return result;
|
||||
}
|
||||
auto* module = GetContainingModule(reinterpret_cast<void*>(segment_addr));
|
||||
if (module == nullptr || function_starts.empty()) {
|
||||
return result;
|
||||
|
|
@ -1262,8 +1351,11 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
|||
starts.erase(unique_end, starts.end());
|
||||
|
||||
std::unique_lock lock {module->mutex};
|
||||
const size_t trampoline_begin = module->trampoline_gen.getSize();
|
||||
std::vector<ReciprocalSquareRootSite> reciprocal_sqrt_sites;
|
||||
const size_t trampoline_begin = module->trampoline_gen.getSize();
|
||||
bool analyze_red_zone = protect_memory;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
analyze_red_zone |= emulate_rsqrt;
|
||||
#endif
|
||||
for (size_t function_index = 0; function_index < starts.size(); ++function_index) {
|
||||
const uintptr_t function_start = starts[function_index];
|
||||
const uintptr_t function_end =
|
||||
|
|
@ -1274,7 +1366,9 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
|||
|
||||
++result.function_count;
|
||||
auto function = DecodeFunction(function_start, function_end, segment_addr, segment_end);
|
||||
AnalyzeRedZoneLiveness(function);
|
||||
if (analyze_red_zone) {
|
||||
AnalyzeRedZoneLiveness(function);
|
||||
}
|
||||
result.instruction_count += function.instructions.size();
|
||||
|
||||
std::map<uintptr_t, InstructionRewrite> rewrite_sites;
|
||||
|
|
@ -1287,21 +1381,16 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
|||
CollectRedZoneMemoryInstructions(function, rewrite_sites, result);
|
||||
}
|
||||
if (emulate_rsqrt) {
|
||||
CollectReciprocalSquareRoots(function, rewrite_sites, reciprocal_sqrt_sites);
|
||||
CollectReciprocalSquareRoots(function, rewrite_sites, result);
|
||||
}
|
||||
if (!rewrite_sites.empty()) {
|
||||
RelocateRedZoneInstructions(module, function, rewrite_sites, result);
|
||||
RelocateGuestInstructions(module, function, rewrite_sites, result);
|
||||
TrapUnrelocatedReciprocalSquareRoots(*module, function, rewrite_sites, result);
|
||||
}
|
||||
}
|
||||
// Preserve valid instruction encodings throughout CFG analysis and relocation.
|
||||
// Only now turn reciprocal roots into traps, including the relocated copies.
|
||||
const auto trampoline_addr =
|
||||
reinterpret_cast<u64>(module->trampoline_gen.getCode()) + trampoline_begin;
|
||||
const auto trampoline_size = module->trampoline_gen.getSize() - trampoline_begin;
|
||||
if (emulate_rsqrt) {
|
||||
result.reciprocal_sqrt_instruction_count = ApplyReciprocalSquareRootPatches(
|
||||
*module, reciprocal_sqrt_sites, trampoline_addr, trampoline_size);
|
||||
}
|
||||
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
|
||||
if (trampoline_size != 0) {
|
||||
Common::VirtualMemory::FlushInstructionCache(trampoline_addr, trampoline_size);
|
||||
|
|
@ -1311,7 +1400,8 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
|
|||
|
||||
#else
|
||||
|
||||
RedZonePatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool) {
|
||||
GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool,
|
||||
bool) {
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
@ -1492,9 +1582,9 @@ bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_fram
|
|||
return true;
|
||||
}
|
||||
|
||||
void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
|
||||
uint64_t trampoline_area_size) {
|
||||
#if defined(_WIN32)
|
||||
void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
|
||||
void* trampoline_area_ptr, uint64_t trampoline_area_size) {
|
||||
#if !defined(__APPLE__)
|
||||
EXIT_IF(module_ptr == nullptr || module_size == 0 || trampoline_area_ptr == nullptr ||
|
||||
trampoline_area_size == 0);
|
||||
const auto module_addr = reinterpret_cast<u64>(module_ptr);
|
||||
|
|
@ -1511,8 +1601,8 @@ void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* tr
|
|||
#endif
|
||||
}
|
||||
|
||||
void UnregisterRedZonePatchModule(void* module_ptr) {
|
||||
#if defined(_WIN32)
|
||||
void UnregisterGuestInstructionPatchModule(void* module_ptr) {
|
||||
#if !defined(__APPLE__)
|
||||
g_patch_modules.erase(reinterpret_cast<u64>(module_ptr));
|
||||
#else
|
||||
(void)module_ptr;
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#ifndef KYTY_LOADER_GUEST_INSTRUCTION_PATCHER_H_
|
||||
#define KYTY_LOADER_GUEST_INSTRUCTION_PATCHER_H_
|
||||
|
||||
#include "common/common.h"
|
||||
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
namespace Loader {
|
||||
|
||||
struct GuestInstructionPatchResult {
|
||||
uint64_t function_count = 0;
|
||||
uint64_t instruction_count = 0;
|
||||
uint64_t red_zone_function_count = 0;
|
||||
uint64_t memory_instruction_count = 0;
|
||||
uint64_t short_memory_instruction_count = 0;
|
||||
uint64_t patched_memory_instruction_count = 0;
|
||||
uint64_t stack_dependent_memory_instruction_count = 0;
|
||||
uint64_t control_flow_memory_instruction_count = 0;
|
||||
uint64_t unrelocatable_memory_instruction_count = 0;
|
||||
uint64_t indirect_red_zone_function_count = 0;
|
||||
uint64_t reciprocal_sqrt_instruction_count = 0;
|
||||
uint64_t patched_reciprocal_sqrt_instruction_count = 0;
|
||||
uint64_t trapped_reciprocal_sqrt_instruction_count = 0;
|
||||
};
|
||||
|
||||
void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
|
||||
void* trampoline_area_ptr, uint64_t trampoline_area_size);
|
||||
void UnregisterGuestInstructionPatchModule(void* module_ptr);
|
||||
|
||||
// Apply enabled instruction fixes using native trampolines or safe trap fallbacks.
|
||||
GuestInstructionPatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size,
|
||||
std::span<const uintptr_t> function_starts,
|
||||
bool protect_memory, bool emulate_rsqrt);
|
||||
|
||||
bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
|
||||
std::vector<uintptr_t>* function_starts);
|
||||
|
||||
} // namespace Loader
|
||||
|
||||
#endif /* KYTY_LOADER_GUEST_INSTRUCTION_PATCHER_H_ */
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#ifndef KYTY_LOADER_RED_ZONE_PATCHER_H_
|
||||
#define KYTY_LOADER_RED_ZONE_PATCHER_H_
|
||||
|
||||
#include "common/common.h"
|
||||
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
namespace Loader {
|
||||
|
||||
struct RedZonePatchResult {
|
||||
uint64_t function_count = 0;
|
||||
uint64_t instruction_count = 0;
|
||||
uint64_t red_zone_function_count = 0;
|
||||
uint64_t memory_instruction_count = 0;
|
||||
uint64_t short_memory_instruction_count = 0;
|
||||
uint64_t patched_memory_instruction_count = 0;
|
||||
uint64_t stack_dependent_memory_instruction_count = 0;
|
||||
uint64_t control_flow_memory_instruction_count = 0;
|
||||
uint64_t unrelocatable_memory_instruction_count = 0;
|
||||
uint64_t indirect_red_zone_function_count = 0;
|
||||
uint64_t reciprocal_sqrt_instruction_count = 0;
|
||||
};
|
||||
|
||||
void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
|
||||
uint64_t trampoline_area_size);
|
||||
void UnregisterRedZonePatchModule(void* module_ptr);
|
||||
|
||||
// Analyze and protect fault sites before replacing reciprocal roots with traps.
|
||||
RedZonePatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size,
|
||||
std::span<const uintptr_t> function_starts,
|
||||
bool protect_memory, bool emulate_rsqrt);
|
||||
|
||||
bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
|
||||
std::vector<uintptr_t>* function_starts);
|
||||
|
||||
} // namespace Loader
|
||||
|
||||
#endif /* KYTY_LOADER_RED_ZONE_PATCHER_H_ */
|
||||
|
|
@ -17,8 +17,8 @@
|
|||
#include "kernel/pthread.h"
|
||||
#include "loader/elf.h"
|
||||
#include "loader/gamePatch.h"
|
||||
#include "loader/guestInstructionPatcher.h"
|
||||
#include "loader/jit.h"
|
||||
#include "loader/redZonePatcher.h"
|
||||
#include "loader/symbolDatabase.h"
|
||||
#include "loader/x64InstructionEmulator.h"
|
||||
|
||||
|
|
@ -1812,38 +1812,39 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||
uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize();
|
||||
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
|
||||
program->mapped_size = program->base_size_aligned + tls_handler_size;
|
||||
#if !defined(__APPLE__)
|
||||
const bool emulate_rsqrt = Config::AmdCpuEnabled();
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
const bool protect_memory_faults = Config::RedZoneProtectionEnabled();
|
||||
const bool use_red_zone_protection = protect_memory_faults || emulate_rsqrt;
|
||||
constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
|
||||
if (use_red_zone_protection) {
|
||||
EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
|
||||
program->mapped_size += RED_ZONE_TRAMPOLINE_SIZE;
|
||||
}
|
||||
#else
|
||||
const bool emulate_rsqrt = false;
|
||||
#endif
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
const bool protect_memory_faults = Config::RedZoneProtectionEnabled();
|
||||
#else
|
||||
const bool protect_memory_faults = false;
|
||||
#endif
|
||||
const bool patch_guest_instructions = protect_memory_faults || emulate_rsqrt;
|
||||
|
||||
constexpr uint64_t INSTRUCTION_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
|
||||
if (patch_guest_instructions) {
|
||||
EXIT_IF(INSTRUCTION_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
|
||||
program->mapped_size += INSTRUCTION_TRAMPOLINE_SIZE;
|
||||
}
|
||||
|
||||
program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
|
||||
g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
|
||||
Common::PathToString(program->file_name.filename()).c_str());
|
||||
EXIT_IF(program->base_vaddr == 0);
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (use_red_zone_protection) {
|
||||
program->red_zone_trampoline_vaddr = program->base_vaddr + program->base_size_aligned;
|
||||
program->red_zone_trampoline_size = RED_ZONE_TRAMPOLINE_SIZE;
|
||||
RegisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr),
|
||||
program->base_size_aligned,
|
||||
reinterpret_cast<void*>(program->red_zone_trampoline_vaddr),
|
||||
program->red_zone_trampoline_size);
|
||||
if (patch_guest_instructions) {
|
||||
const auto trampoline_addr = program->base_vaddr + program->base_size_aligned;
|
||||
program->instruction_trampoline_size = INSTRUCTION_TRAMPOLINE_SIZE;
|
||||
RegisterGuestInstructionPatchModule(
|
||||
reinterpret_cast<void*>(program->base_vaddr), program->base_size_aligned,
|
||||
reinterpret_cast<void*>(trampoline_addr), program->instruction_trampoline_size);
|
||||
}
|
||||
#endif
|
||||
if (!is_shared) {
|
||||
program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
program->tls.handler_vaddr += program->red_zone_trampoline_size;
|
||||
#endif
|
||||
program->tls.handler_vaddr =
|
||||
program->base_vaddr + program->base_size_aligned + program->instruction_trampoline_size;
|
||||
}
|
||||
|
||||
g_desired_base_addr += CODE_BASE_INCR * (1 + program->mapped_size / CODE_BASE_INCR);
|
||||
|
|
@ -1863,10 +1864,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||
}
|
||||
|
||||
std::vector<std::pair<uint64_t, uint64_t>> executable_segments;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
uint64_t eh_frame_header_addr = 0;
|
||||
uint64_t eh_frame_header_size = 0;
|
||||
#endif
|
||||
|
||||
for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) {
|
||||
if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) {
|
||||
|
|
@ -1925,52 +1924,60 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
|
|||
program->proc_param_vaddr = phdr[i].p_vaddr + program->base_vaddr;
|
||||
}
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (use_red_zone_protection && phdr[i].p_type == PT_GNU_EH_FRAME) {
|
||||
if (patch_guest_instructions && phdr[i].p_type == PT_GNU_EH_FRAME) {
|
||||
eh_frame_header_addr = phdr[i].p_vaddr + program->base_vaddr;
|
||||
eh_frame_header_size = phdr[i].p_memsz;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
std::vector<uintptr_t> function_starts;
|
||||
if (use_red_zone_protection) {
|
||||
if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
|
||||
&function_starts)) {
|
||||
LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n",
|
||||
Common::PathToString(program->file_name).c_str());
|
||||
if (patch_guest_instructions) {
|
||||
std::vector<uintptr_t> function_starts;
|
||||
const bool have_function_starts =
|
||||
DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
|
||||
&function_starts) &&
|
||||
!function_starts.empty();
|
||||
const auto module_name = Common::PathToString(program->file_name.filename());
|
||||
if (!have_function_starts) {
|
||||
Log::WriteToConsoleAndLog(
|
||||
fmt::format("{}: {} not patched (function boundaries unavailable)\n",
|
||||
emulate_rsqrt ? "AMD CPU compatibility" : "Guest red-zone protection",
|
||||
module_name));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
for (const auto& [segment_addr, segment_size]: executable_segments) {
|
||||
uint64_t reciprocal_sqrt_count = 0;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (use_red_zone_protection) {
|
||||
const auto result =
|
||||
PatchGuestInstructions(segment_addr, segment_size, function_starts,
|
||||
protect_memory_faults, emulate_rsqrt);
|
||||
LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
|
||||
", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
|
||||
", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
|
||||
Common::PathToString(program->file_name.filename()).c_str(), result.function_count,
|
||||
result.red_zone_function_count, result.memory_instruction_count,
|
||||
result.patched_memory_instruction_count, result.short_memory_instruction_count,
|
||||
result.stack_dependent_memory_instruction_count,
|
||||
result.control_flow_memory_instruction_count,
|
||||
result.unrelocatable_memory_instruction_count);
|
||||
reciprocal_sqrt_count = result.reciprocal_sqrt_instruction_count;
|
||||
GuestInstructionPatchResult totals {};
|
||||
for (const auto& [segment_addr, segment_size]: executable_segments) {
|
||||
const auto result = PatchGuestInstructions(segment_addr, segment_size, function_starts,
|
||||
protect_memory_faults, emulate_rsqrt);
|
||||
totals.reciprocal_sqrt_instruction_count += result.reciprocal_sqrt_instruction_count;
|
||||
totals.patched_reciprocal_sqrt_instruction_count +=
|
||||
result.patched_reciprocal_sqrt_instruction_count;
|
||||
totals.trapped_reciprocal_sqrt_instruction_count +=
|
||||
result.trapped_reciprocal_sqrt_instruction_count;
|
||||
if (protect_memory_faults) {
|
||||
LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
|
||||
", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
|
||||
", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
|
||||
module_name.c_str(),
|
||||
result.function_count, result.red_zone_function_count,
|
||||
result.memory_instruction_count, result.patched_memory_instruction_count,
|
||||
result.short_memory_instruction_count,
|
||||
result.stack_dependent_memory_instruction_count,
|
||||
result.control_flow_memory_instruction_count,
|
||||
result.unrelocatable_memory_instruction_count);
|
||||
}
|
||||
}
|
||||
#else
|
||||
if (emulate_rsqrt) {
|
||||
reciprocal_sqrt_count =
|
||||
X64InstructionEmulator::PatchReciprocalSquareRoots(segment_addr, segment_size);
|
||||
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
|
||||
}
|
||||
#endif
|
||||
if (reciprocal_sqrt_count != 0) {
|
||||
LOGF("Guest VRSQRTPS emulation: %s, instructions=%" PRIu64 "\n",
|
||||
Common::PathToString(program->file_name.filename()).c_str(), reciprocal_sqrt_count);
|
||||
if (emulate_rsqrt && have_function_starts) {
|
||||
const auto patched = totals.patched_reciprocal_sqrt_instruction_count +
|
||||
totals.trapped_reciprocal_sqrt_instruction_count;
|
||||
const auto skipped = totals.reciprocal_sqrt_instruction_count - patched;
|
||||
const char* status = totals.reciprocal_sqrt_instruction_count == 0
|
||||
? "no matching instructions"
|
||||
: patched == 0 ? "not patched"
|
||||
: skipped != 0 ? "partially patched"
|
||||
: "patched";
|
||||
Log::WriteToConsoleAndLog(fmt::format(
|
||||
"AMD CPU compatibility: {} {} (VRSQRTPS: native={}, trapped={}, skipped={})\n",
|
||||
module_name, status, totals.patched_reciprocal_sqrt_instruction_count,
|
||||
totals.trapped_reciprocal_sqrt_instruction_count, skipped));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1999,11 +2006,9 @@ void RuntimeLinker::DeleteProgram(Program* p) {
|
|||
|
||||
if (program->base_vaddr != 0 || program->mapped_size != 0) {
|
||||
EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (program->red_zone_trampoline_size != 0) {
|
||||
UnregisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr));
|
||||
if (program->instruction_trampoline_size != 0) {
|
||||
UnregisterGuestInstructionPatchModule(reinterpret_cast<void*>(program->base_vaddr));
|
||||
}
|
||||
#endif
|
||||
EXIT_IF(
|
||||
!Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -123,10 +123,7 @@ struct Program {
|
|||
uint64_t base_size = 0;
|
||||
uint64_t base_size_aligned = 0;
|
||||
uint64_t mapped_size = 0;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
uint64_t red_zone_trampoline_vaddr = 0;
|
||||
uint64_t red_zone_trampoline_size = 0;
|
||||
#endif
|
||||
uint64_t instruction_trampoline_size = 0;
|
||||
std::unique_ptr<SymbolDatabase> export_symbols;
|
||||
std::unique_ptr<SymbolDatabase> import_symbols;
|
||||
ThreadLocalStorage tls;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
#include "common/common.h"
|
||||
|
||||
#include <Zydis/Zydis.h>
|
||||
#include <bit>
|
||||
#include <cstring>
|
||||
#if !defined(__APPLE__)
|
||||
|
|
@ -713,46 +712,6 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) {
|
|||
|
||||
#endif
|
||||
|
||||
bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction,
|
||||
const ZydisDecodedOperand* operands) {
|
||||
return instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS &&
|
||||
instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX &&
|
||||
instruction.raw.vex.offset == 0 && operands[0].size == 128 &&
|
||||
operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER;
|
||||
}
|
||||
|
||||
uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) {
|
||||
uint64_t patched = 0;
|
||||
#if !defined(__APPLE__)
|
||||
ZydisDecoder decoder {};
|
||||
if (!ZYAN_SUCCESS(
|
||||
ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64))) {
|
||||
return 0;
|
||||
}
|
||||
for (uint64_t offset = 0; offset < size;) {
|
||||
auto* code = reinterpret_cast<uint8_t*>(address + offset);
|
||||
ZydisDecodedInstruction instruction {};
|
||||
ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT] {};
|
||||
if (!ZYAN_SUCCESS(
|
||||
ZydisDecoderDecodeFull(&decoder, code, size - offset, &instruction, operands))) {
|
||||
++offset;
|
||||
continue;
|
||||
}
|
||||
if (IsReciprocalSquareRoot(instruction, operands)) {
|
||||
// vvvv is reserved (must be 1111b). Clear one bit to route this
|
||||
// otherwise intact instruction through the illegal-instruction emulator.
|
||||
code[instruction.raw.vex.size - 1] &= ~0x08u;
|
||||
++patched;
|
||||
}
|
||||
offset += instruction.length;
|
||||
}
|
||||
#else
|
||||
(void)address;
|
||||
(void)size;
|
||||
#endif
|
||||
return patched;
|
||||
}
|
||||
|
||||
bool TryEmulate(void* native_context) {
|
||||
if (native_context == nullptr) {
|
||||
return false;
|
||||
|
|
@ -769,11 +728,8 @@ bool TryEmulate(void* native_context) {
|
|||
Context context {static_cast<ucontext_t*>(native_context)};
|
||||
#endif
|
||||
#if !defined(__APPLE__)
|
||||
if (TryEmulateReciprocalSquareRoot(context)) {
|
||||
return true;
|
||||
}
|
||||
return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
|
||||
TryEmulateShaNi(context);
|
||||
return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) ||
|
||||
TryEmulateSse4a(context) || TryEmulateShaNi(context);
|
||||
#else
|
||||
return TryEmulateSse4a(context);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -1,14 +1,8 @@
|
|||
#ifndef KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
|
||||
#define KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
|
||||
|
||||
#include <Zydis/DecoderTypes.h>
|
||||
#include <cstdint>
|
||||
|
||||
namespace Loader::X64InstructionEmulator {
|
||||
|
||||
[[nodiscard]] bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction,
|
||||
const ZydisDecodedOperand* operands);
|
||||
uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size);
|
||||
[[nodiscard]] bool TryEmulate(void* native_context);
|
||||
|
||||
} // namespace Loader::X64InstructionEmulator
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
#include "kernel/memory.h"
|
||||
#include "kernel/pthread.h"
|
||||
#include "libs/errno.h"
|
||||
#include "loader/redZonePatcher.h"
|
||||
#include "loader/guestInstructionPatcher.h"
|
||||
#include "loader/runtimeLinker.h"
|
||||
#include "loader/systemContent.h"
|
||||
#include "loader/x64InstructionEmulator.h"
|
||||
|
|
@ -232,15 +232,15 @@ void TestWindowsGuestRedZoneStaticPatcher() {
|
|||
DWORD old_protection = 0;
|
||||
Check(test, VirtualProtect(g_red_zone_fault_page, 0x1000, PAGE_NOACCESS, &old_protection) != FALSE,
|
||||
"failed to reset fault page protection");
|
||||
Loader::RegisterRedZonePatchModule(reinterpret_cast<void*>(mapping), CODE_SIZE,
|
||||
reinterpret_cast<void*>(mapping + CODE_SIZE),
|
||||
TRAMPOLINE_SIZE);
|
||||
Loader::RegisterGuestInstructionPatchModule(
|
||||
reinterpret_cast<void*>(mapping), CODE_SIZE, reinterpret_cast<void*>(mapping + CODE_SIZE),
|
||||
TRAMPOLINE_SIZE);
|
||||
const std::array<uintptr_t, 1> function_starts = {static_cast<uintptr_t>(mapping)};
|
||||
const auto result = Loader::PatchGuestInstructions(
|
||||
mapping, code.size(), function_starts, true, false);
|
||||
const bool patched_preserved = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 1;
|
||||
|
||||
Loader::UnregisterRedZonePatchModule(reinterpret_cast<void*>(mapping));
|
||||
Loader::UnregisterGuestInstructionPatchModule(reinterpret_cast<void*>(mapping));
|
||||
RemoveVectoredExceptionHandler(handler);
|
||||
VirtualFree(g_red_zone_fault_page, 0, MEM_RELEASE);
|
||||
g_red_zone_fault_page = nullptr;
|
||||
|
|
@ -2861,11 +2861,7 @@ void ReciprocalSquareRootHandler(int, siginfo_t*, void* context) {
|
|||
void TestPackedReciprocalSquareRoot() {
|
||||
const char* test = "PackedReciprocalSquareRoot";
|
||||
constexpr uint64_t code_size = 0x4000;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
constexpr uint64_t allocation_size = code_size * 2;
|
||||
#else
|
||||
constexpr uint64_t allocation_size = code_size;
|
||||
#endif
|
||||
const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
|
||||
0x902000000, allocation_size, Common::VirtualMemory::Mode::ExecuteReadWrite, "rsqrt_test");
|
||||
Check(test, mapping != 0, "failed to allocate instruction test code");
|
||||
|
|
@ -2885,12 +2881,12 @@ void TestPackedReciprocalSquareRoot() {
|
|||
if (handler != nullptr) {
|
||||
RemoveVectoredExceptionHandler(handler);
|
||||
}
|
||||
Loader::UnregisterRedZonePatchModule(reinterpret_cast<void*>(mapping));
|
||||
#else
|
||||
if (installed) {
|
||||
sigaction(SIGILL, &previous, nullptr);
|
||||
}
|
||||
#endif
|
||||
Loader::UnregisterGuestInstructionPatchModule(reinterpret_cast<void*>(mapping));
|
||||
Libs::LibKernel::Memory::FreeGuestMemory(mapping, size);
|
||||
}
|
||||
} restore {mapping, allocation_size, _mm_getcsr()};
|
||||
|
|
@ -2907,14 +2903,21 @@ void TestPackedReciprocalSquareRoot() {
|
|||
#endif
|
||||
using GuestFunction = void(KYTY_SYSV_ABI*)(const uint32_t*, uint32_t*);
|
||||
const auto function = reinterpret_cast<GuestFunction>(mapping);
|
||||
const std::array<uintptr_t, 1> function_starts {mapping};
|
||||
const auto register_module = [&](uint64_t trampoline_size) {
|
||||
Loader::RegisterGuestInstructionPatchModule(
|
||||
reinterpret_cast<void*>(mapping), code_size,
|
||||
reinterpret_cast<void*>(mapping + code_size), trampoline_size);
|
||||
};
|
||||
const auto refine = [](float estimate) {
|
||||
return (estimate * 0.5f) * std::fma(-estimate, estimate, 3.0f);
|
||||
};
|
||||
constexpr uint64_t red_zone_sentinel = 0x1122334455667788ull;
|
||||
std::array<uint32_t, 16> input {};
|
||||
std::array<uint32_t, 48> output {};
|
||||
std::array<uint32_t, 74> output {};
|
||||
input.fill(0x3f800000);
|
||||
for (const auto registers: {std::array {2, 1}, std::array {10, 9}, std::array {1, 1}}) {
|
||||
for (const auto registers: {std::array {2, 1}, std::array {10, 9}, std::array {1, 1},
|
||||
std::array {0, 1}, std::array {2, 0}, std::array {0, 0}}) {
|
||||
const auto destination = registers[0];
|
||||
const auto source = registers[1];
|
||||
input[0] = 0x3f800000;
|
||||
|
|
@ -2924,8 +2927,8 @@ void TestPackedReciprocalSquareRoot() {
|
|||
expected = 0x3f000000;
|
||||
}
|
||||
Xbyak::CodeGenerator code(code_size, reinterpret_cast<void*>(mapping));
|
||||
if (source == 9) {
|
||||
code.vmovups(Xbyak::Ymm(1), code.ptr[code.rdi + 32]);
|
||||
for (int reg = 0; reg < 3; ++reg) {
|
||||
code.vmovups(Xbyak::Ymm(reg), code.ptr[code.rdi + 32]);
|
||||
}
|
||||
code.vmovups(Xbyak::Ymm(source), code.ptr[code.rdi]);
|
||||
if (destination != source) {
|
||||
|
|
@ -2935,18 +2938,32 @@ void TestPackedReciprocalSquareRoot() {
|
|||
code.mov(code.rax, red_zone_sentinel ^ offset);
|
||||
code.mov(code.qword[code.rsp - offset], code.rax);
|
||||
}
|
||||
code.cmp(code.rdi, code.rdi);
|
||||
code.stc();
|
||||
code.vrsqrtps(Xbyak::Xmm(destination), Xbyak::Xmm(source));
|
||||
const uint64_t instruction_count = destination == 2 && source == 1 ? 2 : 1;
|
||||
if (instruction_count == 2) {
|
||||
// Adjacent four-byte instructions share one relocation span.
|
||||
code.vrsqrtps(Xbyak::Xmm(destination), Xbyak::Xmm(source));
|
||||
}
|
||||
code.lea(code.rsp, code.ptr[code.rsp - 128]);
|
||||
code.pushfq();
|
||||
code.pop(code.rax);
|
||||
code.lea(code.rsp, code.ptr[code.rsp + 128]);
|
||||
code.mov(code.qword[code.rsi + 288], code.rax);
|
||||
for (uint32_t offset = 8; offset <= 128; offset += 8) {
|
||||
code.mov(code.rax, code.qword[code.rsp - offset]);
|
||||
code.mov(code.qword[code.rsi + 64 + offset - 8], code.rax);
|
||||
}
|
||||
code.vmovups(code.ptr[code.rsi], Xbyak::Ymm(destination));
|
||||
code.vmovups(code.ptr[code.rsi + 32], Xbyak::Ymm(source));
|
||||
for (int reg = 0; reg < 3; ++reg) {
|
||||
code.vmovups(code.ptr[code.rsi + 192 + reg * 32], Xbyak::Ymm(reg));
|
||||
}
|
||||
code.vzeroupper();
|
||||
code.ret();
|
||||
#if defined(__linux__)
|
||||
Check(test, Xbyak::GetError() == 0, "failed to generate reciprocal-root fixture");
|
||||
const std::vector<uint8_t> original(code.getCode(), code.getCurr());
|
||||
#endif
|
||||
Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()),
|
||||
"failed to flush generated instruction test code");
|
||||
function(input.data(), output.data());
|
||||
|
|
@ -2965,88 +2982,193 @@ void TestPackedReciprocalSquareRoot() {
|
|||
Check(test, red_zone_intact(), "native instruction corrupted the guest red zone");
|
||||
Check(test, std::isfinite(native) && std::abs(native - std::bit_cast<float>(expected)) < 0.001f,
|
||||
"native reciprocal root is outside its error bound");
|
||||
register_module(code_size);
|
||||
const std::vector<uint8_t> original_trampoline(code.getCode() + code_size,
|
||||
code.getCode() + allocation_size);
|
||||
const auto disabled = Loader::PatchGuestInstructions(
|
||||
mapping, code.getSize(), function_starts, false, false);
|
||||
Check(test, Xbyak::GetError() == 0 && disabled.reciprocal_sqrt_instruction_count == 0 &&
|
||||
disabled.patched_reciprocal_sqrt_instruction_count == 0 &&
|
||||
disabled.trapped_reciprocal_sqrt_instruction_count == 0 &&
|
||||
std::equal(original.begin(), original.end(), code.getCode()) &&
|
||||
std::equal(original_trampoline.begin(), original_trampoline.end(),
|
||||
code.getCode() + code_size),
|
||||
"disabled instruction patching modified executable code or trampolines");
|
||||
if (destination == 2 && source == 1) {
|
||||
// Fail initially, after the RIP-relative constant reference, and inside its literal.
|
||||
for (const uint64_t capacity: {1, 48, 80}) {
|
||||
register_module(capacity);
|
||||
const auto exhausted = Loader::PatchGuestInstructions(
|
||||
mapping, code.getSize(), function_starts, false, true);
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
Loader::RegisterRedZonePatchModule(reinterpret_cast<void*>(mapping), code_size,
|
||||
reinterpret_cast<void*>(mapping + code_size), code_size);
|
||||
const std::array<uintptr_t, 1> function_starts {mapping};
|
||||
// The extended-register case also relocates ordinary memory accesses while
|
||||
// red-zone data is live, exercising both enabled patchers in one function.
|
||||
const bool protect_memory = source == 9;
|
||||
const auto patched = Loader::PatchGuestInstructions(
|
||||
mapping, code.getSize(), function_starts, protect_memory, true);
|
||||
Check(test, patched.reciprocal_sqrt_instruction_count == 1 &&
|
||||
patched.unrelocatable_memory_instruction_count == 0 &&
|
||||
(protect_memory ? patched.patched_memory_instruction_count > 0
|
||||
: patched.patched_memory_instruction_count == 0),
|
||||
"instruction pass lost reciprocal root or memory patch coverage");
|
||||
const uint64_t expected_traps = capacity == 1 ? 0 : instruction_count;
|
||||
#else
|
||||
Check(test, Loader::X64InstructionEmulator::PatchReciprocalSquareRoots(mapping, code.getSize()) == 1,
|
||||
"instruction pass did not patch exactly one packed reciprocal root");
|
||||
const auto* patched = reinterpret_cast<const uint8_t*>(mapping);
|
||||
size_t changed = 0;
|
||||
for (size_t i = 0; i < original.size(); ++i) {
|
||||
changed += original[i] != patched[i];
|
||||
}
|
||||
Check(test, changed == 1, "instruction pass changed unrelated code bytes");
|
||||
Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.getSize()),
|
||||
"failed to flush patched instruction test code");
|
||||
const uint64_t expected_traps = instruction_count;
|
||||
#endif
|
||||
const auto before = g_rsqrt_traps;
|
||||
function(input.data(), output.data());
|
||||
Check(test, g_rsqrt_traps == before + 1, "patched instruction did not execute its handler");
|
||||
Check(test, red_zone_intact(), "patched instruction corrupted the guest red zone");
|
||||
Check(test, output[0] == expected, "patched instruction read or wrote the wrong register");
|
||||
if (destination != source) {
|
||||
Check(test, std::equal(input.begin(), input.begin() + 4, output.begin() + 8),
|
||||
"source lower XMM lanes were corrupted");
|
||||
}
|
||||
if (source != 9) {
|
||||
Check(test, refine(refine(std::bit_cast<float>(output[0]))) == 1.0f,
|
||||
"identity quaternion normalization drifted below one");
|
||||
}
|
||||
for (size_t i = 0; i < 4; ++i) {
|
||||
Check(test, output[4 + i] == 0, "128-bit VEX destination retained upper YMM lanes");
|
||||
const auto expected_upper = destination == source ? 0u : input[4 + i];
|
||||
Check(test, output[12 + i] == expected_upper, "source upper YMM lanes were corrupted");
|
||||
}
|
||||
std::printf("[host] rsqrt xmm%d,xmm%d native=%08x patched=%08x\n",
|
||||
destination, source, std::bit_cast<uint32_t>(native), output[0]);
|
||||
if (destination != 2) {
|
||||
continue;
|
||||
}
|
||||
constexpr std::array<std::array<uint32_t, 8>, 4> cases {{
|
||||
{0x00000000, 0x80000000, 0x00000001, 0x80000001,
|
||||
0x7f800000, 0xff800000, 0x7f800000, 0xff800000},
|
||||
{0x7f800000, 0xff800000, 0xbf800000, 0x7fc12345,
|
||||
0x00000000, 0xffc00000, 0xffc00000, 0x7fc12345},
|
||||
{0x7f812345, 0xff812345, 0x40800000, 0x40000000,
|
||||
0x7fc12345, 0xffc12345, 0x3f000000, 0x3f3504f3},
|
||||
{0x00800000, 0x7f7fffff, 0x3f800000, 0x41800000,
|
||||
0x5f000000, 0x1f800000, 0x3f800000, 0x3e800000},
|
||||
}};
|
||||
for (const auto& values: cases) {
|
||||
std::copy_n(values.begin(), 4, input.begin());
|
||||
for (uint32_t controls = 0; controls < 8; ++controls) {
|
||||
const uint32_t mxcsr = 0x1fa1u | ((controls & 3u) << 13u) | ((controls & 4u) << 4u);
|
||||
_mm_setcsr(mxcsr);
|
||||
Check(test, exhausted.reciprocal_sqrt_instruction_count == instruction_count &&
|
||||
exhausted.patched_reciprocal_sqrt_instruction_count == 0 &&
|
||||
exhausted.trapped_reciprocal_sqrt_instruction_count == expected_traps,
|
||||
"exhausted reciprocal-root generation lost fallback coverage");
|
||||
Check(test, Xbyak::GetError() == 0,
|
||||
"failed trampoline generation leaked Xbyak's thread-local error");
|
||||
if (expected_traps == 0) {
|
||||
Check(test, std::equal(original.begin(), original.end(), code.getCode()),
|
||||
"unsafe fallback modified a live guest red zone site");
|
||||
}
|
||||
const auto traps_before = g_rsqrt_traps;
|
||||
function(input.data(), output.data());
|
||||
const auto result_mxcsr = _mm_getcsr();
|
||||
_mm_setcsr(restore.mxcsr);
|
||||
Check(test, red_zone_intact(), "special-value trap corrupted the guest red zone");
|
||||
Check(test, result_mxcsr == mxcsr, "instruction changed MXCSR controls or exception flags");
|
||||
Check(test, std::equal(output.begin(), output.begin() + 4, values.begin() + 4),
|
||||
"special values or round-independent reciprocal roots differ from ISA semantics");
|
||||
Check(test, static_cast<uint64_t>(g_rsqrt_traps - traps_before) == expected_traps &&
|
||||
red_zone_intact() &&
|
||||
output[0] == (expected_traps ? expected : std::bit_cast<uint32_t>(native)),
|
||||
"exhaustion fallback changed guest state or failed to execute");
|
||||
std::memcpy(reinterpret_cast<void*>(mapping), original.data(), original.size());
|
||||
}
|
||||
register_module(code_size);
|
||||
}
|
||||
// Memory protection alone must retain the host's original reciprocal-root result.
|
||||
const bool protect_memory = source == 9;
|
||||
if (protect_memory) {
|
||||
const auto memory_only = Loader::PatchGuestInstructions(
|
||||
mapping, code.getSize(), function_starts, true, false);
|
||||
Check(test, Xbyak::GetError() == 0 &&
|
||||
memory_only.reciprocal_sqrt_instruction_count == 0 &&
|
||||
memory_only.patched_reciprocal_sqrt_instruction_count == 0 &&
|
||||
memory_only.trapped_reciprocal_sqrt_instruction_count == 0 &&
|
||||
memory_only.patched_memory_instruction_count > 0,
|
||||
"memory protection enabled reciprocal-root replacement");
|
||||
function(input.data(), output.data());
|
||||
Check(test, output[0] == std::bit_cast<uint32_t>(native) && red_zone_intact(),
|
||||
"memory protection changed native reciprocal-root behavior");
|
||||
std::memcpy(reinterpret_cast<void*>(mapping), original.data(), original.size());
|
||||
register_module(code_size);
|
||||
}
|
||||
for (const bool trap_fallback: {false, true}) {
|
||||
if (trap_fallback) {
|
||||
std::memcpy(reinterpret_cast<void*>(mapping), original.data(), original.size());
|
||||
register_module(48);
|
||||
}
|
||||
const bool protect_memory_sites = protect_memory && !trap_fallback;
|
||||
const auto patched = Loader::PatchGuestInstructions(
|
||||
mapping, code.getSize(), function_starts, protect_memory_sites, true);
|
||||
Check(test, Xbyak::GetError() == 0 &&
|
||||
patched.reciprocal_sqrt_instruction_count == instruction_count &&
|
||||
patched.patched_reciprocal_sqrt_instruction_count ==
|
||||
(trap_fallback ? 0 : instruction_count) &&
|
||||
patched.trapped_reciprocal_sqrt_instruction_count ==
|
||||
(trap_fallback ? instruction_count : 0) &&
|
||||
patched.unrelocatable_memory_instruction_count == 0 &&
|
||||
(protect_memory_sites ? patched.patched_memory_instruction_count > 0
|
||||
: patched.patched_memory_instruction_count == 0),
|
||||
"instruction pass lost reciprocal root or memory patch coverage");
|
||||
const auto traps_before = g_rsqrt_traps;
|
||||
function(input.data(), output.data());
|
||||
Check(test, static_cast<uint64_t>(g_rsqrt_traps - traps_before) ==
|
||||
(trap_fallback ? instruction_count : 0),
|
||||
"reciprocal-root execution did not match its native/fallback patch count");
|
||||
Check(test, (output[72] & 0x8d5u) == 0x45u, "patched instruction corrupted RFLAGS");
|
||||
for (int reg = 0; reg < 3; ++reg) {
|
||||
if (reg != destination && reg != source) {
|
||||
Check(test, std::equal(input.begin() + 8, input.end(), output.begin() + 48 + reg * 8),
|
||||
"patched instruction corrupted an unrelated YMM register");
|
||||
}
|
||||
}
|
||||
Check(test, red_zone_intact(), "patched instruction corrupted the guest red zone");
|
||||
Check(test, output[0] == expected, "patched instruction read or wrote the wrong register");
|
||||
if (destination != source) {
|
||||
Check(test, std::equal(input.begin(), input.begin() + 4, output.begin() + 8),
|
||||
"source lower XMM lanes were corrupted");
|
||||
}
|
||||
if (source != 9) {
|
||||
Check(test, refine(refine(std::bit_cast<float>(output[0]))) == 1.0f,
|
||||
"identity quaternion normalization drifted below one");
|
||||
}
|
||||
for (size_t i = 0; i < 4; ++i) {
|
||||
Check(test, output[4 + i] == 0, "128-bit VEX destination retained upper YMM lanes");
|
||||
const auto expected_upper = destination == source ? 0u : input[4 + i];
|
||||
Check(test, output[12 + i] == expected_upper, "source upper YMM lanes were corrupted");
|
||||
}
|
||||
std::printf("[host] rsqrt xmm%d,xmm%d %s native=%08x patched=%08x\n",
|
||||
destination, source, trap_fallback ? "fallback" : "replacement",
|
||||
std::bit_cast<uint32_t>(native), output[0]);
|
||||
if (destination != 2) {
|
||||
continue;
|
||||
}
|
||||
constexpr std::array<std::array<uint32_t, 8>, 4> cases {{
|
||||
{0x00000000, 0x80000000, 0x00000001, 0x80000001,
|
||||
0x7f800000, 0xff800000, 0x7f800000, 0xff800000},
|
||||
{0x7f800000, 0xff800000, 0xbf800000, 0x7fc12345,
|
||||
0x00000000, 0xffc00000, 0xffc00000, 0x7fc12345},
|
||||
{0x7f812345, 0xff812345, 0x40800000, 0x40000000,
|
||||
0x7fc12345, 0xffc12345, 0x3f000000, 0x3f3504f3},
|
||||
{0x00800000, 0x7f7fffff, 0x3f800000, 0x41800000,
|
||||
0x5f000000, 0x1f800000, 0x3f800000, 0x3e800000},
|
||||
}};
|
||||
for (const auto& values: cases) {
|
||||
std::copy_n(values.begin(), 4, input.begin());
|
||||
for (uint32_t controls = 0; controls < 8; ++controls) {
|
||||
const uint32_t mxcsr = 0x1fa1u | ((controls & 3u) << 13u) | ((controls & 4u) << 4u);
|
||||
_mm_setcsr(mxcsr);
|
||||
function(input.data(), output.data());
|
||||
const auto result_mxcsr = _mm_getcsr();
|
||||
_mm_setcsr(restore.mxcsr);
|
||||
Check(test, red_zone_intact(), "special-value replacement corrupted the guest red zone");
|
||||
Check(test, result_mxcsr == mxcsr, "instruction changed MXCSR controls or exception flags");
|
||||
Check(test, std::equal(output.begin(), output.begin() + 4, values.begin() + 4),
|
||||
"special values or round-independent reciprocal roots differ from ISA semantics");
|
||||
}
|
||||
}
|
||||
input.fill(0x3f800000);
|
||||
}
|
||||
input.fill(0x3f800000);
|
||||
}
|
||||
#if defined(__linux__)
|
||||
std::array<uint8_t, 16> unknown {0x0f, 0x0b};
|
||||
// An indirect branch prevents borrowing neighboring bytes for this four-byte site.
|
||||
Xbyak::CodeGenerator unsupported_code(code_size, reinterpret_cast<void*>(mapping));
|
||||
Xbyak::Label continuation;
|
||||
unsupported_code.vmovups(unsupported_code.ymm1, unsupported_code.ptr[unsupported_code.rdi]);
|
||||
unsupported_code.lea(unsupported_code.rax,
|
||||
unsupported_code.ptr[unsupported_code.rip + continuation]);
|
||||
const auto trap_offset = unsupported_code.getSize();
|
||||
unsupported_code.vrsqrtps(unsupported_code.xmm1, unsupported_code.xmm1);
|
||||
unsupported_code.jmp(unsupported_code.rax);
|
||||
unsupported_code.L(continuation);
|
||||
unsupported_code.vmovups(unsupported_code.ptr[unsupported_code.rsi], unsupported_code.ymm1);
|
||||
unsupported_code.vzeroupper();
|
||||
unsupported_code.ret();
|
||||
Check(test, Xbyak::GetError() == 0, "failed to generate unsupported-relocation fixture");
|
||||
std::vector<uint8_t> expected_fallback(unsupported_code.getCode(), unsupported_code.getCurr());
|
||||
expected_fallback[trap_offset + 1] &= ~0x08u;
|
||||
register_module(code_size);
|
||||
const auto unsupported = Loader::PatchGuestInstructions(
|
||||
mapping, unsupported_code.getSize(), function_starts, false, true);
|
||||
Check(test, Xbyak::GetError() == 0 && unsupported.reciprocal_sqrt_instruction_count == 1 &&
|
||||
unsupported.patched_reciprocal_sqrt_instruction_count == 0 &&
|
||||
unsupported.trapped_reciprocal_sqrt_instruction_count == 1 &&
|
||||
std::equal(expected_fallback.begin(), expected_fallback.end(),
|
||||
unsupported_code.getCode()),
|
||||
"unsafe native relocation did not select the in-place trap fallback");
|
||||
const auto traps_before = g_rsqrt_traps;
|
||||
function(input.data(), output.data());
|
||||
Check(test, g_rsqrt_traps == traps_before + 1 && output[0] == 0x3f800000 &&
|
||||
std::all_of(output.begin() + 4, output.begin() + 8,
|
||||
[](uint32_t lane) { return lane == 0; }),
|
||||
"unsafe native relocation fallback did not execute correctly");
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
CONTEXT context {};
|
||||
auto& instruction_pointer = context.Rip;
|
||||
#else
|
||||
ucontext_t context {};
|
||||
context.uc_mcontext.gregs[REG_RIP] = reinterpret_cast<greg_t>(unknown.data());
|
||||
Check(test, !Loader::X64InstructionEmulator::TryEmulate(&context), "unrecognized UD2 was swallowed");
|
||||
_libc_fpstate fpstate {};
|
||||
context.uc_mcontext.fpregs = &fpstate;
|
||||
auto& instruction_pointer = context.uc_mcontext.gregs[REG_RIP];
|
||||
#endif
|
||||
std::array<uint8_t, 16> unknown {0x0f, 0x0b};
|
||||
instruction_pointer = reinterpret_cast<uintptr_t>(unknown.data());
|
||||
Check(test, !Loader::X64InstructionEmulator::TryEmulate(&context), "unrecognized UD2 was swallowed");
|
||||
std::array<uint8_t, 16> unmarked_rsqrt {0xc5, 0xf8, 0x52, 0xd1};
|
||||
instruction_pointer = reinterpret_cast<uintptr_t>(unmarked_rsqrt.data());
|
||||
Check(test, !Loader::X64InstructionEmulator::TryEmulate(&context) &&
|
||||
instruction_pointer == reinterpret_cast<uintptr_t>(unmarked_rsqrt.data()),
|
||||
"unmarked native reciprocal root was incorrectly emulated");
|
||||
#if defined(__linux__)
|
||||
const auto emulate = [&](std::array<uint8_t, 16> instruction, size_t length) {
|
||||
context.uc_mcontext.gregs[REG_RIP] = reinterpret_cast<greg_t>(instruction.data());
|
||||
Check(test, Loader::X64InstructionEmulator::TryEmulate(&context), "existing instruction emulation was rejected");
|
||||
|
|
|
|||
Loading…
Reference in New Issue