shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

loader: enable the AMD CPU instruction patcher on macOS (#1014) 

* loader: enable AMD CPU instruction patching on macOS

The guest instruction patcher was compiled out on macOS and runtimeLinker forced emulate_amd off,
so under Rosetta every SSE4a EXTRQ/INSERTQ took the SIGILL path (about 18 us each). Build the
patcher on all hosts and let --amd-cpu enable it on macOS like on Windows and Linux. The host
feature probe already reports no SSE4a under Rosetta, so EXTRQ becomes a native trampoline and the
remaining fallbacks keep using the x64 instruction emulator through the host exception handler.

* tests: macos_sse4a covers the guest instruction patcher on macOS

Patch EXTRQ/INSERTQ blobs in an RWX area with a trampoline page behind it, with the host features of
Rosetta (no SSE4a/RDPID/CLWB), and compare against the SIGILL emulator: EXTRQ must become a native
trampoline without SIGILL, INSERTQ stays a trap handled by the emulator, and nothing is touched
without emulate_amd.

* loader: emulate the patcher's VRSQRTPS trap and MWAITX on macOS

The AMD CPU patcher marks VRSQRTPS it cannot relocate by clearing a reserved
VEX.vvvv bit, which raises #UD on every host including Rosetta. The macOS build
left TryEmulateReciprocalSquareRoot and TryEmulateMonitorxMwaitx out, so the
guest died with an unhandled SIGILL at the first such site.

Both are plain SSE2/libc, so enable them on macOS. The Darwin context also
gains ClearUpperYmm so the VEX.128 result zeroes YMM[255:128] when the thread
has the AVX context. macos_sse4a covers the trapped VRSQRTPS and MWAITX.

* tests: macos_sse4a runs a plain VEX.128 VRSQRTPS through the patcher

The existing case started from an instruction that was already marked as a trap. The
new cases feed the ordinary encoding to PatchGuestInstructions and compare the result
with the emulated reference, both through a native trampoline and, when the instruction
cannot be relocated, through the trap mark and the signal emulator.

* loader: compact macOS instruction patcher coverage

Share the emulator dispatch chain while keeping SHA emulation limited to non-Apple hosts and preserving handler order. Remove a duplicate VRSQRTPS signal test already covered by both native and trapped patcher cases, plus blank lines left by removed platform guards.

---------

Co-authored-by: nmzik <Nmzik@mail.ru>
This commit is contained in:
Bartłomiej Wołyńczyk 2026-10-06 19:55:11 +02:00 committed by GitHub
parent 4852721005
commit 35c6e8fff6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 244 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -719,8 +719,9 @@ if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
add_executable(macos_sse4a_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/macosSse4aTests.cpp"
"${KYTY_SOURCE_DIR}/loader/x64InstructionEmulator.cpp"
"${KYTY_SOURCE_DIR}/loader/guestInstructionPatcher.cpp"
)
target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis)
target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis xbyak::xbyak)
target_include_directories(macos_sse4a_tests PRIVATE ${inc_headers})
if(BUILD_TESTING)
add_test(NAME macos_sse4a COMMAND $<TARGET_FILE:macos_sse4a_tests>)

30
src/loader/guestInstructionPatcher.cpp
View File

@ -21,10 +21,8 @@
#include <set>
#include <unordered_set>
#include <vector>
#if !defined(__APPLE__)
#include <xbyak/xbyak.h>
#include <xbyak/xbyak_util.h>
#endif
#ifdef min
#undef min
@ -33,9 +31,7 @@
#undef max
#endif
#if !defined(__APPLE__)
using namespace Xbyak::util;
#endif
namespace Loader {
@ -52,8 +48,6 @@ using s64 = int64_t;
constexpr size_t NearJumpSize = 5;
#if !defined(__APPLE__)
struct PatchModule {
std::mutex mutex {};
u8* start = nullptr;
@ -1573,19 +1567,6 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment
return result;
}
#else
GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
return {};
}
GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool,
GuestInstructionHostFeatures) {
return {};
}
#endif
namespace {
constexpr uint8_t DW_EH_PE_FORMAT_MASK = 0x0f;
@ -1763,7 +1744,6 @@ bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_fram
void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
void* trampoline_area_ptr, uint64_t trampoline_area_size) {
#if !defined(__APPLE__)
EXIT_IF(module_ptr == nullptr || module_size == 0 || trampoline_area_ptr == nullptr ||
trampoline_area_size == 0);
const auto module_addr = reinterpret_cast<u64>(module_ptr);
@ -1772,20 +1752,10 @@ void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
std::forward_as_tuple(static_cast<u8*>(module_ptr), module_size,
static_cast<u8*>(trampoline_area_ptr),
trampoline_area_size));
#else
(void)module_ptr;
(void)module_size;
(void)trampoline_area_ptr;
(void)trampoline_area_size;
#endif
}
void UnregisterGuestInstructionPatchModule(void* module_ptr) {
#if !defined(__APPLE__)
g_patch_modules.erase(reinterpret_cast<u64>(module_ptr));
#else
(void)module_ptr;
#endif
}
#undef ASSERT

4
src/loader/runtimeLinker.cpp
View File

@ -1670,11 +1670,7 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize();
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
program->mapped_size = program->base_size_aligned + tls_handler_size;
#if !defined(__APPLE__)
const bool emulate_amd = Config::AmdCpuEnabled();
#else
const bool emulate_amd = false;
#endif
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
const bool protect_memory_faults = Config::RedZoneProtectionEnabled();
#else

23
src/loader/x64InstructionEmulator.cpp
View File

@ -12,6 +12,7 @@
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#include <windows.h> // IWYU pragma: keep
#elif defined(__APPLE__)
#include <sched.h>
#include <sys/ucontext.h>
#else
#include <sched.h>
@ -468,6 +469,16 @@ struct Context {
default: return nullptr;
}
}
// The kernel hands AVX threads the longer context that carries YMM[255:128]; a plain SSE
// context has no such state to clear.
void ClearUpperYmm(uint8_t index) const {
if (index >= 16 || native->uc_mcsize < sizeof(_STRUCT_MCONTEXT_AVX64)) {
return;
}
auto* avx = reinterpret_cast<_STRUCT_MCONTEXT_AVX64*>(native->uc_mcontext);
auto* ymmh = &avx->__fs.__fpu_ymmh0;
ymmh[index] = {};
}
#else
ucontext_t* native;
@ -703,8 +714,6 @@ static bool TryEmulateCpuExtensions(Context& context) {
return true;
}
#if !defined(__APPLE__)
static bool TryEmulateMonitorxMwaitx(Context& context) {
const auto* rip = reinterpret_cast<const uint8_t*>(context.Rip());
if (rip[0] != 0x0f || rip[1] != 0x01 || (rip[2] != 0xfa && rip[2] != 0xfb)) {
@ -790,8 +799,6 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) {
return true;
}
#endif
bool TryEmulate(void* native_context) {
if (native_context == nullptr) {
return false;
@ -807,12 +814,12 @@ bool TryEmulate(void* native_context) {
#else
Context context {static_cast<ucontext_t*>(native_context)};
#endif
#if !defined(__APPLE__)
return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) ||
TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context);
#else
return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context);
TryEmulateSse4a(context) ||
#if !defined(__APPLE__)
TryEmulateShaNi(context) ||
#endif
TryEmulateCpuExtensions(context);
}
} // namespace Loader::X64InstructionEmulator

227
tests/macosSse4aTests.cpp
View File

@ -1,6 +1,8 @@
#include "common/hostException.h"
#include "loader/guestInstructionPatcher.h"
#include "loader/x64InstructionEmulator.h"
#include <array>
#include <atomic>
#include <bit>
#include <cstddef>
@ -82,6 +84,53 @@ constexpr uint8_t kUd2[] = {
0xc3, // ret
};
// VRSQRTPS xmm1, xmm0 as the AMD CPU patcher leaves it when it cannot relocate the instruction: a
// reserved VEX.vvvv bit is cleared (c5 f8 -> c5 f0), which raises #UD on every host. Rosetta aborted
// the guest on exactly this encoding until the emulator handled it there too.
constexpr uint8_t kTrappedVrsqrtpsXmm1Xmm0[] = {
0x0f, 0x10, 0x06, // movups (%rsi), %xmm0
0xc5, 0xfc, 0x10, 0x0e, // vmovups ymm1, [rsi]: dirty the upper half of ymm1
0xc5, 0xf0, 0x52, 0xc8, // vrsqrtps xmm1, xmm0 with the vvvv trap mark
0x0f, 0x11, 0x0f, // movups %xmm1, (%rdi)
0xc4, 0xe3, 0x7d, 0x19, 0x4f, 0x10, 0x01, // vextractf128 [rdi + 16], ymm1, 1
0xc5, 0xf8, 0x77, // vzeroupper
0xc3, // ret
};
// The same sequence with the ordinary VEX.128 VRSQRTPS encoding (vvvv = 1111), as it appears in guest
// code before the patcher marks it as a trap.
constexpr uint8_t kPlainVrsqrtpsXmm1Xmm0[] = {
0x0f, 0x10, 0x06, // movups (%rsi), %xmm0
0xc5, 0xfc, 0x10, 0x0e, // vmovups ymm1, [rsi]: dirty the upper half of ymm1
0xc5, 0xf8, 0x52, 0xc8, // vrsqrtps xmm1, xmm0
0x0f, 0x11, 0x0f, // movups %xmm1, (%rdi)
0xc4, 0xe3, 0x7d, 0x19, 0x4f, 0x10, 0x01, // vextractf128 [rdi + 16], ymm1, 1
0xc5, 0xf8, 0x77, // vzeroupper
0xc3, // ret
};
// The same again behind a never taken indirect branch. A function with an indirect branch does not
// let the patcher relocate neighbours, and the 4-byte VRSQRTPS cannot hold a jump on its own, so it
// is left in place with the trap mark.
constexpr uint8_t kPlainVrsqrtpsIndirectBranch[] = {
0x31, 0xc0, // xor %eax, %eax
0x85, 0xc0, // test %eax, %eax
0x74, 0x02, // jz over
0xff, 0xe0, // jmp *%rax (never taken)
0x0f, 0x10, 0x06, // movups (%rsi), %xmm0
0xc5, 0xfc, 0x10, 0x0e, // vmovups ymm1, [rsi]: dirty the upper half of ymm1
0xc5, 0xf8, 0x52, 0xc8, // vrsqrtps xmm1, xmm0
0x0f, 0x11, 0x0f, // movups %xmm1, (%rdi)
0xc4, 0xe3, 0x7d, 0x19, 0x4f, 0x10, 0x01, // vextractf128 [rdi + 16], ymm1, 1
0xc5, 0xf8, 0x77, // vzeroupper
0xc3, // ret
};
constexpr uint8_t kMwaitx[] = {
0x0f, 0x01, 0xfb, // mwaitx
0xc3, // ret
};
constexpr size_t kPageSize = 4096;
std::atomic<int> g_illegal_hits {0};
@ -233,6 +282,13 @@ void TestInsertqRegister() {
}
}
void TestMwaitxEmulated() {
auto fn = MapCode<VoidBlobFn>(kMwaitx);
const int before = IllegalHits();
fn();
Check(IllegalHits() > before, "host raises SIGILL for MWAITX");
}
void TestUnknownInstructionRefused() {
g_refused.store(false, std::memory_order_relaxed);
g_refuse_skip.store(2, std::memory_order_relaxed);
@ -244,6 +300,170 @@ void TestUnknownInstructionRefused() {
g_refuse_skip.store(0, std::memory_order_relaxed);
}
// The loader rewrites SSE4a instructions into native trampolines when the patcher is enabled. The
// trampoline area lives right behind the code page, like it does behind a mapped guest module.
struct PatchedCode {
void *code = nullptr;
Loader::GuestInstructionPatchResult result {};
};
template <size_t N>
PatchedCode MapPatchedCode(const uint8_t (&blob)[N], bool emulate_amd) {
void *area = ::mmap(nullptr, 2 * kPageSize, PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANON, -1, 0);
Check(area != MAP_FAILED, "map RWX code and trampoline area");
std::memcpy(area, blob, N);
auto *trampoline = static_cast<uint8_t *>(area) + kPageSize;
Loader::RegisterGuestInstructionPatchModule(area, kPageSize, trampoline, kPageSize);
const std::array<uintptr_t, 1> function_starts {reinterpret_cast<uintptr_t>(area)};
// A host without SSE4a, RDPID and CLWB, which is what Rosetta presents.
const Loader::GuestInstructionHostFeatures host {false, false, false};
PatchedCode patched;
patched.code = area;
patched.result = Loader::PatchGuestInstructions(reinterpret_cast<uintptr_t>(area), N, function_starts,
false, emulate_amd, host);
Loader::UnregisterGuestInstructionPatchModule(area);
return patched;
}
using PatchCounts = Loader::InstructionPatchCounts Loader::GuestInstructionPatchResult::*;
// Runs the blob through the SIGILL emulator (reference) and through the patched copy, which must
// produce the same bytes. EXTRQ becomes a native trampoline without a single SIGILL. INSERTQ is
// marked as a trap by the patcher and keeps going through the signal emulator.
template <typename Fn, size_t N, typename Invoke>
void CheckPatchedMatchesEmulator(const uint8_t (&blob)[N], PatchCounts counts, bool native,
Invoke invoke, const char *text) {
uint64_t reference[2] {};
const int before_reference = IllegalHits();
invoke(MapCode<Fn>(blob), reference);
Check(IllegalHits() > before_reference, text);
const auto unpatched = MapPatchedCode(blob, false);
Check((unpatched.result.*counts).found == 0, "patcher stays off without emulate_amd");
uint64_t unpatched_out[2] {};
const int before_off = IllegalHits();
invoke(reinterpret_cast<Fn>(unpatched.code), unpatched_out);
Check(IllegalHits() > before_off, "unpatched copy still raises SIGILL");
const auto patched = MapPatchedCode(blob, true);
Check((patched.result.*counts).found == 1, text);
Check((patched.result.*counts).native == (native ? 1u : 0u), "native trampoline count");
Check((patched.result.*counts).trapped == (native ? 0u : 1u), "trapped (signal path) count");
uint64_t out[2] {};
const int before = IllegalHits();
invoke(reinterpret_cast<Fn>(patched.code), out);
if (native) {
Check(IllegalHits() == before, "patched code raises no SIGILL");
} else {
Check(IllegalHits() > before, "trapped instruction still goes through the signal emulator");
}
Check(out[0] == reference[0] && out[1] == reference[1], "patched result equals emulated result");
}
void TestPatchedExtrqImmediate() {
CheckPatchedMatchesEmulator<BlobFn>(
kExtrqXmm2Len40Index0, &Loader::GuestInstructionPatchResult::extrq, true,
[](BlobFn fn, uint64_t *out) { fn(out, 0.0, 0.0, std::bit_cast<double>(0xaabbccddeeff1122ull)); },
"EXTRQ immediate found by the patcher");
}
void TestPatchedExtrqRexHigh() {
CheckPatchedMatchesEmulator<LoadBlobFn>(
kExtrqXmm10Len40Index0, &Loader::GuestInstructionPatchResult::extrq, true,
[](LoadBlobFn fn, uint64_t *out) {
const uint64_t in[2] {0xaabbccddeeff1122ull, 0x5555555555555555ull};
fn(out, in);
},
"REX EXTRQ found by the patcher");
}
void TestPatchedInsertqImmediate() {
CheckPatchedMatchesEmulator<BlobFn>(
kInsertqXmm2Xmm0Len32Index0, &Loader::GuestInstructionPatchResult::insertq, false,
[](BlobFn fn, uint64_t *out) {
fn(out, std::bit_cast<double>(0x1122334455667788ull), 0.0,
std::bit_cast<double>(0x0123456789abcdefull));
},
"INSERTQ immediate found by the patcher");
}
void TestPatchedInsertqIndexedRexHigh() {
CheckPatchedMatchesEmulator<PairBlobFn>(
kInsertqXmm10Xmm9Len16Index8, &Loader::GuestInstructionPatchResult::insertq, false,
[](PairBlobFn fn, uint64_t *out) {
const uint64_t src[2] {0x8877665544332211ull, 0xdeadbeefcafebabeull};
const uint64_t dest[2] {0xaaaaaaaaaaaaaaaaull, 0xbbbbbbbbbbbbbbbbull};
fn(out, src, dest);
},
"indexed REX INSERTQ found by the patcher");
}
void TestPatchedInsertqRegister() {
CheckPatchedMatchesEmulator<PairBlobFn>(
kInsertqXmm10Xmm9, &Loader::GuestInstructionPatchResult::insertq, false,
[](PairBlobFn fn, uint64_t *out) {
const uint64_t src[2] {0x0123456789abcdefull, 0xdeadbeef0000c4c8ull};
const uint64_t dest[2] {0x8877665544332211ull, 0xfedcba9876543210ull};
fn(out, src, dest);
},
"register INSERTQ found by the patcher");
}
// The ordinary VEX.128 VRSQRTPS must be found by the patcher and give the reference result, either
// through a native trampoline (no SIGILL) or, when it cannot be relocated, marked as a trap (the
// vvvv bit is cleared in place) and emulated from the signal handler.
template <size_t N>
void CheckPatchedReciprocalSquareRoot(const uint8_t (&blob)[N], bool native) {
const uint32_t in_bits[8] {std::bit_cast<uint32_t>(4.0f), 0x00000000u, 0xbf800000u, 0x7f800000u,
0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
const auto Run = [&](LoadBlobFn fn, uint32_t (&out_bits)[8]) {
fn(reinterpret_cast<uint64_t *>(out_bits), reinterpret_cast<const uint64_t *>(in_bits));
};
uint32_t reference[8] {};
const int before_reference = IllegalHits();
Run(MapCode<LoadBlobFn>(kTrappedVrsqrtpsXmm1Xmm0), reference);
Check(IllegalHits() > before_reference, "host raises SIGILL for the pre-marked VRSQRTPS");
const auto unpatched = MapPatchedCode(blob, false);
Check(unpatched.result.reciprocal_sqrt.found == 0, "patcher leaves VRSQRTPS alone without emulate_amd");
const auto patched = MapPatchedCode(blob, true);
const auto &counts = patched.result.reciprocal_sqrt;
Check(counts.found == 1, "patcher finds the ordinary VEX.128 VRSQRTPS");
Check(counts.native == (native ? 1u : 0u) && counts.trapped == (native ? 0u : 1u),
"VRSQRTPS native trampoline / trap count");
const auto *code = static_cast<const uint8_t *>(patched.code);
size_t site = 0;
while (site + 2 < N && !(blob[site] == 0xc5 && blob[site + 1] == 0xf8 && blob[site + 2] == 0x52)) {
++site;
}
Check(site + 2 < N, "locate VRSQRTPS in the test blob");
if (!native) {
Check(code[site] == 0xc5 && code[site + 1] == 0xf0 && code[site + 2] == 0x52,
"the trap mark clears a reserved VEX.vvvv bit (c5 f8 -> c5 f0)");
} else {
Check(code[site] != 0xc5 || code[site + 1] != 0xf8, "native trampoline replaced the VRSQRTPS");
}
uint32_t out[8] {};
const int before = IllegalHits();
Run(reinterpret_cast<LoadBlobFn>(patched.code), out);
if (native) {
Check(IllegalHits() == before, "natively patched VRSQRTPS raises no SIGILL");
} else {
Check(IllegalHits() > before, "marked VRSQRTPS goes through the signal emulator");
}
Check(std::memcmp(out, reference, sizeof(out)) == 0, "patched VRSQRTPS result equals the emulated reference");
Check(std::bit_cast<float>(out[0]) == 0.5f && out[1] == 0x7f800000u && out[2] == 0xffc00000u && out[3] == 0,
"patched VRSQRTPS produces the exact reciprocal square roots");
Check(out[4] == 0 && out[5] == 0 && out[6] == 0 && out[7] == 0, "patched VEX.128 form zeroes the upper half");
}
void TestPatchedReciprocalSquareRoot() {
CheckPatchedReciprocalSquareRoot(kPlainVrsqrtpsXmm1Xmm0, true);
CheckPatchedReciprocalSquareRoot(kPlainVrsqrtpsIndirectBranch, false);
}
} // namespace
int main() {
@ -255,7 +475,14 @@ int main() {
TestRexHighRegisterExtrq();
TestRexHighRegisterInsertqWithIndex();
TestInsertqRegister();
TestMwaitxEmulated();
TestUnknownInstructionRefused();
TestPatchedExtrqImmediate();
TestPatchedExtrqRexHigh();
TestPatchedInsertqImmediate();
TestPatchedInsertqIndexedRexHigh();
TestPatchedInsertqRegister();
TestPatchedReciprocalSquareRoot();
std::printf("macosSse4aTests: all passed\n");
return 0;
}