diff --git a/CMakeLists.txt b/CMakeLists.txt index 86682f1e..a3fc47da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -719,8 +719,9 @@ if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR add_executable(macos_sse4a_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/macosSse4aTests.cpp" "${KYTY_SOURCE_DIR}/loader/x64InstructionEmulator.cpp" + "${KYTY_SOURCE_DIR}/loader/guestInstructionPatcher.cpp" ) - target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis) + target_link_libraries(macos_sse4a_tests common fmt::fmt Zydis::Zydis xbyak::xbyak) target_include_directories(macos_sse4a_tests PRIVATE ${inc_headers}) if(BUILD_TESTING) add_test(NAME macos_sse4a COMMAND $) diff --git a/src/loader/guestInstructionPatcher.cpp b/src/loader/guestInstructionPatcher.cpp index 83069bfd..0ab7de67 100644 --- a/src/loader/guestInstructionPatcher.cpp +++ b/src/loader/guestInstructionPatcher.cpp @@ -21,10 +21,8 @@ #include #include #include -#if !defined(__APPLE__) #include #include -#endif #ifdef min #undef min @@ -33,9 +31,7 @@ #undef max #endif -#if !defined(__APPLE__) using namespace Xbyak::util; -#endif namespace Loader { @@ -52,8 +48,6 @@ using s64 = int64_t; constexpr size_t NearJumpSize = 5; -#if !defined(__APPLE__) - struct PatchModule { std::mutex mutex {}; u8* start = nullptr; @@ -1573,19 +1567,6 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment return result; } -#else - -GuestInstructionHostFeatures GetGuestInstructionHostFeatures() { - return {}; -} - -GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span, bool, bool, - GuestInstructionHostFeatures) { - return {}; -} - -#endif - namespace { constexpr uint8_t DW_EH_PE_FORMAT_MASK = 0x0f; @@ -1763,7 +1744,6 @@ bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_fram void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr, uint64_t trampoline_area_size) { -#if !defined(__APPLE__) EXIT_IF(module_ptr == nullptr || module_size == 0 || trampoline_area_ptr == nullptr || trampoline_area_size == 0); const auto module_addr = reinterpret_cast(module_ptr); @@ -1772,20 +1752,10 @@ void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size, std::forward_as_tuple(static_cast(module_ptr), module_size, static_cast(trampoline_area_ptr), trampoline_area_size)); -#else - (void)module_ptr; - (void)module_size; - (void)trampoline_area_ptr; - (void)trampoline_area_size; -#endif } void UnregisterGuestInstructionPatchModule(void* module_ptr) { -#if !defined(__APPLE__) g_patch_modules.erase(reinterpret_cast(module_ptr)); -#else - (void)module_ptr; -#endif } #undef ASSERT diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp index 3114f239..f69487df 100644 --- a/src/loader/runtimeLinker.cpp +++ b/src/loader/runtimeLinker.cpp @@ -1670,11 +1670,7 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize(); EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned); program->mapped_size = program->base_size_aligned + tls_handler_size; -#if !defined(__APPLE__) const bool emulate_amd = Config::AmdCpuEnabled(); -#else - const bool emulate_amd = false; -#endif #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS const bool protect_memory_faults = Config::RedZoneProtectionEnabled(); #else diff --git a/src/loader/x64InstructionEmulator.cpp b/src/loader/x64InstructionEmulator.cpp index 55fa9798..7500c364 100644 --- a/src/loader/x64InstructionEmulator.cpp +++ b/src/loader/x64InstructionEmulator.cpp @@ -12,6 +12,7 @@ #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #include // IWYU pragma: keep #elif defined(__APPLE__) +#include #include #else #include @@ -468,6 +469,16 @@ struct Context { default: return nullptr; } } + // The kernel hands AVX threads the longer context that carries YMM[255:128]; a plain SSE + // context has no such state to clear. + void ClearUpperYmm(uint8_t index) const { + if (index >= 16 || native->uc_mcsize < sizeof(_STRUCT_MCONTEXT_AVX64)) { + return; + } + auto* avx = reinterpret_cast<_STRUCT_MCONTEXT_AVX64*>(native->uc_mcontext); + auto* ymmh = &avx->__fs.__fpu_ymmh0; + ymmh[index] = {}; + } #else ucontext_t* native; @@ -703,8 +714,6 @@ static bool TryEmulateCpuExtensions(Context& context) { return true; } -#if !defined(__APPLE__) - static bool TryEmulateMonitorxMwaitx(Context& context) { const auto* rip = reinterpret_cast(context.Rip()); if (rip[0] != 0x0f || rip[1] != 0x01 || (rip[2] != 0xfa && rip[2] != 0xfb)) { @@ -790,8 +799,6 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) { return true; } -#endif - bool TryEmulate(void* native_context) { if (native_context == nullptr) { return false; @@ -807,12 +814,12 @@ bool TryEmulate(void* native_context) { #else Context context {static_cast(native_context)}; #endif -#if !defined(__APPLE__) return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) || - TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context); -#else - return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context); + TryEmulateSse4a(context) || +#if !defined(__APPLE__) + TryEmulateShaNi(context) || #endif + TryEmulateCpuExtensions(context); } } // namespace Loader::X64InstructionEmulator diff --git a/tests/macosSse4aTests.cpp b/tests/macosSse4aTests.cpp index 4e7b502f..89674761 100644 --- a/tests/macosSse4aTests.cpp +++ b/tests/macosSse4aTests.cpp @@ -1,6 +1,8 @@ #include "common/hostException.h" +#include "loader/guestInstructionPatcher.h" #include "loader/x64InstructionEmulator.h" +#include #include #include #include @@ -82,6 +84,53 @@ constexpr uint8_t kUd2[] = { 0xc3, // ret }; +// VRSQRTPS xmm1, xmm0 as the AMD CPU patcher leaves it when it cannot relocate the instruction: a +// reserved VEX.vvvv bit is cleared (c5 f8 -> c5 f0), which raises #UD on every host. Rosetta aborted +// the guest on exactly this encoding until the emulator handled it there too. +constexpr uint8_t kTrappedVrsqrtpsXmm1Xmm0[] = { + 0x0f, 0x10, 0x06, // movups (%rsi), %xmm0 + 0xc5, 0xfc, 0x10, 0x0e, // vmovups ymm1, [rsi]: dirty the upper half of ymm1 + 0xc5, 0xf0, 0x52, 0xc8, // vrsqrtps xmm1, xmm0 with the vvvv trap mark + 0x0f, 0x11, 0x0f, // movups %xmm1, (%rdi) + 0xc4, 0xe3, 0x7d, 0x19, 0x4f, 0x10, 0x01, // vextractf128 [rdi + 16], ymm1, 1 + 0xc5, 0xf8, 0x77, // vzeroupper + 0xc3, // ret +}; + +// The same sequence with the ordinary VEX.128 VRSQRTPS encoding (vvvv = 1111), as it appears in guest +// code before the patcher marks it as a trap. +constexpr uint8_t kPlainVrsqrtpsXmm1Xmm0[] = { + 0x0f, 0x10, 0x06, // movups (%rsi), %xmm0 + 0xc5, 0xfc, 0x10, 0x0e, // vmovups ymm1, [rsi]: dirty the upper half of ymm1 + 0xc5, 0xf8, 0x52, 0xc8, // vrsqrtps xmm1, xmm0 + 0x0f, 0x11, 0x0f, // movups %xmm1, (%rdi) + 0xc4, 0xe3, 0x7d, 0x19, 0x4f, 0x10, 0x01, // vextractf128 [rdi + 16], ymm1, 1 + 0xc5, 0xf8, 0x77, // vzeroupper + 0xc3, // ret +}; + +// The same again behind a never taken indirect branch. A function with an indirect branch does not +// let the patcher relocate neighbours, and the 4-byte VRSQRTPS cannot hold a jump on its own, so it +// is left in place with the trap mark. +constexpr uint8_t kPlainVrsqrtpsIndirectBranch[] = { + 0x31, 0xc0, // xor %eax, %eax + 0x85, 0xc0, // test %eax, %eax + 0x74, 0x02, // jz over + 0xff, 0xe0, // jmp *%rax (never taken) + 0x0f, 0x10, 0x06, // movups (%rsi), %xmm0 + 0xc5, 0xfc, 0x10, 0x0e, // vmovups ymm1, [rsi]: dirty the upper half of ymm1 + 0xc5, 0xf8, 0x52, 0xc8, // vrsqrtps xmm1, xmm0 + 0x0f, 0x11, 0x0f, // movups %xmm1, (%rdi) + 0xc4, 0xe3, 0x7d, 0x19, 0x4f, 0x10, 0x01, // vextractf128 [rdi + 16], ymm1, 1 + 0xc5, 0xf8, 0x77, // vzeroupper + 0xc3, // ret +}; + +constexpr uint8_t kMwaitx[] = { + 0x0f, 0x01, 0xfb, // mwaitx + 0xc3, // ret +}; + constexpr size_t kPageSize = 4096; std::atomic g_illegal_hits {0}; @@ -233,6 +282,13 @@ void TestInsertqRegister() { } } +void TestMwaitxEmulated() { + auto fn = MapCode(kMwaitx); + const int before = IllegalHits(); + fn(); + Check(IllegalHits() > before, "host raises SIGILL for MWAITX"); +} + void TestUnknownInstructionRefused() { g_refused.store(false, std::memory_order_relaxed); g_refuse_skip.store(2, std::memory_order_relaxed); @@ -244,6 +300,170 @@ void TestUnknownInstructionRefused() { g_refuse_skip.store(0, std::memory_order_relaxed); } +// The loader rewrites SSE4a instructions into native trampolines when the patcher is enabled. The +// trampoline area lives right behind the code page, like it does behind a mapped guest module. +struct PatchedCode { + void *code = nullptr; + Loader::GuestInstructionPatchResult result {}; +}; + +template +PatchedCode MapPatchedCode(const uint8_t (&blob)[N], bool emulate_amd) { + void *area = ::mmap(nullptr, 2 * kPageSize, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANON, -1, 0); + Check(area != MAP_FAILED, "map RWX code and trampoline area"); + std::memcpy(area, blob, N); + auto *trampoline = static_cast(area) + kPageSize; + Loader::RegisterGuestInstructionPatchModule(area, kPageSize, trampoline, kPageSize); + const std::array function_starts {reinterpret_cast(area)}; + // A host without SSE4a, RDPID and CLWB, which is what Rosetta presents. + const Loader::GuestInstructionHostFeatures host {false, false, false}; + PatchedCode patched; + patched.code = area; + patched.result = Loader::PatchGuestInstructions(reinterpret_cast(area), N, function_starts, + false, emulate_amd, host); + Loader::UnregisterGuestInstructionPatchModule(area); + return patched; +} + +using PatchCounts = Loader::InstructionPatchCounts Loader::GuestInstructionPatchResult::*; + +// Runs the blob through the SIGILL emulator (reference) and through the patched copy, which must +// produce the same bytes. EXTRQ becomes a native trampoline without a single SIGILL. INSERTQ is +// marked as a trap by the patcher and keeps going through the signal emulator. +template +void CheckPatchedMatchesEmulator(const uint8_t (&blob)[N], PatchCounts counts, bool native, + Invoke invoke, const char *text) { + uint64_t reference[2] {}; + const int before_reference = IllegalHits(); + invoke(MapCode(blob), reference); + Check(IllegalHits() > before_reference, text); + + const auto unpatched = MapPatchedCode(blob, false); + Check((unpatched.result.*counts).found == 0, "patcher stays off without emulate_amd"); + uint64_t unpatched_out[2] {}; + const int before_off = IllegalHits(); + invoke(reinterpret_cast(unpatched.code), unpatched_out); + Check(IllegalHits() > before_off, "unpatched copy still raises SIGILL"); + + const auto patched = MapPatchedCode(blob, true); + Check((patched.result.*counts).found == 1, text); + Check((patched.result.*counts).native == (native ? 1u : 0u), "native trampoline count"); + Check((patched.result.*counts).trapped == (native ? 0u : 1u), "trapped (signal path) count"); + uint64_t out[2] {}; + const int before = IllegalHits(); + invoke(reinterpret_cast(patched.code), out); + if (native) { + Check(IllegalHits() == before, "patched code raises no SIGILL"); + } else { + Check(IllegalHits() > before, "trapped instruction still goes through the signal emulator"); + } + Check(out[0] == reference[0] && out[1] == reference[1], "patched result equals emulated result"); +} + +void TestPatchedExtrqImmediate() { + CheckPatchedMatchesEmulator( + kExtrqXmm2Len40Index0, &Loader::GuestInstructionPatchResult::extrq, true, + [](BlobFn fn, uint64_t *out) { fn(out, 0.0, 0.0, std::bit_cast(0xaabbccddeeff1122ull)); }, + "EXTRQ immediate found by the patcher"); +} + +void TestPatchedExtrqRexHigh() { + CheckPatchedMatchesEmulator( + kExtrqXmm10Len40Index0, &Loader::GuestInstructionPatchResult::extrq, true, + [](LoadBlobFn fn, uint64_t *out) { + const uint64_t in[2] {0xaabbccddeeff1122ull, 0x5555555555555555ull}; + fn(out, in); + }, + "REX EXTRQ found by the patcher"); +} + +void TestPatchedInsertqImmediate() { + CheckPatchedMatchesEmulator( + kInsertqXmm2Xmm0Len32Index0, &Loader::GuestInstructionPatchResult::insertq, false, + [](BlobFn fn, uint64_t *out) { + fn(out, std::bit_cast(0x1122334455667788ull), 0.0, + std::bit_cast(0x0123456789abcdefull)); + }, + "INSERTQ immediate found by the patcher"); +} + +void TestPatchedInsertqIndexedRexHigh() { + CheckPatchedMatchesEmulator( + kInsertqXmm10Xmm9Len16Index8, &Loader::GuestInstructionPatchResult::insertq, false, + [](PairBlobFn fn, uint64_t *out) { + const uint64_t src[2] {0x8877665544332211ull, 0xdeadbeefcafebabeull}; + const uint64_t dest[2] {0xaaaaaaaaaaaaaaaaull, 0xbbbbbbbbbbbbbbbbull}; + fn(out, src, dest); + }, + "indexed REX INSERTQ found by the patcher"); +} + +void TestPatchedInsertqRegister() { + CheckPatchedMatchesEmulator( + kInsertqXmm10Xmm9, &Loader::GuestInstructionPatchResult::insertq, false, + [](PairBlobFn fn, uint64_t *out) { + const uint64_t src[2] {0x0123456789abcdefull, 0xdeadbeef0000c4c8ull}; + const uint64_t dest[2] {0x8877665544332211ull, 0xfedcba9876543210ull}; + fn(out, src, dest); + }, + "register INSERTQ found by the patcher"); +} + +// The ordinary VEX.128 VRSQRTPS must be found by the patcher and give the reference result, either +// through a native trampoline (no SIGILL) or, when it cannot be relocated, marked as a trap (the +// vvvv bit is cleared in place) and emulated from the signal handler. +template +void CheckPatchedReciprocalSquareRoot(const uint8_t (&blob)[N], bool native) { + const uint32_t in_bits[8] {std::bit_cast(4.0f), 0x00000000u, 0xbf800000u, 0x7f800000u, + 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + const auto Run = [&](LoadBlobFn fn, uint32_t (&out_bits)[8]) { + fn(reinterpret_cast(out_bits), reinterpret_cast(in_bits)); + }; + uint32_t reference[8] {}; + const int before_reference = IllegalHits(); + Run(MapCode(kTrappedVrsqrtpsXmm1Xmm0), reference); + Check(IllegalHits() > before_reference, "host raises SIGILL for the pre-marked VRSQRTPS"); + + const auto unpatched = MapPatchedCode(blob, false); + Check(unpatched.result.reciprocal_sqrt.found == 0, "patcher leaves VRSQRTPS alone without emulate_amd"); + + const auto patched = MapPatchedCode(blob, true); + const auto &counts = patched.result.reciprocal_sqrt; + Check(counts.found == 1, "patcher finds the ordinary VEX.128 VRSQRTPS"); + Check(counts.native == (native ? 1u : 0u) && counts.trapped == (native ? 0u : 1u), + "VRSQRTPS native trampoline / trap count"); + const auto *code = static_cast(patched.code); + size_t site = 0; + while (site + 2 < N && !(blob[site] == 0xc5 && blob[site + 1] == 0xf8 && blob[site + 2] == 0x52)) { + ++site; + } + Check(site + 2 < N, "locate VRSQRTPS in the test blob"); + if (!native) { + Check(code[site] == 0xc5 && code[site + 1] == 0xf0 && code[site + 2] == 0x52, + "the trap mark clears a reserved VEX.vvvv bit (c5 f8 -> c5 f0)"); + } else { + Check(code[site] != 0xc5 || code[site + 1] != 0xf8, "native trampoline replaced the VRSQRTPS"); + } + uint32_t out[8] {}; + const int before = IllegalHits(); + Run(reinterpret_cast(patched.code), out); + if (native) { + Check(IllegalHits() == before, "natively patched VRSQRTPS raises no SIGILL"); + } else { + Check(IllegalHits() > before, "marked VRSQRTPS goes through the signal emulator"); + } + Check(std::memcmp(out, reference, sizeof(out)) == 0, "patched VRSQRTPS result equals the emulated reference"); + Check(std::bit_cast(out[0]) == 0.5f && out[1] == 0x7f800000u && out[2] == 0xffc00000u && out[3] == 0, + "patched VRSQRTPS produces the exact reciprocal square roots"); + Check(out[4] == 0 && out[5] == 0 && out[6] == 0 && out[7] == 0, "patched VEX.128 form zeroes the upper half"); +} + +void TestPatchedReciprocalSquareRoot() { + CheckPatchedReciprocalSquareRoot(kPlainVrsqrtpsXmm1Xmm0, true); + CheckPatchedReciprocalSquareRoot(kPlainVrsqrtpsIndirectBranch, false); +} + } // namespace int main() { @@ -255,7 +475,14 @@ int main() { TestRexHighRegisterExtrq(); TestRexHighRegisterInsertqWithIndex(); TestInsertqRegister(); + TestMwaitxEmulated(); TestUnknownInstructionRefused(); + TestPatchedExtrqImmediate(); + TestPatchedExtrqRexHigh(); + TestPatchedInsertqImmediate(); + TestPatchedInsertqIndexedRexHigh(); + TestPatchedInsertqRegister(); + TestPatchedReciprocalSquareRoot(); std::printf("macosSse4aTests: all passed\n"); return 0; }