diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 18d528c73..428b0f2be 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -40,6 +40,8 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/StubBodyBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha256Operands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha256Lowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -102,6 +104,8 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/StubBodyBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha256Operands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha256Lowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -146,6 +150,8 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/StubBodyBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha256Operands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/Sha256Lowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp b/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp new file mode 100644 index 000000000..82b15ed41 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp @@ -0,0 +1,19 @@ +#ifndef CODEGEN_X86_CLZEROLOWERING_HPP +#define CODEGEN_X86_CLZEROLOWERING_HPP + +#include +#include +#include +#include + +namespace Codegen { + +class ClzeroLowering { +public: + void EmitOutOfLine(StubBodyBuilder& body, const ClzeroOperands& operands) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const ClzeroOperands& operands, std::span trailing = {}) const; +}; + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/ClzeroOperands.hpp b/core/relinker/codegen/include/codegen/x86/ClzeroOperands.hpp new file mode 100644 index 000000000..d690c73c3 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ClzeroOperands.hpp @@ -0,0 +1,17 @@ +#ifndef CODEGEN_X86_CLZEROOPERANDS_HPP +#define CODEGEN_X86_CLZEROOPERANDS_HPP + +#include +#include + +namespace Codegen { + +struct ClzeroOperands { + bool AddressSize32; +}; + +[[nodiscard]] ClzeroOperands DecodeClzero(const std::uint8_t* data, std::size_t length); + +} + +#endif diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 887d4154f..603915253 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,15 @@ namespace Codegen { namespace { +bool _isPrefixRun(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + return std::all_of(data, data + length, [](const std::uint8_t b) { + return (b >= RexMin && b <= RexMax) || b == PrefixLock || b == PrefixRepne || b == PrefixRep || + b == PrefixSegCs || b == PrefixSegSs || b == PrefixSegDs || b == PrefixSegEs || + b == PrefixSegFs || b == PrefixSegGs || b == PrefixOperandSize || b == PrefixAddressSize; + }); +} + template auto _atFileOffset(const Domain::FileByteOffset base, const TOperation& operation) { try { @@ -110,6 +120,11 @@ void Amd64OnlyConverter::_convertSegment( for (const auto& item : pending) { if (consumed.contains(item.Index)) continue; + if (item.Index > 0) { + const auto& previous = matches[item.Index - 1]; + if (previous.Offset + previous.Length == item.Instruction.Offset && _isPrefixRun(seg.data() + previous.Offset, previous.Length)) + throw CodegenException("AMD-only instruction follows prefixes the decoder split from it", ph.Offset + previous.Offset); + } const auto& match = item.Instruction; const auto& substitution = item.Substitution; const auto fileOffset = static_cast(ph.Offset + match.Offset); @@ -142,7 +157,7 @@ void Amd64OnlyConverter::_convertSegment( if (amdOnly && trailingBytes == 0) { sequence.push_back(bytes); consumed.insert(next); - } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) { + } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget || _isPrefixRun(bytes.data(), bytes.size())) { throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset); } else { trailingBytes += following.Length; diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 19dff1ba2..273444fa8 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -51,6 +53,11 @@ Amd64OnlyMatch _inPlace(const Entry& entry, const std::size_t length, std::vecto return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } +bool _isClzeroOpcode(const DecodedInstruction& instr) { + const auto pos = instr.OpcodeOffset(); + return pos + 2 < instr.Length && instr.Data[pos] == X64OpcodeConstants::TwoByteOpcodeEscape && instr.Data[pos + 1] == X64OpcodeConstants::TwoByteGrp7 && instr.Data[pos + 2] == 0xFC; +} + bool _validWait(const DecodedInstruction& instr) { const auto opcode = instr.Data + instr.OpcodeOffset(); return instr.Length <= kMaxInstructionLength && std::find(instr.Data, opcode, X64OpcodeConstants::PrefixLock) == opcode; @@ -72,10 +79,12 @@ public: private: Sse4aLowering _lowering; Sha256Lowering _sha256Lowering; + ClzeroLowering _clzeroLowering; [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const; [[nodiscard]] Amd64OnlyMatch _matchSha256(const DecodedInstruction& instr, std::span trailing) const; + [[nodiscard]] Amd64OnlyMatch _matchClzero(const DecodedInstruction& instr, std::span trailing) const; }; Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const { @@ -108,6 +117,11 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSha256(const DecodedInstructio return Amd64OnlyMatch{_sha256Entry(operands).Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } +Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchClzero(const DecodedInstruction& instr, std::span trailing) const { + auto body = _clzeroLowering.LowerOutOfLine(DecodeClzero(instr.Data, instr.Length), trailing); + return Amd64OnlyMatch{kClzero.Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; +} + std::optional Amd64OnlyInstructionMatcher::MatchSequence( std::span> instructions, std::span trailing @@ -128,6 +142,11 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( if (name == nullptr) name = _sha256Entry(operands).Name; _sha256Lowering.EmitOutOfLine(body, operands); + } else if (instr.IsClzero()) { + const auto operands = DecodeClzero(instr.Data, instr.Length); + if (name == nullptr) + name = kClzero.Name; + _clzeroLowering.EmitOutOfLine(body, operands); } else { return std::nullopt; } @@ -166,6 +185,9 @@ std::optional Amd64OnlyInstructionMatcher::Match( return _validWait(instr) ? _inPlace(kMwaitx, length, std::vector(kPause.Bytes, kPause.Bytes + kPause.Size)) : _unsupported(kMwaitx, length); if (instr.IsClzero()) + return _matchClzero(instr, trailing); + + if (_isClzeroOpcode(instr)) return _unsupported(kClzero, length); if (instr.IsRdpru()) diff --git a/core/relinker/codegen/src/x86/ClzeroLowering.cpp b/core/relinker/codegen/src/x86/ClzeroLowering.cpp new file mode 100644 index 000000000..c4a1ced26 --- /dev/null +++ b/core/relinker/codegen/src/x86/ClzeroLowering.cpp @@ -0,0 +1,51 @@ +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kPrefixPacked = 0x66; +constexpr std::uint8_t kScratch = 0; +constexpr std::uint64_t kLineMask = ~std::uint64_t{63}; +constexpr std::array kPushRcx = {0x51}; +constexpr std::array kPopRcx = {0x59}; +constexpr std::array kMovqXmm0Rax = {0x66, 0x48, 0x0F, 0x6E, 0xC0}; +constexpr std::array kMovdXmm0Eax = {0x66, 0x0F, 0x6E, 0xC0}; +constexpr std::array kMovqRcxXmm0 = {0x66, 0x48, 0x0F, 0x7E, 0xC1}; +constexpr std::array kMovntdqRcx = {0x66, 0x0F, 0xE7, 0x01}; +constexpr std::array kMovntdqRcx16 = {0x66, 0x0F, 0xE7, 0x41, 0x10}; +constexpr std::array kMovntdqRcx32 = {0x66, 0x0F, 0xE7, 0x41, 0x20}; +constexpr std::array kMovntdqRcx48 = {0x66, 0x0F, 0xE7, 0x41, 0x30}; + +} + +void ClzeroLowering::EmitOutOfLine(StubBodyBuilder& body, const ClzeroOperands& operands) const { + StubConstant lineMask{}; + for (std::size_t byte = 0; byte < 8; ++byte) + lineMask[byte] = static_cast(kLineMask >> (byte * 8)); + body.Spill(kScratch); + body.Raw(kPushRcx); + if (operands.AddressSize32) + body.Raw(kMovdXmm0Eax); + else + body.Raw(kMovqXmm0Rax); + body.RipOperand({0x0F, 0xDB}, kScratch, lineMask); + body.Raw(kMovqRcxXmm0); + body.Sse(kPrefixPacked, {0x0F, 0xEF}, kScratch, kScratch); + body.Raw(kMovntdqRcx); + body.Raw(kMovntdqRcx16); + body.Raw(kMovntdqRcx32); + body.Raw(kMovntdqRcx48); + body.Raw(kPopRcx); + body.Restore(kScratch); +} + +LoweredBody ClzeroLowering::LowerOutOfLine(const ClzeroOperands& operands, std::span trailing) const { + StubBodyBuilder body; + EmitOutOfLine(body, operands); + body.Raw(trailing); + return body.Finish(); +} + +} diff --git a/core/relinker/codegen/src/x86/ClzeroOperands.cpp b/core/relinker/codegen/src/x86/ClzeroOperands.cpp new file mode 100644 index 000000000..2f5de2b7c --- /dev/null +++ b/core/relinker/codegen/src/x86/ClzeroOperands.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +namespace Codegen { + +using namespace X64OpcodeConstants; + +namespace { + +constexpr std::size_t kMaxInstructionLength = 15; + +} + +ClzeroOperands DecodeClzero(const std::uint8_t* data, const std::size_t length) { + if (length > kMaxInstructionLength) { + throw CodegenException("CLZERO longer than 15 bytes"); + } + + ClzeroOperands operands{}; + std::size_t pos = 0; + + while (pos < length) { + const std::uint8_t b = data[pos]; + if (b == PrefixLock) { + throw CodegenException("CLZERO with a LOCK prefix"); + } + if (b == PrefixSegFs || b == PrefixSegGs) { + throw CodegenException("CLZERO with an FS or GS segment override"); + } + if (b == PrefixOperandSize || b == PrefixRepne || b == PrefixRep) { + throw CodegenException("Not a CLZERO instruction"); + } + if (b == PrefixAddressSize) { + operands.AddressSize32 = true; + } else if (b != PrefixSegCs && b != PrefixSegSs && b != PrefixSegDs && b != PrefixSegEs) { + break; + } + pos += 1; + } + + if (pos < length && data[pos] >= RexMin && data[pos] <= RexMax) { + pos += 1; + } + + if (pos + 3 != length || data[pos] != TwoByteOpcodeEscape || data[pos + 1] != TwoByteGrp7 || data[pos + 2] != 0xFC) { + throw CodegenException("Not a CLZERO instruction"); + } + + return operands; +} + +} diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index ef164d12c..a7fe93b40 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -20,6 +22,9 @@ #include #endif #include +#include +#include +#include #include #include #include @@ -97,6 +102,14 @@ const Bytes kInsertqWordBody = { 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x00, 0x01, 0x08, 0x09, 0x04, 0x05, 0x06, 0x07, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; +const Bytes kClzeroBody = { + 0x48, 0x8D, 0xA4, 0x24, 0x70, 0xFF, 0xFF, 0xFF, 0xF3, 0x0F, 0x7F, 0x04, 0x24, 0x51, 0x66, 0x48, + 0x0F, 0x6E, 0xC0, 0x66, 0x0F, 0xDB, 0x05, 0x35, 0x00, 0x00, 0x00, 0x66, 0x48, 0x0F, 0x7E, 0xC1, + 0x66, 0x0F, 0xEF, 0xC0, 0x66, 0x0F, 0xE7, 0x01, 0x66, 0x0F, 0xE7, 0x41, 0x10, 0x66, 0x0F, 0xE7, + 0x41, 0x20, 0x66, 0x0F, 0xE7, 0x41, 0x30, 0x59, 0xF3, 0x0F, 0x6F, 0x04, 0x24, 0x48, 0x8D, 0xA4, + 0x24, 0x90, 0x00, 0x00, 0x00, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, + 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + void decoderLengths() { const Codegen::X64InstructionDecoder decoder; const std::vector instructions = { @@ -151,6 +164,18 @@ void sha256Operands() { require(Codegen::DecodedInstruction{rounds.data(), rounds.size()}.IsShaNi(), "SHA256RNDS2 is not recognised as SHA-NI"); } +void clzeroOperands() { + const auto decode = [](const Bytes& bytes) { return Codegen::DecodeClzero(bytes.data(), bytes.size()); }; + require(!decode({0x0F, 0x01, 0xFC}).AddressSize32 && decode({0x67, 0x0F, 0x01, 0xFC}).AddressSize32, "CLZERO address size was decoded incorrectly"); + require(!decode({0x3E, 0x48, 0x0F, 0x01, 0xFC}).AddressSize32, "CLZERO with a DS override and REX was not decoded"); + requireFailure([&] { (void)decode({0xF0, 0x0F, 0x01, 0xFC}); }, "LOCK CLZERO was accepted"); + requireFailure([&] { (void)decode({0x64, 0x0F, 0x01, 0xFC}); }, "FS-relative CLZERO was accepted"); + requireFailure([&] { (void)decode({0x65, 0x0F, 0x01, 0xFC}); }, "GS-relative CLZERO was accepted"); + Bytes overlong(13, 0x2E); + overlong.insert(overlong.end(), {0x0F, 0x01, 0xFC}); + requireFailure([&] { (void)decode(overlong); }, "CLZERO longer than 15 bytes was accepted"); +} + void matcherSubstitutions() { const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); const auto match = [&](const Bytes& bytes) { return matcher->Match(bytes.data(), bytes.size()); }; @@ -182,7 +207,13 @@ void matcherSubstitutions() { const auto overlong = match(overlongMonitorx); require(overlong && overlong->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "MONITORX longer than 15 bytes was replaced instead of failing"); const auto clzero = match({0x0F, 0x01, 0xFC}); - require(clzero && clzero->Lowering == Codegen::Amd64OnlyLowering::Unsupported && clzero->InstructionName == "CLZERO", "CLZERO was not reported as unsupported"); + require(clzero && clzero->Lowering == Codegen::Amd64OnlyLowering::Trampoline && clzero->InstructionName == "CLZERO", "CLZERO was not lowered through a stub"); + for (const std::uint8_t prefix : {std::uint8_t{0x66}, std::uint8_t{0xF2}, std::uint8_t{0xF3}}) { + const auto prefixedClzero = match({prefix, 0x0F, 0x01, 0xFC}); + require(prefixedClzero && prefixedClzero->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "CLZERO with a 66, F2 or F3 prefix was not reported as unsupported"); + } + const auto rdpru = match({0x0F, 0x01, 0xFD}); + require(rdpru && rdpru->Lowering == Codegen::Amd64OnlyLowering::Unsupported && rdpru->InstructionName == "RDPRU", "RDPRU was not reported as unsupported"); const auto registerForm = match({0x66, 0x0F, 0x79, 0xCA}); require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Trampoline && registerForm->InstructionName == "EXTRQ register form", "EXTRQ register form was not lowered through a stub"); const auto insertqRegisterForm = match({0xF2, 0x0F, 0x79, 0xCA}); @@ -224,6 +255,9 @@ void goldenBodies() { inPlace({0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x00}, {0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00}); inPlace({0xF2, 0x0F, 0x78, 0xC8, 0x20, 0x00}, {0x66, 0x0F, 0x3A, 0x0E, 0xC8, 0x03}); inPlace({0xF2, 0x45, 0x0F, 0x78, 0xC8, 0x10, 0x00}, {0x66, 0x45, 0x0F, 0x3A, 0x0E, 0xC8, 0x01}); + const Bytes clzeroSite = {0x0F, 0x01, 0xFC}; + const auto clzero = Codegen::ClzeroLowering{}.LowerOutOfLine(Codegen::DecodeClzero(clzeroSite.data(), clzeroSite.size())); + require(clzero.Bytes == kClzeroBody && clzero.ReturnBranchOffset == 69, "CLZERO stub differs from the golden encoding"); const auto highRegisters = Codegen::DecodeSse4a(kInsertqHighSite.data(), kInsertqHighSite.size()); const auto generic = lowering.LowerOutOfLine(Codegen::Sse4aOperands{true, false, 9, 4, 5, 3}); require(generic.Bytes[0] == 0x48 && generic.Bytes.size() % 16 == 0 && generic.ReturnBranchOffset < generic.Bytes.size(), "Generic INSERTQ body does not start with the red-zone skip"); @@ -266,12 +300,12 @@ void converterSegment() { auto branchInside = file; branchInside[0x207] = 0x02; requireFailure([&] { (void)converter->Convert(branchInside, {segmentHeader(20)}); }, "Branch into an AMD-only instruction was accepted"); - auto clzero = file; - clzero[0x20F] = 0x0F; - clzero[0x210] = 0x01; - clzero[0x211] = 0xFC; - clzero[0x212] = 0x90; - requireFailure([&] { (void)converter->Convert(clzero, {segmentHeader(20)}); }, "CLZERO was silently kept"); + auto rdpru = file; + rdpru[0x20F] = 0x0F; + rdpru[0x210] = 0x01; + rdpru[0x211] = 0xFD; + rdpru[0x212] = 0x90; + requireFailure([&] { (void)converter->Convert(rdpru, {segmentHeader(20)}); }, "RDPRU was silently kept"); auto registerForm = file; const Bytes extrqRegister = {0x66, 0x0F, 0x79, 0xCA}; std::copy(extrqRegister.begin(), extrqRegister.end(), registerForm.begin() + 0x20F); @@ -326,6 +360,53 @@ void converterMonitorWait() { require(result.Bytes == expected, "MONITORX/MWAITX were replaced with the wrong bytes"); } +void converterClzero() { + const auto converter = Codegen::MakeAmd64OnlyConverter(); + Bytes file(0x300, 0xCC); + const Bytes text = { + 0x0F, 0x01, 0xFC, + 0x48, 0x83, 0xC0, 0x40, + 0x0F, 0x01, 0xFC, + 0x0F, 0x01, 0xFC, + 0x90, + 0xC3}; + std::copy(text.begin(), text.end(), file.begin() + 0x200); + const auto result = converter->Convert(file, {segmentHeader(text.size())}); + require(result.Trampolines.size() == 2 && result.Reports.size() == 2 && result.Bytes == file, "CLZERO sites were not lowered through stubs"); + const auto& advance = result.Trampolines[0]; + require(advance.Offset == 0x200 && advance.Length == 7 && result.Reports[0].InstructionName == "CLZERO", "CLZERO did not absorb the following instruction"); + const Bytes add = {0x48, 0x83, 0xC0, 0x40}; + require(Bytes(advance.Body.begin() + static_cast(advance.ReturnBranchOffset - add.size()), advance.Body.begin() + static_cast(advance.ReturnBranchOffset)) == add, "Absorbed instruction does not run before the return jump"); + const auto& pair = result.Trampolines[1]; + require(pair.Offset == 0x207 && pair.Length == 6 && result.Reports[1].InstructionName == "CLZERO", "Consecutive CLZERO sites were not lowered into one stub"); + auto beforeReturn = file; + beforeReturn[0x203] = 0xC3; + requireFailure([&] { (void)converter->Convert(beforeReturn, {segmentHeader(text.size())}); }, "Short CLZERO followed by a return was relocated"); + auto segmentRelative = file; + const Bytes fsClzero = {0x64, 0x0F, 0x01, 0xFC}; + std::copy(fsClzero.begin(), fsClzero.end(), segmentRelative.begin() + 0x20A); + require(failureOffset([&] { (void)converter->Convert(segmentRelative, {segmentHeader(text.size())}); }, "FS-relative CLZERO was accepted") == 0x20A, "CLZERO operand failure does not carry the file offset"); +} + +void converterSplitPrefixes() { + const auto converter = Codegen::MakeAmd64OnlyConverter(); + for (const Bytes& instruction : {Bytes{0x48, 0x64, 0x0F, 0x01, 0xFC}, Bytes{0x48, 0x67, 0x0F, 0x01, 0xFC}, Bytes{0x48, 0xF0, 0x0F, 0x01, 0xFC}, Bytes{0x48, 0xF0, 0x0F, 0x38, 0xCB, 0xCA}}) { + Bytes file(0x300, 0xCC); + Bytes text = instruction; + text.insert(text.end(), {0x90, 0x90, 0xC3}); + std::copy(text.begin(), text.end(), file.begin() + 0x200); + require(failureOffset([&] { (void)converter->Convert(file, {segmentHeader(text.size())}); }, "AMD-only instruction after a split REX and prefix was accepted") == 0x200, "Split prefix failure does not carry the file offset"); + } + for (const Bytes& following : {Bytes{0x48, 0x64, 0x8B, 0x00}, Bytes{0x48, 0x67, 0x8B, 0x00}, Bytes{0x48, 0xF0, 0x01, 0x00}}) { + Bytes file(0x300, 0xCC); + Bytes text = {0x0F, 0x01, 0xFC}; + text.insert(text.end(), following.begin(), following.end()); + text.push_back(0xC3); + std::copy(text.begin(), text.end(), file.begin() + 0x200); + require(failureOffset([&] { (void)converter->Convert(file, {segmentHeader(text.size())}); }, "Split REX and prefix were moved into a stub") == 0x203, "Split prefix failure after a short site does not carry the file offset"); + } +} + void converterFailureOffsets() { const auto converter = Codegen::MakeAmd64OnlyConverter(); const auto file = segmentFixture(); @@ -339,10 +420,10 @@ void converterFailureOffsets() { auto movntsRegister = file; movntsRegister[0x212] = 0xC1; require(failureOffset([&] { (void)converter->Convert(movntsRegister, {segmentHeader(20)}); }, "MOVNTSS register form was accepted") == 0x20F, "MOVNTSS failure does not carry the file offset"); - auto clzero = file; - const Bytes clzeroBytes = {0x0F, 0x01, 0xFC, 0x90}; - std::copy(clzeroBytes.begin(), clzeroBytes.end(), clzero.begin() + 0x20F); - require(failureOffset([&] { (void)converter->Convert(clzero, {segmentHeader(20)}); }, "CLZERO was accepted") == 0x20F, "Unsupported instruction failure does not carry the file offset"); + auto rdpru = file; + const Bytes rdpruBytes = {0x0F, 0x01, 0xFD, 0x90}; + std::copy(rdpruBytes.begin(), rdpruBytes.end(), rdpru.begin() + 0x20F); + require(failureOffset([&] { (void)converter->Convert(rdpru, {segmentHeader(20)}); }, "RDPRU was accepted") == 0x20F, "Unsupported instruction failure does not carry the file offset"); } Bytes elfFixture(const Bytes& text) { @@ -540,9 +621,123 @@ void sha256Execution() { require(runRegisterFormStub(same, state, state) == sha256Reference(opcode, state, state, kStubScratch), "SHA-256 stub with equal operands computed the wrong result"); } } + +struct ClzeroRun { + std::uint64_t Rax; + std::uint64_t Rcx; + std::uint64_t Flags; + std::uint64_t Xmm[16][2]; +}; + +ClzeroRun runClzeroBody(const Codegen::Amd64OnlyMatch& match, const std::uint64_t rax, const std::uint64_t (&xmmIn)[16][2]) { + require(match.Lowering == Codegen::Amd64OnlyLowering::Trampoline, "CLZERO stub was not produced"); + auto body = match.StubBody; + const auto ret = body.size(); + body.push_back(0xC3); + const auto displacement = static_cast(ret - (match.ReturnBranchOffset + 5)); + std::memcpy(body.data() + match.ReturnBranchOffset + 1, &displacement, sizeof(displacement)); + void* code = mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + require(code != MAP_FAILED, "cannot map executable memory for the stub"); + std::memcpy(code, body.data(), body.size()); + ClzeroRun run{}; + asm volatile( + "movdqu 0x00(%[xmmIn]), %%xmm0\n\t" + "movdqu 0x10(%[xmmIn]), %%xmm1\n\t" + "movdqu 0x20(%[xmmIn]), %%xmm2\n\t" + "movdqu 0x30(%[xmmIn]), %%xmm3\n\t" + "movdqu 0x40(%[xmmIn]), %%xmm4\n\t" + "movdqu 0x50(%[xmmIn]), %%xmm5\n\t" + "movdqu 0x60(%[xmmIn]), %%xmm6\n\t" + "movdqu 0x70(%[xmmIn]), %%xmm7\n\t" + "movdqu 0x80(%[xmmIn]), %%xmm8\n\t" + "movdqu 0x90(%[xmmIn]), %%xmm9\n\t" + "movdqu 0xa0(%[xmmIn]), %%xmm10\n\t" + "movdqu 0xb0(%[xmmIn]), %%xmm11\n\t" + "movdqu 0xc0(%[xmmIn]), %%xmm12\n\t" + "movdqu 0xd0(%[xmmIn]), %%xmm13\n\t" + "movdqu 0xe0(%[xmmIn]), %%xmm14\n\t" + "movdqu 0xf0(%[xmmIn]), %%xmm15\n\t" + "mov %[raxIn], %%rax\n\t" + "movabs $0x1122334455667788, %%rcx\n\t" + "sub $128, %%rsp\n\t" + "pushq $0x8D7\n\t" + "popfq\n\t" + "call *%[code]\n\t" + "pushfq\n\t" + "popq %[flags]\n\t" + "add $128, %%rsp\n\t" + "mov %%rax, %[raxOut]\n\t" + "mov %%rcx, %[rcxOut]\n\t" + "movdqu %%xmm0, 0x00(%[xmmOut])\n\t" + "movdqu %%xmm1, 0x10(%[xmmOut])\n\t" + "movdqu %%xmm2, 0x20(%[xmmOut])\n\t" + "movdqu %%xmm3, 0x30(%[xmmOut])\n\t" + "movdqu %%xmm4, 0x40(%[xmmOut])\n\t" + "movdqu %%xmm5, 0x50(%[xmmOut])\n\t" + "movdqu %%xmm6, 0x60(%[xmmOut])\n\t" + "movdqu %%xmm7, 0x70(%[xmmOut])\n\t" + "movdqu %%xmm8, 0x80(%[xmmOut])\n\t" + "movdqu %%xmm9, 0x90(%[xmmOut])\n\t" + "movdqu %%xmm10, 0xa0(%[xmmOut])\n\t" + "movdqu %%xmm11, 0xb0(%[xmmOut])\n\t" + "movdqu %%xmm12, 0xc0(%[xmmOut])\n\t" + "movdqu %%xmm13, 0xd0(%[xmmOut])\n\t" + "movdqu %%xmm14, 0xe0(%[xmmOut])\n\t" + "movdqu %%xmm15, 0xf0(%[xmmOut])\n\t" + : [flags] "=&r"(run.Flags), [raxOut] "=&r"(run.Rax), [rcxOut] "=&r"(run.Rcx) + : [xmmIn] "r"(xmmIn), [xmmOut] "r"(run.Xmm), [raxIn] "r"(rax), [code] "r"(code) + : "rax", "rcx", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "memory", "cc"); + munmap(code, 4096); + return run; +} + +void clzeroExecution() { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + const Bytes plain = {0x0F, 0x01, 0xFC}; + const Bytes addressSize32 = {0x67, 0x0F, 0x01, 0xFC}; + const Bytes add = {0x48, 0x83, 0xC0, 0x40}; + const std::vector> pair = {plain, plain}; + struct Case { + std::optional Match; + bool Low; + std::uint64_t Junk; + std::uint64_t Advance; + const char* Name; + }; + const std::vector cases = { + {matcher->Match(plain.data(), plain.size()), false, 0, 0, "CLZERO stub"}, + {matcher->Match(addressSize32.data(), addressSize32.size()), true, 0x5A5A5A5A00000000ull, 0, "67h CLZERO stub"}, + {matcher->Match(plain.data(), plain.size(), add), false, 0, 64, "CLZERO stub with a trailing add"}, + {matcher->MatchSequence(pair, {}), false, 0, 0, "CLZERO sequence stub"}}; + std::uint64_t xmmIn[16][2]; + for (unsigned reg = 0; reg < 16; ++reg) { + xmmIn[reg][0] = 0x0101010101010101ull * (reg + 1); + xmmIn[reg][1] = ~xmmIn[reg][0]; + } + for (const auto& item : cases) { + require(item.Match.has_value(), item.Name); + auto* buffer = static_cast(mmap(nullptr, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | (item.Low ? MAP_32BIT : 0), -1, 0)); + require(buffer != MAP_FAILED, "cannot map the CLZERO buffer"); + for (const std::size_t offset : {std::size_t{0x80}, std::size_t{0xAD}, std::size_t{0xFF}}) { + std::memset(buffer, 0xA5, 4096); + const auto address = reinterpret_cast(buffer + offset); + const auto run = runClzeroBody(*item.Match, address | item.Junk, xmmIn); + const auto line = offset & ~std::size_t{63}; + for (std::size_t index = 0; index < 4096; ++index) + require(buffer[index] == (index >= line && index < line + 64 ? 0x00 : 0xA5), "CLZERO stub did not clear exactly the addressed line"); + require(run.Rax == (address | item.Junk) + item.Advance && run.Rcx == 0x1122334455667788ull, "CLZERO stub changed rax or rcx"); + for (unsigned reg = 0; reg < 16; ++reg) + require(run.Xmm[reg][0] == xmmIn[reg][0] && run.Xmm[reg][1] == xmmIn[reg][1], "CLZERO stub clobbered an xmm register"); + if (item.Advance == 0) + require((run.Flags & 0x8D5) == (0x8D7 & 0x8D5), "CLZERO stub changed RFLAGS"); + } + munmap(buffer, 4096); + } +} #else void registerFormExecution() {} void sha256Execution() {} +void clzeroExecution() {} #endif void scannerZeroTail() { @@ -558,13 +753,17 @@ int main() { decoderLengths(); sse4aOperands(); sha256Operands(); + clzeroOperands(); matcherSubstitutions(); goldenBodies(); registerFormExecution(); sha256Execution(); + clzeroExecution(); converterSegment(); converterSha256(); converterMonitorWait(); + converterClzero(); + converterSplitPrefixes(); converterFailureOffsets(); linuxPlacement(); scannerZeroTail(); diff --git a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp index e5f46b7ec..2a48ce66d 100644 --- a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp +++ b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -343,6 +344,55 @@ void executeSha256(const Harness& harness, const Codegen::IAmd64OnlyInstructionM } } +void clzeroExecution(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, std::mt19937_64& random) { + const Bytes plain = {0x0F, 0x01, 0xFC}; + const Bytes addressSize32 = {0x67, 0x0F, 0x01, 0xFC}; + const std::vector> pair = {plain, plain}; + constexpr std::size_t size = 3 * 4096; + auto* high = static_cast(VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)); + std::uint8_t* low = nullptr; + for (std::uintptr_t hint = 0x10000000; low == nullptr && hint < 0x80000000; hint += 0x10000000) + low = static_cast(VirtualAlloc(reinterpret_cast(hint), size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE)); + require(high != nullptr && low != nullptr, "Cannot allocate the CLZERO buffers"); + struct ClzeroCase { + std::optional Match; + std::uint8_t* Buffer; + std::uint64_t Junk; + std::string Name; + }; + const std::vector cases = { + {matcher.Match(plain.data(), plain.size()), high, 0, "clzero"}, + {matcher.Match(addressSize32.data(), addressSize32.size()), low, 0x5A5A5A5A00000000ull, "67h clzero"}, + {matcher.MatchSequence(pair, {}), high, 0, "clzero; clzero"}}; + const std::array flags = {0x202, 0x203, 0x246, 0xAC7}; + for (const auto& item : cases) { + require(item.Match.has_value() && item.Match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "CLZERO was not lowered through a stub: " + item.Name); + for (int image = 0; image < 64; ++image) { + const auto address = reinterpret_cast(item.Buffer) + 64 + random() % (size - 128); + Bytes body = {0x48, 0xB8}; + for (std::size_t index = 0; index < 8; ++index) body.push_back(static_cast((address | item.Junk) >> (index * 8))); + body.insert(body.end(), {0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00}); + const auto prefix = body.size(); + body.insert(body.end(), item.Match->StubBody.begin(), item.Match->StubBody.end()); + State input{}; + for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast(random()); + input.FlagsIn = flags[static_cast(image) % flags.size()]; + State state = input; + std::memset(item.Buffer, 0xA5, size); + harness.Run(body, prefix + item.Match->ReturnBranchOffset, state); + ++g_executions; + const auto line = address & ~std::uint64_t{63}; + for (std::size_t index = 0; index < size; ++index) { + const auto at = reinterpret_cast(item.Buffer + index); + require(item.Buffer[index] == (at >= line && at < line + 64 ? 0x00 : 0xA5), "Lowered CLZERO did not clear exactly the addressed line: " + item.Name); + } + requireEnvironment(input, state, 0, item.Name); + } + } + VirtualFree(high, 0, MEM_RELEASE); + VirtualFree(low, 0, MEM_RELEASE); +} + void cpuExecution() { const Harness harness; const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); @@ -367,6 +417,7 @@ void cpuExecution() { for (auto& step : steps) step = {static_cast(0xCB + random() % 3), static_cast(random() % 16), static_cast(random() % 16)}; executeSha256(harness, *matcher, steps, random, 2); } + clzeroExecution(harness, *matcher, random); } Bytes elfFixture(const Bytes& text) { diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 962857a2d..4f89d4097 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -36,7 +36,8 @@ Throughout the project, every function at every stage either **does exactly what - The executable file that [relinker](../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. - [libSceJpegEnc](../../core/libs/prx/libSceJpegEnc/Export.cpp) encodes 4:2:2 sampling requests with 4:2:0 chroma subsampling, and grayscale requests as a 3-component JPEG with neutral chroma instead of a single-component one: the [JPEG encoder](../../core/Decoder/Jpeg/src/Jpeg.cpp) (stb) only produces 3-component 4:2:0 and 4:4:4 images. - [Relinker](../../core/relinker/elfpatcher/src) doesn't add an icon to the generated executable. This should be done without adding dependencies (only standard). -- `--to-intel` does not lower CLZERO/RDPRU/MCOMMIT; the [matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. The SHA-1 instructions of SHA-NI (SHA1RNDS4, SHA1NEXTE, SHA1MSG1, SHA1MSG2) are not substituted, and SHA-256 instructions with a memory operand fail the relink. +- `--to-intel` does not lower RDPRU/MCOMMIT; the [matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. The SHA-1 instructions of SHA-NI (SHA1RNDS4, SHA1NEXTE, SHA1MSG1, SHA1MSG2) are not substituted, and SHA-256 instructions with a memory operand fail the relink. +- The [instruction decoder](../../core/relinker/codegen/src/x86/X64InstructionDecoder.cpp) takes the byte after a REX prefix as the opcode, so a REX followed by a legacy or another REX prefix is split off as a separate instruction, while the CPU ignores that REX and applies the following prefixes to the next instruction. `--to-intel` fails the relink when such a fragment directly precedes an AMD-only instruction or would be moved into a stub. A split F2/F3 prefix that makes the next instruction MOVNTSS/MOVNTSD is not recognised: `48 F3 0F 2B 00` is scanned as `[48 F3][0F 2B 00]` and kept unchanged (split SSE4a forms already fail in the scanner). - The length-changing path of the [instruction rewriter](../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust VEX/0F38/0F3A RIP-relative operands, data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it.