diff --git a/core/relinker/codegen/include/codegen/x86/Sha256Operands.hpp b/core/relinker/codegen/include/codegen/x86/Sha256Operands.hpp index 6483112e9..f4c922b88 100644 --- a/core/relinker/codegen/include/codegen/x86/Sha256Operands.hpp +++ b/core/relinker/codegen/include/codegen/x86/Sha256Operands.hpp @@ -1,8 +1,10 @@ #ifndef CODEGEN_X86_SHA256OPERANDS_HPP #define CODEGEN_X86_SHA256OPERANDS_HPP +#include #include #include +#include namespace Codegen { @@ -16,6 +18,7 @@ struct Sha256Operands { Sha256Operation Operation; std::uint8_t Destination; std::uint8_t Source; + std::optional Memory; }; [[nodiscard]] Sha256Operands DecodeSha256(const std::uint8_t* data, std::size_t length); diff --git a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp index 7cebc4d55..942f7db56 100644 --- a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp +++ b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp @@ -17,6 +17,18 @@ struct LoweredBody { using StubConstant = std::array; +struct MemoryOperand { + std::vector Prefixes; + std::uint8_t RexIndexBase; + std::uint8_t Mod; + std::uint8_t Rm; + std::uint8_t Sib; + std::int32_t Displacement; + bool StackBase; +}; + +[[nodiscard]] MemoryOperand DecodeMemoryOperand(const std::uint8_t* data, std::size_t length, std::size_t modRmOffset, std::uint8_t rex, std::vector prefixes); + void EmitSse(std::vector& out, std::uint8_t prefix, std::initializer_list opcode, std::uint8_t dst, std::uint8_t src); void EmitShiftImm(std::vector& out, std::uint8_t extension, std::uint8_t reg, std::uint8_t imm); @@ -28,6 +40,7 @@ public: void ShiftImm(std::uint8_t extension, std::uint8_t reg, std::uint8_t imm); void ShiftDwordImm(std::uint8_t extension, std::uint8_t reg, std::uint8_t imm); void RipOperand(std::initializer_list opcode, std::uint8_t reg, const StubConstant& constant); + void Load(std::uint8_t reg, const MemoryOperand& operand); void Spill(std::uint8_t reg); void Restore(std::uint8_t reg); void Raw(std::span bytes); @@ -43,6 +56,7 @@ private: std::vector _bytes; std::vector _fixups; std::vector _constants; + std::size_t _stackDepth = 0; }; } diff --git a/core/relinker/codegen/src/x86/Sha256Lowering.cpp b/core/relinker/codegen/src/x86/Sha256Lowering.cpp index 981ecb425..462436d0b 100644 --- a/core/relinker/codegen/src/x86/Sha256Lowering.cpp +++ b/core/relinker/codegen/src/x86/Sha256Lowering.cpp @@ -182,6 +182,16 @@ void _emitMessage2(StubBodyBuilder& body, const Sha256Operands& operands) { } void Sha256Lowering::EmitOutOfLine(StubBodyBuilder& body, const Sha256Operands& operands) const { + if (operands.Memory) { + auto loaded = operands; + loaded.Memory.reset(); + loaded.Source = _scratch<1>({operands.Operation, operands.Destination, operands.Destination, {}})[0]; + body.Spill(loaded.Source); + body.Load(loaded.Source, *operands.Memory); + EmitOutOfLine(body, loaded); + body.Restore(loaded.Source); + return; + } switch (operands.Operation) { case Sha256Operation::Rnds2: _emitRounds(body, operands); diff --git a/core/relinker/codegen/src/x86/Sha256Operands.cpp b/core/relinker/codegen/src/x86/Sha256Operands.cpp index 74ba3607e..625d1d295 100644 --- a/core/relinker/codegen/src/x86/Sha256Operands.cpp +++ b/core/relinker/codegen/src/x86/Sha256Operands.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include namespace Codegen { @@ -9,6 +11,7 @@ using namespace X64OpcodeConstants; Sha256Operands DecodeSha256(const std::uint8_t* data, const std::size_t length) { std::size_t pos = 0; std::uint8_t rex = 0; + std::vector prefixes; while (pos < length) { const std::uint8_t b = data[pos]; @@ -25,6 +28,8 @@ Sha256Operands DecodeSha256(const std::uint8_t* data, const std::size_t length) b != PrefixSegEs && b != PrefixSegFs && b != PrefixSegGs) { break; } + if (b != PrefixLock) + prefixes.push_back(b); rex = 0; pos += 1; } @@ -49,11 +54,11 @@ Sha256Operands DecodeSha256(const std::uint8_t* data, const std::size_t length) } const std::uint8_t modrm = data[pos + 3]; - if (((modrm >> ModRmModShift) & ModRmModMask) != ModRmModRegister) { - throw CodegenException("SHA-256 instruction with a memory operand"); - } - operands.Destination = static_cast(((modrm >> ModRmRegShift) & ModRmRegMask) | (((rex & 0x4) != 0) ? 8 : 0)); + if (((modrm >> ModRmModShift) & ModRmModMask) != ModRmModRegister) { + operands.Memory = DecodeMemoryOperand(data, length, pos + 3, rex, std::move(prefixes)); + return operands; + } operands.Source = static_cast((modrm & ModRmRmMask) | (((rex & 0x1) != 0) ? 8 : 0)); return operands; } diff --git a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp index 2ab183011..0fef6e627 100644 --- a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp +++ b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp @@ -1,5 +1,8 @@ #include #include +#include +#include +#include namespace Codegen { @@ -11,6 +14,7 @@ constexpr std::uint8_t kPrefixPacked = 0x66; constexpr std::uint8_t kPrefixScalar = 0xF3; constexpr std::uint8_t kRexBase = 0x40; constexpr std::uint8_t kRexR = 0x04; +constexpr std::uint8_t kRexX = 0x02; constexpr std::uint8_t kRexB = 0x01; constexpr std::uint8_t kModRmRegister = 0xC0; constexpr std::uint8_t kModRmRip = 0x05; @@ -18,6 +22,7 @@ constexpr std::uint8_t kModRmRspBase = 0x04; constexpr std::uint8_t kSibRsp = 0x24; constexpr std::uint8_t kShiftDwords = 0x72; constexpr std::uint8_t kShiftQwords = 0x73; +constexpr std::uint8_t kMovdqu = 0x6F; std::uint8_t _rex(const std::uint8_t reg, const std::uint8_t rm) { return static_cast(kRexBase | ((reg & 8) != 0 ? kRexR : 0) | ((rm & 8) != 0 ? kRexB : 0)); @@ -32,6 +37,15 @@ void _emit(std::vector& out, const std::uint8_t prefix, const std: out.push_back(modrm); } +std::size_t _displacementSize(const std::uint8_t mod, const std::uint8_t rm, const std::uint8_t sib) { + using namespace X64OpcodeConstants; + if (mod == ModRmModDisp8) + return Disp8Size; + if (mod == ModRmModDisp32 || (mod == ModRmModIndirect && rm == ModRmRmSibPresent && (sib & SibBaseMask) == SibBaseDisp32)) + return Disp32Size; + return 0; +} + void _shiftImm(std::vector& out, const std::uint8_t opcode, const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { _emit(out, kPrefixPacked, 0, reg, {0x0F, opcode}, static_cast(kModRmRegister | (extension << 3) | (reg & 7))); out.push_back(imm); @@ -39,6 +53,33 @@ void _shiftImm(std::vector& out, const std::uint8_t opcode, const } +MemoryOperand DecodeMemoryOperand(const std::uint8_t* data, const std::size_t length, const std::size_t modRmOffset, const std::uint8_t rex, std::vector prefixes) { + using namespace X64OpcodeConstants; + if (modRmOffset >= length) + throw CodegenException("Memory operand truncated before its ModRM byte"); + const auto modrm = data[modRmOffset]; + MemoryOperand operand{std::move(prefixes), static_cast(rex & (kRexX | kRexB)), static_cast((modrm >> ModRmModShift) & ModRmModMask), static_cast(modrm & ModRmRmMask), 0, 0, false}; + if (operand.Mod == ModRmModRegister) + throw CodegenException("Register operand where a memory operand was expected"); + if (operand.Mod == ModRmModIndirect && operand.Rm == ModRmRmRipRelative) + throw CodegenException("RIP-relative memory operand cannot move into a stub"); + auto pos = modRmOffset + 1; + if (operand.Rm == ModRmRmSibPresent) { + if (pos >= length) + throw CodegenException("Memory operand truncated before its SIB byte"); + operand.Sib = data[pos++]; + operand.StackBase = (operand.Sib & SibBaseMask) == ModRmRmSibPresent && (rex & kRexB) == 0; + } + const auto size = _displacementSize(operand.Mod, operand.Rm, operand.Sib); + if (pos + size > length) + throw CodegenException("Memory operand truncated in its displacement"); + if (size == Disp8Size) + operand.Displacement = static_cast(data[pos]); + for (std::size_t index = 0; size == Disp32Size && index < size; ++index) + operand.Displacement = static_cast(static_cast(operand.Displacement) | (static_cast(data[pos + index]) << (index * 8))); + return operand; +} + void EmitSse(std::vector& out, const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { _emit(out, prefix, dst, src, opcode, static_cast(kModRmRegister | ((dst & 7) << 3) | (src & 7))); } @@ -79,7 +120,27 @@ void StubBodyBuilder::RipOperand(const std::initializer_list opcod _constants.push_back(constant); } +void StubBodyBuilder::Load(const std::uint8_t reg, const MemoryOperand& operand) { + using namespace X64OpcodeConstants; + const auto displacement = static_cast(operand.Displacement) + static_cast(operand.StackBase ? _stackDepth : 0); + if (displacement > std::numeric_limits::max()) + throw CodegenException("Stack displacement does not fit after spilling"); + const auto mod = operand.StackBase && _stackDepth != 0 ? ModRmModDisp32 : operand.Mod; + _bytes.insert(_bytes.end(), operand.Prefixes.begin(), operand.Prefixes.end()); + _bytes.push_back(kPrefixScalar); + const auto rex = static_cast(kRexBase | ((reg & 8) != 0 ? kRexR : 0) | operand.RexIndexBase); + if (rex != kRexBase) + _bytes.push_back(rex); + _bytes.insert(_bytes.end(), {TwoByteOpcodeEscape, kMovdqu, static_cast((mod << ModRmModShift) | ((reg & 7) << ModRmRegShift) | operand.Rm)}); + if (operand.Rm == ModRmRmSibPresent) + _bytes.push_back(operand.Sib); + const auto value = static_cast(static_cast(displacement)); + for (std::size_t index = 0; index < _displacementSize(mod, operand.Rm, operand.Sib); ++index) + _bytes.push_back(static_cast(value >> (index * 8))); +} + void StubBodyBuilder::Spill(const std::uint8_t reg) { + _stackDepth += kRedZoneSpillFrame; _bytes.insert(_bytes.end(), kLeaRspBelowRedZone.Bytes, kLeaRspBelowRedZone.Bytes + kLeaRspBelowRedZone.Size); _emit(_bytes, kPrefixScalar, reg, 0, {0x0F, 0x7F}, static_cast(((reg & 7) << 3) | kModRmRspBase)); _bytes.push_back(kSibRsp); @@ -89,6 +150,7 @@ void StubBodyBuilder::Restore(const std::uint8_t reg) { _emit(_bytes, kPrefixScalar, reg, 0, {0x0F, 0x6F}, static_cast(((reg & 7) << 3) | kModRmRspBase)); _bytes.push_back(kSibRsp); _bytes.insert(_bytes.end(), kLeaRspRestore.Bytes, kLeaRspRestore.Bytes + kLeaRspRestore.Size); + _stackDepth -= kRedZoneSpillFrame; } void StubBodyBuilder::Raw(std::span bytes) { diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 1ac2dbcb4..4bd58800d 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -193,7 +193,16 @@ void sha256Operands() { check({0x0F, 0x38, 0xCB, 0xCA}, Codegen::Sha256Operation::Rnds2, 1, 2); check({0x45, 0x0F, 0x38, 0xCC, 0xE1}, Codegen::Sha256Operation::Msg1, 12, 9); check({0x44, 0x0F, 0x38, 0xCD, 0xC0}, Codegen::Sha256Operation::Msg2, 8, 0); - requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x08}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "SHA-256 memory operand was accepted"); + const Bytes memory = {0x65, 0x67, 0x44, 0x0F, 0x38, 0xCD, 0x54, 0x8C, 0xF0}; + const auto decoded = Codegen::DecodeSha256(memory.data(), memory.size()); + require(decoded.Operation == Codegen::Sha256Operation::Msg2 && decoded.Destination == 10 && decoded.Memory && decoded.Memory->Prefixes == Bytes{0x65, 0x67} && decoded.Memory->Mod == 1 && decoded.Memory->Rm == 4 && decoded.Memory->Sib == 0x8C && decoded.Memory->Displacement == -16 && decoded.Memory->StackBase, "SHA-256 memory operand was decoded incorrectly"); + const Bytes r12Base = {0x41, 0x0F, 0x38, 0xCC, 0x14, 0x24}; + require(!Codegen::DecodeSha256(r12Base.data(), r12Base.size()).Memory->StackBase, "R12 base was decoded as RSP"); + const Bytes absolute = {0x0F, 0x38, 0xCC, 0x14, 0x25, 0x78, 0x56, 0x34, 0x12}; + const auto noBase = Codegen::DecodeSha256(absolute.data(), absolute.size()); + require(!noBase.Memory->StackBase && noBase.Memory->Displacement == 0x12345678, "SIB without a base was decoded incorrectly"); + requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x15, 0, 0, 0, 0}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "SHA-256 RIP-relative operand was accepted"); + requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x54, 0x24}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "Truncated SHA-256 memory operand was accepted"); requireFailure([] { const Bytes bytes = {0x66, 0x0F, 0x38, 0xCB, 0xCA}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "Prefixed 0F 38 CB was decoded as SHA-256"); requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xC9, 0xCA}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "SHA-1 was decoded as SHA-256"); check({0x41, 0x2E, 0x0F, 0x38, 0xCC, 0xCA}, Codegen::Sha256Operation::Msg1, 1, 2); @@ -425,7 +434,12 @@ void converterSha256() { requireFailure([&] { (void)converter->Convert(beforeReturn, {segmentHeader(text.size())}); }, "Short SHA-256 instruction followed by a return was relocated"); auto memoryForm = file; memoryForm[0x20F] = 0x28; - require(failureOffset([&] { (void)converter->Convert(memoryForm, {segmentHeader(text.size())}); }, "SHA-256 memory form was accepted") == 0x20C, "SHA-256 operand failure does not carry the file offset"); + const auto memoryResult = converter->Convert(memoryForm, {segmentHeader(text.size())}); + require(memoryResult.Trampolines.size() == 2 && memoryResult.Reports[1].InstructionName == "SHA256MSG2", "SHA-256 memory form was not lowered through a stub"); + auto ripRelative = file; + const Bytes ripMessage = {0x0F, 0x38, 0xCD, 0x2D, 0x00, 0x00, 0x00, 0x00, 0xC3}; + std::copy(ripMessage.begin(), ripMessage.end(), ripRelative.begin() + 0x20C); + require(failureOffset([&] { (void)converter->Convert(ripRelative, {segmentHeader(text.size() + 3)}); }, "SHA-256 RIP-relative form was accepted") == 0x20C, "SHA-256 operand failure does not carry the file offset"); } void converterSha1() { @@ -748,20 +762,29 @@ std::array runRegisterFormStub(const Bytes& site, const std::u alignas(16) std::uint64_t out[2] = {}; alignas(16) std::uint64_t scratchIn[2] = {kStubScratch[0], kStubScratch[1]}; alignas(16) std::uint64_t scratchOut[2] = {}; + alignas(16) std::uint64_t spareIn[2] = {kStubScratch[1], kStubScratch[0]}; + alignas(16) std::uint64_t spareOut[2] = {}; asm volatile( "movdqu (%[scratch]), %%xmm0\n\t" + "movdqu (%[spare]), %%xmm1\n\t" "movdqu (%[dst]), %%xmm2\n\t" "movdqu (%[ctl]), %%xmm5\n\t" + "mov %[ctl], %%rcx\n\t" + "mov %[ctl], %%r12\n\t" + "mov $16, %%eax\n\t" "sub $128, %%rsp\n\t" + "movdqu %%xmm5, 0x10(%%rsp)\n\t" "call *%[code]\n\t" "add $128, %%rsp\n\t" "movdqu %%xmm2, (%[out])\n\t" "movdqu %%xmm0, (%[scratchOut])\n\t" + "movdqu %%xmm1, (%[spareOut])\n\t" : - : [scratch] "r"(scratchIn), [dst] "r"(destinationIn), [ctl] "r"(sourceIn), [code] "r"(code), [out] "r"(out), [scratchOut] "r"(scratchOut) - : "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "memory", "cc"); + : [scratch] "r"(scratchIn), [spare] "r"(spareIn), [dst] "r"(destinationIn), [ctl] "r"(sourceIn), [code] "r"(code), [out] "r"(out), [scratchOut] "r"(scratchOut), [spareOut] "r"(spareOut) + : "rax", "rcx", "r12", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "memory", "cc"); munmap(code, 4096); require(scratchOut[0] == scratchIn[0] && scratchOut[1] == scratchIn[1], "Register form stub clobbered a scratch register"); + require(spareOut[0] == spareIn[0] && spareOut[1] == spareIn[1], "Stub clobbered xmm1"); return {out[0], out[1]}; } @@ -851,6 +874,8 @@ void sha256Execution() { const Bytes same = {0x0F, 0x38, opcode, 0xD2}; require(runRegisterFormStub(distinct, state, words) == sha256Reference(opcode, state, words, kStubScratch), "SHA-256 stub computed the wrong result"); require(runRegisterFormStub(same, state, state) == sha256Reference(opcode, state, state, kStubScratch), "SHA-256 stub with equal operands computed the wrong result"); + for (const auto& memory : {Bytes{0x0F, 0x38, opcode, 0x11}, Bytes{0x0F, 0x38, opcode, 0x54, 0x24, 0x18}, Bytes{0x0F, 0x38, opcode, 0x54, 0x04, 0x08}, Bytes{0x2E, 0x41, 0x0F, 0x38, opcode, 0x14, 0x24}, Bytes{0x0F, 0x38, opcode, 0x91, 0x00, 0x00, 0x00, 0x00}}) + require(runRegisterFormStub(memory, state, words) == sha256Reference(opcode, state, words, kStubScratch), "SHA-256 memory form stub computed the wrong result"); } }