diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 78548a1e4..2bd7a16ad 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -54,8 +54,6 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstructio Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const { const auto operands = DecodeSse4a(instr.Data, instr.Length); - if (operands.RegisterForm && operands.Insertq) - return _unsupported(registerFormEntry, instr.Length); if (!operands.RegisterForm && trailing.empty()) { if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; @@ -76,10 +74,7 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( const DecodedInstruction instr{instruction.data(), instruction.size()}; if (!instr.IsExtrq() && !instr.IsInsertq()) return std::nullopt; - const auto operands = DecodeSse4a(instr.Data, instr.Length); - if (operands.RegisterForm && operands.Insertq) - return std::nullopt; - sequence.push_back(operands); + sequence.push_back(DecodeSse4a(instr.Data, instr.Length)); } const auto& first = sequence.front(); const auto& name = first.RegisterForm ? (first.Insertq ? kInsertqRegisterForm.Name : kExtrqRegisterForm.Name) : (first.Insertq ? kInsertq.Name : kExtrq.Name); diff --git a/core/relinker/codegen/src/x86/Sse4aLowering.cpp b/core/relinker/codegen/src/x86/Sse4aLowering.cpp index d183ec9bb..2b88bb5bd 100644 --- a/core/relinker/codegen/src/x86/Sse4aLowering.cpp +++ b/core/relinker/codegen/src/x86/Sse4aLowering.cpp @@ -24,6 +24,7 @@ constexpr std::uint8_t kModRmRip = 0x05; constexpr std::uint8_t kModRmRspBase = 0x04; constexpr std::uint8_t kSibRsp = 0x24; constexpr std::uint8_t kShiftRight = 2; +constexpr std::uint8_t kShiftRightBytes = 3; constexpr std::uint8_t kShiftLeft = 6; constexpr std::uint8_t kFieldBits = 64; @@ -125,10 +126,52 @@ private: std::vector _constants; }; +void _emitInsertqRegisterForm(BodyBuilder& body, const Sse4aOperands& operands) { + const auto dst = operands.Destination; + const auto src = operands.Source; + std::array scratch{}; + for (std::uint8_t reg = 0, found = 0; found < scratch.size(); ++reg) + if (reg != dst && reg != src) scratch[found++] = reg; + const auto control = scratch[0]; + const auto index = scratch[1]; + const auto hole = scratch[2]; + Constant fieldMask{}; + fieldMask[0] = kFieldBits - 1; + Constant one{}; + one[0] = 1; + body.Spill(control); + body.Spill(index); + body.Spill(hole); + body.Sse(kPrefixPacked, {0x0F, 0x6F}, control, src); + body.ShiftImm(kShiftRightBytes, control, 8); + body.Sse(kPrefixPacked, {0x0F, 0x6F}, index, control); + body.ShiftImm(kShiftRight, index, 8); + body.RipOperand({0x0F, 0xDB}, index, fieldMask); + body.RipOperand({0x0F, 0xDB}, control, fieldMask); + body.RipOperand({0x0F, 0xEF}, control, fieldMask); + body.RipOperand({0x0F, 0xD4}, control, one); + body.RipOperand({0x0F, 0xDB}, control, fieldMask); + body.Sse(kPrefixPacked, {0x0F, 0x76}, hole, hole); + body.Sse(kPrefixScalar, {0x0F, 0x7E}, hole, hole); + body.Sse(kPrefixPacked, {0x0F, 0xD3}, hole, control); + body.Sse(kPrefixPacked, {0x0F, 0xF3}, hole, index); + body.Sse(kPrefixPacked, {0x0F, 0x6F}, control, src); + body.Sse(kPrefixPacked, {0x0F, 0xF3}, control, index); + body.Sse(kPrefixPacked, {0x0F, 0xEF}, control, dst); + body.Sse(kPrefixPacked, {0x0F, 0xDB}, control, hole); + body.Sse(kPrefixPacked, {0x0F, 0xEF}, dst, control); + body.Sse(kPrefixScalar, {0x0F, 0x7E}, dst, dst); + body.Restore(hole); + body.Restore(index); + body.Restore(control); +} + void _emitOutOfLine(BodyBuilder& body, const Sse4aOperands& operands) { if (operands.RegisterForm) { - if (operands.Insertq) - throw CodegenException("INSERTQ register form has no Intel lowering"); + if (operands.Insertq) { + _emitInsertqRegisterForm(body, operands); + return; + } const auto dst = operands.Destination; const auto src = operands.Source; std::array scratch{}; diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 3d2c49ccd..8ae7e433a 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -145,7 +145,7 @@ void matcherSubstitutions() { const auto registerForm = match({0x66, 0x0F, 0x79, 0xCA}); require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Trampoline && registerForm->InstructionName == "EXTRQ register form", "EXTRQ register form was not lowered through a stub"); const auto insertqRegisterForm = match({0xF2, 0x0F, 0x79, 0xCA}); - require(insertqRegisterForm && insertqRegisterForm->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "INSERTQ register form was not reported as unsupported"); + require(insertqRegisterForm && insertqRegisterForm->Lowering == Codegen::Amd64OnlyLowering::Trampoline && insertqRegisterForm->InstructionName == "INSERTQ register form", "INSERTQ register form was not lowered through a stub"); require(!match({0x66, 0x0F, 0x2B, 0x07}) && !match({0x0F, 0x2B, 0x07}) && !match({0x48, 0x8B, 0x05, 0, 0, 0, 0}), "Ordinary instruction was matched"); const auto stub = match(kInsertqHighSite); require(stub && stub->Lowering == Codegen::Amd64OnlyLowering::Trampoline && stub->StubBody == kInsertqHighBody && stub->ReturnBranchOffset == 15 && stub->InstructionName == "INSERTQ", "INSERTQ was not lowered through a stub"); @@ -182,7 +182,8 @@ void goldenBodies() { const auto generic = lowering.LowerOutOfLine(Codegen::Sse4aOperands{true, false, 9, 4, 5, 3}); require(generic.Bytes[0] == 0x48 && generic.Bytes.size() % 16 == 0 && generic.ReturnBranchOffset < generic.Bytes.size(), "Generic INSERTQ body does not start with the red-zone skip"); (void)highRegisters; - requireFailure([&] { (void)lowering.LowerOutOfLine(Codegen::Sse4aOperands{true, true, 1, 2, 0, 0}); }, "Register form was lowered out of line"); + const auto insertqRegisterForm = lowering.LowerOutOfLine(Codegen::Sse4aOperands{true, true, 1, 2, 0, 0}); + require(insertqRegisterForm.Bytes[0] == 0x48 && insertqRegisterForm.Bytes.size() % 16 == 0 && insertqRegisterForm.ReturnBranchOffset < insertqRegisterForm.Bytes.size(), "INSERTQ register form body does not start with the red-zone skip"); } Bytes segmentFixture() { @@ -344,10 +345,17 @@ std::uint64_t extrqReference(std::uint64_t value, std::uint64_t control) { return length == 0 ? shifted : shifted & ((std::uint64_t{1} << length) - 1); } -std::uint64_t runExtrqStub(const Bytes& site, std::uint64_t destination, std::uint64_t control) { +std::uint64_t insertqReference(std::uint64_t destination, std::uint64_t value, std::uint64_t control) { + const auto length = static_cast(control & 0x3f); + const auto index = static_cast((control >> 8) & 0x3f); + const auto mask = length == 0 ? ~std::uint64_t{0} : ((std::uint64_t{1} << length) - 1); + return (destination & ~(mask << index)) | ((value & mask) << index); +} + +std::uint64_t runRegisterFormStub(const Bytes& site, const std::uint64_t (&destination)[2], const std::uint64_t (&source)[2]) { const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); const auto match = matcher->Match(site.data(), site.size()); - require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "EXTRQ register form stub was not produced"); + require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "Register form stub was not produced"); auto body = match->StubBody; const auto ret = body.size(); body.push_back(0xC3); @@ -356,12 +364,11 @@ std::uint64_t runExtrqStub(const Bytes& site, std::uint64_t destination, std::ui void* code = mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); require(code != MAP_FAILED, "cannot map executable memory for the stub"); std::memcpy(code, body.data(), body.size()); - alignas(16) std::uint64_t destinationIn[2] = {destination, 0x1122334455667788ull}; - alignas(16) std::uint64_t controlIn[2] = {control, 0}; + alignas(16) std::uint64_t destinationIn[2] = {destination[0], destination[1]}; + alignas(16) std::uint64_t sourceIn[2] = {source[0], source[1]}; alignas(16) std::uint64_t out[2] = {}; alignas(16) std::uint64_t scratchIn[2] = {0x0123456789abcdefull, 0xfedcba9876543210ull}; alignas(16) std::uint64_t scratchOut[2] = {}; - const bool same = site[3] == 0xD2; asm volatile( "movdqu (%[scratch]), %%xmm0\n\t" "movdqu (%[dst]), %%xmm2\n\t" @@ -372,21 +379,27 @@ std::uint64_t runExtrqStub(const Bytes& site, std::uint64_t destination, std::ui "movdqu %%xmm2, (%[out])\n\t" "movdqu %%xmm0, (%[scratchOut])\n\t" : - : [scratch] "r"(scratchIn), [dst] "r"(same ? controlIn : destinationIn), [ctl] "r"(controlIn), [code] "r"(code), [out] "r"(out), [scratchOut] "r"(scratchOut) + : [scratch] "r"(scratchIn), [dst] "r"(destinationIn), [ctl] "r"(sourceIn), [code] "r"(code), [out] "r"(out), [scratchOut] "r"(scratchOut) : "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "memory", "cc"); munmap(code, 4096); - require(scratchOut[0] == scratchIn[0] && scratchOut[1] == scratchIn[1], "EXTRQ stub clobbered a scratch register"); + require(scratchOut[0] == scratchIn[0] && scratchOut[1] == scratchIn[1], "Register form stub clobbered a scratch register"); return out[0]; } void registerFormExecution() { - const Bytes distinct = {0x66, 0x0F, 0x79, 0xD5}; - const Bytes same = {0x66, 0x0F, 0x79, 0xD2}; + const Bytes extrqDistinct = {0x66, 0x0F, 0x79, 0xD5}; + const Bytes extrqSame = {0x66, 0x0F, 0x79, 0xD2}; + const Bytes insertqDistinct = {0xF2, 0x0F, 0x79, 0xD5}; + const Bytes insertqSame = {0xF2, 0x0F, 0x79, 0xD2}; const std::uint64_t value = 0x9e3779b97f4a7c15ull; - for (const auto [length, index] : {std::pair{8u, 4u}, {0u, 0u}, {40u, 20u}, {63u, 1u}, {1u, 63u}, {16u, 48u}}) { + const std::uint64_t destination = 0x0f1e2d3c4b5a6978ull; + for (const auto [length, index] : {std::pair{8u, 4u}, {0u, 0u}, {40u, 20u}, {63u, 1u}, {1u, 63u}, {16u, 48u}, {1u, 0u}, {32u, 32u}}) { const auto control = static_cast(length) | (static_cast(index) << 8) | 0xffffc000ull; - require(runExtrqStub(distinct, value, control) == extrqReference(value, control), "EXTRQ register form stub computed the wrong field"); - require(runExtrqStub(same, control, control) == extrqReference(control, control), "EXTRQ register form stub with equal operands computed the wrong field"); + require(runRegisterFormStub(extrqDistinct, {value, 0x1122334455667788ull}, {control, 0}) == extrqReference(value, control), "EXTRQ register form stub computed the wrong field"); + require(runRegisterFormStub(extrqSame, {control, 0}, {control, 0}) == extrqReference(control, control), "EXTRQ register form stub with equal operands computed the wrong field"); + const auto insertqControl = control | 0xC0ull; + require(runRegisterFormStub(insertqDistinct, {destination, 0x1122334455667788ull}, {value, insertqControl}) == insertqReference(destination, value, insertqControl), "INSERTQ register form stub computed the wrong field"); + require(runRegisterFormStub(insertqSame, {value, insertqControl}, {value, insertqControl}) == insertqReference(value, value, insertqControl), "INSERTQ register form stub with equal operands computed the wrong field"); } } #else diff --git a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp index 96c4e1160..82f084e06 100644 --- a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp +++ b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp @@ -166,6 +166,7 @@ std::uint64_t libcReferenceLow(const Codegen::Sse4aOperands& operands, const Sta } sse4a::Instruction instruction; instruction.op = operands.Insertq ? sse4a::Op::Insertq : sse4a::Op::Extrq; + instruction.registerForm = operands.RegisterForm; instruction.destination = operands.Destination; instruction.source = operands.Source; instruction.length = static_cast(operands.Length == 64 ? 0 : operands.Length); @@ -174,34 +175,37 @@ std::uint64_t libcReferenceLow(const Codegen::Sse4aOperands& operands, const Sta return static_cast(context.FltSave.XmmRegisters[operands.Destination].Low); } -Bytes encode(const bool insertq, const std::uint8_t dst, const std::uint8_t src, const std::uint8_t length, const std::uint8_t index) { - Bytes bytes = {static_cast(insertq ? 0xF2 : 0x66)}; - const std::uint8_t reg = insertq ? dst : 0; - const std::uint8_t rm = insertq ? src : dst; - const auto rex = static_cast(0x40 | (reg >= 8 ? 4 : 0) | (rm >= 8 ? 1 : 0)); - if (rex != 0x40) bytes.push_back(rex); - bytes.insert(bytes.end(), {0x0F, 0x78, static_cast(0xC0 | ((reg & 7) << 3) | (rm & 7)), static_cast(length == 64 ? 0 : length), index}); - return bytes; -} - struct Case { bool Insertq; std::uint8_t Destination; std::uint8_t Source; std::uint8_t Length; std::uint8_t Index; + bool RegisterForm = false; }; +Bytes encode(const Case& item) { + Bytes bytes = {static_cast(item.Insertq ? 0xF2 : 0x66)}; + const std::uint8_t reg = item.Insertq || item.RegisterForm ? item.Destination : 0; + const std::uint8_t rm = item.Insertq || item.RegisterForm ? item.Source : item.Destination; + const auto rex = static_cast(0x40 | (reg >= 8 ? 4 : 0) | (rm >= 8 ? 1 : 0)); + if (rex != 0x40) bytes.push_back(rex); + bytes.insert(bytes.end(), {0x0F, static_cast(item.RegisterForm ? 0x79 : 0x78), static_cast(0xC0 | ((reg & 7) << 3) | (rm & 7))}); + if (!item.RegisterForm) bytes.insert(bytes.end(), {static_cast(item.Length == 64 ? 0 : item.Length), item.Index}); + return bytes; +} + std::string describe(const Case& item) { - return std::string(item.Insertq ? "insertq xmm" : "extrq xmm") + std::to_string(item.Destination) + (item.Insertq ? ", xmm" + std::to_string(item.Source) : "") + ", " + std::to_string(item.Length) + ", " + std::to_string(item.Index); + const bool source = item.Insertq || item.RegisterForm; + return std::string(item.Insertq ? "insertq xmm" : "extrq xmm") + std::to_string(item.Destination) + (source ? ", xmm" + std::to_string(item.Source) : "") + (item.RegisterForm ? " (register form)" : "") + ", " + std::to_string(item.Length) + ", " + std::to_string(item.Index); } std::size_t g_executions = 0; void executeCase(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, const Case& item, std::mt19937_64& random, const int images) { - const auto site = encode(item.Insertq, item.Destination, item.Source, item.Length, item.Index); + const auto site = encode(item); const auto match = matcher.Match(site.data(), site.size()); - require(match.has_value() && match->Lowering != Codegen::Amd64OnlyLowering::Unsupported, "Immediate-form SSE4a instruction was not lowered: " + describe(item)); + require(match.has_value() && match->Lowering != Codegen::Amd64OnlyLowering::Unsupported, "SSE4a instruction was not lowered: " + describe(item)); Bytes body; std::size_t returnBranchOffset = 0; if (match->Lowering == Codegen::Amd64OnlyLowering::InPlace) { @@ -215,15 +219,26 @@ void executeCase(const Harness& harness, const Codegen::IAmd64OnlyInstructionMat require(body.size() % 16 == 0 || body.size() == returnBranchOffset + 5, "Stub body with constants is not padded to 16 bytes: " + describe(item)); } const auto operands = Codegen::DecodeSse4a(site.data(), site.size()); + auto field = operands; + if (operands.RegisterForm) { + field.Length = item.Length; + field.Index = item.Index; + } + const std::size_t controlOffset = operands.Insertq ? 8 : 0; const std::array flags = {0x202, 0x203, 0x246, 0xAC7}; for (int image = 0; image < images; ++image) { State input{}; for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast(random()); + if (operands.RegisterForm) { + auto& control = input.Xmm[operands.Source]; + control[controlOffset] = static_cast((control[controlOffset] & 0xC0) | (item.Length == 64 ? 0 : item.Length)); + control[controlOffset + 1] = static_cast((control[controlOffset + 1] & 0xC0) | item.Index); + } input.FlagsIn = flags[static_cast(image) % flags.size()]; State state = input; harness.Run(body, returnBranchOffset, state); ++g_executions; - const auto expected = referenceLow(operands, input); + const auto expected = referenceLow(field, input); require(expected == libcReferenceLow(operands, input), "Transcribed reference disagrees with the libc emulation: " + describe(item)); require(low(state.Xmm[operands.Destination]) == expected, "Lowered sequence computed the wrong field: " + describe(item)); for (unsigned reg = 0; reg < 16; ++reg) { @@ -251,6 +266,8 @@ void cpuExecution() { for (std::uint8_t index = 0; index + length <= 64; ++index) { for (const auto& [dst, src] : pairs) executeCase(harness, *matcher, {true, dst, src, length, index}, random, 2); for (const std::uint8_t dst : std::array{0, 3, 9, 15}) executeCase(harness, *matcher, {false, dst, dst, length, index}, random, 2); + const auto& [dst, src] = pairs[(length * 65u + index) % pairs.size()]; + executeCase(harness, *matcher, {true, dst, src, length, index, true}, random, 2); } } } diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 50606a96d..cf4ee0927 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -34,7 +34,7 @@ Throughout the project, every function at every stage either **does exactly what - The executable file that [relinker](../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. - [libSceJpegEnc](../../core/libs/prx/libSceJpegEnc/Export.cpp) encodes 4:2:2 sampling requests with 4:2:0 chroma subsampling, and grayscale requests as a 3-component JPEG with neutral chroma instead of a single-component one: the [JPEG encoder](../../core/Decoder/Jpeg/src/Jpeg.cpp) (stb) only produces 3-component 4:2:0 and 4:4:4 images. - [Relinker](../../core/relinker/elfpatcher/src) doesn't add an icon to the generated executable. This should be done without adding dependencies (only standard). -- `--to-intel` does not lower the register form of INSERTQ (`F2 0F 79`) nor MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [lowering](../../core/relinker/codegen/src/x86/Sse4aLowering.cpp) fails the relink instead. SHA-NI is not substituted. +- `--to-intel` does not lower MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. SHA-NI is not substituted. - The length-changing path of the [instruction rewriter](../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust VEX/0F38/0F3A RIP-relative operands, data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it.