diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp index b7ee6f081..31118179e 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp @@ -49,6 +49,8 @@ inline constexpr Entry kMovntss = {"MOVNTSS", nullptr, 0}; inline constexpr Entry kMovntsd = {"MOVNTSD", nullptr, 0}; inline constexpr Entry kVrcpps = {"VRCPPS", nullptr, 0}; inline constexpr Entry kVrsqrtps = {"VRSQRTPS", nullptr, 0}; +inline constexpr Entry kRcpps = {"RCPPS", nullptr, 0}; +inline constexpr Entry kRsqrtps = {"RSQRTPS", nullptr, 0}; inline constexpr Entry kSha256rnds2 = {"SHA256RNDS2", nullptr, 0}; inline constexpr Entry kSha256msg1 = {"SHA256MSG1", nullptr, 0}; inline constexpr Entry kSha256msg2 = {"SHA256MSG2", nullptr, 0}; diff --git a/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp b/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp index 31102844e..ca3820623 100644 --- a/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp +++ b/core/relinker/codegen/include/codegen/x86/ReciprocalOperands.hpp @@ -16,9 +16,10 @@ struct ReciprocalOperands { ReciprocalOperation Operation; std::uint8_t Destination; std::uint8_t Source; + bool Vex; }; -[[nodiscard]] std::optional DecodeVexReciprocal(const std::uint8_t* data, std::size_t length); +[[nodiscard]] std::optional DecodeReciprocal(const std::uint8_t* data, std::size_t length); } diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 35b1e0cc8..5d33e8f82 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -41,6 +41,8 @@ const Entry& _sse4aEntry(const Sse4aOperands& operands) { } const Entry& _reciprocalEntry(const ReciprocalOperands& operands) { + if (!operands.Vex) + return operands.Operation == ReciprocalOperation::ReciprocalSquareRoot ? kRsqrtps : kRcpps; return operands.Operation == ReciprocalOperation::ReciprocalSquareRoot ? kVrsqrtps : kVrcpps; } @@ -239,7 +241,7 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( if (name == nullptr) name = kClzero.Name; _clzeroLowering.EmitOutOfLine(body, operands); - } else if (const auto reciprocal = DecodeVexReciprocal(instr.Data, instr.Length)) { + } else if (const auto reciprocal = DecodeReciprocal(instr.Data, instr.Length)) { if (name == nullptr) name = _reciprocalEntry(*reciprocal).Name; _reciprocalLowering.EmitOutOfLine(body, *reciprocal); @@ -254,7 +256,7 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( } _moveTrailing(body, trailing); const DecodedInstruction first{instructions.front().data(), instructions.front().size()}; - const bool optional = DecodeVexReciprocal(first.Data, first.Length).has_value() || (_target == Amd64OnlyTarget::Rosetta && first.IsRdpid()); + const bool optional = DecodeReciprocal(first.Data, first.Length).has_value() || (_target == Amd64OnlyTarget::Rosetta && first.IsRdpid()); return _trampoline(name, instructions.front().size(), body.Finish(), optional); } @@ -289,7 +291,7 @@ std::optional Amd64OnlyInstructionMatcher::Match( if (instr.IsMwaitx()) return _validWait(instr) ? _inPlace(kMwaitx, length, std::vector(kPause.Bytes, kPause.Bytes + kPause.Size)) : _unsupported(kMwaitx, length); - if (const auto reciprocal = DecodeVexReciprocal(data, length)) { + if (const auto reciprocal = DecodeReciprocal(data, length)) { return _trampoline(_reciprocalEntry(*reciprocal).Name, length, _outOfLine(length, trailing, [&](StubBodyBuilder& body) { _reciprocalLowering.EmitOutOfLine(body, *reciprocal); }), true); } diff --git a/core/relinker/codegen/src/x86/ReciprocalLowering.cpp b/core/relinker/codegen/src/x86/ReciprocalLowering.cpp index 8f6d794b4..d01588ae8 100644 --- a/core/relinker/codegen/src/x86/ReciprocalLowering.cpp +++ b/core/relinker/codegen/src/x86/ReciprocalLowering.cpp @@ -46,11 +46,15 @@ void ReciprocalLowering::EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOp body.SsePlain({0x0F, kSqrtps}, scratch, operands.Source); body.RipOperand({0x0F, kMovaps}, dst, _ones()); body.SsePlain({0x0F, kDivps}, dst, scratch); - _vexMovaps(body, dst, dst); + if (operands.Vex) + _vexMovaps(body, dst, dst); } else { body.RipOperand({0x0F, kMovaps}, scratch, _ones()); body.SsePlain({0x0F, kDivps}, scratch, operands.Source); - _vexMovaps(body, dst, scratch); + if (operands.Vex) + _vexMovaps(body, dst, scratch); + else + body.SsePlain({0x0F, kMovaps}, dst, scratch); } body.Restore(scratch); } diff --git a/core/relinker/codegen/src/x86/ReciprocalOperands.cpp b/core/relinker/codegen/src/x86/ReciprocalOperands.cpp index c9e203768..a985003b5 100644 --- a/core/relinker/codegen/src/x86/ReciprocalOperands.cpp +++ b/core/relinker/codegen/src/x86/ReciprocalOperands.cpp @@ -12,14 +12,15 @@ constexpr std::uint8_t kOpcodeRcpps = 0x53; } -std::optional DecodeVexReciprocal(const std::uint8_t* data, const std::size_t length) { +std::optional DecodeReciprocal(const std::uint8_t* data, const std::size_t length) { using namespace X64OpcodeConstants; - if (length < 4) + if (length < 3) return std::nullopt; bool registerExtension = false; bool rmExtension = false; std::uint8_t payload = 0; std::size_t opcodeOffset = 0; + bool vex = true; if (data[0] == OneByteVex2 && length == 4) { registerExtension = (data[1] & 0x80) == 0; payload = data[1]; @@ -31,13 +32,21 @@ std::optional DecodeVexReciprocal(const std::uint8_t* data, rmExtension = (data[1] & 0x20) == 0; payload = data[2]; opcodeOffset = 3; + } else if (data[0] == TwoByteOpcodeEscape && length == 3) { + opcodeOffset = 1; + vex = false; + } else if (data[0] >= RexMin && data[0] <= RexMax && length == 4 && data[1] == TwoByteOpcodeEscape) { + registerExtension = (data[0] & 0x04) != 0; + rmExtension = (data[0] & 0x01) != 0; + opcodeOffset = 2; + vex = false; } else { return std::nullopt; } const auto vvvv = static_cast((payload >> 3) & 0x0F); const bool wide = (payload & 0x04) != 0; const auto pp = static_cast(payload & 0x03); - if (vvvv != kVexUnusedRegister || wide || pp != 0) + if (vex && (vvvv != kVexUnusedRegister || wide || pp != 0)) return std::nullopt; const auto opcode = data[opcodeOffset]; if (opcode != kOpcodeRsqrtps && opcode != kOpcodeRcpps) @@ -49,6 +58,7 @@ std::optional DecodeVexReciprocal(const std::uint8_t* data, operands.Operation = opcode == kOpcodeRsqrtps ? ReciprocalOperation::ReciprocalSquareRoot : ReciprocalOperation::Reciprocal; operands.Destination = static_cast(((modrm >> ModRmRegShift) & ModRmRegMask) | (registerExtension ? 8 : 0)); operands.Source = static_cast((modrm & ModRmRmMask) | (rmExtension ? 8 : 0)); + operands.Vex = vex; return operands; } diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index dba57fd9f..719788893 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -700,17 +700,23 @@ void converterRosetta() { } void reciprocalOperands() { - const auto vex2 = Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x52, 0xD5}.data(), 4); + const auto vex2 = Codegen::DecodeReciprocal(Bytes{0xC5, 0xF8, 0x52, 0xD5}.data(), 4); require(vex2 && vex2->Operation == Codegen::ReciprocalOperation::ReciprocalSquareRoot && vex2->Destination == 2 && vex2->Source == 5, "VEX2 VRSQRTPS was not decoded"); - const auto vex2High = Codegen::DecodeVexReciprocal(Bytes{0xC5, 0x78, 0x53, 0xC1}.data(), 4); + const auto vex2High = Codegen::DecodeReciprocal(Bytes{0xC5, 0x78, 0x53, 0xC1}.data(), 4); require(vex2High && vex2High->Operation == Codegen::ReciprocalOperation::Reciprocal && vex2High->Destination == 8 && vex2High->Source == 1, "VEX2 VRCPPS with a high destination was not decoded"); - const auto vex3 = Codegen::DecodeVexReciprocal(Bytes{0xC4, 0x41, 0x78, 0x52, 0xC9}.data(), 5); + const auto vex3 = Codegen::DecodeReciprocal(Bytes{0xC4, 0x41, 0x78, 0x52, 0xC9}.data(), 5); require(vex3 && vex3->Destination == 9 && vex3->Source == 9, "VEX3 VRSQRTPS with high registers was not decoded"); - require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xFC, 0x52, 0xD5}.data(), 4), "256-bit VRSQRTPS must stay native"); - require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xFA, 0x52, 0xD5}.data(), 4), "VRSQRTSS must stay native"); - require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x52, 0x10}.data(), 4), "Memory form must stay native"); - require(!Codegen::DecodeVexReciprocal(Bytes{0xC5, 0xF8, 0x51, 0xD5}.data(), 4), "VSQRTPS is not a reciprocal"); - require(!Codegen::DecodeVexReciprocal(Bytes{0x0F, 0x52, 0xD5}.data(), 3), "Legacy RSQRTPS must stay native"); + require(!Codegen::DecodeReciprocal(Bytes{0xC5, 0xFC, 0x52, 0xD5}.data(), 4), "256-bit VRSQRTPS must stay native"); + require(!Codegen::DecodeReciprocal(Bytes{0xC5, 0xFA, 0x52, 0xD5}.data(), 4), "VRSQRTSS must stay native"); + require(!Codegen::DecodeReciprocal(Bytes{0xC5, 0xF8, 0x52, 0x10}.data(), 4), "Memory form must stay native"); + require(!Codegen::DecodeReciprocal(Bytes{0xC5, 0xF8, 0x51, 0xD5}.data(), 4), "VSQRTPS is not a reciprocal"); + require(vex2->Vex && vex2High->Vex && vex3->Vex, "VEX reciprocals must clear the upper vector bits"); + const auto legacy = Codegen::DecodeReciprocal(Bytes{0x0F, 0x52, 0xD5}.data(), 3); + require(legacy && !legacy->Vex && legacy->Operation == Codegen::ReciprocalOperation::ReciprocalSquareRoot && legacy->Destination == 2 && legacy->Source == 5, "Legacy RSQRTPS was not decoded"); + const auto legacyHigh = Codegen::DecodeReciprocal(Bytes{0x45, 0x0F, 0x53, 0xC9}.data(), 4); + require(legacyHigh && !legacyHigh->Vex && legacyHigh->Operation == Codegen::ReciprocalOperation::Reciprocal && legacyHigh->Destination == 9 && legacyHigh->Source == 9, "Legacy RCPPS with high registers was not decoded"); + for (const Bytes& unsupported : {Bytes{0xF3, 0x0F, 0x52, 0xD5}, Bytes{0xF3, 0x0F, 0x53, 0xD5}, Bytes{0x0F, 0x52, 0x10}, Bytes{0x0F, 0x53, 0x10}, Bytes{0x0F, 0x51, 0xD5}, Bytes{0x0F, 0x52}, Bytes{0x45, 0x0F, 0x53}}) + require(!Codegen::DecodeReciprocal(unsupported.data(), unsupported.size()), "An unsupported legacy reciprocal encoding was accepted"); } void converterReciprocal() { @@ -744,6 +750,27 @@ void converterReciprocal() { require(moved.Trampolines.size() == 1 && moved.KeptCount == 0 && moved.Bytes == ripRelative && moved.Trampolines[0].Length == 10, "VRSQRTPS did not absorb the following RIP-relative load"); const auto& movedSite = moved.Trampolines[0]; require(movedSite.Relocations.size() == 1 && movedSite.Relocations[0].DisplacementOffset == movedSite.ReturnBranchOffset - 4 && movedSite.Relocations[0].InstructionEnd == movedSite.ReturnBranchOffset && movedSite.Relocations[0].SiteTarget == 10 + 0x10, "Absorbed RIP-relative load has the wrong relocation"); + for (const std::uint8_t opcode : {std::uint8_t{0x52}, std::uint8_t{0x53}}) { + const std::string name = opcode == 0x52 ? "RSQRTPS" : "RCPPS"; + for (const Bytes& instruction : {Bytes{0x0F, opcode, 0xD5}, Bytes{0x45, 0x0F, opcode, 0xC9}}) { + auto legacyFile = file; + Bytes legacyText = instruction; + legacyText.insert(legacyText.end(), {0x90, 0x90, 0xC3}); + std::copy(legacyText.begin(), legacyText.end(), legacyFile.begin() + 0x200); + const auto legacy = converter->Convert(legacyFile, {segmentHeader(legacyText.size())}); + require(legacy.Trampolines.size() == 1 && legacy.KeptCount == 0 && legacy.Reports[0].InstructionName == name && legacy.Trampolines[0].Length == 5, "Legacy reciprocal did not absorb enough instructions for a jump"); + legacyText = instruction; + legacyText.push_back(0xC3); + std::copy(legacyText.begin(), legacyText.end(), legacyFile.begin() + 0x200); + const auto kept = converter->Convert(legacyFile, {segmentHeader(legacyText.size())}); + require(kept.Trampolines.empty() && kept.KeptCount == 1 && kept.Bytes == legacyFile && kept.Reports[0].InstructionName == name, "Legacy reciprocal before a return must stay native"); + } + } + auto sequenceFile = file; + const Bytes sequenceText = {0x0F, 0x52, 0xD5, 0xC5, 0xF8, 0x53, 0xC1, 0xC3}; + std::copy(sequenceText.begin(), sequenceText.end(), sequenceFile.begin() + 0x200); + const auto sequence = converter->Convert(sequenceFile, {segmentHeader(sequenceText.size())}); + require(sequence.Trampolines.size() == 1 && sequence.KeptCount == 0 && sequence.Trampolines[0].Length == 7 && sequence.Reports[0].InstructionName == "RSQRTPS", "Mixed legacy and VEX reciprocals were not lowered together"); } void converterStrayRex() { @@ -1037,10 +1064,10 @@ void reciprocalExecution() { for (const auto& lanes : inputs) { const std::uint64_t source[2] = {packLanes(lanes, 0), packLanes(lanes, 2)}; const std::uint64_t destination[2] = {0x1111111111111111ull, 0x2222222222222222ull}; - for (const bool squareRoot : {true, false}) { + for (const bool squareRoot : {true, false}) for (const bool vex : {false, true}) { const std::uint8_t opcode = squareRoot ? 0x52 : 0x53; - const Bytes distinct = {0xC5, 0xF8, opcode, 0xD5}; - const Bytes same = {0xC5, 0xF8, opcode, 0xD2}; + const Bytes distinct = vex ? Bytes{0xC5, 0xF8, opcode, 0xD5} : Bytes{0x0F, opcode, 0xD5}; + const Bytes same = vex ? Bytes{0xC5, 0xF8, opcode, 0xD2} : Bytes{0x0F, opcode, 0xD2}; const auto distinctOut = runRegisterFormStub(distinct, destination, source); const auto sameOut = runRegisterFormStub(same, source, source); float distinctLanes[4]; @@ -1049,8 +1076,8 @@ void reciprocalExecution() { std::memcpy(sameLanes, sameOut.data(), sizeof(sameLanes)); for (std::size_t lane = 0; lane < 4; ++lane) { const float expected = squareRoot ? 1.0f / std::sqrt(lanes[lane]) : 1.0f / lanes[lane]; - require(sameLane(expected, distinctLanes[lane]), "VRSQRTPS/VRCPPS stub is not correctly rounded"); - require(sameLane(expected, sameLanes[lane]), "VRSQRTPS/VRCPPS stub with equal operands is not correctly rounded"); + require(sameLane(expected, distinctLanes[lane]), "Packed reciprocal stub is not correctly rounded"); + require(sameLane(expected, sameLanes[lane]), "Packed reciprocal stub with equal operands is not correctly rounded"); } } } diff --git a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp index f88d3d93f..c726b2958 100644 --- a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp +++ b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,7 @@ struct alignas(16) State { std::uint64_t RspBefore; std::uint64_t RspAfter; std::uint8_t RedZone[128]; + std::uint8_t Ymm[16][32]; }; constexpr std::uint64_t kCanary = 0xA5C3E17B9D24F608ull; @@ -94,6 +96,11 @@ void movdquXmmRsp(Bytes& code, const std::uint8_t reg, const bool store, const s emitU32(code, offset); } +void vmovdquYmmRbx(Bytes& code, const std::uint8_t reg, const bool store) { + emit(code, {0xC4, static_cast(reg >= 8 ? 0x61 : 0xE1), 0x7E, static_cast(store ? 0x7F : 0x6F), static_cast(0x83 | ((reg & 7) << 3))}); + emitU32(code, static_cast(offsetof(State, Ymm)) + reg * 32); +} + class Harness { public: Harness() : _memory(VirtualAlloc(nullptr, kSize, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE)) { @@ -104,7 +111,7 @@ public: VirtualFree(_memory, 0, MEM_RELEASE); } - void Run(const Bytes& body, const std::size_t returnBranchOffset, State& state) const { + void Run(const Bytes& body, const std::size_t returnBranchOffset, State& state, const bool wide = false) const { Bytes code; emit(code, {0x53, 0x55, 0x56, 0x57, 0x41, 0x54, 0x41, 0x55, 0x41, 0x56, 0x41, 0x57, 0x48, 0x81, 0xEC, 0xA8, 0x00, 0x00, 0x00}); for (std::uint8_t reg = 6; reg < 16; ++reg) movdquXmmRsp(code, reg, true, (reg - 6) * 16); @@ -115,7 +122,12 @@ public: for (int slot = 0; slot < 16; ++slot) emit(code, {0x48, 0x89, 0x44, 0x24, static_cast(-128 + slot * 8)}); emit(code, {0x48, 0x89, 0xA3}); emitU32(code, kRspBeforeOffset); - for (std::uint8_t reg = 0; reg < 16; ++reg) movdquXmmRbx(code, reg, false); + for (std::uint8_t reg = 0; reg < 16; ++reg) { + if (wide) + vmovdquYmmRbx(code, reg, false); + else + movdquXmmRbx(code, reg, false); + } code.push_back(0xE9); const auto jumpOffset = code.size(); emitU32(code, 0); @@ -129,6 +141,8 @@ public: emit(code, {0x9C, 0x8F, 0x83}); emitU32(code, kFlagsOutOffset); for (std::uint8_t reg = 0; reg < 16; ++reg) movdquXmmRbx(code, reg, true); + if (wide) + for (std::uint8_t reg = 0; reg < 16; ++reg) vmovdquYmmRbx(code, reg, true); for (std::uint8_t reg = 6; reg < 16; ++reg) movdquXmmRsp(code, reg, false, (reg - 6) * 16); emit(code, {0x48, 0x81, 0xC4, 0xA8, 0x00, 0x00, 0x00, 0x41, 0x5F, 0x41, 0x5E, 0x41, 0x5D, 0x41, 0x5C, 0x5F, 0x5E, 0x5D, 0x5B, 0xC3}); code.resize(Io::AlignUp(code.size(), std::size_t{16}), 0xCC); @@ -219,6 +233,69 @@ void requireEnvironment(const State& input, const State& state, const std::uint1 } } +void reciprocalExecution() { + const Harness harness; + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + const bool wide = __builtin_cpu_supports("avx"); + const float inputs[][4] = { + {1.0f, 4.0f, 0.25f, 2.0f}, + {0.0f, -0.0f, INFINITY, -1.0f}, + {1.00000012f, 0.99999994f, 3.0e-38f, 1.0e30f}, + {NAN, -INFINITY, -4.0f, 16.0f}}; + for (const bool vex : {false, true}) { + if (vex && !wide) + continue; + for (const std::uint8_t opcode : {std::uint8_t{0x52}, std::uint8_t{0x53}}) { + for (const std::uint8_t destination : {std::uint8_t{0}, std::uint8_t{2}, std::uint8_t{9}, std::uint8_t{15}}) { + for (const std::uint8_t source : {destination, std::uint8_t{0}, std::uint8_t{5}, std::uint8_t{14}}) { + Bytes site; + if (vex) { + emit(site, {0xC4, static_cast((destination >= 8 ? 0 : 0x80) | 0x40 | (source >= 8 ? 0 : 0x20) | 1), 0x78}); + } else { + const auto rex = static_cast(0x40 | (destination >= 8 ? 4 : 0) | (source >= 8 ? 1 : 0)); + if (rex != 0x40) + site.push_back(rex); + site.push_back(0x0F); + } + emit(site, {opcode, static_cast(0xC0 | ((destination & 7) << 3) | (source & 7))}); + const auto match = matcher->Match(site.data(), site.size()); + require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "Packed reciprocal was not lowered"); + for (const auto& lanes : inputs) { + State input{}; + for (std::size_t reg = 0; reg < 16; ++reg) { + for (std::size_t byte = 0; byte < 16; ++byte) { + input.Xmm[reg][byte] = static_cast(reg * 16 + byte); + input.Ymm[reg][byte + 16] = static_cast(reg * 16 + byte + 1); + } + } + std::memcpy(input.Xmm[source], lanes, sizeof(lanes)); + for (std::size_t reg = 0; reg < 16; ++reg) + std::memcpy(input.Ymm[reg], input.Xmm[reg], 16); + input.FlagsIn = 0xAC7; + State state = input; + harness.Run(match->StubBody, match->ReturnBranchOffset, state, wide); + ++g_executions; + float actual[4]; + std::memcpy(actual, state.Xmm[destination], sizeof(actual)); + for (std::size_t lane = 0; lane < 4; ++lane) { + const float expected = opcode == 0x52 ? 1.0f / std::sqrt(lanes[lane]) : 1.0f / lanes[lane]; + require(std::isnan(expected) ? std::isnan(actual[lane]) : std::memcmp(&expected, &actual[lane], sizeof(float)) == 0, "Packed reciprocal result differs from division and square root"); + } + requireEnvironment(input, state, static_cast(1u << destination), "packed reciprocal"); + if (wide) { + const std::uint8_t zero[16] = {}; + for (std::size_t reg = 0; reg < 16; ++reg) { + const auto* expected = vex && reg == destination ? zero : input.Ymm[reg] + 16; + require(std::memcmp(state.Ymm[reg] + 16, expected, 16) == 0, "Packed reciprocal changed the wrong upper YMM bits"); + } + } + } + } + } + } + } +} + void executeCase(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, const Case& item, std::mt19937_64& random, const int images) { const auto site = encode(item); const auto match = matcher.Match(site.data(), site.size()); @@ -643,6 +720,7 @@ void peBuilder() { int main() { try { cpuExecution(); + reciprocalExecution(); peBuilder(); std::cout << "AMD64-only Windows tests passed (" << g_executions << " lowered sequences executed)\n"; } catch (const std::exception& error) { diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 0c72b62a7..5f243bc03 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -513,7 +513,7 @@ Throughout the project, every function at every stage either **does exactly what - [UINT16_ABGR color exports](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - export format 7 is accepted only into an unsigned integer target, and each 16-bit value reaches the target unchanged; a value beyond a narrower target's range (`R8_UINT`) is undefined in Vulkan, and whether the console clamps or truncates it is not confirmed. SINT16_ABGR (8) throws - `--to-intel` guest module trampolines are covered only by a synthetic relinker test; no game title has been verified with them on Linux or Windows. - `--to-rosetta` lowers what Rosetta lacks on top of `--to-intel` ([matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp)). Rosetta runs AVX2, FMA, BMI1/2, F16C and MOVBE but reports none of them through CPUID, so code that checks CPUID takes its baseline path. RDSEED becomes RDRAND and CLWB a NOP. RDPID returns processor 0, because Rosetta's TSC_AUX is the constant 0xAAAAAAAB; the same value reaches RDTSCP's ECX, which is not converted. ADCX/ADOX fault under Rosetta and are not converted: compilers do not emit them, and code that checks CPUID sees no ADX. The conversions are covered by the converter tests; that the converted code runs under Rosetta was checked on an M5 Max only, outside this repository's CI. -- `--to-intel` replaces the approximate VEX.128 register forms of VRSQRTPS and VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). The 256-bit, scalar, legacy and memory forms keep the native instruction, as does a 4-byte site with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). +- `--to-intel` replaces the approximate legacy SSE and VEX.128 register forms of RSQRTPS/RCPPS and VRSQRTPS/VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). Legacy SSE forms preserve the destination's upper YMM bits; VEX forms clear them. The 256-bit, scalar and memory forms keep the native instruction, as does a site shorter than 5 bytes with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - [sceKeyboardGetKey2Char](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) (libSceKeyboard) translates the 101-key (US) arrangement and the alphanumeric layer of the 106-key (Japanese) one, which follows the JIS layout of the HID usages: the Yen (0x89) and Ro (0x87) keys return `\` (0x5C) unshifted, as JIS X 0201 places the yen sign there; whether the console returns U+00A5 instead is unverified. With the Kana LED (bit 4, the HID position) set the 106-key arrangement throws, as the kana the console returns are unknown. Ctrl and Alt do not change the character. - [sceAudioOut2Set3DLatency](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp) and [sceAudioOut2MasteringInit, sceAudioOut2MasteringTerm and sceAudioOut2MasteringSetParam](../../core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp) (libSceAudioOut) have no effect: there is no 3D rendering or mastering stage. - [sceImeKeyboardGetInfo](../../core/libs/prx/libSceIme/Export.cpp) (libSceIme) returns `0x80bc0023` (no resource id) for every id while a keyboard is open, since the IME keyboard never reports a connected device. [sceImeKeyboardSetMode](../../core/libs/prx/libSceIme/Export.cpp) validates the mode bits but the mode has no effect, as no IME keyboard events are delivered.