feat(relinker): lower register-form EXTRQ and relocate following instructions for short sites

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ButHENCbXXt8h4EZuWUabJ

core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp

@@ -5,6 +5,7 @@
 #include <cstdint>
 #include <memory>
 #include <optional>
+#include <span>
 
 namespace Codegen {
 
@@ -14,7 +15,13 @@ public:
 
     [[nodiscard]] virtual std::optional<Amd64OnlyMatch> Match(
         const std::uint8_t* data,
-        std::size_t length
+        std::size_t length,
+        std::span<const std::uint8_t> trailing = {}
+    ) const = 0;
+
+    [[nodiscard]] virtual std::optional<Amd64OnlyMatch> MatchSequence(
+        std::span<const std::span<const std::uint8_t>> instructions,
+        std::span<const std::uint8_t> trailing
     ) const = 0;
 };
 

core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp

@@ -5,6 +5,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <optional>
+#include <span>
 #include <vector>
 
 namespace Codegen {
@@ -17,7 +18,8 @@ struct LoweredBody {
 class Sse4aLowering {
 public:
     [[nodiscard]] std::optional<std::vector<std::uint8_t>> LowerInPlace(const Sse4aOperands& operands, std::size_t originalLength) const;
-    [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands) const;
+    [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands, std::span<const std::uint8_t> trailing = {}) const;
+    [[nodiscard]] LoweredBody LowerOutOfLine(std::span<const Sse4aOperands> sequence, std::span<const std::uint8_t> trailing) const;
 };
 
 }

core/relinker/codegen/src/Amd64OnlyConverter.cpp

@@ -8,7 +8,9 @@
 #include <algorithm>
 #include <memory>
 #include <set>
+#include <span>
 #include <string>
+#include <vector>
 
 namespace Codegen {
 
@@ -32,6 +34,7 @@ public:
 
 private:
     struct Pending {
+        std::size_t Index;
         InstructionMatch Instruction;
         Amd64OnlyMatch Substitution;
     };
@@ -89,10 +92,11 @@ void Amd64OnlyConverter::_convertSegment(
     const auto matches = _atFileOffset(ph.Offset, [&] { return _scanner->ScanCodeSection(seg, 0, seg.size()); });
 
     std::vector<Pending> pending;
-    for (const auto& match : matches) {
+    for (std::size_t index = 0; index < matches.size(); ++index) {
+        const auto& match = matches[index];
         auto substitution = _atFileOffset(ph.Offset + match.Offset, [&] { return _matcher->Match(seg.data() + match.Offset, match.Length); });
         if (substitution.has_value())
-            pending.push_back({match, std::move(*substitution)});
+            pending.push_back({index, match, std::move(*substitution)});
     }
     if (pending.empty())
         return;
@@ -102,7 +106,10 @@ void Amd64OnlyConverter::_convertSegment(
     });
     const auto branchTargets = needsBranchTargets ? _collectBranchTargets(seg, matches, ph) : std::set<std::uint64_t>{};
 
+    std::set<std::size_t> consumed;
     for (const auto& item : pending) {
+        if (consumed.contains(item.Index))
+            continue;
         const auto& match = item.Instruction;
         const auto& substitution = item.Substitution;
         const auto fileOffset = static_cast<Domain::FileByteOffset>(ph.Offset + match.Offset);
@@ -119,21 +126,48 @@ void Amd64OnlyConverter::_convertSegment(
             break;
         }
         case Amd64OnlyLowering::Trampoline: {
-            if (match.Length < Amd64OnlySubstitutionTable::kJmpRel32.Size)
-                throw CodegenException("AMD-only instruction too short for a jump", fileOffset);
+            std::size_t siteLength = match.Length;
+            auto stub = substitution;
+            if (siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size) {
+                const X64InstructionDecoder decoder;
+                std::vector<std::span<const std::uint8_t>> sequence{std::span<const std::uint8_t>(seg.data() + match.Offset, match.Length)};
+                std::size_t trailingBytes = 0;
+                for (auto next = item.Index + 1; siteLength < Amd64OnlySubstitutionTable::kJmpRel32.Size; ++next) {
+                    if (next >= matches.size() || matches[next].Offset != match.Offset + siteLength)
+                        throw CodegenException("AMD-only instruction too short for a jump and not followed by an instruction", fileOffset);
+                    const auto& following = matches[next];
+                    const std::span<const std::uint8_t> bytes(seg.data() + following.Offset, following.Length);
+                    const auto info = decoder.DecodeInstruction(bytes.data(), bytes.size());
+                    const bool amdOnly = _matcher->Match(bytes.data(), bytes.size()).has_value();
+                    if (amdOnly && trailingBytes == 0) {
+                        sequence.push_back(bytes);
+                        consumed.insert(next);
+                    } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) {
+                        throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset);
+                    } else {
+                        trailingBytes += following.Length;
+                    }
+                    siteLength += following.Length;
+                }
+                const std::span<const std::uint8_t> trailing(seg.data() + match.Offset + siteLength - trailingBytes, trailingBytes);
+                auto relocated = _atFileOffset(fileOffset, [&] { return _matcher->MatchSequence(sequence, trailing); });
+                if (!relocated.has_value() || relocated->Lowering != Amd64OnlyLowering::Trampoline)
+                    throw CodegenException("AMD-only instruction sequence has no out-of-line lowering", fileOffset);
+                stub = std::move(*relocated);
+            }
             const auto hit = branchTargets.upper_bound(address);
-            if (hit != branchTargets.end() && *hit < address + match.Length)
+            if (hit != branchTargets.end() && *hit < address + siteLength)
                 throw CodegenException("Branch enters an AMD-only instruction", ph.Offset + (*hit - ph.MappedAddress));
             const auto begin = seg.begin() + static_cast<std::ptrdiff_t>(match.Offset);
             result.Trampolines.push_back({
                 fileOffset,
                 address,
-                match.Length,
-                std::vector<std::uint8_t>(begin, begin + static_cast<std::ptrdiff_t>(match.Length)),
-                substitution.StubBody,
-                substitution.ReturnBranchOffset
+                siteLength,
+                std::vector<std::uint8_t>(begin, begin + static_cast<std::ptrdiff_t>(siteLength)),
+                stub.StubBody,
+                stub.ReturnBranchOffset
             });
-            replacementLength = substitution.StubBody.size();
+            replacementLength = stub.StubBody.size();
             break;
         }
         case Amd64OnlyLowering::Unsupported:

core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp

@@ -6,6 +6,8 @@
 #include <codegen/x86/X64OpcodeConstants.hpp>
 #include <codegen/CodegenException.hpp>
 #include <memory>
+#include <span>
+#include <vector>
 
 namespace Codegen {
 
@@ -21,14 +23,20 @@ class Amd64OnlyInstructionMatcher : public IAmd64OnlyInstructionMatcher {
 public:
     [[nodiscard]] std::optional<Amd64OnlyMatch> Match(
         const std::uint8_t* data,
-        std::size_t length
+        std::size_t length,
+        std::span<const std::uint8_t> trailing = {}
+    ) const override;
+
+    [[nodiscard]] std::optional<Amd64OnlyMatch> MatchSequence(
+        std::span<const std::span<const std::uint8_t>> instructions,
+        std::span<const std::uint8_t> trailing
     ) const override;
 
 private:
     Sse4aLowering _lowering;
 
     [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const;
-    [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry) const;
+    [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span<const std::uint8_t> trailing) const;
 };
 
 Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const {
@@ -44,19 +52,45 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstructio
     return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0};
 }
 
-Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry) const {
+Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span<const std::uint8_t> trailing) const {
     const auto operands = DecodeSse4a(instr.Data, instr.Length);
-    if (operands.RegisterForm)
+    if (operands.RegisterForm && operands.Insertq)
         return _unsupported(registerFormEntry, instr.Length);
-    if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length))
-        return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0};
-    auto body = _lowering.LowerOutOfLine(operands);
-    return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset};
+    if (!operands.RegisterForm && trailing.empty()) {
+        if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length))
+            return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0};
+    }
+    auto body = _lowering.LowerOutOfLine(operands, trailing);
+    const auto& name = operands.RegisterForm ? registerFormEntry.Name : entry.Name;
+    return Amd64OnlyMatch{name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset};
+}
+
+std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::MatchSequence(
+    std::span<const std::span<const std::uint8_t>> instructions,
+    std::span<const std::uint8_t> trailing
+) const {
+    if (instructions.empty())
+        return std::nullopt;
+    std::vector<Sse4aOperands> sequence;
+    for (const auto& instruction : instructions) {
+        const DecodedInstruction instr{instruction.data(), instruction.size()};
+        if (!instr.IsExtrq() && !instr.IsInsertq())
+            return std::nullopt;
+        const auto operands = DecodeSse4a(instr.Data, instr.Length);
+        if (operands.RegisterForm && operands.Insertq)
+            return std::nullopt;
+        sequence.push_back(operands);
+    }
+    const auto& first = sequence.front();
+    const auto& name = first.RegisterForm ? (first.Insertq ? kInsertqRegisterForm.Name : kExtrqRegisterForm.Name) : (first.Insertq ? kInsertq.Name : kExtrq.Name);
+    auto body = _lowering.LowerOutOfLine(std::span<const Sse4aOperands>(sequence), trailing);
+    return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset};
 }
 
 std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::Match(
     const std::uint8_t* data,
-    std::size_t length
+    std::size_t length,
+    std::span<const std::uint8_t> trailing
 ) const {
     const DecodedInstruction instr{data, length};
 
@@ -67,10 +101,10 @@ std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::Match(
         return _matchMovnts(instr, kMovntsd);
 
     if (instr.IsExtrq())
-        return _matchSse4a(instr, kExtrq, kExtrqRegisterForm);
+        return _matchSse4a(instr, kExtrq, kExtrqRegisterForm, trailing);
 
     if (instr.IsInsertq())
-        return _matchSse4a(instr, kInsertq, kInsertqRegisterForm);
+        return _matchSse4a(instr, kInsertq, kInsertqRegisterForm, trailing);
 
     if (instr.IsMonitorx())
         return _unsupported(kMonitorx, length);

core/relinker/codegen/src/x86/Sse4aLowering.cpp

@@ -3,6 +3,7 @@
 #include <codegen/CodegenException.hpp>
 #include <algorithm>
 #include <array>
+#include <span>
 #include <initializer_list>
 
 namespace Codegen {
@@ -89,6 +90,10 @@ public:
         _bytes.insert(_bytes.end(), kLeaRspRestore.Bytes, kLeaRspRestore.Bytes + kLeaRspRestore.Size);
     }
 
+    void Raw(std::span<const std::uint8_t> bytes) {
+        _bytes.insert(_bytes.end(), bytes.begin(), bytes.end());
+    }
+
     LoweredBody Finish() {
         const auto returnBranchOffset = _bytes.size();
         _bytes.insert(_bytes.end(), kJmpRel32.Bytes, kJmpRel32.Bytes + kJmpRel32.Size);
@@ -120,52 +125,41 @@ private:
     std::vector<Constant> _constants;
 };
 
-}
-
-std::optional<std::vector<std::uint8_t>> Sse4aLowering::LowerInPlace(const Sse4aOperands& operands, const std::size_t originalLength) const {
-    if (operands.RegisterForm)
-        return std::nullopt;
-    const auto length = operands.Length;
-    const auto index = operands.Index;
-    const auto dst = operands.Destination;
-    const auto src = operands.Source;
-    std::vector<std::uint8_t> sequence;
-    if (operands.Insertq) {
-        if (dst == src && index == 0) {
-        } else if (length == kFieldBits && index == 0) {
-            _sse(sequence, kPrefixScalar, {0x0F, 0x7E}, dst, src);
-        } else if (index == 0 && length % 16 == 0) {
-            _sse(sequence, kPrefixPacked, {0x0F, 0x3A, 0x0E}, dst, src);
-            sequence.push_back(static_cast<std::uint8_t>((1u << (length / 16)) - 1));
-        } else {
-            return std::nullopt;
-        }
-    } else {
-        if (index == 0 && length == kFieldBits) {
-        } else if (index + length == kFieldBits) {
-            _shiftImm(sequence, kShiftRight, dst, index);
-        } else if (index == 0 && (length == 8 || length == 16 || length == 32)) {
-            const std::uint8_t opcode = length == 8 ? 0x32 : (length == 16 ? 0x34 : 0x35);
-            _sse(sequence, kPrefixPacked, {0x0F, 0x38, opcode}, dst, dst);
-        } else {
-            return std::nullopt;
-        }
+void _emitOutOfLine(BodyBuilder& body, const Sse4aOperands& operands) {
+    if (operands.RegisterForm) {
+        if (operands.Insertq)
+            throw CodegenException("INSERTQ register form has no Intel lowering");
+        const auto dst = operands.Destination;
+        const auto src = operands.Source;
+        std::array<std::uint8_t, 2> scratch{};
+        for (std::uint8_t reg = 0, found = 0; found < scratch.size(); ++reg)
+            if (reg != dst && reg != src) scratch[found++] = reg;
+        Constant fieldMask{};
+        fieldMask[0] = kFieldBits - 1;
+        Constant one{};
+        one[0] = 1;
+        body.Spill(scratch[0]);
+        body.Spill(scratch[1]);
+        body.Sse(kPrefixPacked, {0x0F, 0x6F}, scratch[0], src);
+        body.ShiftImm(kShiftRight, scratch[0], 8);
+        body.RipOperand({0x0F, 0xDB}, scratch[0], fieldMask);
+        body.Sse(kPrefixPacked, {0x0F, 0x6F}, scratch[1], src);
+        body.RipOperand({0x0F, 0xDB}, scratch[1], fieldMask);
+        body.RipOperand({0x0F, 0xEF}, scratch[1], fieldMask);
+        body.RipOperand({0x0F, 0xD4}, scratch[1], one);
+        body.RipOperand({0x0F, 0xDB}, scratch[1], fieldMask);
+        body.Sse(kPrefixPacked, {0x0F, 0xD3}, dst, scratch[0]);
+        body.Sse(kPrefixPacked, {0x0F, 0xF3}, dst, scratch[1]);
+        body.Sse(kPrefixPacked, {0x0F, 0xD3}, dst, scratch[1]);
+        body.Restore(scratch[1]);
+        body.Restore(scratch[0]);
+        return;
     }
-    if (sequence.size() > originalLength)
-        return std::nullopt;
-    _nopFill(sequence, originalLength - sequence.size());
-    return sequence;
-}
-
-LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands) const {
-    if (operands.RegisterForm)
-        throw CodegenException("EXTRQ/INSERTQ register form has no Intel lowering");
     const auto length = operands.Length;
     const auto index = operands.Index;
     const auto dst = operands.Destination;
     const auto src = operands.Source;
     const bool byteAligned = length % 8 == 0 && index % 8 == 0;
-    BodyBuilder body;
     if (!operands.Insertq) {
         if (byteAligned) {
             Constant mask;
@@ -208,6 +202,54 @@ LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands) const {
         body.Sse(kPrefixPacked, {0x0F, 0xEF}, dst, scratch);
         body.Restore(scratch);
     }
+}
+
+}
+
+std::optional<std::vector<std::uint8_t>> Sse4aLowering::LowerInPlace(const Sse4aOperands& operands, const std::size_t originalLength) const {
+    if (operands.RegisterForm)
+        return std::nullopt;
+    const auto length = operands.Length;
+    const auto index = operands.Index;
+    const auto dst = operands.Destination;
+    const auto src = operands.Source;
+    std::vector<std::uint8_t> sequence;
+    if (operands.Insertq) {
+        if (dst == src && index == 0) {
+        } else if (length == kFieldBits && index == 0) {
+            _sse(sequence, kPrefixScalar, {0x0F, 0x7E}, dst, src);
+        } else if (index == 0 && length % 16 == 0) {
+            _sse(sequence, kPrefixPacked, {0x0F, 0x3A, 0x0E}, dst, src);
+            sequence.push_back(static_cast<std::uint8_t>((1u << (length / 16)) - 1));
+        } else {
+            return std::nullopt;
+        }
+    } else {
+        if (index == 0 && length == kFieldBits) {
+        } else if (index + length == kFieldBits) {
+            _shiftImm(sequence, kShiftRight, dst, index);
+        } else if (index == 0 && (length == 8 || length == 16 || length == 32)) {
+            const std::uint8_t opcode = length == 8 ? 0x32 : (length == 16 ? 0x34 : 0x35);
+            _sse(sequence, kPrefixPacked, {0x0F, 0x38, opcode}, dst, dst);
+        } else {
+            return std::nullopt;
+        }
+    }
+    if (sequence.size() > originalLength)
+        return std::nullopt;
+    _nopFill(sequence, originalLength - sequence.size());
+    return sequence;
+}
+
+LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands, std::span<const std::uint8_t> trailing) const {
+    return LowerOutOfLine(std::span<const Sse4aOperands>(&operands, 1), trailing);
+}
+
+LoweredBody Sse4aLowering::LowerOutOfLine(std::span<const Sse4aOperands> sequence, std::span<const std::uint8_t> trailing) const {
+    BodyBuilder body;
+    for (const auto& operands : sequence)
+        _emitOutOfLine(body, operands);
+    body.Raw(trailing);
     return body.Finish();
 }
 

core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp

@@ -11,6 +11,10 @@
 #include <elfpatcher/linux/LinuxElfPatcher.hpp>
 #include <io/ByteWriter.hpp>
 #include <cstring>
+#include <cstdint>
+#ifdef __linux__
+#include <sys/mman.h>
+#endif
 #include <functional>
 #include <iostream>
 #include <stdexcept>
@@ -138,7 +142,9 @@ void matcherSubstitutions() {
     const auto monitorx = match({0x0F, 0x01, 0xFA});
     require(monitorx && monitorx->Lowering == Codegen::Amd64OnlyLowering::Unsupported && monitorx->InstructionName == "MONITORX", "MONITORX was not reported as unsupported");
     const auto registerForm = match({0x66, 0x0F, 0x79, 0xCA});
-    require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "EXTRQ register form was not reported as unsupported");
+    require(registerForm && registerForm->Lowering == Codegen::Amd64OnlyLowering::Trampoline && registerForm->InstructionName == "EXTRQ register form", "EXTRQ register form was not lowered through a stub");
+    const auto insertqRegisterForm = match({0xF2, 0x0F, 0x79, 0xCA});
+    require(insertqRegisterForm && insertqRegisterForm->Lowering == Codegen::Amd64OnlyLowering::Unsupported, "INSERTQ register form was not reported as unsupported");
     require(!match({0x66, 0x0F, 0x2B, 0x07}) && !match({0x0F, 0x2B, 0x07}) && !match({0x48, 0x8B, 0x05, 0, 0, 0, 0}), "Ordinary instruction was matched");
     const auto stub = match(kInsertqHighSite);
     require(stub && stub->Lowering == Codegen::Amd64OnlyLowering::Trampoline && stub->StubBody == kInsertqHighBody && stub->ReturnBranchOffset == 15 && stub->InstructionName == "INSERTQ", "INSERTQ was not lowered through a stub");
@@ -221,7 +227,14 @@ void converterSegment() {
     auto registerForm = file;
     const Bytes extrqRegister = {0x66, 0x0F, 0x79, 0xCA};
     std::copy(extrqRegister.begin(), extrqRegister.end(), registerForm.begin() + 0x20F);
-    requireFailure([&] { (void)converter->Convert(registerForm, {segmentHeader(20)}); }, "EXTRQ register form was silently kept");
+    requireFailure([&] { (void)converter->Convert(registerForm, {segmentHeader(20)}); }, "Short EXTRQ followed by a return was relocated");
+    registerForm[0x213] = 0x90;
+    const auto relocated = converter->Convert(registerForm, {segmentHeader(20)});
+    require(relocated.Trampolines.size() == 2, "Short EXTRQ register form was not lowered through a stub");
+    const auto& shortSite = relocated.Trampolines[1];
+    const Bytes shortOriginal = {0x66, 0x0F, 0x79, 0xCA, 0x90};
+    require(shortSite.Offset == 0x20F && shortSite.Length == 5 && shortSite.OriginalBytes == shortOriginal, "Short EXTRQ site did not absorb the following instruction");
+    require(shortSite.Body[shortSite.ReturnBranchOffset - 1] == 0x90 && shortSite.Body[shortSite.ReturnBranchOffset] == 0xE9, "Absorbed instruction does not run before the return jump");
     requireFailure([&] { (void)converter->Convert(file, {segmentHeader(0x200)}); }, "Segment exceeding the file was accepted");
 }
 
@@ -322,12 +335,70 @@ void linuxPlacement() {
 
 }
 
+#if defined(__linux__) && defined(__x86_64__)
+std::uint64_t extrqReference(std::uint64_t value, std::uint64_t control) {
+    const auto length = static_cast<unsigned>(control & 0x3f);
+    const auto index = static_cast<unsigned>((control >> 8) & 0x3f);
+    const auto shifted = value >> index;
+    return length == 0 ? shifted : shifted & ((std::uint64_t{1} << length) - 1);
+}
+
+std::uint64_t runExtrqStub(const Bytes& site, std::uint64_t destination, std::uint64_t control) {
+    const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher();
+    const auto match = matcher->Match(site.data(), site.size());
+    require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "EXTRQ register form stub was not produced");
+    auto body = match->StubBody;
+    const auto ret = body.size();
+    body.push_back(0xC3);
+    const auto displacement = static_cast<std::int32_t>(ret - (match->ReturnBranchOffset + 5));
+    std::memcpy(body.data() + match->ReturnBranchOffset + 1, &displacement, sizeof(displacement));
+    void* code = mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    require(code != MAP_FAILED, "cannot map executable memory for the stub");
+    std::memcpy(code, body.data(), body.size());
+    alignas(16) std::uint64_t destinationIn[2] = {destination, 0x1122334455667788ull};
+    alignas(16) std::uint64_t controlIn[2] = {control, 0};
+    alignas(16) std::uint64_t out[2] = {};
+    alignas(16) std::uint64_t scratchIn[2] = {0x0123456789abcdefull, 0xfedcba9876543210ull};
+    alignas(16) std::uint64_t scratchOut[2] = {};
+    const bool same = site[3] == 0xD2;
+    asm volatile(
+        "movdqu (%[scratch]), %%xmm0\n\t"
+        "movdqu (%[dst]), %%xmm2\n\t"
+        "movdqu (%[ctl]), %%xmm5\n\t"
+        "sub $128, %%rsp\n\t"
+        "call *%[code]\n\t"
+        "add $128, %%rsp\n\t"
+        "movdqu %%xmm2, (%[out])\n\t"
+        "movdqu %%xmm0, (%[scratchOut])\n\t"
+        :
+        : [scratch] "r"(scratchIn), [dst] "r"(same ? controlIn : destinationIn), [ctl] "r"(controlIn), [code] "r"(code), [out] "r"(out), [scratchOut] "r"(scratchOut)
+        : "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "memory", "cc");
+    munmap(code, 4096);
+    require(scratchOut[0] == scratchIn[0] && scratchOut[1] == scratchIn[1], "EXTRQ stub clobbered a scratch register");
+    return out[0];
+}
+
+void registerFormExecution() {
+    const Bytes distinct = {0x66, 0x0F, 0x79, 0xD5};
+    const Bytes same = {0x66, 0x0F, 0x79, 0xD2};
+    const std::uint64_t value = 0x9e3779b97f4a7c15ull;
+    for (const auto [length, index] : {std::pair{8u, 4u}, {0u, 0u}, {40u, 20u}, {63u, 1u}, {1u, 63u}, {16u, 48u}}) {
+        const auto control = static_cast<std::uint64_t>(length) | (static_cast<std::uint64_t>(index) << 8) | 0xffffc000ull;
+        require(runExtrqStub(distinct, value, control) == extrqReference(value, control), "EXTRQ register form stub computed the wrong field");
+        require(runExtrqStub(same, control, control) == extrqReference(control, control), "EXTRQ register form stub with equal operands computed the wrong field");
+    }
+}
+#else
+void registerFormExecution() {}
+#endif
+
 int main() {
     try {
         decoderLengths();
         sse4aOperands();
         matcherSubstitutions();
         goldenBodies();
+        registerFormExecution();
         converterSegment();
         converterFailureOffsets();
         linuxPlacement();

docs/TechnicalDebt.md

@@ -34,7 +34,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [Shader recompilation](../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../core/shader/recompiler) was written completely independently from [libs/prx](../core/libs/prx).
 - The executable file that [relinker](../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability.
 - [Relinker](../core/relinker/elfpatcher/src) doesn't add an icon to the generated executable. This should be done without adding dependencies (only standard).
-- `--to-intel` does not lower the register forms of EXTRQ/INSERTQ (`0F 79`, 4 or 5 bytes, too short for a jump) nor MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [lowering](../core/relinker/codegen/src/x86/Sse4aLowering.cpp) fails the relink instead. SHA-NI is not substituted.
+- `--to-intel` does not lower the register form of INSERTQ (`F2 0F 79`) nor MONITORX/MWAITX/CLZERO/RDPRU/MCOMMIT; the [lowering](../core/relinker/codegen/src/x86/Sse4aLowering.cpp) fails the relink instead. SHA-NI is not substituted.
 - The length-changing path of the [instruction rewriter](../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust VEX/0F38/0F3A RIP-relative operands, data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length.
 - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test.
 - The libc [SSE4a trap emulation](../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it.