From ce8dbc977fbe01aa35e58ca2efcc4b5b4ef4dc24 Mon Sep 17 00:00:00 2001 From: Achraf Reyani Date: Sat, 10 Oct 2026 14:25:24 +0900 Subject: [PATCH] fix(relinker): decode XOP instructions The x86 decoder had no XOP (AMD 8F escape) support and read 8F E8.. as POP r/m, losing instruction boundaries. Since #2238 roots code addresses taken by RIP-relative LEA, code analysis reaches CPU-dispatched XOP paths, such as the vpperm routines in Stremio Plus (PPSA74126), OpenNOW (PPSA99082) and PPSA99420, and their Windows conversion failed with "Code analysis: overlapping instruction boundaries". Decode 8F followed by a map of 8 or more as a three-byte XOP prefix (8F can only be POP r/m with ModRM.reg 0) with a ModRM operand, an 8-bit immediate in map 8 and a 32-bit immediate in map 0Ah, in both the length and operand decoders. Co-Authored-By: Claude Opus 5.5 --- .../codegen/x86/X64OpcodeConstants.hpp | 4 ++++ .../codegen/src/x86/X64InstructionDecoder.cpp | 23 +++++++++++++++---- .../codegen/tests/Amd64OnlyConverterTests.cpp | 10 +++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp index 969a1fd30..889d0c447 100644 --- a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp +++ b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp @@ -220,6 +220,10 @@ inline constexpr std::size_t Vex2PrefixLength = 2; inline constexpr std::size_t Vex3PrefixLength = 3; inline constexpr std::uint8_t Vex3MapMask = 0x1F; inline constexpr std::uint8_t Vex3Map0F3A = 0x03; +inline constexpr std::uint8_t XopPrefix = 0x8F; +inline constexpr std::uint8_t XopMapMin = 0x08; +inline constexpr std::uint8_t XopMap8 = 0x08; +inline constexpr std::uint8_t XopMapA = 0x0A; } diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index 7a0b2a001..af8eefc1e 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -55,6 +55,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t bool vexPresent = false; bool evexPresent = false; std::uint8_t vexMap = 0; + std::uint8_t xopMap = 0; std::uint8_t threeByteMap = 0; bool threeByteEscape = false; @@ -76,6 +77,12 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t throw CodegenException("Instruction truncated after VEX3 prefix"); } pos += 1; + } else if (opcode == XopPrefix && pos < available && (data[pos] & Vex3MapMask) >= XopMapMin) { + if (pos + 1 >= available) { + throw CodegenException("Instruction truncated after XOP prefix"); + } + xopMap = static_cast(data[pos] & Vex3MapMask); + pos += 2; } else if (opcode == EvexPrefix) { if (pos + 2 >= available) { throw CodegenException("Instruction truncated after EVEX prefix"); @@ -101,7 +108,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t } } - if (vexPresent || evexPresent) { + if (vexPresent || evexPresent || xopMap != 0) { if (pos >= available) { throw CodegenException("Instruction truncated after VEX prefix"); } @@ -114,7 +121,14 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t const bool vectorImmediate = vexMap == 1 && ((opcode >= 0x70 && opcode <= 0x73) || opcode == 0xC2 || opcode == 0xC4 || opcode == 0xC5 || opcode == 0xC6); - if (vexPresent) { + if (xopMap != 0) { + hasModRm = true; + if (xopMap == XopMap8) { + immediateSize = ImmSize8; + } else if (xopMap == XopMapA) { + immediateSize = ImmSize32; + } + } else if (vexPresent) { if (vexMap == 1 && opcode >= VexNoModRmMin && opcode <= VexNoModRmMax) { hasModRm = false; } else { @@ -379,8 +393,9 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( std::uint8_t op = data[pos++]; bool twoByteOpcode = false; - if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix) { - pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 ? 2 : EvexPrefixLength - 1)) + 1; + const bool xop = op == XopPrefix && pos < info.Length && (data[pos] & Vex3MapMask) >= XopMapMin; + if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix || xop) { + pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 || xop ? 2 : EvexPrefixLength - 1)) + 1; if (pos < info.Length) readModRm(pos, false); info.FlowKind = ControlFlowKind::Sequential; diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index c3e4a1da2..b123125f0 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -135,7 +135,10 @@ void decoderLengths() { {0x66, 0x48, 0x69, 0xC0, 1, 2, 3, 4}, {0x66, 0x48, 0x68, 1, 2, 3, 4}, {0x66, 0x48, 0xA9, 1, 2, 3, 4}, {0x66, 0x48, 0xF7, 0xC0, 1, 2, 3, 4}, {0x66, 0x81, 0xC0, 0x34, 0x12}, {0x48, 0x66, 0x81, 0xC0, 0x34, 0x12}, {0x66, 0x41, 0x81, 0xC0, 0x34, 0x12}, {0x66, 0xF7, 0xC0, 0x34, 0x12}, - {0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC}}; + {0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC}, + {0x8F, 0xE8, 0x40, 0xA3, 0xC1, 0x70}, {0x8F, 0xE8, 0xF8, 0xA3, 0x4A, 0x10, 0x60}, {0x8F, 0xE8, 0xC0, 0xA3, 0xBA, 0x70, 0xFF, 0xFF, 0xFF, 0x30}, + {0x8F, 0xE9, 0x78, 0x90, 0xC1}, {0x8F, 0xEA, 0x78, 0x10, 0xC0, 1, 2, 3, 4}, + {0x8F, 0xC0}, {0x41, 0x8F, 0x07}, {0x8F, 0x05, 1, 2, 3, 4}}; Bytes padded; for (const auto& instruction : instructions) { padded = instruction; @@ -156,7 +159,8 @@ const std::vector kRipRelativeVectorLoads = { {0xC4, 0xE2, 0x79, 0x00, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x62, 0xF1, 0xFD, 0x08, 0x6F, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x66, 0x0F, 0x38, 0x00, 0x05, 0x10, 0x00, 0x00, 0x00}, - {0x66, 0x0F, 0x3A, 0x0F, 0x05, 0x10, 0x00, 0x00, 0x00, 0x08}}; + {0x66, 0x0F, 0x3A, 0x0F, 0x05, 0x10, 0x00, 0x00, 0x00, 0x08}, + {0x8F, 0xE8, 0x78, 0xA3, 0x05, 0x10, 0x00, 0x00, 0x00, 0x30}}; const std::vector kRegisterVectorOperations = { {0xC5, 0xF9, 0x6F, 0xC1}, {0x66, 0x0F, 0x38, 0x00, 0xC1}, {0xC5, 0xF8, 0x77}, {0xC4, 0xE2, 0x79, 0x00, 0x00}}; @@ -164,7 +168,7 @@ void decoderRipRelative() { const Codegen::X64InstructionDecoder decoder; for (const auto& instruction : kRipRelativeVectorLoads) { const auto info = decoder.DecodeInstruction(instruction.data(), instruction.size()); - require(info.Length == instruction.size() && info.HasRipRelativeDisp && read(instruction, info.RipRelativeDispOffset) == 0x10, "VEX, EVEX or three-byte RIP-relative operand was not reported"); + require(info.Length == instruction.size() && info.HasRipRelativeDisp && read(instruction, info.RipRelativeDispOffset) == 0x10, "VEX, EVEX, XOP or three-byte RIP-relative operand was not reported"); } for (const auto& instruction : kRegisterVectorOperations) require(!decoder.DecodeInstruction(instruction.data(), instruction.size()).HasRipRelativeDisp, "Vector instruction without a RIP-relative operand was reported as RIP-relative");