shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(relinker): decode XOP instructions 

The x86 decoder had no XOP (AMD 8F escape) support and read 8F E8.. as
POP r/m, losing instruction boundaries. Since #2238 roots code addresses
taken by RIP-relative LEA, code analysis reaches CPU-dispatched XOP
paths, such as the vpperm routines in Stremio Plus (PPSA74126), OpenNOW
(PPSA99082) and PPSA99420, and their Windows conversion failed with
"Code analysis: overlapping instruction boundaries". Decode 8F followed
by a map of 8 or more as a three-byte XOP prefix (8F can only be POP r/m
with ModRM.reg 0) with a ModRM operand, an 8-bit immediate in map 8 and
a 32-bit immediate in map 0Ah, in both the length and operand decoders.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Achraf Reyani 2026-10-10 14:25:24 +09:00 committed by Dean Galvin
parent 23ed501f4a
commit ce8dbc977f
3 changed files with 30 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp
View File

@ -220,6 +220,10 @@ inline constexpr std::size_t Vex2PrefixLength = 2;
inline constexpr std::size_t Vex3PrefixLength = 3;
inline constexpr std::uint8_t Vex3MapMask = 0x1F;
inline constexpr std::uint8_t Vex3Map0F3A = 0x03;
inline constexpr std::uint8_t XopPrefix = 0x8F;
inline constexpr std::uint8_t XopMapMin = 0x08;
inline constexpr std::uint8_t XopMap8 = 0x08;
inline constexpr std::uint8_t XopMapA = 0x0A;
}

23
core/relinker/codegen/src/x86/X64InstructionDecoder.cpp
View File

@ -55,6 +55,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
bool vexPresent = false;
bool evexPresent = false;
std::uint8_t vexMap = 0;
std::uint8_t xopMap = 0;
std::uint8_t threeByteMap = 0;
bool threeByteEscape = false;
@ -76,6 +77,12 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
throw CodegenException("Instruction truncated after VEX3 prefix");
}
pos += 1;
} else if (opcode == XopPrefix && pos < available && (data[pos] & Vex3MapMask) >= XopMapMin) {
if (pos + 1 >= available) {
throw CodegenException("Instruction truncated after XOP prefix");
}
xopMap = static_cast<std::uint8_t>(data[pos] & Vex3MapMask);
pos += 2;
} else if (opcode == EvexPrefix) {
if (pos + 2 >= available) {
throw CodegenException("Instruction truncated after EVEX prefix");
@ -101,7 +108,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
}
}
if (vexPresent || evexPresent) {
if (vexPresent || evexPresent || xopMap != 0) {
if (pos >= available) {
throw CodegenException("Instruction truncated after VEX prefix");
}
@ -114,7 +121,14 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
const bool vectorImmediate = vexMap == 1 && ((opcode >= 0x70 && opcode <= 0x73) || opcode == 0xC2 || opcode == 0xC4 || opcode == 0xC5 || opcode == 0xC6);
if (vexPresent) {
if (xopMap != 0) {
hasModRm = true;
if (xopMap == XopMap8) {
immediateSize = ImmSize8;
} else if (xopMap == XopMapA) {
immediateSize = ImmSize32;
}
} else if (vexPresent) {
if (vexMap == 1 && opcode >= VexNoModRmMin && opcode <= VexNoModRmMax) {
hasModRm = false;
} else {
@ -379,8 +393,9 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction(
std::uint8_t op = data[pos++];
bool twoByteOpcode = false;
if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix) {
pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 ? 2 : EvexPrefixLength - 1)) + 1;
const bool xop = op == XopPrefix && pos < info.Length && (data[pos] & Vex3MapMask) >= XopMapMin;
if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix || xop) {
pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 || xop ? 2 : EvexPrefixLength - 1)) + 1;
if (pos < info.Length)
readModRm(pos, false);
info.FlowKind = ControlFlowKind::Sequential;

10
core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp
View File

@ -135,7 +135,10 @@ void decoderLengths() {
{0x66, 0x48, 0x69, 0xC0, 1, 2, 3, 4}, {0x66, 0x48, 0x68, 1, 2, 3, 4}, {0x66, 0x48, 0xA9, 1, 2, 3, 4},
{0x66, 0x48, 0xF7, 0xC0, 1, 2, 3, 4}, {0x66, 0x81, 0xC0, 0x34, 0x12}, {0x48, 0x66, 0x81, 0xC0, 0x34, 0x12},
{0x66, 0x41, 0x81, 0xC0, 0x34, 0x12}, {0x66, 0xF7, 0xC0, 0x34, 0x12},
{0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC}};
{0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC},
{0x8F, 0xE8, 0x40, 0xA3, 0xC1, 0x70}, {0x8F, 0xE8, 0xF8, 0xA3, 0x4A, 0x10, 0x60}, {0x8F, 0xE8, 0xC0, 0xA3, 0xBA, 0x70, 0xFF, 0xFF, 0xFF, 0x30},
{0x8F, 0xE9, 0x78, 0x90, 0xC1}, {0x8F, 0xEA, 0x78, 0x10, 0xC0, 1, 2, 3, 4},
{0x8F, 0xC0}, {0x41, 0x8F, 0x07}, {0x8F, 0x05, 1, 2, 3, 4}};
Bytes padded;
for (const auto& instruction : instructions) {
padded = instruction;
@ -156,7 +159,8 @@ const std::vector<Bytes> kRipRelativeVectorLoads = {
{0xC4, 0xE2, 0x79, 0x00, 0x05, 0x10, 0x00, 0x00, 0x00},
{0x62, 0xF1, 0xFD, 0x08, 0x6F, 0x05, 0x10, 0x00, 0x00, 0x00},
{0x66, 0x0F, 0x38, 0x00, 0x05, 0x10, 0x00, 0x00, 0x00},
{0x66, 0x0F, 0x3A, 0x0F, 0x05, 0x10, 0x00, 0x00, 0x00, 0x08}};
{0x66, 0x0F, 0x3A, 0x0F, 0x05, 0x10, 0x00, 0x00, 0x00, 0x08},
{0x8F, 0xE8, 0x78, 0xA3, 0x05, 0x10, 0x00, 0x00, 0x00, 0x30}};
const std::vector<Bytes> kRegisterVectorOperations = {
{0xC5, 0xF9, 0x6F, 0xC1}, {0x66, 0x0F, 0x38, 0x00, 0xC1}, {0xC5, 0xF8, 0x77}, {0xC4, 0xE2, 0x79, 0x00, 0x00}};
@ -164,7 +168,7 @@ void decoderRipRelative() {
const Codegen::X64InstructionDecoder decoder;
for (const auto& instruction : kRipRelativeVectorLoads) {
const auto info = decoder.DecodeInstruction(instruction.data(), instruction.size());
require(info.Length == instruction.size() && info.HasRipRelativeDisp && read<std::int32_t>(instruction, info.RipRelativeDispOffset) == 0x10, "VEX, EVEX or three-byte RIP-relative operand was not reported");
require(info.Length == instruction.size() && info.HasRipRelativeDisp && read<std::int32_t>(instruction, info.RipRelativeDispOffset) == 0x10, "VEX, EVEX, XOP or three-byte RIP-relative operand was not reported");
}
for (const auto& instruction : kRegisterVectorOperations)
require(!decoder.DecodeInstruction(instruction.data(), instruction.size()).HasRipRelativeDisp, "Vector instruction without a RIP-relative operand was reported as RIP-relative");