shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(code): support vex prefix and imul decoding

This commit is contained in:
boykopovar 2026-08-04 23:16:33 +03:00
parent aeb9adfb33
commit ab78ae90e2
2 changed files with 62 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
core/codegen/include/codegen/x86/X64OpcodeConstants.hpp
View File

@ -59,6 +59,8 @@ inline constexpr std::uint8_t OneByteIncDecRm8 = 0xFE;
inline constexpr std::uint8_t OneByteGrp5Rm = 0xFF;
inline constexpr std::uint8_t OneByteModRmRangeJMin = 0x88;
inline constexpr std::uint8_t OneByteModRmRangeJMax = 0x8B;
inline constexpr std::uint8_t OneByteImulRm32Imm32 = 0x69;
inline constexpr std::uint8_t OneByteImulRm32Imm8 = 0x6B;
inline constexpr std::uint8_t OneByteImm8AluOrGrp3 = 0xC6;
inline constexpr std::uint8_t OneByteImm8AluCmp = 0x80;
@ -122,6 +124,7 @@ inline constexpr std::uint8_t TwoByteModRmRangeGMax = 0x77;
inline constexpr std::uint8_t TwoByteModRmRangeHMin = 0xD0;
inline constexpr std::uint8_t TwoByteModRmRangeHMax = 0xFE;
inline constexpr std::uint8_t TwoByteNopModRm = 0x1F;
inline constexpr std::uint8_t TwoByteEndbr = 0x1E;
inline constexpr std::size_t ImmSizeNone = 0;
inline constexpr std::size_t ImmSize8 = 1;
@ -131,6 +134,8 @@ inline constexpr std::size_t ImmSize64 = 8;
inline constexpr std::uint8_t ModRmModShift = 6;
inline constexpr std::uint8_t ModRmModMask = 0x03;
inline constexpr std::uint8_t ModRmRegShift = 3;
inline constexpr std::uint8_t ModRmRegMask = 0x07;
inline constexpr std::uint8_t ModRmRmMask = 0x07;
inline constexpr std::uint8_t ModRmModIndirect = 0x00;
inline constexpr std::uint8_t ModRmModDisp8 = 0x01;
@ -139,6 +144,7 @@ inline constexpr std::uint8_t ModRmModRegister = 0x03;
inline constexpr std::uint8_t ModRmRmSibPresent = 0x04;
inline constexpr std::uint8_t ModRmRmRipRelative = 0x05;
inline constexpr std::uint8_t Grp3RegTestMax = 0x01;
inline constexpr std::uint8_t SibBaseMask = 0x07;
inline constexpr std::uint8_t SibBaseDisp32 = 0x05;
@ -147,6 +153,11 @@ inline constexpr std::size_t Disp32Size = 4;
inline constexpr std::size_t Rel32InstructionLength = 5;
inline constexpr std::size_t Vex2PrefixLength = 2;
inline constexpr std::size_t Vex3PrefixLength = 3;
inline constexpr std::uint8_t Vex3MapMask = 0x1F;
inline constexpr std::uint8_t Vex3Map0F3A = 0x03;
}
#endif

57
core/codegen/src/x86/X64InstructionDecoder.cpp
View File

@ -49,8 +49,28 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
std::uint8_t opcode = data[pos];
pos += 1;
bool twoByteOpcode = false;
bool vexPresent = false;
std::uint8_t vexMap = 0;
if (opcode == TwoByteOpcodeEscape) {
if (opcode == OneByteVex2) {
if (pos >= available) {
throw CodegenException("Instruction truncated after VEX2 prefix");
}
vexPresent = true;
vexMap = TwoByteOpcodeEscape;
pos += 1;
} else if (opcode == OneByteVex3) {
if (pos >= available) {
throw CodegenException("Instruction truncated after VEX3 prefix");
}
vexPresent = true;
vexMap = static_cast<std::uint8_t>(data[pos] & Vex3MapMask);
pos += 1;
if (pos >= available) {
throw CodegenException("Instruction truncated after VEX3 prefix");
}
pos += 1;
} else if (opcode == TwoByteOpcodeEscape) {
if (pos >= available) {
throw CodegenException("Instruction truncated after two-byte opcode escape");
}
@ -59,10 +79,23 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
pos += 1;
}
if (vexPresent) {
if (pos >= available) {
throw CodegenException("Instruction truncated after VEX prefix");
}
opcode = data[pos];
pos += 1;
}
bool hasModRm = false;
std::size_t immediateSize = ImmSizeNone;
if (!twoByteOpcode) {
if (vexPresent) {
hasModRm = true;
if (vexMap == Vex3Map0F3A) {
immediateSize = ImmSize8;
}
} else if (!twoByteOpcode) {
if ((opcode >= OneByteModRmRangeAMin && opcode <= OneByteModRmRangeAMax) ||
(opcode >= OneByteModRmRangeBMin && opcode <= OneByteModRmRangeBMax) ||
(opcode >= OneByteModRmRangeCMin && opcode <= OneByteModRmRangeCMax) ||
@ -74,8 +107,10 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
opcode == OneByteArpl || opcode == OneByteMovsxd ||
(opcode >= OneByteModRmRangeIMin && opcode <= OneByteModRmRangeIMax) ||
opcode == OneByteGrp2Rm8Imm8 || opcode == OneByteGrp2RmImm8 ||
opcode == OneByteVex3 || opcode == OneByteVex2 ||
opcode == OneByteMovImm8 || opcode == OneByteMovImm32 ||
opcode == OneByteImm32AluCmp || opcode == OneByteImm8Grp1 ||
opcode == OneByteImm8AluCmp ||
opcode == OneByteImulRm32Imm32 || opcode == OneByteImulRm32Imm8 ||
opcode == OneByteShiftRm8By1 || opcode == OneByteShiftRmBy1 ||
opcode == OneByteShiftRm8ByCl || opcode == OneByteShiftRmByCl ||
opcode == OneByteTestGrp3Rm8 || opcode == OneByteTestGrp3Rm ||
@ -93,7 +128,8 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
opcode == OneBytePushImm8 ||
opcode == OneByteTestAlImm8) {
immediateSize = ImmSize8;
} else if (opcode == OneByteImm32AluCmp ||
} else if (opcode == OneByteImm32AluCmp || opcode == OneByteMovImm32 ||
opcode == OneByteImulRm32Imm32 ||
opcode == OneByteImm32AddEax || opcode == OneByteImm32OrEax ||
opcode == OneByteImm32AdcEax || opcode == OneByteImm32SbbEax ||
opcode == OneByteImm32AndEax || opcode == OneByteImm32SubEax ||
@ -107,7 +143,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
} else {
immediateSize = operandSizeOverride ? ImmSize16 : ImmSize32;
}
} else if (opcode == OneByteImm8Grp1) {
} else if (opcode == OneByteImm8Grp1 || opcode == OneByteImulRm32Imm8) {
immediateSize = ImmSize8;
}
@ -139,7 +175,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
(opcode >= TwoByteModRmRangeFMin && opcode <= TwoByteModRmRangeFMax) ||
(opcode >= TwoByteModRmRangeGMin && opcode <= TwoByteModRmRangeGMax) ||
(opcode >= TwoByteModRmRangeHMin && opcode <= TwoByteModRmRangeHMax) ||
opcode == TwoByteNopModRm) {
opcode == TwoByteNopModRm || opcode == TwoByteEndbr) {
hasModRm = true;
}
}
@ -159,8 +195,17 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t
pos += 1;
const std::uint8_t mod = static_cast<std::uint8_t>((modrm >> ModRmModShift) & ModRmModMask);
const std::uint8_t reg = static_cast<std::uint8_t>((modrm >> ModRmRegShift) & ModRmRegMask);
const std::uint8_t rm = static_cast<std::uint8_t>(modrm & ModRmRmMask);
if (!vexPresent && !twoByteOpcode &&
(opcode == OneByteTestGrp3Rm8 || opcode == OneByteTestGrp3Rm) &&
reg <= Grp3RegTestMax) {
immediateSize = (opcode == OneByteTestGrp3Rm8)
? ImmSize8
: (operandSizeOverride ? ImmSize16 : ImmSize32);
}
if (mod != ModRmModRegister && rm == ModRmRmSibPresent) {
if (pos >= available) {
throw CodegenException("Instruction truncated before SIB byte");