#include #include #include #include #include namespace Codegen { namespace { using namespace Amd64OnlySubstitutionTable; constexpr std::uint8_t kPrefixPacked = 0x66; constexpr std::uint8_t kPrefixScalar = 0xF3; constexpr std::uint8_t kRexBase = 0x40; constexpr std::uint8_t kRexR = 0x04; constexpr std::uint8_t kRexX = 0x02; constexpr std::uint8_t kRexB = 0x01; constexpr std::uint8_t kModRmRegister = 0xC0; constexpr std::uint8_t kModRmRip = 0x05; constexpr std::uint8_t kModRmRspBase = 0x04; constexpr std::uint8_t kSibRsp = 0x24; constexpr std::uint8_t kShiftDwords = 0x72; constexpr std::uint8_t kShiftQwords = 0x73; constexpr std::uint8_t kMovdqu = 0x6F; std::uint8_t _rex(const std::uint8_t reg, const std::uint8_t rm) { return static_cast(kRexBase | ((reg & 8) != 0 ? kRexR : 0) | ((rm & 8) != 0 ? kRexB : 0)); } void _emit(std::vector& out, const std::uint8_t prefix, const std::uint8_t reg, const std::uint8_t rm, const std::initializer_list opcode, const std::uint8_t modrm) { out.push_back(prefix); const auto rex = _rex(reg, rm); if (rex != kRexBase) out.push_back(rex); out.insert(out.end(), opcode.begin(), opcode.end()); out.push_back(modrm); } std::size_t _displacementSize(const std::uint8_t mod, const std::uint8_t rm, const std::uint8_t sib) { using namespace X64OpcodeConstants; if (mod == ModRmModDisp8) return Disp8Size; if (mod == ModRmModDisp32 || (mod == ModRmModIndirect && rm == ModRmRmRipRelative) || (mod == ModRmModIndirect && rm == ModRmRmSibPresent && (sib & SibBaseMask) == SibBaseDisp32)) return Disp32Size; return 0; } void _shiftImm(std::vector& out, const std::uint8_t opcode, const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { _emit(out, kPrefixPacked, 0, reg, {0x0F, opcode}, static_cast(kModRmRegister | (extension << 3) | (reg & 7))); out.push_back(imm); } } MemoryOperand DecodeMemoryOperand(const std::uint8_t* data, const std::size_t length, const std::size_t modRmOffset, const std::uint8_t rex, std::vector prefixes) { using namespace X64OpcodeConstants; if (modRmOffset >= length) throw CodegenException("Memory operand truncated before its ModRM byte"); const auto modrm = data[modRmOffset]; MemoryOperand operand{std::move(prefixes), static_cast(rex & (kRexX | kRexB)), static_cast((modrm >> ModRmModShift) & ModRmModMask), static_cast(modrm & ModRmRmMask), 0, 0, false, 0, false, length}; if (operand.Mod == ModRmModRegister) throw CodegenException("Register operand where a memory operand was expected"); operand.RipRelative = operand.Mod == ModRmModIndirect && operand.Rm == ModRmRmRipRelative; auto pos = modRmOffset + 1; if (operand.Rm == ModRmRmSibPresent) { if (pos >= length) throw CodegenException("Memory operand truncated before its SIB byte"); operand.Sib = data[pos++]; operand.StackBase = (operand.Sib & SibBaseMask) == ModRmRmSibPresent && (rex & kRexB) == 0; } const auto size = _displacementSize(operand.Mod, operand.Rm, operand.Sib); if (pos + size > length) throw CodegenException("Memory operand truncated in its displacement"); operand.EncodedSize = pos + size - modRmOffset; if (size == Disp8Size) operand.Displacement = static_cast(data[pos]); for (std::size_t index = 0; size == Disp32Size && index < size; ++index) operand.Displacement = static_cast(static_cast(operand.Displacement) | (static_cast(data[pos + index]) << (index * 8))); return operand; } void ApplyStubRelocations(const std::span body, const std::span relocations, const std::uint64_t siteAddress, const std::uint64_t stubAddress, const Domain::FileByteOffset failureOffset) { using namespace X64OpcodeConstants; for (const auto& relocation : relocations) { if (relocation.InstructionEnd > body.size() || relocation.DisplacementOffset > relocation.InstructionEnd || relocation.InstructionEnd - relocation.DisplacementOffset < Disp32Size) throw CodegenException("Stub relocation lies outside the stub body", failureOffset); const auto target = siteAddress + static_cast(relocation.SiteTarget); const auto displacement = static_cast(target - (stubAddress + relocation.InstructionEnd)); if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) throw CodegenException("RIP-relative operand is out of rel32 range from its stub", failureOffset); const auto value = static_cast(static_cast(displacement)); for (std::size_t index = 0; index < Disp32Size; ++index) body[relocation.DisplacementOffset + index] = static_cast(value >> (index * 8)); } } void EmitSse(std::vector& out, const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { _emit(out, prefix, dst, src, opcode, static_cast(kModRmRegister | ((dst & 7) << 3) | (src & 7))); } void EmitShiftImm(std::vector& out, const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { _shiftImm(out, kShiftQwords, extension, reg, imm); } void StubBodyBuilder::Sse(const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { EmitSse(_bytes, prefix, opcode, dst, src); } void StubBodyBuilder::SsePlain(const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { const auto rex = _rex(dst, src); if (rex != kRexBase) _bytes.push_back(rex); _bytes.insert(_bytes.end(), opcode.begin(), opcode.end()); _bytes.push_back(static_cast(kModRmRegister | ((dst & 7) << 3) | (src & 7))); } void StubBodyBuilder::SseImm(const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src, const std::uint8_t imm) { EmitSse(_bytes, prefix, opcode, dst, src); _bytes.push_back(imm); } void StubBodyBuilder::ShiftImm(const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { _shiftImm(_bytes, kShiftQwords, extension, reg, imm); } void StubBodyBuilder::ShiftDwordImm(const std::uint8_t extension, const std::uint8_t reg, const std::uint8_t imm) { _shiftImm(_bytes, kShiftDwords, extension, reg, imm); } void StubBodyBuilder::RipOperand(const std::initializer_list opcode, const std::uint8_t reg, const StubConstant& constant) { _emit(_bytes, kPrefixPacked, reg, 0, opcode, static_cast(((reg & 7) << 3) | kModRmRip)); _fixups.push_back({_bytes.size(), _bytes.size() + 4, _constants.size()}); _bytes.insert(_bytes.end(), 4, 0); _constants.push_back(constant); } void StubBodyBuilder::Load(const std::uint8_t reg, const MemoryOperand& operand) { using namespace X64OpcodeConstants; const auto displacement = static_cast(operand.Displacement) + static_cast(operand.StackBase ? _stackDepth : 0); if (displacement > std::numeric_limits::max()) throw CodegenException("Stack displacement does not fit after spilling"); const auto mod = operand.StackBase && _stackDepth != 0 ? ModRmModDisp32 : operand.Mod; _bytes.insert(_bytes.end(), operand.Prefixes.begin(), operand.Prefixes.end()); _bytes.push_back(kPrefixScalar); const auto rex = static_cast(kRexBase | ((reg & 8) != 0 ? kRexR : 0) | operand.RexIndexBase); if (rex != kRexBase) _bytes.push_back(rex); _bytes.insert(_bytes.end(), {TwoByteOpcodeEscape, kMovdqu, static_cast((mod << ModRmModShift) | ((reg & 7) << ModRmRegShift) | operand.Rm)}); if (operand.Rm == ModRmRmSibPresent) _bytes.push_back(operand.Sib); if (operand.RipRelative) { _relocations.push_back({_bytes.size(), _bytes.size() + Disp32Size, static_cast(_siteOffset + operand.InstructionLength) + operand.Displacement}); _bytes.insert(_bytes.end(), Disp32Size, 0); return; } const auto value = static_cast(static_cast(displacement)); for (std::size_t index = 0; index < _displacementSize(mod, operand.Rm, operand.Sib); ++index) _bytes.push_back(static_cast(value >> (index * 8))); } void StubBodyBuilder::Spill(const std::uint8_t reg) { _stackDepth += kRedZoneSpillFrame; _bytes.insert(_bytes.end(), kLeaRspBelowRedZone.Bytes, kLeaRspBelowRedZone.Bytes + kLeaRspBelowRedZone.Size); _emit(_bytes, kPrefixScalar, reg, 0, {0x0F, 0x7F}, static_cast(((reg & 7) << 3) | kModRmRspBase)); _bytes.push_back(kSibRsp); } void StubBodyBuilder::Restore(const std::uint8_t reg) { _emit(_bytes, kPrefixScalar, reg, 0, {0x0F, 0x6F}, static_cast(((reg & 7) << 3) | kModRmRspBase)); _bytes.push_back(kSibRsp); _bytes.insert(_bytes.end(), kLeaRspRestore.Bytes, kLeaRspRestore.Bytes + kLeaRspRestore.Size); _stackDepth -= kRedZoneSpillFrame; } void StubBodyBuilder::Raw(std::span bytes) { _bytes.insert(_bytes.end(), bytes.begin(), bytes.end()); } void StubBodyBuilder::Move(const std::span instruction, const std::optional ripDisplacementOffset) { using namespace X64OpcodeConstants; const auto start = _bytes.size(); Raw(instruction); if (ripDisplacementOffset) { if (*ripDisplacementOffset > instruction.size() || instruction.size() - *ripDisplacementOffset < Disp32Size) throw CodegenException("RIP-relative displacement lies outside its instruction"); std::uint32_t value = 0; for (std::size_t index = 0; index < Disp32Size; ++index) { value |= static_cast(instruction[*ripDisplacementOffset + index]) << (index * 8); _bytes[start + *ripDisplacementOffset + index] = 0; } _relocations.push_back({start + *ripDisplacementOffset, start + instruction.size(), static_cast(_siteOffset + instruction.size()) + static_cast(value)}); } Advance(instruction.size()); } void StubBodyBuilder::Advance(const std::size_t length) { _siteOffset += length; } LoweredBody StubBodyBuilder::Finish() { const auto returnBranchOffset = _bytes.size(); _bytes.insert(_bytes.end(), kJmpRel32.Bytes, kJmpRel32.Bytes + kJmpRel32.Size); std::vector constantOffsets; for (const auto& constant : _constants) { while (_bytes.size() % kStubAlignment != 0) _bytes.push_back(kTrapFill); constantOffsets.push_back(_bytes.size()); _bytes.insert(_bytes.end(), constant.begin(), constant.end()); } for (const auto& fixup : _fixups) { const auto displacement = static_cast(constantOffsets[fixup.ConstantIndex]) - static_cast(fixup.InstructionEnd); const auto value = static_cast(static_cast(displacement)); for (std::size_t index = 0; index < 4; ++index) _bytes[fixup.DisplacementOffset + index] = static_cast(value >> (index * 8)); } return {std::move(_bytes), returnBranchOffset, std::move(_relocations)}; } }