#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace Codegen { namespace { using namespace Amd64OnlySubstitutionTable; constexpr std::size_t kMaxInstructionLength = 15; Amd64OnlyMatch _unsupported(const Entry& entry, const std::size_t length) { return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::Unsupported, {}, {}, 0}; } const Entry& _sse4aEntry(const Sse4aOperands& operands) { if (operands.RegisterForm) return operands.Insertq ? kInsertqRegisterForm : kExtrqRegisterForm; return operands.Insertq ? kInsertq : kExtrq; } const Entry& _reciprocalEntry(const ReciprocalOperands& operands) { return operands.Operation == ReciprocalOperation::ReciprocalSquareRoot ? kVrsqrtps : kVrcpps; } const Entry& _sha256Entry(const Sha256Operands& operands) { switch (operands.Operation) { case Sha256Operation::Rnds2: return kSha256rnds2; case Sha256Operation::Msg1: return kSha256msg1; case Sha256Operation::Msg2: return kSha256msg2; } return kSha256rnds2; } const Entry& _sha1Entry(const Sha1Operands& operands) { switch (operands.Operation) { case Sha1Operation::Rnds4: return kSha1rnds4; case Sha1Operation::Nexte: return kSha1nexte; case Sha1Operation::Msg1: return kSha1msg1; case Sha1Operation::Msg2: return kSha1msg2; } return kSha1rnds4; } Amd64OnlyMatch _inPlace(const Entry& entry, const std::size_t length, std::vector replacement) { while (replacement.size() < length) { const auto& nop = kNops[std::min(length - replacement.size(), std::size(kNops)) - 1]; replacement.insert(replacement.end(), nop.Bytes, nop.Bytes + nop.Size); } return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } bool _isClzeroOpcode(const DecodedInstruction& instr) { const auto pos = instr.OpcodeOffset(); return pos + 2 < instr.Length && instr.Data[pos] == X64OpcodeConstants::TwoByteOpcodeEscape && instr.Data[pos + 1] == X64OpcodeConstants::TwoByteGrp7 && instr.Data[pos + 2] == 0xFC; } bool _validWait(const DecodedInstruction& instr) { const auto opcode = instr.Data + instr.OpcodeOffset(); return instr.Length <= kMaxInstructionLength && std::find(instr.Data, opcode, X64OpcodeConstants::PrefixLock) == opcode; } class Amd64OnlyInstructionMatcher : public IAmd64OnlyInstructionMatcher { public: [[nodiscard]] std::optional Match( const std::uint8_t* data, std::size_t length, std::span trailing = {} ) const override; [[nodiscard]] std::optional MatchSequence( std::span> instructions, std::span trailing ) const override; private: Sse4aLowering _lowering; Sha256Lowering _sha256Lowering; Sha1Lowering _sha1Lowering; ClzeroLowering _clzeroLowering; ReciprocalLowering _reciprocalLowering; [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const; [[nodiscard]] Amd64OnlyMatch _matchSha256(const DecodedInstruction& instr, std::span trailing) const; [[nodiscard]] Amd64OnlyMatch _matchSha1(const DecodedInstruction& instr, std::span trailing) const; [[nodiscard]] Amd64OnlyMatch _matchClzero(const DecodedInstruction& instr, std::span trailing) const; }; Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const { using namespace X64OpcodeConstants; const auto opcodeOffset = instr.OpcodeOffset(); if (opcodeOffset + 2 >= instr.Length) throw CodegenException("MOVNTSS/MOVNTSD truncated before its ModRM byte"); const auto modrm = instr.Data[opcodeOffset + 2]; if (((modrm >> ModRmModShift) & ModRmModMask) == ModRmModRegister) throw CodegenException("MOVNTSS/MOVNTSD with a register operand"); std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[opcodeOffset + 1] = kMovsStoreOpcode; return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const { const auto operands = DecodeSse4a(instr.Data, instr.Length); if (!operands.RegisterForm && trailing.empty()) { if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; } auto body = _lowering.LowerOutOfLine(operands, trailing); const auto& name = operands.RegisterForm ? registerFormEntry.Name : entry.Name; return Amd64OnlyMatch{name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSha256(const DecodedInstruction& instr, std::span trailing) const { const auto operands = DecodeSha256(instr.Data, instr.Length); auto body = _sha256Lowering.LowerOutOfLine(operands, trailing); return Amd64OnlyMatch{_sha256Entry(operands).Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSha1(const DecodedInstruction& instr, std::span trailing) const { const auto operands = DecodeSha1(instr.Data, instr.Length); auto body = _sha1Lowering.LowerOutOfLine(operands, trailing); return Amd64OnlyMatch{_sha1Entry(operands).Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchClzero(const DecodedInstruction& instr, std::span trailing) const { auto body = _clzeroLowering.LowerOutOfLine(DecodeClzero(instr.Data, instr.Length), trailing); return Amd64OnlyMatch{kClzero.Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } std::optional Amd64OnlyInstructionMatcher::MatchSequence( std::span> instructions, std::span trailing ) const { if (instructions.empty()) return std::nullopt; StubBodyBuilder body; const char* name = nullptr; for (const auto& instruction : instructions) { const DecodedInstruction instr{instruction.data(), instruction.size()}; if (instr.IsExtrq() || instr.IsInsertq()) { const auto operands = DecodeSse4a(instr.Data, instr.Length); if (name == nullptr) name = _sse4aEntry(operands).Name; _lowering.EmitOutOfLine(body, operands); } else if (instr.IsSha256()) { const auto operands = DecodeSha256(instr.Data, instr.Length); if (name == nullptr) name = _sha256Entry(operands).Name; _sha256Lowering.EmitOutOfLine(body, operands); } else if (instr.IsSha1()) { const auto operands = DecodeSha1(instr.Data, instr.Length); if (name == nullptr) name = _sha1Entry(operands).Name; _sha1Lowering.EmitOutOfLine(body, operands); } else if (instr.IsClzero()) { const auto operands = DecodeClzero(instr.Data, instr.Length); if (name == nullptr) name = kClzero.Name; _clzeroLowering.EmitOutOfLine(body, operands); } else if (const auto reciprocal = DecodeVexReciprocal(instr.Data, instr.Length)) { if (name == nullptr) name = _reciprocalEntry(*reciprocal).Name; _reciprocalLowering.EmitOutOfLine(body, *reciprocal); } else { return std::nullopt; } } body.Raw(trailing); auto lowered = body.Finish(); const bool optional = DecodeVexReciprocal(instructions.front().data(), instructions.front().size()).has_value(); return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(lowered.Bytes), lowered.ReturnBranchOffset, optional}; } std::optional Amd64OnlyInstructionMatcher::Match( const std::uint8_t* data, std::size_t length, std::span trailing ) const { const DecodedInstruction instr{data, length}; if (instr.IsMovntss()) return _matchMovnts(instr, kMovntss); if (instr.IsMovntsd()) return _matchMovnts(instr, kMovntsd); if (instr.IsExtrq()) return _matchSse4a(instr, kExtrq, kExtrqRegisterForm, trailing); if (instr.IsInsertq()) return _matchSse4a(instr, kInsertq, kInsertqRegisterForm, trailing); if (instr.IsSha256()) return _matchSha256(instr, trailing); if (instr.IsSha1()) return _matchSha1(instr, trailing); if (instr.IsMonitorx()) return _validWait(instr) ? _inPlace(kMonitorx, length, {}) : _unsupported(kMonitorx, length); if (instr.IsMwaitx()) return _validWait(instr) ? _inPlace(kMwaitx, length, std::vector(kPause.Bytes, kPause.Bytes + kPause.Size)) : _unsupported(kMwaitx, length); if (const auto reciprocal = DecodeVexReciprocal(data, length)) { auto body = _reciprocalLowering.LowerOutOfLine(*reciprocal, trailing); return Amd64OnlyMatch{_reciprocalEntry(*reciprocal).Name, length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, true}; } if (instr.IsClzero()) return _matchClzero(instr, trailing); if (_isClzeroOpcode(instr)) return _unsupported(kClzero, length); if (instr.IsRdpru()) return _unsupported(kRdpru, length); if (instr.IsMcommit()) return _unsupported(kMcommit, length); return std::nullopt; } } std::unique_ptr MakeAmd64OnlyInstructionMatcher() { return std::make_unique(); } }