#include #include #include #include #include #include #include #include #include #include namespace Codegen { namespace { using namespace Amd64OnlySubstitutionTable; Amd64OnlyMatch _unsupported(const Entry& entry, const std::size_t length) { return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::Unsupported, {}, {}, 0}; } class Amd64OnlyInstructionMatcher : public IAmd64OnlyInstructionMatcher { public: [[nodiscard]] std::optional Match( const std::uint8_t* data, std::size_t length, std::span trailing = {} ) const override; [[nodiscard]] std::optional MatchSequence( std::span> instructions, std::span trailing ) const override; private: Sse4aLowering _lowering; [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const; }; Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const { using namespace X64OpcodeConstants; const auto opcodeOffset = instr.OpcodeOffset(); if (opcodeOffset + 2 >= instr.Length) throw CodegenException("MOVNTSS/MOVNTSD truncated before its ModRM byte"); const auto modrm = instr.Data[opcodeOffset + 2]; if (((modrm >> ModRmModShift) & ModRmModMask) == ModRmModRegister) throw CodegenException("MOVNTSS/MOVNTSD with a register operand"); std::vector replacement(instr.Data, instr.Data + instr.Length); replacement[opcodeOffset + 1] = kMovsStoreOpcode; return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const { const auto operands = DecodeSse4a(instr.Data, instr.Length); if (!operands.RegisterForm && trailing.empty()) { if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; } auto body = _lowering.LowerOutOfLine(operands, trailing); const auto& name = operands.RegisterForm ? registerFormEntry.Name : entry.Name; return Amd64OnlyMatch{name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } std::optional Amd64OnlyInstructionMatcher::MatchSequence( std::span> instructions, std::span trailing ) const { if (instructions.empty()) return std::nullopt; std::vector sequence; for (const auto& instruction : instructions) { const DecodedInstruction instr{instruction.data(), instruction.size()}; if (!instr.IsExtrq() && !instr.IsInsertq()) return std::nullopt; sequence.push_back(DecodeSse4a(instr.Data, instr.Length)); } const auto& first = sequence.front(); const auto& name = first.RegisterForm ? (first.Insertq ? kInsertqRegisterForm.Name : kExtrqRegisterForm.Name) : (first.Insertq ? kInsertq.Name : kExtrq.Name); auto body = _lowering.LowerOutOfLine(std::span(sequence), trailing); return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; } std::optional Amd64OnlyInstructionMatcher::Match( const std::uint8_t* data, std::size_t length, std::span trailing ) const { const DecodedInstruction instr{data, length}; if (instr.IsMovntss()) return _matchMovnts(instr, kMovntss); if (instr.IsMovntsd()) return _matchMovnts(instr, kMovntsd); if (instr.IsExtrq()) return _matchSse4a(instr, kExtrq, kExtrqRegisterForm, trailing); if (instr.IsInsertq()) return _matchSse4a(instr, kInsertq, kInsertqRegisterForm, trailing); if (instr.IsMonitorx()) return _unsupported(kMonitorx, length); if (instr.IsMwaitx()) return _unsupported(kMwaitx, length); if (instr.IsClzero()) return _unsupported(kClzero, length); if (instr.IsRdpru()) return _unsupported(kRdpru, length); if (instr.IsMcommit()) return _unsupported(kMcommit, length); return std::nullopt; } } std::unique_ptr MakeAmd64OnlyInstructionMatcher() { return std::make_unique(); } }