diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 4e8e1a4ad..ac1ee47ee 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -53,6 +53,8 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -141,6 +143,8 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -304,6 +308,8 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp index 31118179e..7f09e114a 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp @@ -41,6 +41,7 @@ inline constexpr Entry kMcommit = {"MCOMMIT", nullptr, 0}; inline constexpr Entry kRdseed = {"RDSEED", nullptr, 0}; inline constexpr Entry kRdpid = {"RDPID", nullptr, 0}; inline constexpr Entry kClwb = {"CLWB", nullptr, 0}; +inline constexpr Entry kWideStore = {"256-bit store", nullptr, 0}; inline constexpr Entry kExtrq = {"EXTRQ", nullptr, 0}; inline constexpr Entry kInsertq = {"INSERTQ", nullptr, 0}; inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0}; diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp index bcb5df816..e2f594642 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp @@ -35,6 +35,7 @@ struct Amd64OnlyMatch { std::size_t ReturnBranchOffset; bool Optional = false; std::vector Relocations = {}; + bool RelocatableWindow = false; }; struct Amd64OnlySubstitutionReport { diff --git a/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp b/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp index df17eb175..9986e7928 100644 --- a/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp +++ b/core/relinker/codegen/include/codegen/x86/IAmd64OnlyInstructionMatcher.hpp @@ -2,6 +2,7 @@ #define CODEGEN_X86_IAMD64ONLYINSTRUCTIONMATCHER_HPP #include +#include #include #include #include @@ -23,6 +24,11 @@ public: std::span> instructions, std::span trailing ) const = 0; + + virtual void EmitOutOfLine( + StubBodyBuilder& body, + std::span instruction + ) const = 0; }; std::unique_ptr MakeAmd64OnlyInstructionMatcher(Amd64OnlyTarget target = Amd64OnlyTarget::Intel); diff --git a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp index 212eeea54..f2ff3c365 100644 --- a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp +++ b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp @@ -1,6 +1,7 @@ #ifndef CODEGEN_X86_STUBBODYBUILDER_HPP #define CODEGEN_X86_STUBBODYBUILDER_HPP +#include #include #include #include @@ -53,6 +54,10 @@ public: void Restore(std::uint8_t reg); void Raw(std::span bytes); void Move(std::span instruction, std::optional ripDisplacementOffset); + void RipRelative(std::span instruction, std::size_t displacementOffset, std::int64_t target); + void MoveControlFlow(std::span instruction, const DecodedInstructionInfo& info); + void JumpTo(std::int64_t siteTarget); + void Seek(std::int64_t siteOffset); void Advance(std::size_t length); [[nodiscard]] LoweredBody Finish(); @@ -63,12 +68,15 @@ private: std::size_t ConstantIndex; }; + void _rel32(std::initializer_list opcode, std::int64_t siteTarget); + void _pushReturnAddress(std::int64_t siteTarget); + std::vector _bytes; std::vector _fixups; std::vector _constants; std::vector _relocations; std::size_t _stackDepth = 0; - std::size_t _siteOffset = 0; + std::int64_t _siteOffset = 0; }; } diff --git a/core/relinker/codegen/include/codegen/x86/WideStoreLowering.hpp b/core/relinker/codegen/include/codegen/x86/WideStoreLowering.hpp new file mode 100644 index 000000000..7e27e2796 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/WideStoreLowering.hpp @@ -0,0 +1,16 @@ +#ifndef CODEGEN_X86_WIDESTORELOWERING_HPP +#define CODEGEN_X86_WIDESTORELOWERING_HPP + +#include +#include + +namespace Codegen { + +class WideStoreLowering { +public: + void EmitOutOfLine(StubBodyBuilder& body, const WideStoreOperands& operands) const; +}; + +} + +#endif diff --git a/core/relinker/codegen/include/codegen/x86/WideStoreOperands.hpp b/core/relinker/codegen/include/codegen/x86/WideStoreOperands.hpp new file mode 100644 index 000000000..1716af37b --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/WideStoreOperands.hpp @@ -0,0 +1,26 @@ +#ifndef CODEGEN_X86_WIDESTOREOPERANDS_HPP +#define CODEGEN_X86_WIDESTOREOPERANDS_HPP + +#include +#include +#include +#include + +namespace Codegen { + +struct WideStoreHalf { + std::vector Bytes; + std::optional RipDisplacementOffset; + std::int64_t RipTarget = 0; +}; + +struct WideStoreOperands { + WideStoreHalf Low; + WideStoreHalf High; +}; + +[[nodiscard]] std::optional DecodeVexWideStore(const std::uint8_t* data, std::size_t length); + +} + +#endif diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 201630464..2f57bd9fd 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -3,9 +3,13 @@ #include #include #include +#include #include +#include #include +#include #include +#include #include #include #include @@ -24,6 +28,181 @@ auto _atFileOffset(const Domain::FileByteOffset base, const TOperation& operatio } } +constexpr std::int64_t kShortJumpSize = 2; +constexpr std::uint64_t kIslandSearchBytes = 0x100; + +struct Placement { + std::vector Sites; + std::vector Claimed; + std::vector Consumed; + std::size_t ReplacementLength = 0; +}; + +class WindowPlacer { +public: + WindowPlacer( + std::vector& seg, + const std::vector& matches, + const Domain::ProgramHeader& ph, + const std::set& branchTargets, + const std::set& pending, + const std::set& claimed, + const IAmd64OnlyInstructionMatcher& matcher + ) : _seg(seg), _matches(matches), _ph(ph), _branchTargets(branchTargets), _pending(pending), _claimed(claimed), _matcher(matcher) {} + + [[nodiscard]] std::optional Window(const std::size_t index) const { + if (index + 1 < _matches.size() && (_pending.contains(index + 1) || _movable(index + 1, true))) { + if (auto placement = _relocate(index, index, index + 1)) + return placement; + } + if (index > 0 && _movable(index - 1, false)) + return _relocate(index, index - 1, index); + return std::nullopt; + } + + [[nodiscard]] std::optional Island(const std::size_t index) { + const auto store = _address(index); + for (std::size_t distance = 1;; ++distance) { + bool inReach = false; + for (const bool before : {true, false}) { + if (before ? distance > index : index + distance >= _matches.size()) + continue; + const auto start = before ? index - distance : index + distance; + if (_address(start) + kIslandSearchBytes < store || _address(start) > store + kIslandSearchBytes) + continue; + inReach = true; + if (auto placement = _islandAt(index, start)) + return placement; + } + if (!inReach) + return std::nullopt; + } + } + +private: + std::vector& _seg; + const std::vector& _matches; + const Domain::ProgramHeader& _ph; + const std::set& _branchTargets; + const std::set& _pending; + const std::set& _claimed; + const IAmd64OnlyInstructionMatcher& _matcher; + X64InstructionDecoder _decoder; + + [[nodiscard]] std::uint64_t _address(const std::size_t index) const { return _ph.MappedAddress + _matches[index].Offset; } + [[nodiscard]] std::uint64_t _end(const std::size_t index) const { return _address(index) + _matches[index].Length; } + [[nodiscard]] std::span _bytes(const std::size_t index) const { return {_seg.data() + _matches[index].Offset, _matches[index].Length}; } + [[nodiscard]] DecodedInstructionInfo _info(const std::size_t index) const { return _decoder.DecodeInstruction(_seg.data() + _matches[index].Offset, _matches[index].Length); } + + [[nodiscard]] bool _entered(const std::uint64_t start, const std::uint64_t end) const { + const auto hit = _branchTargets.upper_bound(start); + return hit != _branchTargets.end() && *hit < end; + } + + [[nodiscard]] bool _blocksFollowing(const std::size_t index) const { + const auto kind = _info(index).FlowKind; + return kind != ControlFlowKind::Sequential && kind != ControlFlowKind::ConditionalBranch; + } + + [[nodiscard]] bool _movable(const std::size_t index, const bool last) const { + return !_claimed.contains(index) && !_pending.contains(index) && (last || !_blocksFollowing(index)); + } + + [[nodiscard]] TrampolineSite _site(const std::uint64_t start, const std::size_t length, LoweredBody body) const { + const auto offset = static_cast(start - _ph.MappedAddress); + return { + static_cast(_ph.Offset + offset), + static_cast(start), + length, + std::vector(_seg.begin() + static_cast(offset), _seg.begin() + static_cast(offset + length)), + std::move(body.Bytes), + body.ReturnBranchOffset, + std::move(body.Relocations) + }; + } + + [[nodiscard]] std::optional _relocate(const std::size_t index, const std::size_t first, const std::size_t last) const { + if (_entered(_address(first), _end(last))) + return std::nullopt; + StubBodyBuilder body; + Placement placement; + try { + for (auto k = first; k <= last; ++k) { + if (_pending.contains(k)) { + _matcher.EmitOutOfLine(body, _bytes(k)); + if (k != index) + placement.Consumed.push_back(k); + } else { + body.MoveControlFlow(_bytes(k), _info(k)); + } + placement.Claimed.push_back(k); + } + } catch (const CodegenException&) { + return std::nullopt; + } + placement.Sites.push_back(_site(_address(first), static_cast(_end(last) - _address(first)), body.Finish())); + placement.ReplacementLength = placement.Sites.back().Body.size(); + return placement; + } + + [[nodiscard]] std::optional _islandAt(const std::size_t index, const std::size_t start) { + using Amd64OnlySubstitutionTable::kJmpRel32; + std::size_t head = 0; + for (auto last = start; last < _matches.size(); ++last) { + if (!_movable(last, true) || _entered(_address(start), _end(last))) + return std::nullopt; + if (head == 0 && _end(last) - _address(start) >= kJmpRel32.Size) + head = static_cast(_end(last) - _address(start)); + const auto island = _address(start) + head; + const auto reach = static_cast(island) - static_cast(_address(index) + kShortJumpSize); + if (head != 0 && (reach < std::numeric_limits::min() || reach > std::numeric_limits::max())) + return std::nullopt; + if (head != 0 && _end(last) >= island + kJmpRel32.Size) { + try { + return _buildIsland(index, start, last, head); + } catch (const CodegenException&) { + return std::nullopt; + } + } + if (_blocksFollowing(last)) + return std::nullopt; + } + return std::nullopt; + } + + [[nodiscard]] Placement _buildIsland(const std::size_t index, const std::size_t start, const std::size_t last, const std::size_t head) { + using Amd64OnlySubstitutionTable::kJmpRel32; + const auto windowStart = _address(start); + const auto windowEnd = _end(last); + const auto island = windowStart + head; + const auto store = _address(index); + Placement placement; + + StubBodyBuilder moved; + for (auto k = start; k <= last; ++k) { + moved.MoveControlFlow(_bytes(k), _info(k)); + placement.Claimed.push_back(k); + } + if (!_blocksFollowing(last)) + moved.JumpTo(static_cast(windowEnd - windowStart)); + placement.Sites.push_back(_site(windowStart, head, moved.Finish())); + + StubBodyBuilder lowered; + lowered.Seek(static_cast(store) - static_cast(island)); + _matcher.EmitOutOfLine(lowered, _bytes(index)); + lowered.JumpTo(static_cast(_end(index)) - static_cast(island)); + placement.Sites.push_back(_site(island, kJmpRel32.Size, lowered.Finish())); + placement.ReplacementLength = placement.Sites.back().Body.size(); + placement.Claimed.push_back(index); + + const auto offset = _matches[index].Offset; + std::fill_n(_seg.begin() + static_cast(offset), _matches[index].Length, Amd64OnlySubstitutionTable::kTrapFill); + _seg[offset] = X64OpcodeConstants::OneByteJmpRel8; + _seg[offset + 1] = static_cast(static_cast(static_cast(island) - static_cast(store + kShortJumpSize))); + return placement; + } +}; + class Amd64OnlyConverter : public IAmd64OnlyConverter { public: explicit Amd64OnlyConverter(const Amd64OnlyTarget target) : _matcher(MakeAmd64OnlyInstructionMatcher(target)) {} @@ -106,6 +285,10 @@ void Amd64OnlyConverter::_convertSegment( }); const auto branchTargets = needsBranchTargets ? _collectBranchTargets(seg, matches, ph) : std::set{}; + std::set pendingIndices; + for (const auto& item : pending) + pendingIndices.insert(item.Index); + std::set claimed; std::set consumed; for (const auto& item : pending) { if (consumed.contains(item.Index)) @@ -126,6 +309,20 @@ void Amd64OnlyConverter::_convertSegment( break; } case Amd64OnlyLowering::Trampoline: { + if (substitution.RelocatableWindow && match.Length < Amd64OnlySubstitutionTable::kJmpRel32.Size) { + WindowPlacer placer(seg, matches, ph, branchTargets, pendingIndices, claimed, *_matcher); + auto placement = placer.Window(item.Index); + if (!placement) + placement = placer.Island(item.Index); + if (!placement) + throw CodegenException("No room for a jump to the stub of " + substitution.InstructionName + " within its neighbours or a nearby island", fileOffset); + for (auto& site : placement->Sites) + result.Trampolines.push_back(std::move(site)); + claimed.insert(placement->Claimed.begin(), placement->Claimed.end()); + consumed.insert(placement->Consumed.begin(), placement->Consumed.end()); + replacementLength = placement->ReplacementLength; + break; + } std::vector taken; try { std::size_t siteLength = match.Length; @@ -172,6 +369,8 @@ void Amd64OnlyConverter::_convertSegment( }); replacementLength = stub.StubBody.size(); consumed.insert(taken.begin(), taken.end()); + for (auto k = item.Index; k < matches.size() && matches[k].Offset < match.Offset + siteLength; ++k) + claimed.insert(k); } catch (const CodegenException&) { if (!substitution.Optional) throw; diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 5d33e8f82..edf59c514 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -72,8 +74,8 @@ const Entry& _sha1Entry(const Sha1Operands& operands) { return kSha1rnds4; } -Amd64OnlyMatch _trampoline(std::string name, const std::size_t length, LoweredBody body, const bool optional = false) { - return Amd64OnlyMatch{std::move(name), length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, optional, std::move(body.Relocations)}; +Amd64OnlyMatch _trampoline(std::string name, const std::size_t length, LoweredBody body, const bool optional = false, const bool window = false) { + return Amd64OnlyMatch{std::move(name), length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, optional, std::move(body.Relocations), window}; } void _moveTrailing(StubBodyBuilder& body, std::span trailing) { @@ -142,6 +144,11 @@ public: std::span trailing ) const override; + void EmitOutOfLine( + StubBodyBuilder& body, + std::span instruction + ) const override; + private: Amd64OnlyTarget _target; Sse4aLowering _lowering; @@ -149,6 +156,7 @@ private: Sha1Lowering _sha1Lowering; ClzeroLowering _clzeroLowering; ReciprocalLowering _reciprocalLowering; + WideStoreLowering _wideStoreLowering; [[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const; [[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span trailing) const; @@ -156,6 +164,7 @@ private: [[nodiscard]] Amd64OnlyMatch _matchSha1(const DecodedInstruction& instr, std::span trailing) const; [[nodiscard]] Amd64OnlyMatch _matchClzero(const DecodedInstruction& instr, std::span trailing) const; [[nodiscard]] std::optional _matchRosetta(const DecodedInstruction& instr, std::span trailing) const; + [[nodiscard]] const char* _emitOutOfLine(StubBodyBuilder& body, const DecodedInstruction& instr) const; }; Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const { @@ -208,9 +217,52 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _inPlace(kClwb, instr.Length, {}); if (instr.IsRdpid()) return _trampoline(kRdpid.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { body.Raw(_rdpidReplacement(instr)); }), true); + if (const auto wide = DecodeVexWideStore(instr.Data, instr.Length)) + return _trampoline(kWideStore.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _wideStoreLowering.EmitOutOfLine(body, *wide); }), false, true); return std::nullopt; } +const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, const DecodedInstruction& instr) const { + if (instr.IsExtrq() || instr.IsInsertq()) { + const auto operands = DecodeSse4a(instr.Data, instr.Length); + _lowering.EmitOutOfLine(body, operands); + return _sse4aEntry(operands).Name; + } + if (instr.IsSha256()) { + const auto operands = DecodeSha256(instr.Data, instr.Length); + _sha256Lowering.EmitOutOfLine(body, operands); + return _sha256Entry(operands).Name; + } + if (instr.IsSha1()) { + const auto operands = DecodeSha1(instr.Data, instr.Length); + _sha1Lowering.EmitOutOfLine(body, operands); + return _sha1Entry(operands).Name; + } + if (instr.IsClzero()) { + _clzeroLowering.EmitOutOfLine(body, DecodeClzero(instr.Data, instr.Length)); + return kClzero.Name; + } + if (const auto reciprocal = DecodeReciprocal(instr.Data, instr.Length)) { + _reciprocalLowering.EmitOutOfLine(body, *reciprocal); + return _reciprocalEntry(*reciprocal).Name; + } + if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdpid()) { + body.Raw(_rdpidReplacement(instr)); + return kRdpid.Name; + } + if (const auto wide = _target == Amd64OnlyTarget::Rosetta ? DecodeVexWideStore(instr.Data, instr.Length) : std::nullopt) { + _wideStoreLowering.EmitOutOfLine(body, *wide); + return kWideStore.Name; + } + return nullptr; +} + +void Amd64OnlyInstructionMatcher::EmitOutOfLine(StubBodyBuilder& body, const std::span instruction) const { + if (_emitOutOfLine(body, DecodedInstruction{instruction.data(), instruction.size()}) == nullptr) + throw CodegenException("AMD-only instruction has no out-of-line lowering"); + body.Advance(instruction.size()); +} + std::optional Amd64OnlyInstructionMatcher::MatchSequence( std::span> instructions, std::span trailing @@ -221,37 +273,11 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( const char* name = nullptr; for (const auto& instruction : instructions) { const DecodedInstruction instr{instruction.data(), instruction.size()}; - if (instr.IsExtrq() || instr.IsInsertq()) { - const auto operands = DecodeSse4a(instr.Data, instr.Length); - if (name == nullptr) - name = _sse4aEntry(operands).Name; - _lowering.EmitOutOfLine(body, operands); - } else if (instr.IsSha256()) { - const auto operands = DecodeSha256(instr.Data, instr.Length); - if (name == nullptr) - name = _sha256Entry(operands).Name; - _sha256Lowering.EmitOutOfLine(body, operands); - } else if (instr.IsSha1()) { - const auto operands = DecodeSha1(instr.Data, instr.Length); - if (name == nullptr) - name = _sha1Entry(operands).Name; - _sha1Lowering.EmitOutOfLine(body, operands); - } else if (instr.IsClzero()) { - const auto operands = DecodeClzero(instr.Data, instr.Length); - if (name == nullptr) - name = kClzero.Name; - _clzeroLowering.EmitOutOfLine(body, operands); - } else if (const auto reciprocal = DecodeReciprocal(instr.Data, instr.Length)) { - if (name == nullptr) - name = _reciprocalEntry(*reciprocal).Name; - _reciprocalLowering.EmitOutOfLine(body, *reciprocal); - } else if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdpid()) { - if (name == nullptr) - name = kRdpid.Name; - body.Raw(_rdpidReplacement(instr)); - } else { + const auto emitted = _emitOutOfLine(body, instr); + if (emitted == nullptr) return std::nullopt; - } + if (name == nullptr) + name = emitted; body.Advance(instr.Length); } _moveTrailing(body, trailing); diff --git a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp index a1b6e44b6..e68805215 100644 --- a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp +++ b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include namespace Codegen { @@ -150,7 +151,7 @@ void StubBodyBuilder::Load(const std::uint8_t reg, const MemoryOperand& operand) if (operand.Rm == ModRmRmSibPresent) _bytes.push_back(operand.Sib); if (operand.RipRelative) { - _relocations.push_back({_bytes.size(), _bytes.size() + Disp32Size, static_cast(_siteOffset + operand.InstructionLength) + operand.Displacement}); + _relocations.push_back({_bytes.size(), _bytes.size() + Disp32Size, _siteOffset + static_cast(operand.InstructionLength) + operand.Displacement}); _bytes.insert(_bytes.end(), Disp32Size, 0); return; } @@ -189,13 +190,82 @@ void StubBodyBuilder::Move(const std::span instruction, cons value |= static_cast(instruction[*ripDisplacementOffset + index]) << (index * 8); _bytes[start + *ripDisplacementOffset + index] = 0; } - _relocations.push_back({start + *ripDisplacementOffset, start + instruction.size(), static_cast(_siteOffset + instruction.size()) + static_cast(value)}); + _relocations.push_back({start + *ripDisplacementOffset, start + instruction.size(), _siteOffset + static_cast(instruction.size()) + static_cast(value)}); } Advance(instruction.size()); } +void StubBodyBuilder::RipRelative(const std::span instruction, const std::size_t displacementOffset, const std::int64_t target) { + using namespace X64OpcodeConstants; + if (displacementOffset > instruction.size() || instruction.size() - displacementOffset < Disp32Size) + throw CodegenException("RIP-relative displacement lies outside its instruction"); + const auto start = _bytes.size(); + Raw(instruction); + std::fill_n(_bytes.begin() + static_cast(start + displacementOffset), Disp32Size, std::uint8_t{0}); + _relocations.push_back({start + displacementOffset, start + instruction.size(), _siteOffset + target}); +} + +void StubBodyBuilder::MoveControlFlow(const std::span instruction, const DecodedInstructionInfo& info) { + using namespace X64OpcodeConstants; + const auto end = _siteOffset + static_cast(instruction.size()); + if (info.FlowKind == ControlFlowKind::IndirectCall) + throw CodegenException("A register or memory call cannot move into a stub"); + if (info.FlowKind == ControlFlowKind::Call && info.HasRipRelativeDisp) { + _pushReturnAddress(end); + std::vector jump(instruction.begin(), instruction.end()); + auto& modrm = jump[info.RipRelativeDispOffset - 1]; + modrm = static_cast((modrm & ~(ModRmRegMask << ModRmRegShift)) | (4u << ModRmRegShift)); + Move(jump, info.RipRelativeDispOffset); + return; + } + if (!info.HasBranchTarget || info.HasRipRelativeDisp) { + Move(instruction, info.HasRipRelativeDisp ? std::optional(info.RipRelativeDispOffset) : std::nullopt); + return; + } + for (std::size_t index = 0; index < info.OpcodeOffset; ++index) { + const auto prefix = instruction[index]; + if (prefix != PrefixSegCs && prefix != PrefixSegDs && prefix != PrefixRepne && prefix != PrefixAddressSize && (prefix & 0xF0) != kRexBase) + throw CodegenException("A relative branch with this prefix cannot move into a stub"); + } + const auto target = end + info.BranchDisp; + if (info.IsTwoByteOpcode) { + _rel32({TwoByteOpcodeEscape, info.Opcode}, target); + } else if (info.Opcode == OneByteCallRel32) { + _pushReturnAddress(end); + _rel32({OneByteJmpRel32}, target); + } else if (info.Opcode == OneByteJmpRel32 || info.Opcode == OneByteJmpRel8) { + _rel32({OneByteJmpRel32}, target); + } else if (info.Opcode >= OneByteJccRel8Min && info.Opcode <= OneByteJccRel8Max) { + _rel32({TwoByteOpcodeEscape, static_cast(TwoByteJccRel32Min | (info.Opcode & 0x0F))}, target); + } else { + throw CodegenException("A LOOP or JRCXZ branch cannot move into a stub"); + } + Advance(instruction.size()); +} + +void StubBodyBuilder::JumpTo(const std::int64_t siteTarget) { + _rel32({X64OpcodeConstants::OneByteJmpRel32}, siteTarget); +} + +void StubBodyBuilder::Seek(const std::int64_t siteOffset) { + _siteOffset = siteOffset; +} + void StubBodyBuilder::Advance(const std::size_t length) { - _siteOffset += length; + _siteOffset += static_cast(length); +} + +void StubBodyBuilder::_rel32(const std::initializer_list opcode, const std::int64_t siteTarget) { + _bytes.insert(_bytes.end(), opcode); + const auto displacement = _bytes.size(); + _bytes.insert(_bytes.end(), X64OpcodeConstants::Disp32Size, std::uint8_t{0}); + _relocations.push_back({displacement, _bytes.size(), siteTarget}); +} + +void StubBodyBuilder::_pushReturnAddress(const std::int64_t siteTarget) { + _bytes.insert(_bytes.end(), {0x48, 0x8D, 0x64, 0x24, 0xF8, 0x50}); + _rel32({0x48, 0x8D, 0x05}, siteTarget); + _bytes.insert(_bytes.end(), {0x48, 0x89, 0x44, 0x24, 0x08, 0x58}); } LoweredBody StubBodyBuilder::Finish() { diff --git a/core/relinker/codegen/src/x86/WideStoreLowering.cpp b/core/relinker/codegen/src/x86/WideStoreLowering.cpp new file mode 100644 index 000000000..40e22c9a3 --- /dev/null +++ b/core/relinker/codegen/src/x86/WideStoreLowering.cpp @@ -0,0 +1,21 @@ +#include + +namespace Codegen { + +namespace { + +void _emitHalf(StubBodyBuilder& body, const WideStoreHalf& half) { + if (half.RipDisplacementOffset) + body.RipRelative(half.Bytes, *half.RipDisplacementOffset, half.RipTarget); + else + body.Raw(half.Bytes); +} + +} + +void WideStoreLowering::EmitOutOfLine(StubBodyBuilder& body, const WideStoreOperands& operands) const { + _emitHalf(body, operands.Low); + _emitHalf(body, operands.High); +} + +} diff --git a/core/relinker/codegen/src/x86/WideStoreOperands.cpp b/core/relinker/codegen/src/x86/WideStoreOperands.cpp new file mode 100644 index 000000000..478c4108e --- /dev/null +++ b/core/relinker/codegen/src/x86/WideStoreOperands.cpp @@ -0,0 +1,133 @@ +#include +#include +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kVexMap0F = 0x01; +constexpr std::uint8_t kVexLength256 = 0x04; +constexpr std::uint8_t kVexUnusedRegister = 0x0F; +constexpr std::uint8_t kPpNone = 0x00; +constexpr std::uint8_t kPp66 = 0x01; +constexpr std::uint8_t kPpF3 = 0x02; +constexpr std::uint8_t kVextractf128 = 0x19; +constexpr std::uint8_t kVextractf128Payload = 0x7D; +constexpr std::uint8_t kUpperHalf = 0x01; +constexpr std::int64_t kHalfBytes = 16; + +bool _unalignedStore(const std::uint8_t pp, const std::uint8_t opcode) { + return ((pp == kPpNone || pp == kPp66) && opcode == 0x11) || (pp == kPpF3 && opcode == 0x7F); +} + +bool _legacyPrefixAllowed(const std::uint8_t prefix) { + using namespace X64OpcodeConstants; + return prefix == PrefixSegCs || prefix == PrefixSegSs || prefix == PrefixSegDs || prefix == PrefixSegEs || prefix == PrefixSegFs || prefix == PrefixSegGs || prefix == PrefixAddressSize; +} + +void _appendU32(std::vector& bytes, const std::uint32_t value) { + for (std::size_t index = 0; index < X64OpcodeConstants::Disp32Size; ++index) + bytes.push_back(static_cast(value >> (index * 8))); +} + +} + +std::optional DecodeVexWideStore(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + std::size_t pos = 0; + while (pos < length && _legacyPrefixAllowed(data[pos])) + ++pos; + const auto vex = pos; + if (vex + 2 > length) + return std::nullopt; + std::uint8_t inverted = 0; + std::uint8_t payload = 0; + if (data[vex] == OneByteVex2) { + inverted = static_cast((data[vex + 1] & 0x80) | 0x60); + payload = data[vex + 1]; + pos = vex + Vex2PrefixLength; + } else if (data[vex] == OneByteVex3) { + if (vex + Vex3PrefixLength > length || (data[vex + 1] & Vex3MapMask) != kVexMap0F) + return std::nullopt; + inverted = static_cast(data[vex + 1] & 0xE0); + payload = data[vex + 2]; + pos = vex + Vex3PrefixLength; + } else { + return std::nullopt; + } + if ((payload & kVexLength256) == 0 || ((payload >> 3) & 0x0F) != kVexUnusedRegister) + return std::nullopt; + if (pos + 2 > length || !_unalignedStore(static_cast(payload & 0x03), data[pos])) + return std::nullopt; + const auto modRmOffset = pos + 1; + const auto modrm = data[modRmOffset]; + const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); + const auto rm = static_cast(modrm & ModRmRmMask); + if (mod == ModRmModRegister) + return std::nullopt; + auto cursor = modRmOffset + 1; + std::optional sib; + if (rm == ModRmRmSibPresent) { + if (cursor >= length) + return std::nullopt; + sib = data[cursor++]; + } + const bool ripRelative = mod == ModRmModIndirect && rm == ModRmRmRipRelative; + const bool absolute = mod == ModRmModIndirect && sib.has_value() && (*sib & SibBaseMask) == SibBaseDisp32; + const std::size_t displacementSize = mod == ModRmModDisp8 ? Disp8Size : (mod == ModRmModDisp32 || ripRelative || absolute) ? Disp32Size : 0; + if (cursor + displacementSize != length) + return std::nullopt; + std::int64_t displacement = 0; + if (displacementSize == Disp8Size) { + displacement = static_cast(data[cursor]); + } else if (displacementSize == Disp32Size) { + std::uint32_t value = 0; + for (std::size_t index = 0; index < Disp32Size; ++index) + value |= static_cast(data[cursor + index]) << (index * 8); + displacement = static_cast(value); + } + + WideStoreOperands operands; + operands.Low.Bytes.assign(data, data + length); + operands.Low.Bytes[data[vex] == OneByteVex2 ? vex + 1 : vex + 2] &= static_cast(~kVexLength256); + if (ripRelative) { + operands.Low.RipDisplacementOffset = cursor; + operands.Low.RipTarget = static_cast(length) + displacement; + } + + auto& high = operands.High.Bytes; + high.assign(data, data + vex); + high.insert(high.end(), {OneByteVex3, static_cast(inverted | Vex3Map0F3A), kVextractf128Payload, kVextractf128}); + const auto reg = static_cast(modrm & (ModRmRegMask << ModRmRegShift)); + std::int64_t highDisplacement = displacement + kHalfBytes; + std::uint8_t highMod = mod; + std::size_t highDisplacementSize = displacementSize; + if (!ripRelative && !absolute) { + if (mod == ModRmModIndirect || (mod == ModRmModDisp8 && highDisplacement > std::numeric_limits::max())) { + highMod = highDisplacement > std::numeric_limits::max() ? ModRmModDisp32 : ModRmModDisp8; + highDisplacementSize = highMod == ModRmModDisp8 ? Disp8Size : Disp32Size; + } + } + if (!ripRelative && highDisplacement > std::numeric_limits::max()) + throw CodegenException("256-bit store displacement leaves no room for its upper half"); + high.push_back(static_cast((highMod << ModRmModShift) | reg | rm)); + if (sib) + high.push_back(*sib); + const auto highDisplacementOffset = high.size(); + if (highDisplacementSize == Disp8Size) + high.push_back(static_cast(static_cast(highDisplacement))); + else if (ripRelative) + _appendU32(high, 0); + else if (highDisplacementSize == Disp32Size) + _appendU32(high, static_cast(static_cast(highDisplacement))); + high.push_back(kUpperHalf); + if (ripRelative) { + operands.High.RipDisplacementOffset = highDisplacementOffset; + operands.High.RipTarget = static_cast(length) + displacement + kHalfBytes; + } + return operands; +} + +} diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 719788893..16e891044 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -670,6 +671,221 @@ void rosettaSubstitutions() { require(rdpid->ReturnBranchOffset == 6 && Bytes(rdpid->StubBody.begin(), rdpid->StubBody.begin() + 7) == Bytes{0x41, 0xBB, 0, 0, 0, 0, 0xE9}, "RDPID r11 stub does not load 0 before returning"); } +void wideStoreSplit() { + struct Case { + Bytes Original; + Bytes Low; + Bytes High; + const char* Name; + }; + const std::vector cases = { + {{0xC5, 0xFC, 0x11, 0x07}, {0xC5, 0xF8, 0x11, 0x07}, {0xC4, 0xE3, 0x7D, 0x19, 0x47, 0x10, 0x01}, "vmovups (%rdi)"}, + {{0xC4, 0x41, 0x7E, 0x7F, 0x4C, 0x87, 0x70}, {0xC4, 0x41, 0x7A, 0x7F, 0x4C, 0x87, 0x70}, {0xC4, 0x43, 0x7D, 0x19, 0x8C, 0x87, 0x80, 0x00, 0x00, 0x00, 0x01}, "vmovdqu 0x70(%r15,%rax,4)"}, + {{0xC5, 0xFC, 0x11, 0x4C, 0x24, 0xF8}, {0xC5, 0xF8, 0x11, 0x4C, 0x24, 0xF8}, {0xC4, 0xE3, 0x7D, 0x19, 0x4C, 0x24, 0x08, 0x01}, "vmovups -8(%rsp)"}, + {{0xC5, 0xFE, 0x7F, 0x93, 0x78, 0x56, 0x34, 0x12}, {0xC5, 0xFA, 0x7F, 0x93, 0x78, 0x56, 0x34, 0x12}, {0xC4, 0xE3, 0x7D, 0x19, 0x93, 0x88, 0x56, 0x34, 0x12, 0x01}, "vmovdqu 0x12345678(%rbx)"}, + {{0x64, 0xC5, 0xFC, 0x11, 0x04, 0x25, 0x40, 0x00, 0x00, 0x00}, {0x64, 0xC5, 0xF8, 0x11, 0x04, 0x25, 0x40, 0x00, 0x00, 0x00}, {0x64, 0xC4, 0xE3, 0x7D, 0x19, 0x04, 0x25, 0x50, 0x00, 0x00, 0x00, 0x01}, "vmovups %fs:0x40"}, + {{0xC4, 0x41, 0x7C, 0x11, 0x65, 0x00}, {0xC4, 0x41, 0x78, 0x11, 0x65, 0x00}, {0xC4, 0x43, 0x7D, 0x19, 0x65, 0x10, 0x01}, "vmovups (%r13)"}, + {{0xC4, 0xC1, 0x7C, 0x11, 0x47, 0x68}, {0xC4, 0xC1, 0x78, 0x11, 0x47, 0x68}, {0xC4, 0xC3, 0x7D, 0x19, 0x47, 0x78, 0x01}, "vmovups 0x68(%r15)"}}; + for (const auto& item : cases) { + const auto split = Codegen::DecodeVexWideStore(item.Original.data(), item.Original.size()); + require(split && split->Low.Bytes == item.Low && split->High.Bytes == item.High && !split->Low.RipDisplacementOffset && !split->High.RipDisplacementOffset, (std::string("256-bit store was not split into its 128-bit halves: ") + item.Name).c_str()); + } + const Bytes rip = {0xC5, 0xFD, 0x11, 0x1D, 0x00, 0x01, 0x00, 0x00}; + const auto ripSplit = Codegen::DecodeVexWideStore(rip.data(), rip.size()); + require(ripSplit && ripSplit->Low.Bytes == Bytes{0xC5, 0xF9, 0x11, 0x1D, 0x00, 0x01, 0x00, 0x00} && ripSplit->High.Bytes == Bytes{0xC4, 0xE3, 0x7D, 0x19, 0x1D, 0x00, 0x00, 0x00, 0x00, 0x01}, "RIP-relative 256-bit store was not split"); + require(ripSplit->Low.RipDisplacementOffset == 4 && ripSplit->Low.RipTarget == 8 + 0x100 && ripSplit->High.RipDisplacementOffset == 5 && ripSplit->High.RipTarget == 8 + 0x110, "RIP-relative halves do not address the original operand and the one 16 bytes above it"); + for (const auto& other : {Bytes{0xC5, 0xF8, 0x11, 0x07}, Bytes{0xC5, 0xFC, 0x11, 0xC1}, Bytes{0xC5, 0xFC, 0x10, 0x07}, Bytes{0xC4, 0xE2, 0x75, 0x2E, 0x07}, Bytes{0xC5, 0xF4, 0x11, 0x07}, Bytes{0xC4, 0xE3, 0x7D, 0x19, 0x07, 0x01}, Bytes{0xC5, 0xFC, 0x11, 0x04}, Bytes{0xC5, 0xFC, 0x11, 0x47}}) + require(!Codegen::DecodeVexWideStore(other.data(), other.size()).has_value(), "An instruction other than a plain 256-bit store was split"); + for (const auto& aligned : {Bytes{0xC5, 0xFC, 0x29, 0x4C, 0x24, 0xF8}, Bytes{0xC5, 0xFD, 0xE7, 0x93, 0x78, 0x56, 0x34, 0x12}, Bytes{0xC5, 0xFD, 0x7F, 0x1F}, Bytes{0xC5, 0xFC, 0x2B, 0x27}}) + require(!Codegen::DecodeVexWideStore(aligned.data(), aligned.size()).has_value(), "An aligned 256-bit store, which cannot cross a page, was split"); + const Bytes farRip = {0xC5, 0xFC, 0x11, 0x05, 0xF8, 0xFF, 0xFF, 0x7F}; + const auto farRipSplit = Codegen::DecodeVexWideStore(farRip.data(), farRip.size()); + require(farRipSplit && farRipSplit->High.RipTarget == std::int64_t{8} + 0x7FFFFFF8 + 0x10, "A RIP-relative 256-bit store near the end of its displacement range was not split"); + const Bytes farBase = {0xC5, 0xFC, 0x11, 0x80, 0xF8, 0xFF, 0xFF, 0x7F}; + requireFailure([&] { (void)Codegen::DecodeVexWideStore(farBase.data(), farBase.size()); }, "A 256-bit store whose upper half has no 32-bit displacement was kept"); + requireFailure([] { Codegen::StubBodyBuilder body; body.RipRelative(Bytes{0x90}, 0, 0); }, "A RIP-relative displacement outside its instruction was relocated"); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const Bytes store = {0xC4, 0xC1, 0x7C, 0x11, 0x47, 0x68}; + const auto lowered = rosetta->Match(store.data(), store.size()); + require(lowered && lowered->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !lowered->Optional && lowered->RelocatableWindow && lowered->InstructionName == "256-bit store", "A 256-bit store was not lowered through a required stub that may move its neighbours for Rosetta"); + Bytes expectedBody = cases.back().Low; + expectedBody.insert(expectedBody.end(), cases.back().High.begin(), cases.back().High.end()); + require(Bytes(lowered->StubBody.begin(), lowered->StubBody.begin() + static_cast(lowered->ReturnBranchOffset)) == expectedBody, "The 256-bit store stub is not the two halves"); + const std::vector> pair = {std::span(cases[0].Original), std::span(rip)}; + const auto sequence = rosetta->MatchSequence(pair, {}); + require(sequence && sequence->Relocations.size() == 2 && sequence->Relocations[0].SiteTarget == 4 + 8 + 0x100 && sequence->Relocations[1].SiteTarget == 4 + 8 + 0x110, "A RIP-relative 256-bit store after another is not relocated from its place in the site"); + require(!Codegen::MakeAmd64OnlyInstructionMatcher()->Match(store.data(), store.size()).has_value(), "A 256-bit store was split for an Intel host"); + const std::vector> single = {std::span(store)}; + require(!Codegen::MakeAmd64OnlyInstructionMatcher()->MatchSequence(single, {}).has_value(), "A 256-bit store was split in a sequence for an Intel host"); +} + +void converterWideStoreWindows() { + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); + const Bytes store = {0xC5, 0xFC, 0x11, 0x07}; + Bytes halves = {0xC5, 0xF8, 0x11, 0x07, 0xC4, 0xE3, 0x7D, 0x19, 0x47, 0x10, 0x01}; + const auto convert = [&](const Bytes& text) { + Bytes file(0x300 + text.size(), 0xCC); + std::copy(text.begin(), text.end(), file.begin() + 0x200); + return converter->Convert(file, {segmentHeader(text.size())}); + }; + const auto body = [](const Codegen::TrampolineSite& site) { + return Bytes(site.Body.begin(), site.Body.begin() + static_cast(site.ReturnBranchOffset)); + }; + const auto withStore = [&](Bytes before, const Bytes& after) { + before.insert(before.end(), store.begin(), store.end()); + before.insert(before.end(), after.begin(), after.end()); + return before; + }; + + const auto returned = convert(withStore({}, {0xC3})); + Bytes expected = halves; + expected.push_back(0xC3); + require(returned.Trampolines.size() == 1 && returned.KeptCount == 0 && returned.Reports.size() == 1 && returned.Reports[0].InstructionName == "256-bit store", "A 256-bit store followed by a return was not moved"); + require(returned.Trampolines[0].Offset == 0x200 && returned.Trampolines[0].Length == 5 && body(returned.Trampolines[0]) == expected, "A 256-bit store followed by a return did not end its stub with the return"); + + const auto called = convert(withStore({}, {0xE8, 0x10, 0x00, 0x00, 0x00})); + require(called.Trampolines.size() == 1 && called.Trampolines[0].Offset == 0x200 && called.Trampolines[0].Length == 9, "A 256-bit store followed by a call did not take the call"); + const auto& call = called.Trampolines[0]; + expected = halves; + expected.insert(expected.end(), {0x48, 0x8D, 0x64, 0x24, 0xF8, 0x50, 0x48, 0x8D, 0x05, 0, 0, 0, 0, 0x48, 0x89, 0x44, 0x24, 0x08, 0x58, 0xE9, 0, 0, 0, 0}); + require(body(call) == expected, "A moved call does not push its original return address and jump"); + require(call.Relocations.size() == 2 && call.Relocations[0].DisplacementOffset == halves.size() + 9 && call.Relocations[0].SiteTarget == 9 && call.Relocations[1].DisplacementOffset == halves.size() + 20 && call.Relocations[1].SiteTarget == 9 + 0x10, "A moved call returns to or calls the wrong address"); + + const auto branched = convert(withStore({}, {0x75, 0x10, 0x90})); + require(branched.Trampolines.size() == 1 && branched.Trampolines[0].Length == 6, "A 256-bit store followed by a short conditional branch did not take the branch"); + const auto& jcc = branched.Trampolines[0]; + expected = halves; + expected.insert(expected.end(), {0x0F, 0x85, 0, 0, 0, 0}); + require(body(jcc) == expected && jcc.Relocations.size() == 1 && jcc.Relocations[0].SiteTarget == 6 + 0x10, "A moved short conditional branch was not re-encoded with a 32-bit displacement to its target"); + + const auto jumped = convert(withStore({}, {0xEB, 0x10, 0x90})); + expected = halves; + expected.insert(expected.end(), {0xE9, 0, 0, 0, 0}); + require(jumped.Trampolines.size() == 1 && body(jumped.Trampolines[0]) == expected && jumped.Trampolines[0].Relocations.size() == 1 && jumped.Trampolines[0].Relocations[0].SiteTarget == 6 + 0x10, "A moved short jump was not re-encoded with a 32-bit displacement to its target"); + + const auto longBranched = convert(withStore({}, {0x0F, 0x85, 0x10, 0x00, 0x00, 0x00})); + expected = halves; + expected.insert(expected.end(), {0x0F, 0x85, 0, 0, 0, 0}); + require(longBranched.Trampolines.size() == 1 && longBranched.Trampolines[0].Length == 10 && body(longBranched.Trampolines[0]) == expected && longBranched.Trampolines[0].Relocations.size() == 1 && longBranched.Trampolines[0].Relocations[0].SiteTarget == 10 + 0x10, "A moved conditional branch with a 32-bit displacement does not reach its target"); + + const auto entered = convert(withStore({0x48, 0x83, 0xC1, 0x01}, {0xC3, 0xEB, 0xFD})); + expected = {0x48, 0x83, 0xC1, 0x01}; + expected.insert(expected.end(), halves.begin(), halves.end()); + require(entered.Trampolines.size() == 1 && entered.Trampolines[0].Offset == 0x200 && entered.Trampolines[0].Length == 8 && body(entered.Trampolines[0]) == expected, "A 256-bit store before a branch target did not take room from the instruction before it"); + + Bytes adds; + for (int index = 0; index < 4; ++index) + adds.insert(adds.end(), {0x48, 0x83, 0xC1, 0x01}); + Bytes after = {0xC3, 0xEB, 0xFD}; + after.insert(after.end(), adds.begin(), adds.end()); + const auto island = convert(withStore({0xE8, 0x00, 0x00, 0x00, 0x00}, after)); + require(island.Trampolines.size() == 2 && island.KeptCount == 0, "A 256-bit store with no movable neighbours was not lowered through an island"); + const auto& moved = island.Trampolines[0]; + const auto& lowered = island.Trampolines[1]; + expected = adds; + expected.insert(expected.end(), {0xE9, 0, 0, 0, 0}); + require(moved.Offset == 0x20C && moved.Length == 8 && body(moved) == expected && moved.Relocations.size() == 1 && moved.Relocations[0].SiteTarget == 16, "The instructions under the island were not moved and continued after it"); + expected = halves; + expected.insert(expected.end(), {0xE9, 0, 0, 0, 0}); + require(lowered.Offset == 0x214 && lowered.Length == 5 && body(lowered) == expected && lowered.Relocations.size() == 1 && lowered.Relocations[0].SiteTarget == -11, "The island does not store both halves and return after the original store"); + require(Bytes(island.Bytes.begin() + 0x205, island.Bytes.begin() + 0x209) == Bytes{0xEB, 0x0D, 0xCC, 0xCC}, "The 256-bit store was not replaced by a short jump to its island"); + + requireFailure([&] { (void)convert(withStore({0xE8, 0x00, 0x00, 0x00, 0x00}, {0xC3, 0xEB, 0xFD})); }, "A 256-bit store with no room for a stub jump was kept"); + + const auto paired = convert(withStore({}, {0xC5, 0xFC, 0x11, 0x0E, 0xC3})); + expected = halves; + expected.insert(expected.end(), {0xC5, 0xF8, 0x11, 0x0E, 0xC4, 0xE3, 0x7D, 0x19, 0x4E, 0x10, 0x01}); + require(paired.Trampolines.size() == 1 && paired.Trampolines[0].Length == 8 && body(paired.Trampolines[0]) == expected && paired.Reports.size() == 1, "A 256-bit store followed by another was not lowered with it in one stub"); + + const Bytes push = {0x48, 0x8D, 0x64, 0x24, 0xF8, 0x50, 0x48, 0x8D, 0x05, 0, 0, 0, 0, 0x48, 0x89, 0x44, 0x24, 0x08, 0x58}; + const auto ripCalled = convert(withStore({}, {0xFF, 0x15, 0x10, 0x00, 0x00, 0x00})); + expected = halves; + expected.insert(expected.end(), push.begin(), push.end()); + expected.insert(expected.end(), {0xFF, 0x25, 0, 0, 0, 0}); + const auto& ripCall = ripCalled.Trampolines[0]; + require(ripCalled.Trampolines.size() == 1 && ripCall.Length == 10 && body(ripCall) == expected, "A moved call through a RIP-relative pointer does not push its original return address and jump through the pointer"); + require(ripCall.Relocations.size() == 2 && ripCall.Relocations[0].SiteTarget == 10 && ripCall.Relocations[1].DisplacementOffset == halves.size() + 21 && ripCall.Relocations[1].SiteTarget == 10 + 0x10, "A moved call through a RIP-relative pointer returns to or reads the wrong address"); + + const auto ripJumped = convert(withStore({}, {0xFF, 0x25, 0x10, 0x00, 0x00, 0x00})); + expected = halves; + expected.insert(expected.end(), {0xFF, 0x25, 0, 0, 0, 0}); + require(ripJumped.Trampolines.size() == 1 && body(ripJumped.Trampolines[0]) == expected && ripJumped.Trampolines[0].Relocations.size() == 1 && ripJumped.Trampolines[0].Relocations[0].SiteTarget == 10 + 0x10, "A moved jump through a RIP-relative pointer reads the wrong address"); + + const auto relaxed = convert(withStore({}, {0x67, 0xE8, 0x10, 0x00, 0x00, 0x00})); + expected = halves; + expected.insert(expected.end(), push.begin(), push.end()); + expected.insert(expected.end(), {0xE9, 0, 0, 0, 0}); + require(relaxed.Trampolines.size() == 1 && relaxed.Trampolines[0].Length == 10 && body(relaxed.Trampolines[0]) == expected && relaxed.Trampolines[0].Relocations.size() == 2 && relaxed.Trampolines[0].Relocations[1].SiteTarget == 10 + 0x10, "A call with an address-size prefix was not moved like a call"); + + const Bytes add = {0x48, 0x83, 0xC1, 0x01}; + for (const auto& follower : {Bytes{0xFF, 0xD0}, Bytes{0xE3, 0x10}, Bytes{0x66, 0x75, 0x10}, Bytes{0x66, 0x0F, 0xAE, 0x37}}) { + const auto preceded = convert(withStore(add, follower)); + expected = add; + expected.insert(expected.end(), halves.begin(), halves.end()); + require(!preceded.Trampolines.empty() && preceded.Trampolines[0].Offset == 0x200 && preceded.Trampolines[0].Length == 8 && body(preceded.Trampolines[0]) == expected, "A 256-bit store moved a following instruction that cannot leave its place instead of the one before it"); + } + + Bytes throwing = {0xC3, 0xEB, 0xFD}; + for (int index = 0; index < 3; ++index) + throwing.insert(throwing.end(), add.begin(), add.end()); + throwing.insert(throwing.end(), {0xFF, 0xD0}); + throwing.insert(throwing.end(), adds.begin(), adds.end()); + const auto farther = convert(withStore({0xE8, 0x00, 0x00, 0x00, 0x00}, throwing)); + require(farther.Trampolines.size() == 2 && farther.Trampolines[0].Offset == 0x21A && farther.Trampolines[1].Offset == 0x222, "An island that would move a register call was not passed over for the next one"); + require(Bytes(farther.Bytes.begin() + 0x205, farther.Bytes.begin() + 0x207) == Bytes{0xEB, 0x1B}, "The 256-bit store does not jump to the island after the register call"); + + Bytes reentered = {0xFF, 0xD0, 0xEB, 0xF8}; + reentered.insert(reentered.end(), adds.begin(), adds.end()); + const auto beyondStore = convert(withStore(Bytes(adds.begin(), adds.begin() + 8), reentered)); + require(beyondStore.Trampolines.size() == 2 && beyondStore.Trampolines[0].Offset == 0x210 && beyondStore.Trampolines[1].Offset == 0x218, "An island before a 256-bit store that is a branch target took the store with it"); + require(Bytes(beyondStore.Bytes.begin() + 0x208, beyondStore.Bytes.begin() + 0x20A) == Bytes{0xEB, 0x0E}, "The 256-bit store that is a branch target does not jump to the island after it"); + + const Bytes farCall = {0xE8, 0x40, 0x00, 0x00, 0x00}; + Bytes claimedFirst = {0xFF, 0xD0}; + claimedFirst.insert(claimedFirst.end(), adds.begin(), adds.end()); + const auto afterSequence = convert(withStore({0xC5, 0xF8, 0x53, 0xC1, 0x48, 0x83, 0xC1, 0x01}, claimedFirst)); + require(afterSequence.Trampolines.size() == 3 && afterSequence.Trampolines[0].Offset == 0x200 && afterSequence.Trampolines[0].Length == 8 && afterSequence.Trampolines[1].Offset == 0x20E && afterSequence.Trampolines[2].Offset == 0x216, "A 256-bit store moved an instruction already moved into the stub of the instruction before it"); + + Bytes pendingRun = {0xFF, 0xD0, 0x48, 0x83, 0xC1, 0x01, 0xC5, 0xFC, 0x11, 0x47, 0x10}; + pendingRun.insert(pendingRun.end(), adds.begin(), adds.end()); + const auto pendingIsland = convert(withStore(farCall, pendingRun)); + require(pendingIsland.Trampolines.size() == 3 && pendingIsland.Trampolines[0].Offset == 0x214 && pendingIsland.Trampolines[1].Offset == 0x21C && pendingIsland.Trampolines[2].Offset == 0x20F && pendingIsland.Trampolines[2].Length == 5, "An island moved the call before a 256-bit store or another store that still needs its own stub"); + + Bytes enteredRun = {0xFF, 0xD0}; + for (int index = 0; index < 6; ++index) + enteredRun.insert(enteredRun.end(), add.begin(), add.end()); + enteredRun.insert(enteredRun.end(), {0xEB, 0xEE}); + const auto enteredIsland = convert(withStore(farCall, enteredRun)); + require(enteredIsland.Trampolines.size() == 2 && enteredIsland.Trampolines[0].Offset == 0x213 && enteredIsland.Trampolines[1].Offset == 0x21B, "An island was placed over a branch target"); + + Bytes returning = {0xFF, 0xD0}; + for (int index = 0; index < 3; ++index) + returning.insert(returning.end(), add.begin(), add.end()); + returning.push_back(0xC3); + const auto returnIsland = convert(withStore(farCall, returning)); + expected = Bytes(adds.begin(), adds.begin() + 12); + expected.push_back(0xC3); + require(returnIsland.Trampolines.size() == 2 && returnIsland.Trampolines[0].Offset == 0x20B && body(returnIsland.Trampolines[0]) == expected, "Instructions moved for an island that end in a return jump back after it"); + + const auto afterBranch = convert(withStore({0x75, 0x10}, {0xFF, 0xD0})); + expected = {0x0F, 0x85, 0, 0, 0, 0}; + expected.insert(expected.end(), halves.begin(), halves.end()); + require(afterBranch.Trampolines.size() == 1 && afterBranch.Trampolines[0].Offset == 0x200 && afterBranch.Trampolines[0].Length == 6 && body(afterBranch.Trampolines[0]) == expected && afterBranch.Trampolines[0].Relocations[0].SiteTarget == 2 + 0x10, "A 256-bit store after a conditional branch did not take the branch"); + + Bytes far = adds; + far.resize(8); + for (int index = 0; index < 60; ++index) + far.insert(far.end(), {0xFF, 0xD0}); + far.insert(far.end(), adds.begin(), adds.end()); + for (int index = 0; index < 70; ++index) + far.insert(far.end(), {0xFF, 0xD0}); + Bytes tail; + for (int index = 0; index < 11; ++index) + tail.insert(tail.end(), {0xFF, 0xD0}); + tail.insert(tail.end(), {0x90, 0x90}); + requireFailure([&] { (void)convert(withStore(far, tail)); }, "A 256-bit store whose movable neighbours are all beyond a short jump was kept"); +} + void converterRosetta() { const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); Bytes file(0x300, 0xCC); @@ -776,7 +992,7 @@ void converterReciprocal() { void converterStrayRex() { const auto converter = Codegen::MakeAmd64OnlyConverter(); const auto convert = [&](const Bytes& text) { - Bytes file(0x300, 0xCC); + Bytes file(0x300 + text.size(), 0xCC); std::copy(text.begin(), text.end(), file.begin() + 0x200); return std::pair{file, converter->Convert(file, {segmentHeader(text.size())})}; }; @@ -1419,6 +1635,8 @@ int main() { converterClzero(); rosettaSubstitutions(); converterRosetta(); + wideStoreSplit(); + converterWideStoreWindows(); converterReciprocal(); converterStrayRex(); rewriterStrayRex(); diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 0e747853d..1734ede1b 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -71,10 +71,11 @@ int Run(const int argc, char* argv[]) { sourceBytes = std::move(converted.Bytes); trampolines = std::move(converted.Trampolines); std::map stubsByName; + const auto counted = [](const std::string& name) { return name == "VRSQRTPS" || name == "VRCPPS" || name == "256-bit store"; }; for (const auto& report : converted.Reports) { if (report.Lowering == Codegen::Amd64OnlyLowering::Kept) std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) kept: no room for a jump\n"; - else if (report.InstructionName == "VRSQRTPS" || report.InstructionName == "VRCPPS") + else if (counted(report.InstructionName)) ++stubsByName[report.InstructionName]; else std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n";