feat(relinker): split unaligned 256-bit stores for Rosetta
Rosetta 2 ends the process when an unaligned 256-bit store crosses into
a page the store may not write ("rosetta error: unexpectedly need to
EmulateForward on a synchronous exception") instead of raising the
fault. A store inside such a page, and a 128-bit store across one,
fault normally. The macOS write watch keeps watched pages read-only
between collections, so a guest that stores 32 bytes across the start
of a watched page ends the process.
--to-rosetta now lowers VMOVUPS, VMOVUPD and VMOVDQU with a 256-bit
memory destination to a stub that stores the low half with the same
instruction at VEX.L=0 and the high half with VEXTRACTF128 at the
operand plus 16, so each half faults like any other store. RIP-relative
operands are relocated from the instruction's place in the site. A
store whose upper half has no 32-bit displacement fails the conversion.
Aligned stores (VMOVAPS, VMOVAPD, VMOVDQA and the non-temporal forms)
cannot cross a page and are left alone.
Every store is lowered. A 4-byte store has no room for the jump to its
stub, so its stub also takes one neighbouring instruction. The
following one: a return ends the stub, a call pushes its original
return address and jumps, a jump or conditional branch is re-encoded
with a 32-bit displacement, and another 256-bit store is lowered in the
same stub. Otherwise the preceding one, unless it is a call, a jump, a
return or a trap. Register calls, LOOP, JRCXZ and branches with a
prefix that changes them do not move. When neither neighbour can move,
the store becomes a short jump to an island: a run of movable
instructions within reach is moved to its own stub, and the bytes it
frees hold the jump to the store's stub. A store with neither fails the
conversion.
The converter tests compare the halves with clang's encodings for base,
SIB, disp8 overflowing to disp32, disp32, RSP-relative, FS-absolute,
r13-based and RIP-relative operands, and check that aligned stores,
loads, register forms and masked stores are not split. They check the
stub of a short store followed by a return, a call (direct, with an
address-size prefix, and through a RIP-relative pointer), short, near
and RIP-relative jumps, short and near conditional branches and another
store; that a following register call, LOOP, JRCXZ, prefixed branch or
CLWB makes the store take the instruction before it; islands past
instructions already moved, a store that needs its own stub, a branch
target, a register call and the short-jump reach; and that a store with
no room fails.
PPSA12544: 55295 stores split in the executable, 278 of them in the
stub of the store before them; every module converts with none kept.
AI-assisted: yes
This commit is contained in:
fablevoid
parent
34a16b7f08
commit
c763a78b67
|
|
@ -53,6 +53,8 @@ add_executable(relinker
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp
|
||||
|
|
@ -141,6 +143,8 @@ if(WIN32)
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp
|
||||
)
|
||||
target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows)
|
||||
if(BUILD_TESTING)
|
||||
|
|
@ -304,6 +308,8 @@ if(BUILD_TESTING)
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ClzeroLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalOperands.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ inline constexpr Entry kMcommit = {"MCOMMIT", nullptr, 0};
|
|||
inline constexpr Entry kRdseed = {"RDSEED", nullptr, 0};
|
||||
inline constexpr Entry kRdpid = {"RDPID", nullptr, 0};
|
||||
inline constexpr Entry kClwb = {"CLWB", nullptr, 0};
|
||||
inline constexpr Entry kWideStore = {"256-bit store", nullptr, 0};
|
||||
inline constexpr Entry kExtrq = {"EXTRQ", nullptr, 0};
|
||||
inline constexpr Entry kInsertq = {"INSERTQ", nullptr, 0};
|
||||
inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0};
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ struct Amd64OnlyMatch {
|
|||
std::size_t ReturnBranchOffset;
|
||||
bool Optional = false;
|
||||
std::vector<StubRelocation> Relocations = {};
|
||||
bool RelocatableWindow = false;
|
||||
};
|
||||
|
||||
struct Amd64OnlySubstitutionReport {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#define CODEGEN_X86_IAMD64ONLYINSTRUCTIONMATCHER_HPP
|
||||
|
||||
#include <codegen/x86/Amd64OnlySubstitutionTypes.hpp>
|
||||
#include <codegen/x86/StubBodyBuilder.hpp>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
|
@ -23,6 +24,11 @@ public:
|
|||
std::span<const std::span<const std::uint8_t>> instructions,
|
||||
std::span<const std::uint8_t> trailing
|
||||
) const = 0;
|
||||
|
||||
virtual void EmitOutOfLine(
|
||||
StubBodyBuilder& body,
|
||||
std::span<const std::uint8_t> instruction
|
||||
) const = 0;
|
||||
};
|
||||
|
||||
std::unique_ptr<IAmd64OnlyInstructionMatcher> MakeAmd64OnlyInstructionMatcher(Amd64OnlyTarget target = Amd64OnlyTarget::Intel);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#ifndef CODEGEN_X86_STUBBODYBUILDER_HPP
|
||||
#define CODEGEN_X86_STUBBODYBUILDER_HPP
|
||||
|
||||
#include <codegen/CodegenTypes.hpp>
|
||||
#include <codegen/x86/Amd64OnlySubstitutionTypes.hpp>
|
||||
#include <domain/Types.hpp>
|
||||
#include <array>
|
||||
|
|
@ -53,6 +54,10 @@ public:
|
|||
void Restore(std::uint8_t reg);
|
||||
void Raw(std::span<const std::uint8_t> bytes);
|
||||
void Move(std::span<const std::uint8_t> instruction, std::optional<std::size_t> ripDisplacementOffset);
|
||||
void RipRelative(std::span<const std::uint8_t> instruction, std::size_t displacementOffset, std::int64_t target);
|
||||
void MoveControlFlow(std::span<const std::uint8_t> instruction, const DecodedInstructionInfo& info);
|
||||
void JumpTo(std::int64_t siteTarget);
|
||||
void Seek(std::int64_t siteOffset);
|
||||
void Advance(std::size_t length);
|
||||
[[nodiscard]] LoweredBody Finish();
|
||||
|
||||
|
|
@ -63,12 +68,15 @@ private:
|
|||
std::size_t ConstantIndex;
|
||||
};
|
||||
|
||||
void _rel32(std::initializer_list<std::uint8_t> opcode, std::int64_t siteTarget);
|
||||
void _pushReturnAddress(std::int64_t siteTarget);
|
||||
|
||||
std::vector<std::uint8_t> _bytes;
|
||||
std::vector<Fixup> _fixups;
|
||||
std::vector<StubConstant> _constants;
|
||||
std::vector<StubRelocation> _relocations;
|
||||
std::size_t _stackDepth = 0;
|
||||
std::size_t _siteOffset = 0;
|
||||
std::int64_t _siteOffset = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
#ifndef CODEGEN_X86_WIDESTORELOWERING_HPP
|
||||
#define CODEGEN_X86_WIDESTORELOWERING_HPP
|
||||
|
||||
#include <codegen/x86/StubBodyBuilder.hpp>
|
||||
#include <codegen/x86/WideStoreOperands.hpp>
|
||||
|
||||
namespace Codegen {
|
||||
|
||||
class WideStoreLowering {
|
||||
public:
|
||||
void EmitOutOfLine(StubBodyBuilder& body, const WideStoreOperands& operands) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
#ifndef CODEGEN_X86_WIDESTOREOPERANDS_HPP
|
||||
#define CODEGEN_X86_WIDESTOREOPERANDS_HPP
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
namespace Codegen {
|
||||
|
||||
struct WideStoreHalf {
|
||||
std::vector<std::uint8_t> Bytes;
|
||||
std::optional<std::size_t> RipDisplacementOffset;
|
||||
std::int64_t RipTarget = 0;
|
||||
};
|
||||
|
||||
struct WideStoreOperands {
|
||||
WideStoreHalf Low;
|
||||
WideStoreHalf High;
|
||||
};
|
||||
|
||||
[[nodiscard]] std::optional<WideStoreOperands> DecodeVexWideStore(const std::uint8_t* data, std::size_t length);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -3,9 +3,13 @@
|
|||
#include <codegen/CodegenException.hpp>
|
||||
#include <codegen/x86/Amd64OnlySubstitutionTable.hpp>
|
||||
#include <codegen/x86/X64InstructionDecoder.hpp>
|
||||
#include <codegen/x86/X64OpcodeConstants.hpp>
|
||||
#include <codegen/x86/IAmd64OnlyInstructionMatcher.hpp>
|
||||
#include <codegen/x86/StubBodyBuilder.hpp>
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <span>
|
||||
#include <string>
|
||||
|
|
@ -24,6 +28,181 @@ auto _atFileOffset(const Domain::FileByteOffset base, const TOperation& operatio
|
|||
}
|
||||
}
|
||||
|
||||
constexpr std::int64_t kShortJumpSize = 2;
|
||||
constexpr std::uint64_t kIslandSearchBytes = 0x100;
|
||||
|
||||
struct Placement {
|
||||
std::vector<TrampolineSite> Sites;
|
||||
std::vector<std::size_t> Claimed;
|
||||
std::vector<std::size_t> Consumed;
|
||||
std::size_t ReplacementLength = 0;
|
||||
};
|
||||
|
||||
class WindowPlacer {
|
||||
public:
|
||||
WindowPlacer(
|
||||
std::vector<std::uint8_t>& seg,
|
||||
const std::vector<InstructionMatch>& matches,
|
||||
const Domain::ProgramHeader& ph,
|
||||
const std::set<std::uint64_t>& branchTargets,
|
||||
const std::set<std::size_t>& pending,
|
||||
const std::set<std::size_t>& claimed,
|
||||
const IAmd64OnlyInstructionMatcher& matcher
|
||||
) : _seg(seg), _matches(matches), _ph(ph), _branchTargets(branchTargets), _pending(pending), _claimed(claimed), _matcher(matcher) {}
|
||||
|
||||
[[nodiscard]] std::optional<Placement> Window(const std::size_t index) const {
|
||||
if (index + 1 < _matches.size() && (_pending.contains(index + 1) || _movable(index + 1, true))) {
|
||||
if (auto placement = _relocate(index, index, index + 1))
|
||||
return placement;
|
||||
}
|
||||
if (index > 0 && _movable(index - 1, false))
|
||||
return _relocate(index, index - 1, index);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::optional<Placement> Island(const std::size_t index) {
|
||||
const auto store = _address(index);
|
||||
for (std::size_t distance = 1;; ++distance) {
|
||||
bool inReach = false;
|
||||
for (const bool before : {true, false}) {
|
||||
if (before ? distance > index : index + distance >= _matches.size())
|
||||
continue;
|
||||
const auto start = before ? index - distance : index + distance;
|
||||
if (_address(start) + kIslandSearchBytes < store || _address(start) > store + kIslandSearchBytes)
|
||||
continue;
|
||||
inReach = true;
|
||||
if (auto placement = _islandAt(index, start))
|
||||
return placement;
|
||||
}
|
||||
if (!inReach)
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::uint8_t>& _seg;
|
||||
const std::vector<InstructionMatch>& _matches;
|
||||
const Domain::ProgramHeader& _ph;
|
||||
const std::set<std::uint64_t>& _branchTargets;
|
||||
const std::set<std::size_t>& _pending;
|
||||
const std::set<std::size_t>& _claimed;
|
||||
const IAmd64OnlyInstructionMatcher& _matcher;
|
||||
X64InstructionDecoder _decoder;
|
||||
|
||||
[[nodiscard]] std::uint64_t _address(const std::size_t index) const { return _ph.MappedAddress + _matches[index].Offset; }
|
||||
[[nodiscard]] std::uint64_t _end(const std::size_t index) const { return _address(index) + _matches[index].Length; }
|
||||
[[nodiscard]] std::span<const std::uint8_t> _bytes(const std::size_t index) const { return {_seg.data() + _matches[index].Offset, _matches[index].Length}; }
|
||||
[[nodiscard]] DecodedInstructionInfo _info(const std::size_t index) const { return _decoder.DecodeInstruction(_seg.data() + _matches[index].Offset, _matches[index].Length); }
|
||||
|
||||
[[nodiscard]] bool _entered(const std::uint64_t start, const std::uint64_t end) const {
|
||||
const auto hit = _branchTargets.upper_bound(start);
|
||||
return hit != _branchTargets.end() && *hit < end;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool _blocksFollowing(const std::size_t index) const {
|
||||
const auto kind = _info(index).FlowKind;
|
||||
return kind != ControlFlowKind::Sequential && kind != ControlFlowKind::ConditionalBranch;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool _movable(const std::size_t index, const bool last) const {
|
||||
return !_claimed.contains(index) && !_pending.contains(index) && (last || !_blocksFollowing(index));
|
||||
}
|
||||
|
||||
[[nodiscard]] TrampolineSite _site(const std::uint64_t start, const std::size_t length, LoweredBody body) const {
|
||||
const auto offset = static_cast<std::size_t>(start - _ph.MappedAddress);
|
||||
return {
|
||||
static_cast<Domain::FileByteOffset>(_ph.Offset + offset),
|
||||
static_cast<Domain::VirtualAddress>(start),
|
||||
length,
|
||||
std::vector<std::uint8_t>(_seg.begin() + static_cast<std::ptrdiff_t>(offset), _seg.begin() + static_cast<std::ptrdiff_t>(offset + length)),
|
||||
std::move(body.Bytes),
|
||||
body.ReturnBranchOffset,
|
||||
std::move(body.Relocations)
|
||||
};
|
||||
}
|
||||
|
||||
[[nodiscard]] std::optional<Placement> _relocate(const std::size_t index, const std::size_t first, const std::size_t last) const {
|
||||
if (_entered(_address(first), _end(last)))
|
||||
return std::nullopt;
|
||||
StubBodyBuilder body;
|
||||
Placement placement;
|
||||
try {
|
||||
for (auto k = first; k <= last; ++k) {
|
||||
if (_pending.contains(k)) {
|
||||
_matcher.EmitOutOfLine(body, _bytes(k));
|
||||
if (k != index)
|
||||
placement.Consumed.push_back(k);
|
||||
} else {
|
||||
body.MoveControlFlow(_bytes(k), _info(k));
|
||||
}
|
||||
placement.Claimed.push_back(k);
|
||||
}
|
||||
} catch (const CodegenException&) {
|
||||
return std::nullopt;
|
||||
}
|
||||
placement.Sites.push_back(_site(_address(first), static_cast<std::size_t>(_end(last) - _address(first)), body.Finish()));
|
||||
placement.ReplacementLength = placement.Sites.back().Body.size();
|
||||
return placement;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::optional<Placement> _islandAt(const std::size_t index, const std::size_t start) {
|
||||
using Amd64OnlySubstitutionTable::kJmpRel32;
|
||||
std::size_t head = 0;
|
||||
for (auto last = start; last < _matches.size(); ++last) {
|
||||
if (!_movable(last, true) || _entered(_address(start), _end(last)))
|
||||
return std::nullopt;
|
||||
if (head == 0 && _end(last) - _address(start) >= kJmpRel32.Size)
|
||||
head = static_cast<std::size_t>(_end(last) - _address(start));
|
||||
const auto island = _address(start) + head;
|
||||
const auto reach = static_cast<std::int64_t>(island) - static_cast<std::int64_t>(_address(index) + kShortJumpSize);
|
||||
if (head != 0 && (reach < std::numeric_limits<std::int8_t>::min() || reach > std::numeric_limits<std::int8_t>::max()))
|
||||
return std::nullopt;
|
||||
if (head != 0 && _end(last) >= island + kJmpRel32.Size) {
|
||||
try {
|
||||
return _buildIsland(index, start, last, head);
|
||||
} catch (const CodegenException&) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
if (_blocksFollowing(last))
|
||||
return std::nullopt;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
[[nodiscard]] Placement _buildIsland(const std::size_t index, const std::size_t start, const std::size_t last, const std::size_t head) {
|
||||
using Amd64OnlySubstitutionTable::kJmpRel32;
|
||||
const auto windowStart = _address(start);
|
||||
const auto windowEnd = _end(last);
|
||||
const auto island = windowStart + head;
|
||||
const auto store = _address(index);
|
||||
Placement placement;
|
||||
|
||||
StubBodyBuilder moved;
|
||||
for (auto k = start; k <= last; ++k) {
|
||||
moved.MoveControlFlow(_bytes(k), _info(k));
|
||||
placement.Claimed.push_back(k);
|
||||
}
|
||||
if (!_blocksFollowing(last))
|
||||
moved.JumpTo(static_cast<std::int64_t>(windowEnd - windowStart));
|
||||
placement.Sites.push_back(_site(windowStart, head, moved.Finish()));
|
||||
|
||||
StubBodyBuilder lowered;
|
||||
lowered.Seek(static_cast<std::int64_t>(store) - static_cast<std::int64_t>(island));
|
||||
_matcher.EmitOutOfLine(lowered, _bytes(index));
|
||||
lowered.JumpTo(static_cast<std::int64_t>(_end(index)) - static_cast<std::int64_t>(island));
|
||||
placement.Sites.push_back(_site(island, kJmpRel32.Size, lowered.Finish()));
|
||||
placement.ReplacementLength = placement.Sites.back().Body.size();
|
||||
placement.Claimed.push_back(index);
|
||||
|
||||
const auto offset = _matches[index].Offset;
|
||||
std::fill_n(_seg.begin() + static_cast<std::ptrdiff_t>(offset), _matches[index].Length, Amd64OnlySubstitutionTable::kTrapFill);
|
||||
_seg[offset] = X64OpcodeConstants::OneByteJmpRel8;
|
||||
_seg[offset + 1] = static_cast<std::uint8_t>(static_cast<std::int8_t>(static_cast<std::int64_t>(island) - static_cast<std::int64_t>(store + kShortJumpSize)));
|
||||
return placement;
|
||||
}
|
||||
};
|
||||
|
||||
class Amd64OnlyConverter : public IAmd64OnlyConverter {
|
||||
public:
|
||||
explicit Amd64OnlyConverter(const Amd64OnlyTarget target) : _matcher(MakeAmd64OnlyInstructionMatcher(target)) {}
|
||||
|
|
@ -106,6 +285,10 @@ void Amd64OnlyConverter::_convertSegment(
|
|||
});
|
||||
const auto branchTargets = needsBranchTargets ? _collectBranchTargets(seg, matches, ph) : std::set<std::uint64_t>{};
|
||||
|
||||
std::set<std::size_t> pendingIndices;
|
||||
for (const auto& item : pending)
|
||||
pendingIndices.insert(item.Index);
|
||||
std::set<std::size_t> claimed;
|
||||
std::set<std::size_t> consumed;
|
||||
for (const auto& item : pending) {
|
||||
if (consumed.contains(item.Index))
|
||||
|
|
@ -126,6 +309,20 @@ void Amd64OnlyConverter::_convertSegment(
|
|||
break;
|
||||
}
|
||||
case Amd64OnlyLowering::Trampoline: {
|
||||
if (substitution.RelocatableWindow && match.Length < Amd64OnlySubstitutionTable::kJmpRel32.Size) {
|
||||
WindowPlacer placer(seg, matches, ph, branchTargets, pendingIndices, claimed, *_matcher);
|
||||
auto placement = placer.Window(item.Index);
|
||||
if (!placement)
|
||||
placement = placer.Island(item.Index);
|
||||
if (!placement)
|
||||
throw CodegenException("No room for a jump to the stub of " + substitution.InstructionName + " within its neighbours or a nearby island", fileOffset);
|
||||
for (auto& site : placement->Sites)
|
||||
result.Trampolines.push_back(std::move(site));
|
||||
claimed.insert(placement->Claimed.begin(), placement->Claimed.end());
|
||||
consumed.insert(placement->Consumed.begin(), placement->Consumed.end());
|
||||
replacementLength = placement->ReplacementLength;
|
||||
break;
|
||||
}
|
||||
std::vector<std::size_t> taken;
|
||||
try {
|
||||
std::size_t siteLength = match.Length;
|
||||
|
|
@ -172,6 +369,8 @@ void Amd64OnlyConverter::_convertSegment(
|
|||
});
|
||||
replacementLength = stub.StubBody.size();
|
||||
consumed.insert(taken.begin(), taken.end());
|
||||
for (auto k = item.Index; k < matches.size() && matches[k].Offset < match.Offset + siteLength; ++k)
|
||||
claimed.insert(k);
|
||||
} catch (const CodegenException&) {
|
||||
if (!substitution.Optional)
|
||||
throw;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@
|
|||
#include <codegen/x86/Sha1Lowering.hpp>
|
||||
#include <codegen/x86/Sha1Operands.hpp>
|
||||
#include <codegen/x86/StubBodyBuilder.hpp>
|
||||
#include <codegen/x86/WideStoreLowering.hpp>
|
||||
#include <codegen/x86/WideStoreOperands.hpp>
|
||||
#include <codegen/x86/X64InstructionDecoder.hpp>
|
||||
#include <codegen/x86/X64OpcodeConstants.hpp>
|
||||
#include <codegen/CodegenException.hpp>
|
||||
|
|
@ -72,8 +74,8 @@ const Entry& _sha1Entry(const Sha1Operands& operands) {
|
|||
return kSha1rnds4;
|
||||
}
|
||||
|
||||
Amd64OnlyMatch _trampoline(std::string name, const std::size_t length, LoweredBody body, const bool optional = false) {
|
||||
return Amd64OnlyMatch{std::move(name), length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, optional, std::move(body.Relocations)};
|
||||
Amd64OnlyMatch _trampoline(std::string name, const std::size_t length, LoweredBody body, const bool optional = false, const bool window = false) {
|
||||
return Amd64OnlyMatch{std::move(name), length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, optional, std::move(body.Relocations), window};
|
||||
}
|
||||
|
||||
void _moveTrailing(StubBodyBuilder& body, std::span<const std::uint8_t> trailing) {
|
||||
|
|
@ -142,6 +144,11 @@ public:
|
|||
std::span<const std::uint8_t> trailing
|
||||
) const override;
|
||||
|
||||
void EmitOutOfLine(
|
||||
StubBodyBuilder& body,
|
||||
std::span<const std::uint8_t> instruction
|
||||
) const override;
|
||||
|
||||
private:
|
||||
Amd64OnlyTarget _target;
|
||||
Sse4aLowering _lowering;
|
||||
|
|
@ -149,6 +156,7 @@ private:
|
|||
Sha1Lowering _sha1Lowering;
|
||||
ClzeroLowering _clzeroLowering;
|
||||
ReciprocalLowering _reciprocalLowering;
|
||||
WideStoreLowering _wideStoreLowering;
|
||||
|
||||
[[nodiscard]] Amd64OnlyMatch _matchMovnts(const DecodedInstruction& instr, const Entry& entry) const;
|
||||
[[nodiscard]] Amd64OnlyMatch _matchSse4a(const DecodedInstruction& instr, const Entry& entry, const Entry& registerFormEntry, std::span<const std::uint8_t> trailing) const;
|
||||
|
|
@ -156,6 +164,7 @@ private:
|
|||
[[nodiscard]] Amd64OnlyMatch _matchSha1(const DecodedInstruction& instr, std::span<const std::uint8_t> trailing) const;
|
||||
[[nodiscard]] Amd64OnlyMatch _matchClzero(const DecodedInstruction& instr, std::span<const std::uint8_t> trailing) const;
|
||||
[[nodiscard]] std::optional<Amd64OnlyMatch> _matchRosetta(const DecodedInstruction& instr, std::span<const std::uint8_t> trailing) const;
|
||||
[[nodiscard]] const char* _emitOutOfLine(StubBodyBuilder& body, const DecodedInstruction& instr) const;
|
||||
};
|
||||
|
||||
Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchMovnts(const DecodedInstruction& instr, const Entry& entry) const {
|
||||
|
|
@ -208,9 +217,52 @@ std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::_matchRosetta(const D
|
|||
return _inPlace(kClwb, instr.Length, {});
|
||||
if (instr.IsRdpid())
|
||||
return _trampoline(kRdpid.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { body.Raw(_rdpidReplacement(instr)); }), true);
|
||||
if (const auto wide = DecodeVexWideStore(instr.Data, instr.Length))
|
||||
return _trampoline(kWideStore.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _wideStoreLowering.EmitOutOfLine(body, *wide); }), false, true);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, const DecodedInstruction& instr) const {
|
||||
if (instr.IsExtrq() || instr.IsInsertq()) {
|
||||
const auto operands = DecodeSse4a(instr.Data, instr.Length);
|
||||
_lowering.EmitOutOfLine(body, operands);
|
||||
return _sse4aEntry(operands).Name;
|
||||
}
|
||||
if (instr.IsSha256()) {
|
||||
const auto operands = DecodeSha256(instr.Data, instr.Length);
|
||||
_sha256Lowering.EmitOutOfLine(body, operands);
|
||||
return _sha256Entry(operands).Name;
|
||||
}
|
||||
if (instr.IsSha1()) {
|
||||
const auto operands = DecodeSha1(instr.Data, instr.Length);
|
||||
_sha1Lowering.EmitOutOfLine(body, operands);
|
||||
return _sha1Entry(operands).Name;
|
||||
}
|
||||
if (instr.IsClzero()) {
|
||||
_clzeroLowering.EmitOutOfLine(body, DecodeClzero(instr.Data, instr.Length));
|
||||
return kClzero.Name;
|
||||
}
|
||||
if (const auto reciprocal = DecodeReciprocal(instr.Data, instr.Length)) {
|
||||
_reciprocalLowering.EmitOutOfLine(body, *reciprocal);
|
||||
return _reciprocalEntry(*reciprocal).Name;
|
||||
}
|
||||
if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdpid()) {
|
||||
body.Raw(_rdpidReplacement(instr));
|
||||
return kRdpid.Name;
|
||||
}
|
||||
if (const auto wide = _target == Amd64OnlyTarget::Rosetta ? DecodeVexWideStore(instr.Data, instr.Length) : std::nullopt) {
|
||||
_wideStoreLowering.EmitOutOfLine(body, *wide);
|
||||
return kWideStore.Name;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void Amd64OnlyInstructionMatcher::EmitOutOfLine(StubBodyBuilder& body, const std::span<const std::uint8_t> instruction) const {
|
||||
if (_emitOutOfLine(body, DecodedInstruction{instruction.data(), instruction.size()}) == nullptr)
|
||||
throw CodegenException("AMD-only instruction has no out-of-line lowering");
|
||||
body.Advance(instruction.size());
|
||||
}
|
||||
|
||||
std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::MatchSequence(
|
||||
std::span<const std::span<const std::uint8_t>> instructions,
|
||||
std::span<const std::uint8_t> trailing
|
||||
|
|
@ -221,37 +273,11 @@ std::optional<Amd64OnlyMatch> Amd64OnlyInstructionMatcher::MatchSequence(
|
|||
const char* name = nullptr;
|
||||
for (const auto& instruction : instructions) {
|
||||
const DecodedInstruction instr{instruction.data(), instruction.size()};
|
||||
if (instr.IsExtrq() || instr.IsInsertq()) {
|
||||
const auto operands = DecodeSse4a(instr.Data, instr.Length);
|
||||
if (name == nullptr)
|
||||
name = _sse4aEntry(operands).Name;
|
||||
_lowering.EmitOutOfLine(body, operands);
|
||||
} else if (instr.IsSha256()) {
|
||||
const auto operands = DecodeSha256(instr.Data, instr.Length);
|
||||
if (name == nullptr)
|
||||
name = _sha256Entry(operands).Name;
|
||||
_sha256Lowering.EmitOutOfLine(body, operands);
|
||||
} else if (instr.IsSha1()) {
|
||||
const auto operands = DecodeSha1(instr.Data, instr.Length);
|
||||
if (name == nullptr)
|
||||
name = _sha1Entry(operands).Name;
|
||||
_sha1Lowering.EmitOutOfLine(body, operands);
|
||||
} else if (instr.IsClzero()) {
|
||||
const auto operands = DecodeClzero(instr.Data, instr.Length);
|
||||
if (name == nullptr)
|
||||
name = kClzero.Name;
|
||||
_clzeroLowering.EmitOutOfLine(body, operands);
|
||||
} else if (const auto reciprocal = DecodeReciprocal(instr.Data, instr.Length)) {
|
||||
if (name == nullptr)
|
||||
name = _reciprocalEntry(*reciprocal).Name;
|
||||
_reciprocalLowering.EmitOutOfLine(body, *reciprocal);
|
||||
} else if (_target == Amd64OnlyTarget::Rosetta && instr.IsRdpid()) {
|
||||
if (name == nullptr)
|
||||
name = kRdpid.Name;
|
||||
body.Raw(_rdpidReplacement(instr));
|
||||
} else {
|
||||
const auto emitted = _emitOutOfLine(body, instr);
|
||||
if (emitted == nullptr)
|
||||
return std::nullopt;
|
||||
}
|
||||
if (name == nullptr)
|
||||
name = emitted;
|
||||
body.Advance(instr.Length);
|
||||
}
|
||||
_moveTrailing(body, trailing);
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include <codegen/x86/Amd64OnlySubstitutionTable.hpp>
|
||||
#include <codegen/x86/X64OpcodeConstants.hpp>
|
||||
#include <codegen/CodegenException.hpp>
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
|
||||
namespace Codegen {
|
||||
|
|
@ -150,7 +151,7 @@ void StubBodyBuilder::Load(const std::uint8_t reg, const MemoryOperand& operand)
|
|||
if (operand.Rm == ModRmRmSibPresent)
|
||||
_bytes.push_back(operand.Sib);
|
||||
if (operand.RipRelative) {
|
||||
_relocations.push_back({_bytes.size(), _bytes.size() + Disp32Size, static_cast<std::int64_t>(_siteOffset + operand.InstructionLength) + operand.Displacement});
|
||||
_relocations.push_back({_bytes.size(), _bytes.size() + Disp32Size, _siteOffset + static_cast<std::int64_t>(operand.InstructionLength) + operand.Displacement});
|
||||
_bytes.insert(_bytes.end(), Disp32Size, 0);
|
||||
return;
|
||||
}
|
||||
|
|
@ -189,13 +190,82 @@ void StubBodyBuilder::Move(const std::span<const std::uint8_t> instruction, cons
|
|||
value |= static_cast<std::uint32_t>(instruction[*ripDisplacementOffset + index]) << (index * 8);
|
||||
_bytes[start + *ripDisplacementOffset + index] = 0;
|
||||
}
|
||||
_relocations.push_back({start + *ripDisplacementOffset, start + instruction.size(), static_cast<std::int64_t>(_siteOffset + instruction.size()) + static_cast<std::int32_t>(value)});
|
||||
_relocations.push_back({start + *ripDisplacementOffset, start + instruction.size(), _siteOffset + static_cast<std::int64_t>(instruction.size()) + static_cast<std::int32_t>(value)});
|
||||
}
|
||||
Advance(instruction.size());
|
||||
}
|
||||
|
||||
void StubBodyBuilder::RipRelative(const std::span<const std::uint8_t> instruction, const std::size_t displacementOffset, const std::int64_t target) {
|
||||
using namespace X64OpcodeConstants;
|
||||
if (displacementOffset > instruction.size() || instruction.size() - displacementOffset < Disp32Size)
|
||||
throw CodegenException("RIP-relative displacement lies outside its instruction");
|
||||
const auto start = _bytes.size();
|
||||
Raw(instruction);
|
||||
std::fill_n(_bytes.begin() + static_cast<std::ptrdiff_t>(start + displacementOffset), Disp32Size, std::uint8_t{0});
|
||||
_relocations.push_back({start + displacementOffset, start + instruction.size(), _siteOffset + target});
|
||||
}
|
||||
|
||||
void StubBodyBuilder::MoveControlFlow(const std::span<const std::uint8_t> instruction, const DecodedInstructionInfo& info) {
|
||||
using namespace X64OpcodeConstants;
|
||||
const auto end = _siteOffset + static_cast<std::int64_t>(instruction.size());
|
||||
if (info.FlowKind == ControlFlowKind::IndirectCall)
|
||||
throw CodegenException("A register or memory call cannot move into a stub");
|
||||
if (info.FlowKind == ControlFlowKind::Call && info.HasRipRelativeDisp) {
|
||||
_pushReturnAddress(end);
|
||||
std::vector<std::uint8_t> jump(instruction.begin(), instruction.end());
|
||||
auto& modrm = jump[info.RipRelativeDispOffset - 1];
|
||||
modrm = static_cast<std::uint8_t>((modrm & ~(ModRmRegMask << ModRmRegShift)) | (4u << ModRmRegShift));
|
||||
Move(jump, info.RipRelativeDispOffset);
|
||||
return;
|
||||
}
|
||||
if (!info.HasBranchTarget || info.HasRipRelativeDisp) {
|
||||
Move(instruction, info.HasRipRelativeDisp ? std::optional<std::size_t>(info.RipRelativeDispOffset) : std::nullopt);
|
||||
return;
|
||||
}
|
||||
for (std::size_t index = 0; index < info.OpcodeOffset; ++index) {
|
||||
const auto prefix = instruction[index];
|
||||
if (prefix != PrefixSegCs && prefix != PrefixSegDs && prefix != PrefixRepne && prefix != PrefixAddressSize && (prefix & 0xF0) != kRexBase)
|
||||
throw CodegenException("A relative branch with this prefix cannot move into a stub");
|
||||
}
|
||||
const auto target = end + info.BranchDisp;
|
||||
if (info.IsTwoByteOpcode) {
|
||||
_rel32({TwoByteOpcodeEscape, info.Opcode}, target);
|
||||
} else if (info.Opcode == OneByteCallRel32) {
|
||||
_pushReturnAddress(end);
|
||||
_rel32({OneByteJmpRel32}, target);
|
||||
} else if (info.Opcode == OneByteJmpRel32 || info.Opcode == OneByteJmpRel8) {
|
||||
_rel32({OneByteJmpRel32}, target);
|
||||
} else if (info.Opcode >= OneByteJccRel8Min && info.Opcode <= OneByteJccRel8Max) {
|
||||
_rel32({TwoByteOpcodeEscape, static_cast<std::uint8_t>(TwoByteJccRel32Min | (info.Opcode & 0x0F))}, target);
|
||||
} else {
|
||||
throw CodegenException("A LOOP or JRCXZ branch cannot move into a stub");
|
||||
}
|
||||
Advance(instruction.size());
|
||||
}
|
||||
|
||||
void StubBodyBuilder::JumpTo(const std::int64_t siteTarget) {
|
||||
_rel32({X64OpcodeConstants::OneByteJmpRel32}, siteTarget);
|
||||
}
|
||||
|
||||
void StubBodyBuilder::Seek(const std::int64_t siteOffset) {
|
||||
_siteOffset = siteOffset;
|
||||
}
|
||||
|
||||
void StubBodyBuilder::Advance(const std::size_t length) {
|
||||
_siteOffset += length;
|
||||
_siteOffset += static_cast<std::int64_t>(length);
|
||||
}
|
||||
|
||||
void StubBodyBuilder::_rel32(const std::initializer_list<std::uint8_t> opcode, const std::int64_t siteTarget) {
|
||||
_bytes.insert(_bytes.end(), opcode);
|
||||
const auto displacement = _bytes.size();
|
||||
_bytes.insert(_bytes.end(), X64OpcodeConstants::Disp32Size, std::uint8_t{0});
|
||||
_relocations.push_back({displacement, _bytes.size(), siteTarget});
|
||||
}
|
||||
|
||||
void StubBodyBuilder::_pushReturnAddress(const std::int64_t siteTarget) {
|
||||
_bytes.insert(_bytes.end(), {0x48, 0x8D, 0x64, 0x24, 0xF8, 0x50});
|
||||
_rel32({0x48, 0x8D, 0x05}, siteTarget);
|
||||
_bytes.insert(_bytes.end(), {0x48, 0x89, 0x44, 0x24, 0x08, 0x58});
|
||||
}
|
||||
|
||||
LoweredBody StubBodyBuilder::Finish() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
#include <codegen/x86/WideStoreLowering.hpp>
|
||||
|
||||
namespace Codegen {
|
||||
|
||||
namespace {
|
||||
|
||||
void _emitHalf(StubBodyBuilder& body, const WideStoreHalf& half) {
|
||||
if (half.RipDisplacementOffset)
|
||||
body.RipRelative(half.Bytes, *half.RipDisplacementOffset, half.RipTarget);
|
||||
else
|
||||
body.Raw(half.Bytes);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void WideStoreLowering::EmitOutOfLine(StubBodyBuilder& body, const WideStoreOperands& operands) const {
|
||||
_emitHalf(body, operands.Low);
|
||||
_emitHalf(body, operands.High);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
#include <codegen/x86/WideStoreOperands.hpp>
|
||||
#include <codegen/x86/X64OpcodeConstants.hpp>
|
||||
#include <codegen/CodegenException.hpp>
|
||||
#include <limits>
|
||||
|
||||
namespace Codegen {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::uint8_t kVexMap0F = 0x01;
|
||||
constexpr std::uint8_t kVexLength256 = 0x04;
|
||||
constexpr std::uint8_t kVexUnusedRegister = 0x0F;
|
||||
constexpr std::uint8_t kPpNone = 0x00;
|
||||
constexpr std::uint8_t kPp66 = 0x01;
|
||||
constexpr std::uint8_t kPpF3 = 0x02;
|
||||
constexpr std::uint8_t kVextractf128 = 0x19;
|
||||
constexpr std::uint8_t kVextractf128Payload = 0x7D;
|
||||
constexpr std::uint8_t kUpperHalf = 0x01;
|
||||
constexpr std::int64_t kHalfBytes = 16;
|
||||
|
||||
bool _unalignedStore(const std::uint8_t pp, const std::uint8_t opcode) {
|
||||
return ((pp == kPpNone || pp == kPp66) && opcode == 0x11) || (pp == kPpF3 && opcode == 0x7F);
|
||||
}
|
||||
|
||||
bool _legacyPrefixAllowed(const std::uint8_t prefix) {
|
||||
using namespace X64OpcodeConstants;
|
||||
return prefix == PrefixSegCs || prefix == PrefixSegSs || prefix == PrefixSegDs || prefix == PrefixSegEs || prefix == PrefixSegFs || prefix == PrefixSegGs || prefix == PrefixAddressSize;
|
||||
}
|
||||
|
||||
void _appendU32(std::vector<std::uint8_t>& bytes, const std::uint32_t value) {
|
||||
for (std::size_t index = 0; index < X64OpcodeConstants::Disp32Size; ++index)
|
||||
bytes.push_back(static_cast<std::uint8_t>(value >> (index * 8)));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::optional<WideStoreOperands> DecodeVexWideStore(const std::uint8_t* data, const std::size_t length) {
|
||||
using namespace X64OpcodeConstants;
|
||||
std::size_t pos = 0;
|
||||
while (pos < length && _legacyPrefixAllowed(data[pos]))
|
||||
++pos;
|
||||
const auto vex = pos;
|
||||
if (vex + 2 > length)
|
||||
return std::nullopt;
|
||||
std::uint8_t inverted = 0;
|
||||
std::uint8_t payload = 0;
|
||||
if (data[vex] == OneByteVex2) {
|
||||
inverted = static_cast<std::uint8_t>((data[vex + 1] & 0x80) | 0x60);
|
||||
payload = data[vex + 1];
|
||||
pos = vex + Vex2PrefixLength;
|
||||
} else if (data[vex] == OneByteVex3) {
|
||||
if (vex + Vex3PrefixLength > length || (data[vex + 1] & Vex3MapMask) != kVexMap0F)
|
||||
return std::nullopt;
|
||||
inverted = static_cast<std::uint8_t>(data[vex + 1] & 0xE0);
|
||||
payload = data[vex + 2];
|
||||
pos = vex + Vex3PrefixLength;
|
||||
} else {
|
||||
return std::nullopt;
|
||||
}
|
||||
if ((payload & kVexLength256) == 0 || ((payload >> 3) & 0x0F) != kVexUnusedRegister)
|
||||
return std::nullopt;
|
||||
if (pos + 2 > length || !_unalignedStore(static_cast<std::uint8_t>(payload & 0x03), data[pos]))
|
||||
return std::nullopt;
|
||||
const auto modRmOffset = pos + 1;
|
||||
const auto modrm = data[modRmOffset];
|
||||
const auto mod = static_cast<std::uint8_t>((modrm >> ModRmModShift) & ModRmModMask);
|
||||
const auto rm = static_cast<std::uint8_t>(modrm & ModRmRmMask);
|
||||
if (mod == ModRmModRegister)
|
||||
return std::nullopt;
|
||||
auto cursor = modRmOffset + 1;
|
||||
std::optional<std::uint8_t> sib;
|
||||
if (rm == ModRmRmSibPresent) {
|
||||
if (cursor >= length)
|
||||
return std::nullopt;
|
||||
sib = data[cursor++];
|
||||
}
|
||||
const bool ripRelative = mod == ModRmModIndirect && rm == ModRmRmRipRelative;
|
||||
const bool absolute = mod == ModRmModIndirect && sib.has_value() && (*sib & SibBaseMask) == SibBaseDisp32;
|
||||
const std::size_t displacementSize = mod == ModRmModDisp8 ? Disp8Size : (mod == ModRmModDisp32 || ripRelative || absolute) ? Disp32Size : 0;
|
||||
if (cursor + displacementSize != length)
|
||||
return std::nullopt;
|
||||
std::int64_t displacement = 0;
|
||||
if (displacementSize == Disp8Size) {
|
||||
displacement = static_cast<std::int8_t>(data[cursor]);
|
||||
} else if (displacementSize == Disp32Size) {
|
||||
std::uint32_t value = 0;
|
||||
for (std::size_t index = 0; index < Disp32Size; ++index)
|
||||
value |= static_cast<std::uint32_t>(data[cursor + index]) << (index * 8);
|
||||
displacement = static_cast<std::int32_t>(value);
|
||||
}
|
||||
|
||||
WideStoreOperands operands;
|
||||
operands.Low.Bytes.assign(data, data + length);
|
||||
operands.Low.Bytes[data[vex] == OneByteVex2 ? vex + 1 : vex + 2] &= static_cast<std::uint8_t>(~kVexLength256);
|
||||
if (ripRelative) {
|
||||
operands.Low.RipDisplacementOffset = cursor;
|
||||
operands.Low.RipTarget = static_cast<std::int64_t>(length) + displacement;
|
||||
}
|
||||
|
||||
auto& high = operands.High.Bytes;
|
||||
high.assign(data, data + vex);
|
||||
high.insert(high.end(), {OneByteVex3, static_cast<std::uint8_t>(inverted | Vex3Map0F3A), kVextractf128Payload, kVextractf128});
|
||||
const auto reg = static_cast<std::uint8_t>(modrm & (ModRmRegMask << ModRmRegShift));
|
||||
std::int64_t highDisplacement = displacement + kHalfBytes;
|
||||
std::uint8_t highMod = mod;
|
||||
std::size_t highDisplacementSize = displacementSize;
|
||||
if (!ripRelative && !absolute) {
|
||||
if (mod == ModRmModIndirect || (mod == ModRmModDisp8 && highDisplacement > std::numeric_limits<std::int8_t>::max())) {
|
||||
highMod = highDisplacement > std::numeric_limits<std::int8_t>::max() ? ModRmModDisp32 : ModRmModDisp8;
|
||||
highDisplacementSize = highMod == ModRmModDisp8 ? Disp8Size : Disp32Size;
|
||||
}
|
||||
}
|
||||
if (!ripRelative && highDisplacement > std::numeric_limits<std::int32_t>::max())
|
||||
throw CodegenException("256-bit store displacement leaves no room for its upper half");
|
||||
high.push_back(static_cast<std::uint8_t>((highMod << ModRmModShift) | reg | rm));
|
||||
if (sib)
|
||||
high.push_back(*sib);
|
||||
const auto highDisplacementOffset = high.size();
|
||||
if (highDisplacementSize == Disp8Size)
|
||||
high.push_back(static_cast<std::uint8_t>(static_cast<std::int8_t>(highDisplacement)));
|
||||
else if (ripRelative)
|
||||
_appendU32(high, 0);
|
||||
else if (highDisplacementSize == Disp32Size)
|
||||
_appendU32(high, static_cast<std::uint32_t>(static_cast<std::int32_t>(highDisplacement)));
|
||||
high.push_back(kUpperHalf);
|
||||
if (ripRelative) {
|
||||
operands.High.RipDisplacementOffset = highDisplacementOffset;
|
||||
operands.High.RipTarget = static_cast<std::int64_t>(length) + displacement + kHalfBytes;
|
||||
}
|
||||
return operands;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@
|
|||
#include <codegen/x86/ClzeroOperands.hpp>
|
||||
#include <codegen/x86/ClzeroLowering.hpp>
|
||||
#include <codegen/x86/ReciprocalOperands.hpp>
|
||||
#include <codegen/x86/WideStoreOperands.hpp>
|
||||
#include <codegen/x86/StubBodyBuilder.hpp>
|
||||
#include <codegen/x86/DecodedInstruction.hpp>
|
||||
#include <codegen/x86/X64InstructionDecoder.hpp>
|
||||
|
|
@ -670,6 +671,221 @@ void rosettaSubstitutions() {
|
|||
require(rdpid->ReturnBranchOffset == 6 && Bytes(rdpid->StubBody.begin(), rdpid->StubBody.begin() + 7) == Bytes{0x41, 0xBB, 0, 0, 0, 0, 0xE9}, "RDPID r11 stub does not load 0 before returning");
|
||||
}
|
||||
|
||||
void wideStoreSplit() {
|
||||
struct Case {
|
||||
Bytes Original;
|
||||
Bytes Low;
|
||||
Bytes High;
|
||||
const char* Name;
|
||||
};
|
||||
const std::vector<Case> cases = {
|
||||
{{0xC5, 0xFC, 0x11, 0x07}, {0xC5, 0xF8, 0x11, 0x07}, {0xC4, 0xE3, 0x7D, 0x19, 0x47, 0x10, 0x01}, "vmovups (%rdi)"},
|
||||
{{0xC4, 0x41, 0x7E, 0x7F, 0x4C, 0x87, 0x70}, {0xC4, 0x41, 0x7A, 0x7F, 0x4C, 0x87, 0x70}, {0xC4, 0x43, 0x7D, 0x19, 0x8C, 0x87, 0x80, 0x00, 0x00, 0x00, 0x01}, "vmovdqu 0x70(%r15,%rax,4)"},
|
||||
{{0xC5, 0xFC, 0x11, 0x4C, 0x24, 0xF8}, {0xC5, 0xF8, 0x11, 0x4C, 0x24, 0xF8}, {0xC4, 0xE3, 0x7D, 0x19, 0x4C, 0x24, 0x08, 0x01}, "vmovups -8(%rsp)"},
|
||||
{{0xC5, 0xFE, 0x7F, 0x93, 0x78, 0x56, 0x34, 0x12}, {0xC5, 0xFA, 0x7F, 0x93, 0x78, 0x56, 0x34, 0x12}, {0xC4, 0xE3, 0x7D, 0x19, 0x93, 0x88, 0x56, 0x34, 0x12, 0x01}, "vmovdqu 0x12345678(%rbx)"},
|
||||
{{0x64, 0xC5, 0xFC, 0x11, 0x04, 0x25, 0x40, 0x00, 0x00, 0x00}, {0x64, 0xC5, 0xF8, 0x11, 0x04, 0x25, 0x40, 0x00, 0x00, 0x00}, {0x64, 0xC4, 0xE3, 0x7D, 0x19, 0x04, 0x25, 0x50, 0x00, 0x00, 0x00, 0x01}, "vmovups %fs:0x40"},
|
||||
{{0xC4, 0x41, 0x7C, 0x11, 0x65, 0x00}, {0xC4, 0x41, 0x78, 0x11, 0x65, 0x00}, {0xC4, 0x43, 0x7D, 0x19, 0x65, 0x10, 0x01}, "vmovups (%r13)"},
|
||||
{{0xC4, 0xC1, 0x7C, 0x11, 0x47, 0x68}, {0xC4, 0xC1, 0x78, 0x11, 0x47, 0x68}, {0xC4, 0xC3, 0x7D, 0x19, 0x47, 0x78, 0x01}, "vmovups 0x68(%r15)"}};
|
||||
for (const auto& item : cases) {
|
||||
const auto split = Codegen::DecodeVexWideStore(item.Original.data(), item.Original.size());
|
||||
require(split && split->Low.Bytes == item.Low && split->High.Bytes == item.High && !split->Low.RipDisplacementOffset && !split->High.RipDisplacementOffset, (std::string("256-bit store was not split into its 128-bit halves: ") + item.Name).c_str());
|
||||
}
|
||||
const Bytes rip = {0xC5, 0xFD, 0x11, 0x1D, 0x00, 0x01, 0x00, 0x00};
|
||||
const auto ripSplit = Codegen::DecodeVexWideStore(rip.data(), rip.size());
|
||||
require(ripSplit && ripSplit->Low.Bytes == Bytes{0xC5, 0xF9, 0x11, 0x1D, 0x00, 0x01, 0x00, 0x00} && ripSplit->High.Bytes == Bytes{0xC4, 0xE3, 0x7D, 0x19, 0x1D, 0x00, 0x00, 0x00, 0x00, 0x01}, "RIP-relative 256-bit store was not split");
|
||||
require(ripSplit->Low.RipDisplacementOffset == 4 && ripSplit->Low.RipTarget == 8 + 0x100 && ripSplit->High.RipDisplacementOffset == 5 && ripSplit->High.RipTarget == 8 + 0x110, "RIP-relative halves do not address the original operand and the one 16 bytes above it");
|
||||
for (const auto& other : {Bytes{0xC5, 0xF8, 0x11, 0x07}, Bytes{0xC5, 0xFC, 0x11, 0xC1}, Bytes{0xC5, 0xFC, 0x10, 0x07}, Bytes{0xC4, 0xE2, 0x75, 0x2E, 0x07}, Bytes{0xC5, 0xF4, 0x11, 0x07}, Bytes{0xC4, 0xE3, 0x7D, 0x19, 0x07, 0x01}, Bytes{0xC5, 0xFC, 0x11, 0x04}, Bytes{0xC5, 0xFC, 0x11, 0x47}})
|
||||
require(!Codegen::DecodeVexWideStore(other.data(), other.size()).has_value(), "An instruction other than a plain 256-bit store was split");
|
||||
for (const auto& aligned : {Bytes{0xC5, 0xFC, 0x29, 0x4C, 0x24, 0xF8}, Bytes{0xC5, 0xFD, 0xE7, 0x93, 0x78, 0x56, 0x34, 0x12}, Bytes{0xC5, 0xFD, 0x7F, 0x1F}, Bytes{0xC5, 0xFC, 0x2B, 0x27}})
|
||||
require(!Codegen::DecodeVexWideStore(aligned.data(), aligned.size()).has_value(), "An aligned 256-bit store, which cannot cross a page, was split");
|
||||
const Bytes farRip = {0xC5, 0xFC, 0x11, 0x05, 0xF8, 0xFF, 0xFF, 0x7F};
|
||||
const auto farRipSplit = Codegen::DecodeVexWideStore(farRip.data(), farRip.size());
|
||||
require(farRipSplit && farRipSplit->High.RipTarget == std::int64_t{8} + 0x7FFFFFF8 + 0x10, "A RIP-relative 256-bit store near the end of its displacement range was not split");
|
||||
const Bytes farBase = {0xC5, 0xFC, 0x11, 0x80, 0xF8, 0xFF, 0xFF, 0x7F};
|
||||
requireFailure([&] { (void)Codegen::DecodeVexWideStore(farBase.data(), farBase.size()); }, "A 256-bit store whose upper half has no 32-bit displacement was kept");
|
||||
requireFailure([] { Codegen::StubBodyBuilder body; body.RipRelative(Bytes{0x90}, 0, 0); }, "A RIP-relative displacement outside its instruction was relocated");
|
||||
const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta);
|
||||
const Bytes store = {0xC4, 0xC1, 0x7C, 0x11, 0x47, 0x68};
|
||||
const auto lowered = rosetta->Match(store.data(), store.size());
|
||||
require(lowered && lowered->Lowering == Codegen::Amd64OnlyLowering::Trampoline && !lowered->Optional && lowered->RelocatableWindow && lowered->InstructionName == "256-bit store", "A 256-bit store was not lowered through a required stub that may move its neighbours for Rosetta");
|
||||
Bytes expectedBody = cases.back().Low;
|
||||
expectedBody.insert(expectedBody.end(), cases.back().High.begin(), cases.back().High.end());
|
||||
require(Bytes(lowered->StubBody.begin(), lowered->StubBody.begin() + static_cast<std::ptrdiff_t>(lowered->ReturnBranchOffset)) == expectedBody, "The 256-bit store stub is not the two halves");
|
||||
const std::vector<std::span<const std::uint8_t>> pair = {std::span<const std::uint8_t>(cases[0].Original), std::span<const std::uint8_t>(rip)};
|
||||
const auto sequence = rosetta->MatchSequence(pair, {});
|
||||
require(sequence && sequence->Relocations.size() == 2 && sequence->Relocations[0].SiteTarget == 4 + 8 + 0x100 && sequence->Relocations[1].SiteTarget == 4 + 8 + 0x110, "A RIP-relative 256-bit store after another is not relocated from its place in the site");
|
||||
require(!Codegen::MakeAmd64OnlyInstructionMatcher()->Match(store.data(), store.size()).has_value(), "A 256-bit store was split for an Intel host");
|
||||
const std::vector<std::span<const std::uint8_t>> single = {std::span<const std::uint8_t>(store)};
|
||||
require(!Codegen::MakeAmd64OnlyInstructionMatcher()->MatchSequence(single, {}).has_value(), "A 256-bit store was split in a sequence for an Intel host");
|
||||
}
|
||||
|
||||
void converterWideStoreWindows() {
|
||||
const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta);
|
||||
const Bytes store = {0xC5, 0xFC, 0x11, 0x07};
|
||||
Bytes halves = {0xC5, 0xF8, 0x11, 0x07, 0xC4, 0xE3, 0x7D, 0x19, 0x47, 0x10, 0x01};
|
||||
const auto convert = [&](const Bytes& text) {
|
||||
Bytes file(0x300 + text.size(), 0xCC);
|
||||
std::copy(text.begin(), text.end(), file.begin() + 0x200);
|
||||
return converter->Convert(file, {segmentHeader(text.size())});
|
||||
};
|
||||
const auto body = [](const Codegen::TrampolineSite& site) {
|
||||
return Bytes(site.Body.begin(), site.Body.begin() + static_cast<std::ptrdiff_t>(site.ReturnBranchOffset));
|
||||
};
|
||||
const auto withStore = [&](Bytes before, const Bytes& after) {
|
||||
before.insert(before.end(), store.begin(), store.end());
|
||||
before.insert(before.end(), after.begin(), after.end());
|
||||
return before;
|
||||
};
|
||||
|
||||
const auto returned = convert(withStore({}, {0xC3}));
|
||||
Bytes expected = halves;
|
||||
expected.push_back(0xC3);
|
||||
require(returned.Trampolines.size() == 1 && returned.KeptCount == 0 && returned.Reports.size() == 1 && returned.Reports[0].InstructionName == "256-bit store", "A 256-bit store followed by a return was not moved");
|
||||
require(returned.Trampolines[0].Offset == 0x200 && returned.Trampolines[0].Length == 5 && body(returned.Trampolines[0]) == expected, "A 256-bit store followed by a return did not end its stub with the return");
|
||||
|
||||
const auto called = convert(withStore({}, {0xE8, 0x10, 0x00, 0x00, 0x00}));
|
||||
require(called.Trampolines.size() == 1 && called.Trampolines[0].Offset == 0x200 && called.Trampolines[0].Length == 9, "A 256-bit store followed by a call did not take the call");
|
||||
const auto& call = called.Trampolines[0];
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0x48, 0x8D, 0x64, 0x24, 0xF8, 0x50, 0x48, 0x8D, 0x05, 0, 0, 0, 0, 0x48, 0x89, 0x44, 0x24, 0x08, 0x58, 0xE9, 0, 0, 0, 0});
|
||||
require(body(call) == expected, "A moved call does not push its original return address and jump");
|
||||
require(call.Relocations.size() == 2 && call.Relocations[0].DisplacementOffset == halves.size() + 9 && call.Relocations[0].SiteTarget == 9 && call.Relocations[1].DisplacementOffset == halves.size() + 20 && call.Relocations[1].SiteTarget == 9 + 0x10, "A moved call returns to or calls the wrong address");
|
||||
|
||||
const auto branched = convert(withStore({}, {0x75, 0x10, 0x90}));
|
||||
require(branched.Trampolines.size() == 1 && branched.Trampolines[0].Length == 6, "A 256-bit store followed by a short conditional branch did not take the branch");
|
||||
const auto& jcc = branched.Trampolines[0];
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0x0F, 0x85, 0, 0, 0, 0});
|
||||
require(body(jcc) == expected && jcc.Relocations.size() == 1 && jcc.Relocations[0].SiteTarget == 6 + 0x10, "A moved short conditional branch was not re-encoded with a 32-bit displacement to its target");
|
||||
|
||||
const auto jumped = convert(withStore({}, {0xEB, 0x10, 0x90}));
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0xE9, 0, 0, 0, 0});
|
||||
require(jumped.Trampolines.size() == 1 && body(jumped.Trampolines[0]) == expected && jumped.Trampolines[0].Relocations.size() == 1 && jumped.Trampolines[0].Relocations[0].SiteTarget == 6 + 0x10, "A moved short jump was not re-encoded with a 32-bit displacement to its target");
|
||||
|
||||
const auto longBranched = convert(withStore({}, {0x0F, 0x85, 0x10, 0x00, 0x00, 0x00}));
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0x0F, 0x85, 0, 0, 0, 0});
|
||||
require(longBranched.Trampolines.size() == 1 && longBranched.Trampolines[0].Length == 10 && body(longBranched.Trampolines[0]) == expected && longBranched.Trampolines[0].Relocations.size() == 1 && longBranched.Trampolines[0].Relocations[0].SiteTarget == 10 + 0x10, "A moved conditional branch with a 32-bit displacement does not reach its target");
|
||||
|
||||
const auto entered = convert(withStore({0x48, 0x83, 0xC1, 0x01}, {0xC3, 0xEB, 0xFD}));
|
||||
expected = {0x48, 0x83, 0xC1, 0x01};
|
||||
expected.insert(expected.end(), halves.begin(), halves.end());
|
||||
require(entered.Trampolines.size() == 1 && entered.Trampolines[0].Offset == 0x200 && entered.Trampolines[0].Length == 8 && body(entered.Trampolines[0]) == expected, "A 256-bit store before a branch target did not take room from the instruction before it");
|
||||
|
||||
Bytes adds;
|
||||
for (int index = 0; index < 4; ++index)
|
||||
adds.insert(adds.end(), {0x48, 0x83, 0xC1, 0x01});
|
||||
Bytes after = {0xC3, 0xEB, 0xFD};
|
||||
after.insert(after.end(), adds.begin(), adds.end());
|
||||
const auto island = convert(withStore({0xE8, 0x00, 0x00, 0x00, 0x00}, after));
|
||||
require(island.Trampolines.size() == 2 && island.KeptCount == 0, "A 256-bit store with no movable neighbours was not lowered through an island");
|
||||
const auto& moved = island.Trampolines[0];
|
||||
const auto& lowered = island.Trampolines[1];
|
||||
expected = adds;
|
||||
expected.insert(expected.end(), {0xE9, 0, 0, 0, 0});
|
||||
require(moved.Offset == 0x20C && moved.Length == 8 && body(moved) == expected && moved.Relocations.size() == 1 && moved.Relocations[0].SiteTarget == 16, "The instructions under the island were not moved and continued after it");
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0xE9, 0, 0, 0, 0});
|
||||
require(lowered.Offset == 0x214 && lowered.Length == 5 && body(lowered) == expected && lowered.Relocations.size() == 1 && lowered.Relocations[0].SiteTarget == -11, "The island does not store both halves and return after the original store");
|
||||
require(Bytes(island.Bytes.begin() + 0x205, island.Bytes.begin() + 0x209) == Bytes{0xEB, 0x0D, 0xCC, 0xCC}, "The 256-bit store was not replaced by a short jump to its island");
|
||||
|
||||
requireFailure([&] { (void)convert(withStore({0xE8, 0x00, 0x00, 0x00, 0x00}, {0xC3, 0xEB, 0xFD})); }, "A 256-bit store with no room for a stub jump was kept");
|
||||
|
||||
const auto paired = convert(withStore({}, {0xC5, 0xFC, 0x11, 0x0E, 0xC3}));
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0xC5, 0xF8, 0x11, 0x0E, 0xC4, 0xE3, 0x7D, 0x19, 0x4E, 0x10, 0x01});
|
||||
require(paired.Trampolines.size() == 1 && paired.Trampolines[0].Length == 8 && body(paired.Trampolines[0]) == expected && paired.Reports.size() == 1, "A 256-bit store followed by another was not lowered with it in one stub");
|
||||
|
||||
const Bytes push = {0x48, 0x8D, 0x64, 0x24, 0xF8, 0x50, 0x48, 0x8D, 0x05, 0, 0, 0, 0, 0x48, 0x89, 0x44, 0x24, 0x08, 0x58};
|
||||
const auto ripCalled = convert(withStore({}, {0xFF, 0x15, 0x10, 0x00, 0x00, 0x00}));
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), push.begin(), push.end());
|
||||
expected.insert(expected.end(), {0xFF, 0x25, 0, 0, 0, 0});
|
||||
const auto& ripCall = ripCalled.Trampolines[0];
|
||||
require(ripCalled.Trampolines.size() == 1 && ripCall.Length == 10 && body(ripCall) == expected, "A moved call through a RIP-relative pointer does not push its original return address and jump through the pointer");
|
||||
require(ripCall.Relocations.size() == 2 && ripCall.Relocations[0].SiteTarget == 10 && ripCall.Relocations[1].DisplacementOffset == halves.size() + 21 && ripCall.Relocations[1].SiteTarget == 10 + 0x10, "A moved call through a RIP-relative pointer returns to or reads the wrong address");
|
||||
|
||||
const auto ripJumped = convert(withStore({}, {0xFF, 0x25, 0x10, 0x00, 0x00, 0x00}));
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), {0xFF, 0x25, 0, 0, 0, 0});
|
||||
require(ripJumped.Trampolines.size() == 1 && body(ripJumped.Trampolines[0]) == expected && ripJumped.Trampolines[0].Relocations.size() == 1 && ripJumped.Trampolines[0].Relocations[0].SiteTarget == 10 + 0x10, "A moved jump through a RIP-relative pointer reads the wrong address");
|
||||
|
||||
const auto relaxed = convert(withStore({}, {0x67, 0xE8, 0x10, 0x00, 0x00, 0x00}));
|
||||
expected = halves;
|
||||
expected.insert(expected.end(), push.begin(), push.end());
|
||||
expected.insert(expected.end(), {0xE9, 0, 0, 0, 0});
|
||||
require(relaxed.Trampolines.size() == 1 && relaxed.Trampolines[0].Length == 10 && body(relaxed.Trampolines[0]) == expected && relaxed.Trampolines[0].Relocations.size() == 2 && relaxed.Trampolines[0].Relocations[1].SiteTarget == 10 + 0x10, "A call with an address-size prefix was not moved like a call");
|
||||
|
||||
const Bytes add = {0x48, 0x83, 0xC1, 0x01};
|
||||
for (const auto& follower : {Bytes{0xFF, 0xD0}, Bytes{0xE3, 0x10}, Bytes{0x66, 0x75, 0x10}, Bytes{0x66, 0x0F, 0xAE, 0x37}}) {
|
||||
const auto preceded = convert(withStore(add, follower));
|
||||
expected = add;
|
||||
expected.insert(expected.end(), halves.begin(), halves.end());
|
||||
require(!preceded.Trampolines.empty() && preceded.Trampolines[0].Offset == 0x200 && preceded.Trampolines[0].Length == 8 && body(preceded.Trampolines[0]) == expected, "A 256-bit store moved a following instruction that cannot leave its place instead of the one before it");
|
||||
}
|
||||
|
||||
Bytes throwing = {0xC3, 0xEB, 0xFD};
|
||||
for (int index = 0; index < 3; ++index)
|
||||
throwing.insert(throwing.end(), add.begin(), add.end());
|
||||
throwing.insert(throwing.end(), {0xFF, 0xD0});
|
||||
throwing.insert(throwing.end(), adds.begin(), adds.end());
|
||||
const auto farther = convert(withStore({0xE8, 0x00, 0x00, 0x00, 0x00}, throwing));
|
||||
require(farther.Trampolines.size() == 2 && farther.Trampolines[0].Offset == 0x21A && farther.Trampolines[1].Offset == 0x222, "An island that would move a register call was not passed over for the next one");
|
||||
require(Bytes(farther.Bytes.begin() + 0x205, farther.Bytes.begin() + 0x207) == Bytes{0xEB, 0x1B}, "The 256-bit store does not jump to the island after the register call");
|
||||
|
||||
Bytes reentered = {0xFF, 0xD0, 0xEB, 0xF8};
|
||||
reentered.insert(reentered.end(), adds.begin(), adds.end());
|
||||
const auto beyondStore = convert(withStore(Bytes(adds.begin(), adds.begin() + 8), reentered));
|
||||
require(beyondStore.Trampolines.size() == 2 && beyondStore.Trampolines[0].Offset == 0x210 && beyondStore.Trampolines[1].Offset == 0x218, "An island before a 256-bit store that is a branch target took the store with it");
|
||||
require(Bytes(beyondStore.Bytes.begin() + 0x208, beyondStore.Bytes.begin() + 0x20A) == Bytes{0xEB, 0x0E}, "The 256-bit store that is a branch target does not jump to the island after it");
|
||||
|
||||
const Bytes farCall = {0xE8, 0x40, 0x00, 0x00, 0x00};
|
||||
Bytes claimedFirst = {0xFF, 0xD0};
|
||||
claimedFirst.insert(claimedFirst.end(), adds.begin(), adds.end());
|
||||
const auto afterSequence = convert(withStore({0xC5, 0xF8, 0x53, 0xC1, 0x48, 0x83, 0xC1, 0x01}, claimedFirst));
|
||||
require(afterSequence.Trampolines.size() == 3 && afterSequence.Trampolines[0].Offset == 0x200 && afterSequence.Trampolines[0].Length == 8 && afterSequence.Trampolines[1].Offset == 0x20E && afterSequence.Trampolines[2].Offset == 0x216, "A 256-bit store moved an instruction already moved into the stub of the instruction before it");
|
||||
|
||||
Bytes pendingRun = {0xFF, 0xD0, 0x48, 0x83, 0xC1, 0x01, 0xC5, 0xFC, 0x11, 0x47, 0x10};
|
||||
pendingRun.insert(pendingRun.end(), adds.begin(), adds.end());
|
||||
const auto pendingIsland = convert(withStore(farCall, pendingRun));
|
||||
require(pendingIsland.Trampolines.size() == 3 && pendingIsland.Trampolines[0].Offset == 0x214 && pendingIsland.Trampolines[1].Offset == 0x21C && pendingIsland.Trampolines[2].Offset == 0x20F && pendingIsland.Trampolines[2].Length == 5, "An island moved the call before a 256-bit store or another store that still needs its own stub");
|
||||
|
||||
Bytes enteredRun = {0xFF, 0xD0};
|
||||
for (int index = 0; index < 6; ++index)
|
||||
enteredRun.insert(enteredRun.end(), add.begin(), add.end());
|
||||
enteredRun.insert(enteredRun.end(), {0xEB, 0xEE});
|
||||
const auto enteredIsland = convert(withStore(farCall, enteredRun));
|
||||
require(enteredIsland.Trampolines.size() == 2 && enteredIsland.Trampolines[0].Offset == 0x213 && enteredIsland.Trampolines[1].Offset == 0x21B, "An island was placed over a branch target");
|
||||
|
||||
Bytes returning = {0xFF, 0xD0};
|
||||
for (int index = 0; index < 3; ++index)
|
||||
returning.insert(returning.end(), add.begin(), add.end());
|
||||
returning.push_back(0xC3);
|
||||
const auto returnIsland = convert(withStore(farCall, returning));
|
||||
expected = Bytes(adds.begin(), adds.begin() + 12);
|
||||
expected.push_back(0xC3);
|
||||
require(returnIsland.Trampolines.size() == 2 && returnIsland.Trampolines[0].Offset == 0x20B && body(returnIsland.Trampolines[0]) == expected, "Instructions moved for an island that end in a return jump back after it");
|
||||
|
||||
const auto afterBranch = convert(withStore({0x75, 0x10}, {0xFF, 0xD0}));
|
||||
expected = {0x0F, 0x85, 0, 0, 0, 0};
|
||||
expected.insert(expected.end(), halves.begin(), halves.end());
|
||||
require(afterBranch.Trampolines.size() == 1 && afterBranch.Trampolines[0].Offset == 0x200 && afterBranch.Trampolines[0].Length == 6 && body(afterBranch.Trampolines[0]) == expected && afterBranch.Trampolines[0].Relocations[0].SiteTarget == 2 + 0x10, "A 256-bit store after a conditional branch did not take the branch");
|
||||
|
||||
Bytes far = adds;
|
||||
far.resize(8);
|
||||
for (int index = 0; index < 60; ++index)
|
||||
far.insert(far.end(), {0xFF, 0xD0});
|
||||
far.insert(far.end(), adds.begin(), adds.end());
|
||||
for (int index = 0; index < 70; ++index)
|
||||
far.insert(far.end(), {0xFF, 0xD0});
|
||||
Bytes tail;
|
||||
for (int index = 0; index < 11; ++index)
|
||||
tail.insert(tail.end(), {0xFF, 0xD0});
|
||||
tail.insert(tail.end(), {0x90, 0x90});
|
||||
requireFailure([&] { (void)convert(withStore(far, tail)); }, "A 256-bit store whose movable neighbours are all beyond a short jump was kept");
|
||||
}
|
||||
|
||||
void converterRosetta() {
|
||||
const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta);
|
||||
Bytes file(0x300, 0xCC);
|
||||
|
|
@ -776,7 +992,7 @@ void converterReciprocal() {
|
|||
void converterStrayRex() {
|
||||
const auto converter = Codegen::MakeAmd64OnlyConverter();
|
||||
const auto convert = [&](const Bytes& text) {
|
||||
Bytes file(0x300, 0xCC);
|
||||
Bytes file(0x300 + text.size(), 0xCC);
|
||||
std::copy(text.begin(), text.end(), file.begin() + 0x200);
|
||||
return std::pair{file, converter->Convert(file, {segmentHeader(text.size())})};
|
||||
};
|
||||
|
|
@ -1419,6 +1635,8 @@ int main() {
|
|||
converterClzero();
|
||||
rosettaSubstitutions();
|
||||
converterRosetta();
|
||||
wideStoreSplit();
|
||||
converterWideStoreWindows();
|
||||
converterReciprocal();
|
||||
converterStrayRex();
|
||||
rewriterStrayRex();
|
||||
|
|
|
|||
|
|
@ -71,10 +71,11 @@ int Run(const int argc, char* argv[]) {
|
|||
sourceBytes = std::move(converted.Bytes);
|
||||
trampolines = std::move(converted.Trampolines);
|
||||
std::map<std::string, std::size_t> stubsByName;
|
||||
const auto counted = [](const std::string& name) { return name == "VRSQRTPS" || name == "VRCPPS" || name == "256-bit store"; };
|
||||
for (const auto& report : converted.Reports) {
|
||||
if (report.Lowering == Codegen::Amd64OnlyLowering::Kept)
|
||||
std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) kept: no room for a jump\n";
|
||||
else if (report.InstructionName == "VRSQRTPS" || report.InstructionName == "VRCPPS")
|
||||
else if (counted(report.InstructionName))
|
||||
++stubsByName[report.InstructionName];
|
||||
else
|
||||
std::cout << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) -> " << (report.Lowering == Codegen::Amd64OnlyLowering::InPlace ? "in place " : "stub ") << report.ReplacementLength << " bytes\n";
|
||||
|
|
|
|||
Loading…
Reference in New Issue