shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
AnyPS5/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp

732 lines
36 KiB
C++
Raw Blame History

#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include <Sse4aEmulation.hpp>
#include <codegen/IAmd64OnlyConverter.hpp>
#include <codegen/x86/IAmd64OnlyInstructionMatcher.hpp>
#include <codegen/x86/Sse4aOperands.hpp>
#include <elfpatcher/windows/WindowsLoadImage.hpp>
#include <elfpatcher/windows/WindowsPeWriter.hpp>
#include <elfpatcher/windows/WindowsTrampolineBuilder.hpp>
#include <io/BufferUtils.hpp>
#include <array>
#include <cmath>
#include <cstring>
#include <functional>
#include <iostream>
#include <optional>
#include <random>
#include <span>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
using namespace Elfpatcher::Windows;
using Bytes = std::vector<std::uint8_t>;
void require(bool condition, const std::string& message) {
if (!condition) throw std::runtime_error(message);
}
void requireFailure(const std::function<void()>& operation, const char* message) {
try {
operation();
} catch (const Domain::RelinkerException&) {
return;
}
throw std::runtime_error(message);
}
template<typename TValue>
void write(Bytes& bytes, std::size_t offset, TValue value) {
if (offset > bytes.size() || sizeof(value) > bytes.size() - offset) throw std::runtime_error("Test fixture write is out of bounds");
std::memcpy(bytes.data() + offset, &value, sizeof(value));
}
template<typename TValue>
TValue read(const Bytes& bytes, std::size_t offset) {
TValue value;
if (offset > bytes.size() || sizeof(value) > bytes.size() - offset) throw std::runtime_error("Test fixture read is out of bounds");
std::memcpy(&value, bytes.data() + offset, sizeof(value));
return value;
}
struct alignas(16) State {
std::uint8_t Xmm[16][16];
std::uint64_t FlagsIn;
std::uint64_t FlagsOut;
std::uint64_t RspBefore;
std::uint64_t RspAfter;
std::uint8_t RedZone[128];
std::uint8_t Ymm[16][32];
};
constexpr std::uint64_t kCanary = 0xA5C3E17B9D24F608ull;
constexpr std::uint64_t kStatusFlags = 0x8D5;
constexpr std::uint32_t kXmmOffset = 0;
constexpr std::uint32_t kFlagsInOffset = 256;
constexpr std::uint32_t kFlagsOutOffset = 264;
constexpr std::uint32_t kRspBeforeOffset = 272;
constexpr std::uint32_t kRspAfterOffset = 280;
constexpr std::uint32_t kRedZoneOffset = 288;
static_assert(offsetof(State, FlagsIn) == kFlagsInOffset && offsetof(State, FlagsOut) == kFlagsOutOffset && offsetof(State, RspBefore) == kRspBeforeOffset && offsetof(State, RspAfter) == kRspAfterOffset && offsetof(State, RedZone) == kRedZoneOffset, "State layout drifted");
void emit(Bytes& code, std::initializer_list<std::uint8_t> bytes) {
code.insert(code.end(), bytes.begin(), bytes.end());
}
void emitU32(Bytes& code, std::uint32_t value) {
for (std::size_t index = 0; index < 4; ++index) code.push_back(static_cast<std::uint8_t>(value >> (index * 8)));
}
void movdquXmmRbx(Bytes& code, const std::uint8_t reg, const bool store) {
code.push_back(0xF3);
if (reg >= 8) code.push_back(0x44);
emit(code, {0x0F, static_cast<std::uint8_t>(store ? 0x7F : 0x6F), static_cast<std::uint8_t>(0x83 | ((reg & 7) << 3))});
emitU32(code, kXmmOffset + reg * 16);
}
void movdquXmmRsp(Bytes& code, const std::uint8_t reg, const bool store, const std::uint32_t offset) {
code.push_back(0xF3);
if (reg >= 8) code.push_back(0x44);
emit(code, {0x0F, static_cast<std::uint8_t>(store ? 0x7F : 0x6F), static_cast<std::uint8_t>(0x84 | ((reg & 7) << 3)), 0x24});
emitU32(code, offset);
}
void vmovdquYmmRbx(Bytes& code, const std::uint8_t reg, const bool store) {
emit(code, {0xC4, static_cast<std::uint8_t>(reg >= 8 ? 0x61 : 0xE1), 0x7E, static_cast<std::uint8_t>(store ? 0x7F : 0x6F), static_cast<std::uint8_t>(0x83 | ((reg & 7) << 3))});
emitU32(code, static_cast<std::uint32_t>(offsetof(State, Ymm)) + reg * 32);
}
class Harness {
public:
Harness() : _memory(VirtualAlloc(nullptr, kSize, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE)) {
if (_memory == nullptr) throw std::runtime_error("Cannot allocate executable memory");
}
~Harness() {
VirtualFree(_memory, 0, MEM_RELEASE);
}
void Run(const Bytes& body, const std::size_t returnBranchOffset, State& state, const bool wide = false) const {
Bytes code;
emit(code, {0x53, 0x55, 0x56, 0x57, 0x41, 0x54, 0x41, 0x55, 0x41, 0x56, 0x41, 0x57, 0x48, 0x81, 0xEC, 0xA8, 0x00, 0x00, 0x00});
for (std::uint8_t reg = 6; reg < 16; ++reg) movdquXmmRsp(code, reg, true, (reg - 6) * 16);
emit(code, {0x48, 0x89, 0xCB, 0xFF, 0xB3});
emitU32(code, kFlagsInOffset);
emit(code, {0x9D, 0x48, 0xB8});
for (std::size_t index = 0; index < 8; ++index) code.push_back(static_cast<std::uint8_t>(kCanary >> (index * 8)));
for (int slot = 0; slot < 16; ++slot) emit(code, {0x48, 0x89, 0x44, 0x24, static_cast<std::uint8_t>(-128 + slot * 8)});
emit(code, {0x48, 0x89, 0xA3});
emitU32(code, kRspBeforeOffset);
for (std::uint8_t reg = 0; reg < 16; ++reg) {
if (wide)
vmovdquYmmRbx(code, reg, false);
else
movdquXmmRbx(code, reg, false);
}
code.push_back(0xE9);
const auto jumpOffset = code.size();
emitU32(code, 0);
const auto continuation = code.size();
for (int slot = 0; slot < 16; ++slot) {
emit(code, {0x48, 0x8B, 0x44, 0x24, static_cast<std::uint8_t>(-128 + slot * 8), 0x48, 0x89, 0x83});
emitU32(code, kRedZoneOffset + slot * 8);
}
emit(code, {0x48, 0x89, 0xA3});
emitU32(code, kRspAfterOffset);
emit(code, {0x9C, 0x8F, 0x83});
emitU32(code, kFlagsOutOffset);
for (std::uint8_t reg = 0; reg < 16; ++reg) movdquXmmRbx(code, reg, true);
if (wide)
for (std::uint8_t reg = 0; reg < 16; ++reg) vmovdquYmmRbx(code, reg, true);
for (std::uint8_t reg = 6; reg < 16; ++reg) movdquXmmRsp(code, reg, false, (reg - 6) * 16);
emit(code, {0x48, 0x81, 0xC4, 0xA8, 0x00, 0x00, 0x00, 0x41, 0x5F, 0x41, 0x5E, 0x41, 0x5D, 0x41, 0x5C, 0x5F, 0x5E, 0x5D, 0x5B, 0xC3});
code.resize(Io::AlignUp(code.size(), std::size_t{16}), 0xCC);
const auto bodyOffset = code.size();
code.insert(code.end(), body.begin(), body.end());
write<std::int32_t>(code, jumpOffset, static_cast<std::int32_t>(bodyOffset - (jumpOffset + 4)));
write<std::int32_t>(code, bodyOffset + returnBranchOffset + 1, static_cast<std::int32_t>(continuation - (bodyOffset + returnBranchOffset + 5)));
if (code.size() > kSize) throw std::runtime_error("Harness code exceeds its page");
std::memcpy(_memory, code.data(), code.size());
FlushInstructionCache(GetCurrentProcess(), _memory, code.size());
reinterpret_cast<void (*)(State*)>(_memory)(&state);
}
private:
static constexpr std::size_t kSize = 4096;
void* _memory;
};
std::uint64_t low(const std::uint8_t (&lane)[16]) {
std::uint64_t value;
std::memcpy(&value, lane, 8);
return value;
}
std::uint64_t referenceLow(const Codegen::Sse4aOperands& operands, const State& input) {
const auto mask = operands.Length >= 64 ? ~std::uint64_t{0} : ((std::uint64_t{1} << operands.Length) - 1);
const auto destination = low(input.Xmm[operands.Destination]);
if (!operands.Insertq) return (destination >> operands.Index) & mask;
const auto hole = mask << operands.Index;
return (destination & ~hole) | ((low(input.Xmm[operands.Source]) & mask) << operands.Index);
}
std::uint64_t libcReferenceLow(const Codegen::Sse4aOperands& operands, const State& input) {
CONTEXT context{};
for (unsigned reg = 0; reg < 16; ++reg) {
std::memcpy(&context.FltSave.XmmRegisters[reg].Low, input.Xmm[reg], 8);
std::memcpy(&context.FltSave.XmmRegisters[reg].High, input.Xmm[reg] + 8, 8);
}
sse4a::Instruction instruction;
instruction.op = operands.Insertq ? sse4a::Op::Insertq : sse4a::Op::Extrq;
instruction.registerForm = operands.RegisterForm;
instruction.destination = operands.Destination;
instruction.source = operands.Source;
instruction.length = static_cast<std::uint8_t>(operands.Length == 64 ? 0 : operands.Length);
instruction.index = operands.Index;
sse4a::Execute(instruction, context);
return static_cast<std::uint64_t>(context.FltSave.XmmRegisters[operands.Destination].Low);
}
struct Case {
bool Insertq;
std::uint8_t Destination;
std::uint8_t Source;
std::uint8_t Length;
std::uint8_t Index;
bool RegisterForm = false;
};
Bytes encode(const Case& item) {
Bytes bytes = {static_cast<std::uint8_t>(item.Insertq ? 0xF2 : 0x66)};
const std::uint8_t reg = item.Insertq || item.RegisterForm ? item.Destination : 0;
const std::uint8_t rm = item.Insertq || item.RegisterForm ? item.Source : item.Destination;
const auto rex = static_cast<std::uint8_t>(0x40 | (reg >= 8 ? 4 : 0) | (rm >= 8 ? 1 : 0));
if (rex != 0x40) bytes.push_back(rex);
bytes.insert(bytes.end(), {0x0F, static_cast<std::uint8_t>(item.RegisterForm ? 0x79 : 0x78), static_cast<std::uint8_t>(0xC0 | ((reg & 7) << 3) | (rm & 7))});
if (!item.RegisterForm) bytes.insert(bytes.end(), {static_cast<std::uint8_t>(item.Length == 64 ? 0 : item.Length), item.Index});
return bytes;
}
std::string describe(const Case& item) {
const bool source = item.Insertq || item.RegisterForm;
return std::string(item.Insertq ? "insertq xmm" : "extrq xmm") + std::to_string(item.Destination) + (source ? ", xmm" + std::to_string(item.Source) : "") + (item.RegisterForm ? " (register form)" : "") + ", " + std::to_string(item.Length) + ", " + std::to_string(item.Index);
}
std::size_t g_executions = 0;
void requireEnvironment(const State& input, const State& state, const std::uint16_t written, const std::string& description) {
for (unsigned reg = 0; reg < 16; ++reg) {
if (((written >> reg) & 1) != 0) continue;
require(std::memcmp(state.Xmm[reg], input.Xmm[reg], 16) == 0, "Lowered sequence clobbered xmm" + std::to_string(reg) + ": " + description);
}
require(((state.FlagsOut ^ input.FlagsIn) & kStatusFlags) == 0, "Lowered sequence changed RFLAGS: " + description);
require(state.RspAfter == state.RspBefore, "Lowered sequence did not restore rsp: " + description);
for (int slot = 0; slot < 16; ++slot) {
std::uint64_t value;
std::memcpy(&value, state.RedZone + slot * 8, 8);
require(value == kCanary, "Lowered sequence wrote into the red zone: " + description);
}
}
void reciprocalExecution() {
const Harness harness;
const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher();
const bool wide = __builtin_cpu_supports("avx");
const float inputs[][4] = {
{1.0f, 4.0f, 0.25f, 2.0f},
{0.0f, -0.0f, INFINITY, -1.0f},
{1.00000012f, 0.99999994f, 3.0e-38f, 1.0e30f},
{NAN, -INFINITY, -4.0f, 16.0f}};
for (const bool vex : {false, true}) {
if (vex && !wide)
continue;
for (const std::uint8_t opcode : {std::uint8_t{0x52}, std::uint8_t{0x53}}) {
for (const std::uint8_t destination : {std::uint8_t{0}, std::uint8_t{2}, std::uint8_t{9}, std::uint8_t{15}}) {
for (const std::uint8_t source : {destination, std::uint8_t{0}, std::uint8_t{5}, std::uint8_t{14}}) {
Bytes site;
if (vex) {
emit(site, {0xC4, static_cast<std::uint8_t>((destination >= 8 ? 0 : 0x80) | 0x40 | (source >= 8 ? 0 : 0x20) | 1), 0x78});
} else {
const auto rex = static_cast<std::uint8_t>(0x40 | (destination >= 8 ? 4 : 0) | (source >= 8 ? 1 : 0));
if (rex != 0x40)
site.push_back(rex);
site.push_back(0x0F);
}
emit(site, {opcode, static_cast<std::uint8_t>(0xC0 | ((destination & 7) << 3) | (source & 7))});
const auto match = matcher->Match(site.data(), site.size());
require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "Packed reciprocal was not lowered");
for (const auto& lanes : inputs) {
State input{};
for (std::size_t reg = 0; reg < 16; ++reg) {
for (std::size_t byte = 0; byte < 16; ++byte) {
input.Xmm[reg][byte] = static_cast<std::uint8_t>(reg * 16 + byte);
input.Ymm[reg][byte + 16] = static_cast<std::uint8_t>(reg * 16 + byte + 1);
}
}
std::memcpy(input.Xmm[source], lanes, sizeof(lanes));
for (std::size_t reg = 0; reg < 16; ++reg)
std::memcpy(input.Ymm[reg], input.Xmm[reg], 16);
input.FlagsIn = 0xAC7;
State state = input;
harness.Run(match->StubBody, match->ReturnBranchOffset, state, wide);
++g_executions;
float actual[4];
std::memcpy(actual, state.Xmm[destination], sizeof(actual));
for (std::size_t lane = 0; lane < 4; ++lane) {
const float expected = opcode == 0x52 ? 1.0f / std::sqrt(lanes[lane]) : 1.0f / lanes[lane];
require(std::isnan(expected) ? std::isnan(actual[lane]) : std::memcmp(&expected, &actual[lane], sizeof(float)) == 0, "Packed reciprocal result differs from division and square root");
}
requireEnvironment(input, state, static_cast<std::uint16_t>(1u << destination), "packed reciprocal");
if (wide) {
const std::uint8_t zero[16] = {};
for (std::size_t reg = 0; reg < 16; ++reg) {
const auto* expected = vex && reg == destination ? zero : input.Ymm[reg] + 16;
require(std::memcmp(state.Ymm[reg] + 16, expected, 16) == 0, "Packed reciprocal changed the wrong upper YMM bits");
}
}
}
}
}
}
}
}
void executeCase(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, const Case& item, std::mt19937_64& random, const int images) {
const auto site = encode(item);
const auto match = matcher.Match(site.data(), site.size());
require(match.has_value() && match->Lowering != Codegen::Amd64OnlyLowering::Unsupported, "SSE4a instruction was not lowered: " + describe(item));
Bytes body;
std::size_t returnBranchOffset = 0;
if (match->Lowering == Codegen::Amd64OnlyLowering::InPlace) {
require(match->ReplacementBytes.size() == site.size(), "In-place lowering changed the length: " + describe(item));
body = match->ReplacementBytes;
returnBranchOffset = body.size();
body.insert(body.end(), {0xE9, 0, 0, 0, 0});
} else {
body = match->StubBody;
returnBranchOffset = match->ReturnBranchOffset;
require(body.size() % 16 == 0 || body.size() == returnBranchOffset + 5, "Stub body with constants is not padded to 16 bytes: " + describe(item));
}
const auto operands = Codegen::DecodeSse4a(site.data(), site.size());
auto field = operands;
if (operands.RegisterForm) {
field.Length = item.Length;
field.Index = item.Index;
}
const std::size_t controlOffset = operands.Insertq ? 8 : 0;
const std::array<std::uint64_t, 4> flags = {0x202, 0x203, 0x246, 0xAC7};
for (int image = 0; image < images; ++image) {
State input{};
for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast<std::uint8_t>(random());
if (operands.RegisterForm) {
auto& control = input.Xmm[operands.Source];
control[controlOffset] = static_cast<std::uint8_t>((control[controlOffset] & 0xC0) | (item.Length == 64 ? 0 : item.Length));
control[controlOffset + 1] = static_cast<std::uint8_t>((control[controlOffset + 1] & 0xC0) | item.Index);
}
input.FlagsIn = flags[static_cast<std::size_t>(image) % flags.size()];
State state = input;
harness.Run(body, returnBranchOffset, state);
++g_executions;
const auto expected = referenceLow(field, input);
require(expected == libcReferenceLow(operands, input), "Transcribed reference disagrees with the libc emulation: " + describe(item));
require(low(state.Xmm[operands.Destination]) == expected, "Lowered sequence computed the wrong field: " + describe(item));
requireEnvironment(input, state, static_cast<std::uint16_t>(1u << operands.Destination), describe(item));
}
}
struct Sha256Step {
std::uint8_t Opcode;
std::uint8_t Destination;
std::uint8_t Source;
};
Bytes encodeSha256(const Sha256Step& step) {
Bytes bytes;
const auto rex = static_cast<std::uint8_t>(0x40 | (step.Destination >= 8 ? 4 : 0) | (step.Source >= 8 ? 1 : 0));
if (rex != 0x40) bytes.push_back(rex);
bytes.insert(bytes.end(), {0x0F, 0x38, step.Opcode, static_cast<std::uint8_t>(0xC0 | ((step.Destination & 7) << 3) | (step.Source & 7))});
return bytes;
}
std::uint32_t rotr(const std::uint32_t value, const unsigned count) {
return (value >> count) | (value << (32 - count));
}
void sha256Reference(const Sha256Step& step, std::uint8_t (&xmm)[16][16]) {
std::uint32_t a[4];
std::uint32_t b[4];
std::uint32_t k[4];
std::uint32_t r[4];
std::memcpy(a, xmm[step.Destination], sizeof(a));
std::memcpy(b, xmm[step.Source], sizeof(b));
std::memcpy(k, xmm[0], sizeof(k));
const auto sigma0 = [](const std::uint32_t w) { return rotr(w, 7) ^ rotr(w, 18) ^ (w >> 3); };
const auto sigma1 = [](const std::uint32_t w) { return rotr(w, 17) ^ rotr(w, 19) ^ (w >> 10); };
if (step.Opcode == 0xCC) {
for (int lane = 0; lane < 3; ++lane) r[lane] = a[lane] + sigma0(a[lane + 1]);
r[3] = a[3] + sigma0(b[0]);
} else if (step.Opcode == 0xCD) {
r[0] = a[0] + sigma1(b[2]);
r[1] = a[1] + sigma1(b[3]);
r[2] = a[2] + sigma1(r[0]);
r[3] = a[3] + sigma1(r[1]);
} else {
std::uint32_t sa = b[3], sb = b[2], sc = a[3], sd = a[2], se = b[1], sf = b[0], sg = a[1], sh = a[0];
for (int round = 0; round < 2; ++round) {
const auto t1 = sh + (rotr(se, 6) ^ rotr(se, 11) ^ rotr(se, 25)) + ((se & sf) ^ (~se & sg)) + k[round];
const auto t2 = (rotr(sa, 2) ^ rotr(sa, 13) ^ rotr(sa, 22)) + ((sa & sb) ^ (sa & sc) ^ (sb & sc));
sh = sg; sg = sf; sf = se; se = sd + t1; sd = sc; sc = sb; sb = sa; sa = t1 + t2;
}
r[0] = sf;
r[1] = se;
r[2] = sb;
r[3] = sa;
}
std::memcpy(xmm[step.Destination], r, sizeof(r));
}
std::string describeSha256(const std::vector<Sha256Step>& steps) {
std::string text;
for (const auto& step : steps)
text += std::string(step.Opcode == 0xCB ? "sha256rnds2" : step.Opcode == 0xCC ? "sha256msg1" : "sha256msg2") + " xmm" + std::to_string(step.Destination) + ", xmm" + std::to_string(step.Source) + "; ";
return text;
}
void executeSha256(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, const std::vector<Sha256Step>& steps, std::mt19937_64& random, const int images) {
std::vector<Bytes> sites;
for (const auto& step : steps) sites.push_back(encodeSha256(step));
const std::vector<std::span<const std::uint8_t>> spans(sites.begin(), sites.end());
const auto match = steps.size() == 1 ? matcher.Match(sites[0].data(), sites[0].size()) : matcher.MatchSequence(spans, {});
require(match.has_value() && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "SHA-256 was not lowered through a stub: " + describeSha256(steps));
std::uint16_t written = 0;
for (const auto& step : steps) written = static_cast<std::uint16_t>(written | (1u << step.Destination));
const std::array<std::uint64_t, 4> flags = {0x202, 0x203, 0x246, 0xAC7};
for (int image = 0; image < images; ++image) {
State input{};
for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast<std::uint8_t>(random());
input.FlagsIn = flags[static_cast<std::size_t>(image) % flags.size()];
State state = input;
harness.Run(match->StubBody, match->ReturnBranchOffset, state);
++g_executions;
State expected = input;
for (const auto& step : steps) sha256Reference(step, expected.Xmm);
for (unsigned reg = 0; reg < 16; ++reg)
if (((written >> reg) & 1) != 0) require(std::memcmp(state.Xmm[reg], expected.Xmm[reg], 16) == 0, "Lowered SHA-256 computed the wrong xmm" + std::to_string(reg) + ": " + describeSha256(steps));
requireEnvironment(input, state, written, describeSha256(steps));
}
}
struct Sha1Memory {
std::uint8_t ModRm;
Bytes Tail;
std::optional<std::uint8_t> Slot;
const char* Name;
};
const std::array<Sha1Memory, 4> kSha1Memory = {{
{0x03, {}, std::uint8_t{0}, "[rbx]"},
{0x43, {0x70}, std::uint8_t{7}, "[rbx+0x70]"},
{0x83, {0xF0, 0x00, 0x00, 0x00}, std::uint8_t{15}, "[rbx+0xf0]"},
{0x44, {0x24, 0xF0}, std::nullopt, "[rsp-0x10]"}}};
struct Sha1Step {
std::uint8_t Opcode;
std::uint8_t Destination;
std::uint8_t Source;
std::uint8_t Immediate = 0;
bool Memory = false;
};
Bytes encodeSha1(const Sha1Step& step) {
Bytes bytes;
const auto rex = static_cast<std::uint8_t>(0x40 | (step.Destination >= 8 ? 4 : 0) | (!step.Memory && step.Source >= 8 ? 1 : 0));
if (rex != 0x40) bytes.push_back(rex);
bytes.insert(bytes.end(), {0x0F, static_cast<std::uint8_t>(step.Opcode == 0xCC ? 0x3A : 0x38), step.Opcode});
if (step.Memory) {
const auto& memory = kSha1Memory[step.Source];
bytes.push_back(static_cast<std::uint8_t>(memory.ModRm | ((step.Destination & 7) << 3)));
bytes.insert(bytes.end(), memory.Tail.begin(), memory.Tail.end());
} else {
bytes.push_back(static_cast<std::uint8_t>(0xC0 | ((step.Destination & 7) << 3) | (step.Source & 7)));
}
if (step.Opcode == 0xCC) bytes.push_back(step.Immediate);
return bytes;
}
std::uint32_t rotl(const std::uint32_t value, const unsigned count) {
return (value << count) | (value >> (32 - count));
}
void sha1Reference(const Sha1Step& step, const State& input, std::uint8_t (&xmm)[16][16]) {
std::uint32_t x[4];
std::uint32_t y[4];
std::uint32_t r[4];
std::memcpy(x, xmm[step.Destination], sizeof(x));
if (!step.Memory) {
std::memcpy(y, xmm[step.Source], sizeof(y));
} else if (const auto slot = kSha1Memory[step.Source].Slot) {
std::memcpy(y, input.Xmm[*slot], sizeof(y));
} else {
const std::uint64_t canary[2] = {kCanary, kCanary};
std::memcpy(y, canary, sizeof(y));
}
if (step.Opcode == 0xC8) {
r[0] = y[0];
r[1] = y[1];
r[2] = y[2];
r[3] = y[3] + rotl(x[3], 30);
} else if (step.Opcode == 0xC9) {
r[0] = x[0] ^ y[2];
r[1] = x[1] ^ y[3];
r[2] = x[2] ^ x[0];
r[3] = x[3] ^ x[1];
} else if (step.Opcode == 0xCA) {
r[3] = rotl(x[3] ^ y[2], 1);
r[2] = rotl(x[2] ^ y[1], 1);
r[1] = rotl(x[1] ^ y[0], 1);
r[0] = rotl(x[0] ^ r[3], 1);
} else {
constexpr std::uint32_t keys[4] = {0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6};
const auto function = step.Immediate & 3;
std::uint32_t a = x[3], b = x[2], c = x[1], d = x[0], e = 0;
for (int round = 0; round < 4; ++round) {
std::uint32_t f = b ^ c ^ d;
if (function == 0) f = (b & c) ^ (~b & d);
if (function == 2) f = (b & c) ^ (b & d) ^ (c & d);
const auto next = f + rotl(a, 5) + y[3 - round] + e + keys[function];
e = d; d = c; c = rotl(b, 30); b = a; a = next;
}
r[0] = d;
r[1] = c;
r[2] = b;
r[3] = a;
}
std::memcpy(xmm[step.Destination], r, sizeof(r));
}
std::string describeSha1(const std::vector<Sha1Step>& steps) {
std::string text;
for (const auto& step : steps) {
text += std::string(step.Opcode == 0xCC ? "sha1rnds4" : step.Opcode == 0xC8 ? "sha1nexte" : step.Opcode == 0xC9 ? "sha1msg1" : "sha1msg2") + " xmm" + std::to_string(step.Destination) + ", ";
text += step.Memory ? std::string(kSha1Memory[step.Source].Name) : "xmm" + std::to_string(step.Source);
if (step.Opcode == 0xCC) text += ", " + std::to_string(step.Immediate);
text += "; ";
}
return text;
}
void executeSha1(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, const std::vector<Sha1Step>& steps, std::mt19937_64& random, const int images) {
std::vector<Bytes> sites;
for (const auto& step : steps) sites.push_back(encodeSha1(step));
const std::vector<std::span<const std::uint8_t>> spans(sites.begin(), sites.end());
const auto match = steps.size() == 1 ? matcher.Match(sites[0].data(), sites[0].size()) : matcher.MatchSequence(spans, {});
require(match.has_value() && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "SHA-1 was not lowered through a stub: " + describeSha1(steps));
std::uint16_t written = 0;
for (const auto& step : steps) written = static_cast<std::uint16_t>(written | (1u << step.Destination));
const std::array<std::uint64_t, 4> flags = {0x202, 0x203, 0x246, 0xAC7};
for (int image = 0; image < images; ++image) {
State input{};
for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast<std::uint8_t>(random());
input.FlagsIn = flags[static_cast<std::size_t>(image) % flags.size()];
State state = input;
harness.Run(match->StubBody, match->ReturnBranchOffset, state);
++g_executions;
State expected = input;
for (const auto& step : steps) sha1Reference(step, input, expected.Xmm);
for (unsigned reg = 0; reg < 16; ++reg)
if (((written >> reg) & 1) != 0) require(std::memcmp(state.Xmm[reg], expected.Xmm[reg], 16) == 0, "Lowered SHA-1 computed the wrong xmm" + std::to_string(reg) + ": " + describeSha1(steps));
requireEnvironment(input, state, written, describeSha1(steps));
}
}
void sha1Execution(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, std::mt19937_64& random) {
const std::array<std::uint8_t, 4> opcodes = {0xCC, 0xC8, 0xC9, 0xCA};
for (const auto opcode : opcodes) {
for (std::uint8_t dst = 0; dst < 16; ++dst) {
for (std::uint8_t src = 0; src < 16; ++src)
executeSha1(harness, matcher, {{opcode, dst, src, static_cast<std::uint8_t>(random())}}, random, 2);
for (std::uint8_t memory = 0; memory < kSha1Memory.size(); ++memory)
for (std::uint8_t function = 0; function < 4; ++function)
executeSha1(harness, matcher, {{opcode, dst, memory, static_cast<std::uint8_t>((random() & 0xFC) | function), true}}, random, 2);
}
}
for (int sequence = 0; sequence < 256; ++sequence) {
std::vector<Sha1Step> steps(2 + random() % 2);
for (auto& step : steps) {
const bool memory = random() % 2 == 0;
step = {opcodes[random() % opcodes.size()], static_cast<std::uint8_t>(random() % 16), static_cast<std::uint8_t>(random() % (memory ? kSha1Memory.size() : 16)), static_cast<std::uint8_t>(random()), memory};
}
executeSha1(harness, matcher, steps, random, 2);
}
}
void clzeroExecution(const Harness& harness, const Codegen::IAmd64OnlyInstructionMatcher& matcher, std::mt19937_64& random) {
const Bytes plain = {0x0F, 0x01, 0xFC};
const Bytes addressSize32 = {0x67, 0x0F, 0x01, 0xFC};
const std::vector<std::span<const std::uint8_t>> pair = {plain, plain};
constexpr std::size_t size = 3 * 4096;
auto* high = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
std::uint8_t* low = nullptr;
for (std::uintptr_t hint = 0x10000000; low == nullptr && hint < 0x80000000; hint += 0x10000000)
low = static_cast<std::uint8_t*>(VirtualAlloc(reinterpret_cast<void*>(hint), size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
require(high != nullptr && low != nullptr, "Cannot allocate the CLZERO buffers");
struct ClzeroCase {
std::optional<Codegen::Amd64OnlyMatch> Match;
std::uint8_t* Buffer;
std::uint64_t Junk;
std::string Name;
};
const std::vector<ClzeroCase> cases = {
{matcher.Match(plain.data(), plain.size()), high, 0, "clzero"},
{matcher.Match(addressSize32.data(), addressSize32.size()), low, 0x5A5A5A5A00000000ull, "67h clzero"},
{matcher.MatchSequence(pair, {}), high, 0, "clzero; clzero"}};
const std::array<std::uint64_t, 4> flags = {0x202, 0x203, 0x246, 0xAC7};
for (const auto& item : cases) {
require(item.Match.has_value() && item.Match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "CLZERO was not lowered through a stub: " + item.Name);
for (int image = 0; image < 64; ++image) {
const auto address = reinterpret_cast<std::uint64_t>(item.Buffer) + 64 + random() % (size - 128);
Bytes body = {0x48, 0xB8};
for (std::size_t index = 0; index < 8; ++index) body.push_back(static_cast<std::uint8_t>((address | item.Junk) >> (index * 8)));
body.insert(body.end(), {0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00});
const auto prefix = body.size();
body.insert(body.end(), item.Match->StubBody.begin(), item.Match->StubBody.end());
State input{};
for (auto& lane : input.Xmm) for (auto& byte : lane) byte = static_cast<std::uint8_t>(random());
input.FlagsIn = flags[static_cast<std::size_t>(image) % flags.size()];
State state = input;
std::memset(item.Buffer, 0xA5, size);
harness.Run(body, prefix + item.Match->ReturnBranchOffset, state);
++g_executions;
const auto line = address & ~std::uint64_t{63};
for (std::size_t index = 0; index < size; ++index) {
const auto at = reinterpret_cast<std::uint64_t>(item.Buffer + index);
require(item.Buffer[index] == (at >= line && at < line + 64 ? 0x00 : 0xA5), "Lowered CLZERO did not clear exactly the addressed line: " + item.Name);
}
requireEnvironment(input, state, 0, item.Name);
}
}
VirtualFree(high, 0, MEM_RELEASE);
VirtualFree(low, 0, MEM_RELEASE);
}
void cpuExecution() {
const Harness harness;
const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher();
std::mt19937_64 random(0x5EED);
const std::vector<Case> sites = {{false, 3, 3, 8, 40}, {true, 3, 3, 8, 8}, {true, 1, 0, 8, 0}, {true, 9, 4, 16, 16}, {true, 3, 4, 16, 16}};
for (const auto& site : sites) executeCase(harness, *matcher, site, random, 64);
const std::vector<std::pair<std::uint8_t, std::uint8_t>> pairs = {{3, 3}, {1, 0}, {9, 4}, {3, 4}, {15, 15}, {0, 15}, {8, 8}};
for (std::uint8_t length = 1; length <= 64; ++length) {
for (std::uint8_t index = 0; index + length <= 64; ++index) {
for (const auto& [dst, src] : pairs) executeCase(harness, *matcher, {true, dst, src, length, index}, random, 2);
for (const std::uint8_t dst : std::array<std::uint8_t, 4>{0, 3, 9, 15}) executeCase(harness, *matcher, {false, dst, dst, length, index}, random, 2);
const auto& [dst, src] = pairs[(length * 65u + index) % pairs.size()];
executeCase(harness, *matcher, {true, dst, src, length, index, true}, random, 2);
}
}
for (const std::uint8_t opcode : {std::uint8_t{0xCB}, std::uint8_t{0xCC}, std::uint8_t{0xCD}})
for (std::uint8_t dst = 0; dst < 16; ++dst)
for (std::uint8_t src = 0; src < 16; ++src)
executeSha256(harness, *matcher, {{opcode, dst, src}}, random, 2);
for (int sequence = 0; sequence < 256; ++sequence) {
std::vector<Sha256Step> steps(2 + random() % 2, Sha256Step{});
for (auto& step : steps) step = {static_cast<std::uint8_t>(0xCB + random() % 3), static_cast<std::uint8_t>(random() % 16), static_cast<std::uint8_t>(random() % 16)};
executeSha256(harness, *matcher, steps, random, 2);
}
clzeroExecution(harness, *matcher, random);
sha1Execution(harness, *matcher, random);
}
Bytes elfFixture(const Bytes& text) {
Bytes bytes(0x400);
bytes[0] = 0x7F;
bytes[1] = 'E';
bytes[2] = 'L';
bytes[3] = 'F';
bytes[4] = 2;
bytes[5] = 1;
bytes[6] = 1;
write<std::uint16_t>(bytes, 16, 3);
write<std::uint16_t>(bytes, 18, 62);
write<std::uint64_t>(bytes, 24, 0x1000);
write<std::uint64_t>(bytes, 32, 64);
write<std::uint16_t>(bytes, 54, 56);
write<std::uint16_t>(bytes, 56, 2);
std::fill(bytes.begin() + 0x200, bytes.begin() + 0x300, 0xCC);
std::copy(text.begin(), text.end(), bytes.begin() + 0x200);
return bytes;
}
std::vector<Domain::ProgramHeader> elfHeaders() {
return {{1, 5, 0x200, 0x1000, 0, 0x100, 0x100, 0x1000}, {1, 6, 0x300, 0x2000, 0, 0x100, 0x100, 0x1000}};
}
void peBuilder() {
const Bytes site = {0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x08};
Bytes text = {0xEB, 0x06};
text.insert(text.end(), site.begin(), site.end());
text.push_back(0xC3);
const auto source = elfFixture(text);
const auto headers = elfHeaders();
const auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(source, {headers[0]});
require(converted.Trampolines.size() == 1, "PE fixture conversion did not produce one stub");
const WindowsLoadImage image(converted.Bytes, headers);
auto sections = image.BuildSections();
auto nextRva = image.GetEndRva();
const auto stubSectionRva = nextRva;
WindowsTrampolineBuilder().Build(converted.Trampolines, image, sections, nextRva);
require(sections.size() == 3 && sections.back().Name == ".amdstub" && sections.back().Rva == stubSectionRva && (sections.back().Characteristics & SectionExecute) != 0 && nextRva == AlignRva(stubSectionRva + sections.back().Data.size()), "Stub section was not appended");
const auto& text0 = sections[0];
const auto siteRva = LoadRva + 2;
require(text0.Data[2] == 0xE9 && text0.Data[7] == 0x90 && text0.Data[8] == 0xC3, "PE site was not replaced by a jump");
const auto stubRva = static_cast<std::uint32_t>(static_cast<std::int64_t>(siteRva + 5) + read<std::int32_t>(text0.Data, 3));
require(stubRva == stubSectionRva && stubRva % 16 == 0, "PE jump does not land on the 16-aligned stub");
const auto& body = sections.back().Data;
require(body.size() == 32 && body[0] == 0x66 && body[9] == 0xE9 && body[16] == 0x00 && body[17] == 0x00 && body[18] == 0x02, "PE stub body is wrong");
require(static_cast<std::int64_t>(stubRva + 9 + 5) + read<std::int32_t>(body, 10) == siteRva + 6, "PE stub does not return past the site");
require(read<std::int32_t>(body, 5) == 7 && (stubRva + 16) % 16 == 0, "PE mask constant is not 16-aligned");
std::array<PeDirectory, 16> directories{};
const auto file = WindowsPeWriter().Write(sections, LoadRva, directories);
require(file.size() > 0x400, "PE writer rejected the stub section");
require(read<std::uint16_t>(file, 0x96) == 0x22, "PE writer marked a fixed image as relocation-stripped");
auto altered = converted.Bytes;
altered[0x205] = 0xDC;
requireFailure([&] {
const WindowsLoadImage alteredImage(altered, headers);
auto alteredSections = alteredImage.BuildSections();
auto rva = alteredImage.GetEndRva();
WindowsTrampolineBuilder().Build(converted.Trampolines, alteredImage, alteredSections, rva);
}, "Changed PE site bytes were accepted");
requireFailure([&] {
auto unmapped = converted.Trampolines;
unmapped[0].Address = 0x5000;
auto freshSections = image.BuildSections();
auto rva = image.GetEndRva();
WindowsTrampolineBuilder().Build(unmapped, image, freshSections, rva);
}, "Unmapped PE site was accepted");
auto untouched = image.BuildSections();
auto untouchedRva = image.GetEndRva();
WindowsTrampolineBuilder().Build({}, image, untouched, untouchedRva);
require(untouched.size() == 2 && untouchedRva == image.GetEndRva(), "Empty site list changed the image");
}
}
int main() {
try {
cpuExecution();
reciprocalExecution();
peBuilder();
std::cout << "AMD64-only Windows tests passed (" << g_executions << " lowered sequences executed)\n";
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink