diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp new file mode 100644 index 000000000..d0a599877 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp @@ -0,0 +1,304 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Record = 16; +constexpr std::uint32_t Forms = 4; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint64_t NodeBytes = 64; +constexpr std::uint32_t LastNode = 6; +constexpr std::uint32_t Invalid = 0xffffffffu; +constexpr std::uint32_t BoxSort = 0x80000000u; +constexpr std::uint32_t TriangleId = 0x00abc123u; +constexpr std::array Box32Children{0x1000u, 0x1001u, 0x1002u, 0x1003u}; +constexpr std::array Box16Children{0x2000u, 0x2001u, 0x2002u, 0x2003u}; +constexpr std::array PastEndChildren{0x3000u, 0x3001u, 0x3002u, 0x3003u}; +constexpr std::array FormNames{"image_bvh64_intersect_ray", "image_bvh_intersect_ray", "image_bvh64_intersect_ray a16", "image_bvh_intersect_ray a16"}; + +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +constexpr std::array MakeCode() { + std::array code{}; + std::size_t at = 0; + code[at++] = 0x34020086u; + for (std::uint32_t dword = 0; dword < Record; ++dword) { + code[at++] = 0xe0301000u | (dword * 4u); + code[at++] = 0x80000001u | ((4u + dword) << 8u); + } + code[at++] = 0xbf8c3f70u; + code[at++] = 0xf19c9f01u; + code[at++] = 0x00031405u; + code[at++] = 0xf1989f07u; + code[at++] = 0x00021804u; + code[at++] = 0x0a090807u; + code[at++] = 0x0e0d0c0bu; + code[at++] = 0x0000100fu; + code[at++] = 0xf19c9f05u; + code[at++] = 0x40031c05u; + code[at++] = 0x09080706u; + code[at++] = 0x1312110au; + code[at++] = 0xf1989f05u; + code[at++] = 0x40022004u; + code[at++] = 0x0a090807u; + code[at++] = 0x00131211u; + for (std::uint32_t dword = 0; dword < Record; ++dword) { + code[at++] = 0xe0701000u | (dword * 4u); + code[at++] = 0x80010001u | ((20u + dword) << 8u); + } + code[at++] = 0xbf810000u; + return code; +} + +alignas(256) constexpr std::array Code = MakeCode(); + +struct Ray { + float extent; + std::array origin; + std::array direction; + std::array inverse; +}; + +struct Case { + std::uint32_t node; + Ray ray; + std::array expected; +}; + +constexpr float Infinity = std::numeric_limits::infinity(); +constexpr std::array Diagonal{0.25f, 0.5f, 1.0f}; +constexpr std::array DiagonalInverse{4.0f, 2.0f, 1.0f}; +constexpr Ray Long{100.0f, {0.0f, 0.0f, 0.0f}, Diagonal, DiagonalInverse}; +constexpr Ray Short{2.0f, {0.0f, 0.0f, 0.0f}, Diagonal, DiagonalInverse}; +constexpr Ray Forward{100.0f, {0.25f, 0.25f, 0.0f}, {0.0f, 0.0f, 1.0f}, {Infinity, Infinity, 1.0f}}; + +constexpr std::uint32_t Pointer(std::uint32_t node, std::uint32_t type) { + return (node << 3u) | type; +} + +constexpr std::array Cases{{ + {Pointer(2, 5), Long, {Box32Children[2], Box32Children[0], Invalid, Invalid}}, + {Pointer(4, 4), Long, {Box16Children[2], Box16Children[0], Invalid, Invalid}}, + {Pointer(1, 0), Forward, {0xc0a00000u, 0xbf800000u, TriangleId, 1u}}, + {Pointer(1, 1), Forward, {0x7f800000u, 0x3f800000u, TriangleId, 0u}}, + {Pointer(1, 7), Long, {Invalid, Invalid, Invalid, Invalid}}, + {Pointer(6, 5), Long, {Invalid, Invalid, Invalid, Invalid}}, + {Pointer(7, 0), Forward, {Invalid, Invalid, Invalid, Invalid}}, + {Pointer(2, 5), Short, {Box32Children[2], Invalid, Invalid, Invalid}}, +}}; + +constexpr std::array, 4> Boxes{{ + {1.0f, 2.0f, 4.0f, 2.0f, 4.0f, 8.0f}, + {10.0f, 0.0f, 0.0f, 11.0f, 1.0f, 1.0f}, + {0.25f, 0.5f, 1.0f, 0.5f, 1.0f, 2.0f}, + {-2.0f, -4.0f, -8.0f, -1.0f, -2.0f, -4.0f}, +}}; + +constexpr std::array TriangleVertices{0.0f, 0.0f, 5.0f, 1.0f, 0.0f, 5.0f, 0.0f, 1.0f, 5.0f, 1.0f, 1.0f, 5.0f}; + +std::uint32_t Bits(float value) { + return std::bit_cast(value); +} + +std::uint32_t Half(float value) { + const auto bits = Bits(value); + const auto sign = (bits >> 16u) & 0x8000u; + const auto exponent = (bits >> 23u) & 0xffu; + const auto mantissa = bits & 0x7fffffu; + if (exponent == 0u && mantissa == 0u) return sign; + if (exponent == 0xffu && mantissa == 0u) return sign | 0x7c00u; + Require(exponent >= 113u && exponent <= 142u && (mantissa & 0x1fffu) == 0u, "bvh64 intersect ray: a test value is not a float16"); + return sign | ((exponent - 112u) << 10u) | (mantissa >> 13u); +} + +class GuestBlock { +public: + GuestBlock() { + constexpr std::uintptr_t hint = std::uintptr_t{1} << 40u; +#ifdef _WIN32 + for (std::uintptr_t attempt = 0; block == nullptr && attempt < 16u; ++attempt) { + block = static_cast(VirtualAlloc(reinterpret_cast(hint + (attempt << 32u)), BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } + if (block == nullptr) block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + void* mapped = mmap(reinterpret_cast(hint), BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + block = mapped == MAP_FAILED ? nullptr : static_cast(mapped); +#endif + Require(block != nullptr, "bvh64 intersect ray: cannot allocate the guest block"); + std::memset(block, 0, BlockBytes); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + munmap(block, BlockBytes); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint64_t Address() const { return static_cast(reinterpret_cast(block)); } + + void Write(std::uint32_t node, std::uint32_t dword, std::uint32_t value) { + std::memcpy(block + node * NodeBytes + dword * 4u, &value, sizeof(value)); + } + +private: + std::uint8_t* block = nullptr; +}; + +void WriteBox32(GuestBlock& guest, std::uint32_t node, const std::array& children) { + for (std::uint32_t child = 0; child < 4u; ++child) { + guest.Write(node, child, children[child]); + for (std::uint32_t index = 0; index < 6u; ++index) guest.Write(node, 4u + child * 6u + index, Bits(Boxes[child][index])); + } +} + +void WriteBox16(GuestBlock& guest, std::uint32_t node, const std::array& children) { + for (std::uint32_t child = 0; child < 4u; ++child) { + guest.Write(node, child, children[child]); + for (std::uint32_t index = 0; index < 6u; index += 2u) { + const auto position = child * 6u + index; + guest.Write(node, 4u + position / 2u, Half(Boxes[child][index]) | (Half(Boxes[child][index + 1u]) << 16u)); + } + } +} + +void WriteTriangles(GuestBlock& guest, std::uint32_t node) { + for (std::uint32_t index = 0; index < TriangleVertices.size(); ++index) guest.Write(node, index, Bits(TriangleVertices[index])); + guest.Write(node, 15u, TriangleId); +} + +void FillNodes(GuestBlock& guest) { + WriteTriangles(guest, 1u); + WriteBox32(guest, 2u, Box32Children); + WriteBox16(guest, 4u, Box16Children); + WriteBox32(guest, 6u, PastEndChildren); + WriteTriangles(guest, 7u); +} + +void FillInput(std::uint64_t nodeBias) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto& test = Cases[tid % Cases.size()]; + const auto& ray = test.ray; + const std::uint64_t node64 = nodeBias + test.node; + auto* words = &Input[tid * Record]; + words[0] = test.node; + words[1] = static_cast(node64); + words[2] = static_cast(node64 >> 32u); + words[3] = Bits(ray.extent); + for (std::uint32_t axis = 0; axis < 3u; ++axis) { + words[4u + axis] = Bits(ray.origin[axis]); + words[7u + axis] = Bits(ray.direction[axis]); + words[10u + axis] = Bits(ray.inverse[axis]); + } + words[13] = Half(ray.direction[0]) | (Half(ray.direction[1]) << 16u); + words[14] = Half(ray.direction[2]) | (Half(ray.inverse[0]) << 16u); + words[15] = Half(ray.inverse[1]) | (Half(ray.inverse[2]) << 16u); + } +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; +} + +std::array BvhDescriptor(std::uint64_t base, std::uint64_t lastNode) { + return {static_cast(base >> 8u), static_cast((base >> 40u) & 0xffu) | BoxSort, static_cast(lastNode), static_cast((lastNode >> 32u) & 0x3ffu)}; +} + +void Run(AgcDriver::VulkanDevice& device, const GuestBlock& guest, bool addressInPointer) { + const auto address = guest.Address(); + FillInput(addressInPointer ? address >> 3u : 0u); + Output.fill(0xdeadbeefu); + std::vector userData(16, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + const auto narrow = BvhDescriptor(address, LastNode); + const auto wide = addressInPointer ? BvhDescriptor(0u, address / NodeBytes + LastNode) : narrow; + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + std::copy(narrow.begin(), narrow.end(), userData.begin() + 8); + std::copy(wide.begin(), wide.end(), userData.begin() + 12); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(bool addressInPointer) { + const std::string mode = addressInPointer ? "address in the node pointer" : "base in the descriptor"; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto& expected = Cases[tid % Cases.size()].expected; + for (std::uint32_t form = 0; form < Forms; ++form) { + for (std::uint32_t dword = 0; dword < 4u; ++dword) { + const auto actual = Output[tid * Record + form * 4u + dword]; + Require(actual == expected[dword], std::string(FormNames[form]) + ", " + mode + ": thread " + std::to_string(tid) + " dword " + std::to_string(dword) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[dword])); + } + } + } +} + +} + +int main() { + try { + if (ShaderRecompiler::RayTracingStrict() || ShaderRecompiler::RayTracingMiss()) { + std::puts("skipped, APS5_RAYTRACING replaces the node test"); + return VulkanTestSkipped; + } + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestBlock guest; + Require((guest.Address() >> 35u) != 0u, "bvh64 intersect ray: the guest block is not above 32 GiB, the node pointer would fit 32 bits"); + FillNodes(guest); + for (const bool addressInPointer : {true, false}) { + Run(*device, guest, addressInPointer); + Check(addressInPointer); + } + std::puts("bvh64 intersect ray tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/tests/RayTracingMiss.cpp b/core/libs/tests/RayTracingMiss.cpp index 803f62a95..5ca726847 100644 --- a/core/libs/tests/RayTracingMiss.cpp +++ b/core/libs/tests/RayTracingMiss.cpp @@ -6,12 +6,20 @@ using namespace ShaderRecompiler; static void Require(bool value) { if (!value) throw std::runtime_error("ray tracing miss regression"); } -int main() { - const std::array code{0xF1981F01u, 0x00000000u}; +struct Form { + std::uint32_t word0; + std::uint32_t word1; + RdnaOpcode opcode; + std::uint32_t addressDwords; +}; +static void Check(const Form& form) { + const std::array code{form.word0, form.word1}; const RdnaInstruction instruction = DecodeRdnaMimg(0u, code, 0u); - Require(instruction.op == RdnaOpcode::ImageBvhIntersectRay); + Require(instruction.op == form.opcode); Require(instruction.family == RdnaInstructionFamily::MIMG); Require(IsImageOpcode(instruction.op)); + Require(instruction.imageAddressComponents == form.addressDwords); + Require(GetRdnaImageAddressDwordCount(instruction.imageSampleFlags, instruction.imageAddressComponents) == form.addressDwords); IrProgram program; auto& block = program.CreateBlock(); program.SetEntryBlock(block); @@ -26,8 +34,14 @@ int main() { Require(thrown); } else { context.TranslateInstruction(instruction); - if (RayTracingMiss()) { - for (auto* value : block.Instructions()) Require(value->Opcode() != IrOpcode::ImageBvhIntersectRay); - } + bool emitted = false; + for (auto* value : block.Instructions()) emitted = emitted || value->Opcode() == IrOpcode::ImageBvhIntersectRay; + Require(emitted != RayTracingMiss()); } } +int main() { + Check({0xF1981F01u, 0x00000000u, RdnaOpcode::ImageBvhIntersectRay, 11u}); + Check({0xF1981F01u, 0x40000000u, RdnaOpcode::ImageBvhIntersectRay, 8u}); + Check({0xF19C1F01u, 0x00000000u, RdnaOpcode::ImageBvh64IntersectRay, 12u}); + Check({0xF19C1F01u, 0x40000000u, RdnaOpcode::ImageBvh64IntersectRay, 9u}); +} diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 9d9ef50cb..e646263f8 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -56,6 +56,7 @@ std::uint32_t estimatedSpirvWords(const RdnaInstruction& instruction) { case RdnaOpcode::DsPermuteB32: return 1200u; case RdnaOpcode::ImageBvhIntersectRay: + case RdnaOpcode::ImageBvh64IntersectRay: return 2000u; default: break; diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp index 5ed43f202..0eae47c8d 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp @@ -892,6 +892,7 @@ enum class RdnaOpcode : std::uint16_t { ImageGather4LO, ImageGather4O, ImageBvhIntersectRay, + ImageBvh64IntersectRay, VInterpP1F32, VInterpP2F32, VInterpMovF32, diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp index a019dd480..0a44d6361 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp @@ -111,6 +111,7 @@ constexpr ImageOpcodeInfo imageOpcodes[] = { {0x0eu, RdnaOpcode::ImageGetResinfo, nullptr, 0, false, false, false}, {0x60u, RdnaOpcode::ImageGetLod, nullptr, 0, false, false, false}, {0xe6u, RdnaOpcode::ImageBvhIntersectRay, "image_bvh_intersect_ray", 0, false, false, false}, + {0xe7u, RdnaOpcode::ImageBvh64IntersectRay, "image_bvh64_intersect_ray", 0, false, false, false}, }; const ImageOpcodeInfo& lookupOpcode(std::uint32_t opcode) { @@ -306,8 +307,9 @@ RdnaInstruction DecodeRdnaMimg(std::uint32_t programCounter, std::span origin{}; std::array direction{}; std::array inverse{}; for (std::uint32_t axis = 0; axis < 3u; ++axis) { - origin[axis] = asFloat(component(2u + axis)); + origin[axis] = asFloat(component(ray + 1u + axis)); if (!a16) { - direction[axis] = asFloat(component(5u + axis)); - inverse[axis] = asFloat(component(8u + axis)); + direction[axis] = asFloat(component(ray + 4u + axis)); + inverse[axis] = asFloat(component(ray + 7u + axis)); continue; } const auto half = [&](std::uint32_t index) { - const auto packed = component(5u + index / 2u); + const auto packed = component(ray + 4u + index / 2u); return EmitF16BitsToF32(state, index % 2u == 0u ? op(spv::OpBitwiseAnd, u32, packed, uint(0xffffu)) : op(spv::OpShiftRightLogical, u32, packed, uint(16u))); }; direction[axis] = half(axis); diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp index 76cc041ea..2f0bf639b 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp @@ -61,6 +61,7 @@ bool TranslationContext::imageBvhIntersectRay(const RdnaInstruction& inst) { memory.kind = ResourceKind::Global; memory.dataDwords = 4u; memory.imageSampleFlags = inst.imageA16 ? RdnaImageSampleFlagA16 : 0u; + memory.addressIsFull = inst.op == RdnaOpcode::ImageBvh64IntersectRay; IrValue* descriptor = constructU32x4(inst.source1, 4u); IrValue* address = makeImageAddress(inst, inst.source0); IrValue& result = ir.Emit(IrOpcode::ImageBvhIntersectRay, IrOpcodeType(IrOpcode::ImageBvhIntersectRay), {descriptor, address, &ir.GetExec()}, addMemoryInfo(memory, inst.programCounter)); diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp index 1031f7aa8..e5dee40b3 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp @@ -432,6 +432,7 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { case RdnaOpcode::ImageGetLod: return imageGetLod(inst); case RdnaOpcode::ImageBvhIntersectRay: + case RdnaOpcode::ImageBvh64IntersectRay: return imageBvhIntersectRay(inst); default: