diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp index 1389054f..fb590f9e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp @@ -226,13 +226,13 @@ void DefineBvhIntersect(EmitterState& s) { if (!s.requirements.bvh) return; const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeScalarU64(s); const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3); - const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, u, f, vec3, vec3, vec3); + const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3); s.bvh_intersect_function = s.builder.AllocateId(); s.builder.AddName(s.bvh_intersect_function, "bvh_intersect"); s.builder.AddFunction(spv::OpFunction, vec4, s.bvh_intersect_function, spv::FunctionControlMaskNone, signature); std::array args; - const std::array types {vec4, u, f, vec3, vec3, vec3}; + const std::array types {vec4, wide, f, vec3, vec3, vec3}; for (uint32_t i = 0; i < args.size(); ++i) { args[i] = s.builder.AllocateId(); s.builder.AddFunction(spv::OpFunctionParameter, types[i], args[i]); @@ -253,7 +253,7 @@ void DefineBvhIntersect(EmitterState& s) { const auto shr = [&](uint32_t value, uint32_t shift) { return Binary(s, spv::OpShiftRightLogical, u, value, ConstantU32(s, shift)); }; - const auto kind = and_bits(node, 7); + const auto kind = and_bits(Unary(s, spv::OpUConvert, u, node), 7); const auto triangle = Binary(s, spv::OpULessThan, b, kind, ConstantU32(s, 4)); const auto full = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 5)); const auto half = Binary(s, spv::OpIEqual, b, kind, ConstantU32(s, 4)); @@ -262,7 +262,7 @@ void DefineBvhIntersect(EmitterState& s) { const auto grow = and_bits(shr(words[1], 23), 0xff); const auto base = Binary(s, spv::OpShiftLeftLogical, wide, DeviceAddressFromWords(s, words[0], and_bits(words[1], 0xff)), ConstantDeviceAddress(s, 8)); - const auto index = Unary(s, spv::OpUConvert, wide, shr(node, 3)); + const auto index = Binary(s, spv::OpShiftRightLogical, wide, node, ConstantDeviceAddress(s, 3)); const auto last_index = Binary(s, spv::OpIAdd, wide, index, Select(s, wide, full, ConstantDeviceAddress(s, 1), ConstantDeviceAddress(s, 0))); const auto address = Binary(s, spv::OpIAdd, wide, base, @@ -325,17 +325,23 @@ void DefineBvhIntersect(EmitterState& s) { uint32_t EmitBvhIntersect(ValueEmitContext& ctx, const IR::Inst& inst) { auto& s = ctx.state; const auto* ray = ctx.ImageAddress(inst.Arg(1)); - if (ray == nullptr || ray->NumArgs() < 11 || s.bvh_intersect_function == 0) - ctx.Fail(inst, "BVH intersection requires eleven address components and its shared function"); + const auto node_words = inst.Flags(); + if (ray == nullptr || (node_words != 1u && node_words != 2u) || + ray->NumArgs() < node_words + 10u || s.bvh_intersect_function == 0) + ctx.Fail(inst, "BVH intersection requires its node pointer, ray components and shared function"); const auto vector = [&](uint32_t first) { Vec3 values; for (uint32_t i = 0; i < 3; ++i) values[i] = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, first + i)); return Vector(s, TypeF32Vector(s, 3), values); }; - const auto descriptor = ctx.Arg(inst, 0), node = ctx.Arg(*ray, 0); - const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, 1)); - const auto origin = vector(2), direction = vector(5), inverse = vector(8); + const auto descriptor = ctx.Arg(inst, 0); + const auto node = node_words == 2u + ? DeviceAddressFromWords(s, ctx.Arg(*ray, 0), ctx.Arg(*ray, 1)) + : Unary(s, spv::OpUConvert, TypeScalarU64(s), ctx.Arg(*ray, 0)); + const auto extent = Unary(s, spv::OpBitcast, TypeF32(s), ctx.Arg(*ray, node_words)); + const auto origin = vector(node_words + 1u), direction = vector(node_words + 4u), + inverse = vector(node_words + 7u); return EmitValueOrDefaultIfCondition(s, ctx.Arg(inst, 2), TypeU32Vector(s, 4), ConstantU32CompositeZero(s, 4), [&] { const auto result = s.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 1810a4db..1c08d461 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -228,7 +228,8 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim case 0x09u: return Opcode::IMAGE_STORE_MIP; case 0x0eu: return Opcode::IMAGE_GET_RESINFO; case 0x60u: return Opcode::IMAGE_GET_LOD; - case 0xe6u: return Opcode::IMAGE_BVH_INTERSECT_RAY; + case 0xe6u: + case 0xe7u: return Opcode::IMAGE_BVH_INTERSECT_RAY; default: return Opcode::UNSUPPORTED; } } @@ -258,6 +259,7 @@ uint32_t DecodeMimgAddressComponents(uint32_t opcode, ImageDimension dimension, switch (opcode) { case 0xe6u: return 11u; + case 0xe7u: return 12u; case 0x0eu: return 1u; case 0x01u: case 0x09u: return ImageCoordComponents(dimension) + 1u; @@ -379,10 +381,10 @@ void DecodeMimg(uint32_t pc, std::span code, uint32_t word_index if (d16 && !supports_d16) { SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode does not support D16 data"); } - if (opcode == 0xe6u && + if (inst.opcode == Opcode::IMAGE_BVH_INTERSECT_RAY && (a16 || !r128 || inst.dmask != 0xfu || (nsa_dwords != 0u && nsa_dwords != 3u))) { SetUnsupported(inst, Family::MIMG, opcode, - "BVH intersection requires eleven full-float ray DWORDs and R128/dmask:0xf"); + "BVH intersection requires full-float ray DWORDs and R128/dmask:0xf"); } DecodeVectorGpr(vdata, inst.dst); diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index bc96c529..5872deb6 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -905,7 +905,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { switch (inst.opcode) { case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: { const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect, - {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()}); + {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()}, + inst.image_address_components - 10u); for (uint32_t component = 0; component < 4; ++component) { WriteOperand(OffsetOperand(inst.dst, component), ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(component)})); diff --git a/tests/ShaderRayTracingGpuTests.inc b/tests/ShaderRayTracingGpuTests.inc index 6ba16f1a..f2bdb2a2 100644 --- a/tests/ShaderRayTracingGpuTests.inc +++ b/tests/ShaderRayTracingGpuTests.inc @@ -83,13 +83,15 @@ TestCase FlatStackApertures(u32 wave_size) { return test; } -TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) { +TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0, bool wide = false) { TestCase test; test.name = barycentrics ? (sorted ? "BvhBarycentricSorted" : "BvhBarycentricUnsorted") : (sorted ? "BvhTriangleIdSorted" : "BvhTriangleIdUnsorted"); if (grow != 0) test.name = "BvhBoxExitGrowth"; + if (wide) test.name = "Bvh64HighNodePointers"; constexpr u32 NodeStart = 1024, OutputStart = 512, LaneCount = 32; constexpr uint64_t GuestBase = 0x10000; + const u32 node_base = wide ? 1u << 29 : 0; const auto bits = [](float value) { return std::bit_cast(value); }; constexpr u32 Infinity = 0x7f800000, Invalid = UINT32_MAX, Flag = 0x04040404; test.initial.resize(NodeStart + 258 * 16); @@ -183,6 +185,19 @@ TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) { } else if (lane == 29) { data[input] = (255 << 3) | 5; } + if (wide) { + // Keep the same small backing while placing nodes beyond the 32-bit pointer range. + std::copy_backward(data.begin() + input + 1, data.begin() + input + 11, + data.begin() + input + 12); + data[input + 1] = 1; + if (lane == 14 || lane == 15) { + data[input] = lane == 14 ? 0 : 5; + // Neither an oversized index nor a shifted-address wrap may read node zero. + data[input + 1] = lane == 14 ? 0x00001000 : 0x20000000; + expected[lane] = lane == 14 ? std::array{Infinity, bits(1), Invalid, 0} + : std::array{Invalid, Invalid, Invalid, Invalid}; + } + } } test.expected = test.initial; for (u32 lane = 0; lane < LaneCount; ++lane) { @@ -198,29 +213,31 @@ TestCase BvhIntersections(bool barycentrics, bool sorted, u32 grow = 0) { // The resource is a GPU value, not a materialized host image descriptor. AppendSMovLiteral(&code, 8, GuestBase >> 8); AppendSMovLiteral(&code, 9, (sorted ? 0x80000000 : 0) | (grow << 23)); - AppendSMovLiteral(&code, 10, 256); + AppendSMovLiteral(&code, 10, node_base + 256); AppendSMovLiteral(&code, 11, barycentrics ? 0x81000000 : 0x80000000); code.push_back(EncodeVop2(0x1a, 1, InlineU32(6), 0)); for (u32 offset : {0u, 4u, 8u}) { - code.insert(code.end(), {EncodeMubuf0(offset == 8 ? 0x0f : 0x0e, offset * 4), + code.insert(code.end(), {EncodeMubuf0(offset == 8 && !wide ? 0x0f : 0x0e, offset * 4), EncodeMubuf1(4 + offset, 0, 1)}); } - code.insert(code.end(), {EncodeMimg0(0xe6, 15, 0, false, 0, true), + code.insert(code.end(), {EncodeMimg0(wide ? 0xe7 : 0xe6, 15, 0, false, 0, true), EncodeMimg1(20, 4, 2)}); code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0)); for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 20 + i, 2, OutputStart + i); // NSA addressing, the FP32 second-block bound, and inactive destination lanes. - AppendSMovLiteral(&code, 10, 2); + AppendSMovLiteral(&code, 10, node_base + 2); for (u32 i = 0; i < 4; ++i) AppendVMovLiteral(&code, 24 + i, 0x12345678 + i); AppendSMovLiteral(&code, 126, 0x000fffff); - code.insert(code.end(), {EncodeMimg0(0xe6, 15, 3, false, 0, true), - EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, 0x00000e0d}); + code.insert(code.end(), {EncodeMimg0(wide ? 0xe7 : 0xe6, 15, 3, false, 0, true), + EncodeMimg1(24, 4, 2), 0x08070605, 0x0c0b0a09, + wide ? 0x000f0e0du : 0x00000e0du}); code.push_back(EncodeSop1(0x04, 126, 193)); for (u32 i = 0; i < 4; ++i) AppendStoreVgprAtLaneDwordOffset(&code, 24 + i, 2, OutputStart + LaneCount * 4 + i); AppendEnd(&code); - test.bda_mappings = {{GuestBase, NodeStart * sizeof(u32)}, - {GuestBase + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}}; + const auto nodes_address = GuestBase + (uint64_t{node_base} << 6); + test.bda_mappings = {{nodes_address, NodeStart * sizeof(u32)}, + {nodes_address + 0x4000, (NodeStart + 257 * 16) * sizeof(u32)}}; test.compute_info.wave_size = LaneCount; test.compute_info.threads_num[0] = LaneCount; test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; diff --git a/tests/ShaderRayTracingTests.inc b/tests/ShaderRayTracingTests.inc index dd79ee70..79f40329 100644 --- a/tests/ShaderRayTracingTests.inc +++ b/tests/ShaderRayTracingTests.inc @@ -5,10 +5,15 @@ void TestRayTracingInstructions() { EncodeMubuf0(0x1e, 0, false), EncodeMubuf1(16, 0, 0), 0xf1989f07u, 0x00071001u, 0x3d3e3c15u, 0x38095851u, 0x00003a39u, EncodeMubuf0(0x1e, 16, false), EncodeMubuf1(16, 0, 0), + 0xf19c9f01u, 0x00081014u, + EncodeMubuf0(0x1e, 32, false), EncodeMubuf1(16, 0, 0), + // Captured 016c1ab44ee6a526: pointer v2:v7 and final inverse component v5. + 0xf19c9f07u, 0x00090202u, 0x28263607u, 0x221f1e29u, 0x00050403u, + EncodeMubuf0(0x1e, 48, false), EncodeMubuf1(2, 0, 0), EncodeSopp(0x01)}; D::Program decoded; D::DecodeProgram(code, decoded); - Check(decoded.instructions.size() == 5 && + Check(decoded.instructions.size() == 9 && decoded.instructions.back().opcode == D::Opcode::S_ENDPGM, "decoder stopped before instructions following BVH intersections"); for (const auto index : {0u, 2u}) { @@ -17,10 +22,20 @@ void TestRayTracingInstructions() { inst.image_address_components == 11 && inst.data_dwords == 4 && inst.image_r128, "BVH32 instruction lost its ray or descriptor width"); } + for (const auto index : {4u, 6u}) { + const auto &inst = decoded.instructions[index]; + Check(inst.opcode == D::Opcode::IMAGE_BVH_INTERSECT_RAY && inst.opcode_id == 0xe7 && + inst.image_address_components == 12 && inst.data_dwords == 4 && inst.image_r128, + "BVH64 instruction lost its full node pointer or ray width"); + } + Check(decoded.instructions[6].word_count == 5 && + decoded.instructions[6].image_nsa_addr[0] == 7 && + decoded.instructions[6].image_nsa_addr[10] == 5, + "captured BVH64 NSA pointer or final ray component was not decoded"); Check(decoded.instructions[2].image_nsa_addr[9] == 58, "BVH NSA ray components were not decoded"); const auto result = RecompileForTest(code, MakeCompileOptions(ShaderType::Compute)); - Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 2 && + Check(CountSourceOccurrences(result.ir_dump, "BvhIntersect") == 4 && result.program.info.uses_dma && result.program.info.images.empty() && result.program.info.samplers.empty(), "BVH intersections did not use GPU descriptors and raw guest memory"); @@ -31,20 +46,22 @@ void TestRayTracingInstructions() { Check(decoded.instructions.size() == 2 && decoded.instructions[0].opcode == D::Opcode::S_MOV_B32, "BVH-shaped scalar literal was decoded as an instruction"); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - for (const auto words : {std::array{0xf19c9f01u, 0x00081014u}, // BVH64 - std::array{0xf1989f01u, 0x40081014u}, // A16 + for (const auto words : {std::array{0xf19c9f01u, 0x40081014u}, // BVH64 A16 + std::array{0xf1989f01u, 0x40081014u}, // BVH32 A16 std::array{0xf1981f01u, 0x00081014u}, // not R128 std::array{0xf1989f01u, 0x80081014u}}) { // D16 const std::array unsupported{words[0], words[1], EncodeSopp(0x01)}; ExpectFatal([&]() { (void)RecompileForTest(unsupported, MakeCompileOptions(ShaderType::Compute)); }, "unsupported BVH encoding was silently accepted"); } - for (const auto nsa : {1u, 2u}) { - std::array truncated{0xf1989f01u | (nsa << 1), 0x00081014u}; - truncated[2 + nsa] = EncodeSopp(0x01); - ExpectFatal([&]() { - (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute)); - }, "BVH NSA instruction accepted fewer than eleven ray addresses"); + for (const auto encoding : {0xf1989f01u, 0xf19c9f01u}) { + for (const auto nsa : {1u, 2u}) { + std::array truncated{encoding | (nsa << 1), 0x00081014u}; + truncated[2 + nsa] = EncodeSopp(0x01); + ExpectFatal([&]() { + (void)RecompileForTest(std::span(truncated).first(3 + nsa), MakeCompileOptions(ShaderType::Compute)); + }, "BVH NSA instruction accepted an incomplete ray address"); + } } #endif } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ca3c78b5..4fda1f55 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -33248,6 +33248,7 @@ std::vector MakeCases() { cases.push_back(FlatStackApertures(32)); cases.push_back(FlatStackApertures(64)); cases.push_back(BvhIntersections(true, true, 1)); + cases.push_back(BvhIntersections(true, true, 0, true)); for (bool barycentrics : {false, true}) { for (bool sorted : {false, true}) { cases.push_back(BvhIntersections(barycentrics, sorted)); @@ -38481,6 +38482,7 @@ int main(int argc, char **argv) { RunCase(&vulkan, FlatStackApertures(32)); RunCase(&vulkan, FlatStackApertures(64)); RunCase(&vulkan, BvhIntersections(true, true, 1)); + RunCase(&vulkan, BvhIntersections(true, true, 0, true)); for (bool barycentrics : {false, true}) { for (bool sorted : {false, true}) { RunCase(&vulkan, BvhIntersections(barycentrics, sorted));