From 20e57994cf1ea30e78bc2b9f235d2ecae18d2f88 Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 11 Sep 2026 19:10:48 +0400 Subject: [PATCH] shader: reuse native emitters and simplify memory dispatch --- .../backend/spirv/spirvEmitterAlu.cpp | 454 +++++--------- .../backend/spirv/spirvEmitterAluHelpers.cpp | 144 ----- .../backend/spirv/spirvEmitterFlow.cpp | 82 ++- .../backend/spirv/spirvEmitterInstructions.h | 287 ++++----- .../backend/spirv/spirvEmitterInternal.h | 78 +-- .../backend/spirv/spirvEmitterMemory.cpp | 553 ++++++++---------- .../spirv/spirvEmitterMemoryHelpers.cpp | 14 - .../backend/spirv/spirvEmitterProgram.cpp | 30 +- tests/ShaderRecompilerComputeTests.cpp | 39 ++ tests/shaderCfgTests.cpp | 33 ++ 10 files changed, 721 insertions(+), 993 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp index 63ed452f..fde9f073 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp @@ -41,32 +41,6 @@ uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_ return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path); } -uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto low_pair = state.builder.AllocateId(); - const auto low = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low}); - state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0}); - state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1}); - const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high); - const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry); - return MakePair(state, low, high); -} - -uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low); - const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low); - const auto borrow_u32 = - Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0)); - const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high); - const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32); - return MakePair(state, low, high); -} - uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) { const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state); const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state); @@ -84,58 +58,42 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high; } -uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { - const auto lhs = ExtractPair(state, lhs_value); - const auto rhs = ExtractPair(state, rhs_value); - const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low); - const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false); - const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high); - const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low); - return MakePair(state, low, - Binary(state, spv::OpIAdd, TypeU32(state), - Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2)); -} - -uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { - const auto pair = ExtractPair(state, value); - uint32_t low = 0; - uint32_t high = 0; +uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) { + const auto pair = ExtractPair(state, value); + const auto amount = EmitAndConstant(state, shift, 63u); + const auto word_shift = EmitAndConstant(state, amount, 31u); + const auto at_least_32 = + Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32)); + const auto nonzero = + Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0)); + const auto carry_count = EmitAndConstant( + state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u); if (opcode == spv::OpShiftLeftLogical) { - EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high); - } else if (opcode == spv::OpShiftRightLogical) { - EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high); - } else { - const auto amount = - Binary(state, spv::OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63)); - const auto word_shift = - Binary(state, spv::OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31)); - const auto at_least_32 = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, - ConstantU32(state, 32)); - const auto nonzero = - Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0)); - const auto high_shifted = - Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift); - const auto low_shifted = - Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift); - const auto carry_count = - Binary(state, spv::OpBitwiseAnd, TypeU32(state), - Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), - ConstantU32(state, 31)); + const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift); const auto carry = Select(state, TypeU32(state), nonzero, - Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count), + Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count), ConstantU32(state, 0)); - const auto low_below_32 = - Binary(state, spv::OpBitwiseOr, TypeU32(state), low_shifted, carry); - const auto negative = - Binary(state, spv::OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0)); - const auto sign_fill = Select(state, TypeU32(state), negative, - ConstantU32(state, 0xffffffffu), ConstantU32(state, 0)); + const auto high = + Binary(state, spv::OpBitwiseOr, TypeU32(state), + Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry); return MakePair(state, - Select(state, TypeU32(state), at_least_32, high_shifted, low_below_32), - Select(state, TypeU32(state), at_least_32, sign_fill, high_shifted)); + Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low), + Select(state, TypeU32(state), at_least_32, low, high)); } - return MakePair(state, low, high); + const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift); + const auto carry = + Select(state, TypeU32(state), nonzero, + Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count), + ConstantU32(state, 0)); + const auto low = Binary( + state, spv::OpBitwiseOr, TypeU32(state), + Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry); + const auto fill = opcode == spv::OpShiftRightArithmetic + ? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31)) + : ConstantU32(state, 0); + return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low), + Select(state, TypeU32(state), at_least_32, fill, high)); } uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { @@ -176,26 +134,10 @@ uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t valu : ConstantU32(state, 0); const auto low = shift == 32u ? pair.high : Binary(state, opcode, TypeU32(state), pair.high, - ConstantU32(state, shift - 32u)); + ConstantU32(state, shift - 32u)); return MakePair(state, low, high); } -uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) { - const auto pair = ExtractPair(state, value); - const auto high_i = state.builder.AllocateId(); - const auto low_i = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high}); - state.builder.AddFunction( - {spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low}); - const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i); - const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i); - const auto high_nonzero = - Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0)); - return Select(state, TypeU32(state), high_nonzero, - Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low); -} - uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value, bool max_value) { const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value) @@ -217,19 +159,6 @@ uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, b return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value); } -uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) { - const auto min_ab = EmitMinMaxF32Value(state, a, b, false); - const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false); - const auto max_ab = EmitMinMaxF32Value(state, a, b, true); - const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false); - const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true); - const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state), - EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan); - const auto any_nan = - Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan); - return Select(state, TypeF32(state), any_nan, min3, median); -} - uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode, std::initializer_list args) { const auto result = state.builder.AllocateId(); @@ -271,121 +200,127 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) { return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high); } -uint32_t EmitPackHalf(EmitterState& state, uint32_t src) { - return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {src}); -} - } // namespace - -uint32_t EmitConvertU16U32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitNative(ctx, arg0, ConstantU32(state, 0xffffu)); +uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) { + const auto min_ab = EmitMinMaxF32Value(state, a, b, false); + const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false); + const auto max_ab = EmitMinMaxF32Value(state, a, b, true); + const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false); + const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true); + const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state), + EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan); + const auto any_nan = + Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan); + return Select(state, TypeF32(state), any_nan, min3, median); } -uint32_t EmitConvertU8U32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitNative(ctx, arg0, ConstantU32(state, 0xffu)); +uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) { + const auto pair = ExtractPair(state, value); + const auto high_i = state.builder.AllocateId(); + const auto low_i = state.builder.AllocateId(); + state.builder.AddFunction( + {spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high}); + state.builder.AddFunction( + {spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low}); + const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i); + const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i); + const auto high_nonzero = + Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0)); + return Select(state, TypeU32(state), high_nonzero, + Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low); } -uint32_t EmitConvertF16F32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - const auto pair = state.builder.AllocateId(); +uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low); + const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false); + const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high); + const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low); + return MakePair(state, low, + Binary(state, spv::OpIAdd, TypeU32(state), + Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2)); +} + +uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low); + const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low); + const auto borrow_u32 = + Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0)); + const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high); + const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32); + return MakePair(state, low, high); +} + +uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto low_pair = state.builder.AllocateId(); + const auto low = state.builder.AllocateId(); + const auto carry = state.builder.AllocateId(); + state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low}); + state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0}); + state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1}); + const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high); + const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry); + return MakePair(state, low, high); +} + +uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) { + return EmitNative(state, arg0, ConstantU32(state, 0xffffu)); +} + +uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0) { + return EmitNative(state, arg0, ConstantU32(state, 0xffu)); +} + +uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0) { + const auto pair = state.builder.AllocateId(); state.builder.AddFunction( {spv::OpCompositeConstruct, TypeF32Vector(state, 2), pair, arg0, ConstantF32(state, 0)}); - return EmitPackHalf(state, pair); + return EmitPackHalf2x16(state, pair); } -uint32_t EmitConvertF32F16(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitF16BitsToF32(state, arg0); -} - -uint32_t EmitConvertS32F32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0) { return EmitF32ToU32(state, arg0, true); } -uint32_t EmitConvertU32F32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) { return EmitF32ToU32(state, arg0, false); } -uint32_t EmitConvertF32S32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0) { const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0); - return EmitNative(ctx, signed_value); + return EmitNative(state, signed_value); } -uint32_t EmitCompositeExtractU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) { - return EmitNative(ctx, arg0, arg1.U32()); +uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) { + return EmitNative(state, arg0, arg1.U32()); } -uint32_t EmitPackHalf2x16(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {arg0}); -} - -uint32_t EmitPackSnorm2x16(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeU32(state), GLSLstd450PackSnorm2x16, {arg0}); -} - -uint32_t EmitPackUnorm2x16(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeU32(state), GLSLstd450PackUnorm2x16, {arg0}); -} - -uint32_t EmitPackFloat2x16Rtz(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; - const auto low = EmitF32ToF16RtzBits(state, arg0); - const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), - EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16)); +uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1) { + const auto low = EmitF32ToF16RtzBits(state, arg0); + const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), + EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16)); return Binary(state, spv::OpBitwiseOr, TypeU32(state), low, high); } -uint32_t EmitFPAbs32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitFAbsValue(state, arg0); -} - -uint32_t EmitFPNeg32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitFNegateValue(state, arg0); -} - -uint32_t EmitFPSaturate32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0) { return EmitExt(state, TypeF32(state), GLSLstd450FClamp, {arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)}); } -uint32_t EmitIAdd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; - return EmitAdd64(state, arg0, arg1); -} - -uint32_t EmitISub64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; - return EmitSub64(state, arg0, arg1); -} - -uint32_t EmitIMul64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; - return EmitMul64(state, arg0, arg1); -} - -uint32_t EmitSMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMulHigh(state, arg0, arg1, true); } -uint32_t EmitUMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMulHigh(state, arg0, arg1, false); } -uint32_t EmitIAbs32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) { const auto value = arg0; const auto neg = Unary(state, spv::OpSNegate, TypeU32(state), value); const auto negative = @@ -414,225 +349,148 @@ uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Va : EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1)); } -uint32_t EmitBitwiseAnd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1); } -uint32_t EmitBitCount64(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0)); +uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) { + const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0)); return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high); } -uint32_t EmitFindILsb32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0) { const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindILsb, {arg0}); return Unary(state, spv::OpBitcast, TypeU32(state), value); } -uint32_t EmitFindUMsb32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0) { const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindUMsb, {arg0}); return Unary(state, spv::OpBitcast, TypeU32(state), value); } -uint32_t EmitFindUMsb64(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitFindMsb64(state, arg0); -} - -uint32_t EmitSMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMinMaxI32Value(state, arg0, arg1, false); } -uint32_t EmitSMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMinMaxI32Value(state, arg0, arg1, true); } -uint32_t EmitUMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMinMaxU32Value(state, arg0, arg1, false); } -uint32_t EmitUMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMinMaxU32Value(state, arg0, arg1, true); } -uint32_t EmitSMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitMinMax3(state, arg0, arg1, arg2, true, false); } -uint32_t EmitSMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitMinMax3(state, arg0, arg1, arg2, true, true); } -uint32_t EmitUMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitMinMax3(state, arg0, arg1, arg2, false, false); } -uint32_t EmitUMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitMinMax3(state, arg0, arg1, arg2, false, true); } -uint32_t EmitSMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitMed3(state, arg0, arg1, arg2, true); } -uint32_t EmitUMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitMed3(state, arg0, arg1, arg2, false); } -uint32_t EmitIEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return CompareEqual64(state, arg0, arg1, false); } -uint32_t EmitINotEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return CompareEqual64(state, arg0, arg1, true); } -uint32_t EmitULessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan); } -uint32_t EmitSLessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan); } -uint32_t EmitUGreaterThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) { return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan); } -uint32_t EmitFPIsNan32(ValueEmitContext& ctx, uint32_t arg0) { - return EmitNative(ctx, arg0, arg0); +uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) { + return EmitNative(state, arg0, arg0); } -uint32_t EmitFPCmpClass32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; - return EmitClassMaskF32(state, arg0, arg1); -} - -uint32_t EmitFPFma32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; - return EmitExt(state, TypeF32(state), GLSLstd450Fma, {arg0, arg1, arg2}); -} - -uint32_t EmitFPMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMinMaxF32Value(state, arg0, arg1, false); } -uint32_t EmitFPMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) { return EmitMinMaxF32Value(state, arg0, arg1, true); } -uint32_t EmitFPMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitFMinMax3(state, arg0, arg1, arg2, false); } -uint32_t EmitFPMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; +uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) { return EmitFMinMax3(state, arg0, arg1, arg2, true); } -uint32_t EmitFPMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) { - auto& state = ctx.state; - return EmitFMed3(state, arg0, arg1, arg2); -} - -uint32_t EmitFPRecip32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0) { const auto source = EmitFlushF32DenormToSignedZero(state, arg0); return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), source); } -uint32_t EmitFPRecipIFlag32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0) { // Integer-to-float inputs used by IFLAG cannot be denormal. return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), arg0); } -uint32_t EmitFPRecipSqrt32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0) { return EmitExt(state, TypeF32(state), GLSLstd450InverseSqrt, {EmitFlushF32DenormToSignedZero(state, arg0)}); } -uint32_t EmitFPSqrt(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0) { return EmitExt(state, TypeF32(state), GLSLstd450Sqrt, {EmitFlushF32DenormToSignedZero(state, arg0)}); } -uint32_t EmitFPExp2(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0) { return EmitExt(state, TypeF32(state), GLSLstd450Exp2, {EmitFlushF32DenormToSignedZero(state, arg0)}); } -uint32_t EmitFPLog2(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; +uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0) { return EmitExt(state, TypeF32(state), GLSLstd450Log2, {EmitFlushF32DenormToSignedZero(state, arg0)}); } -uint32_t EmitFPLdexp(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) { - auto& state = ctx.state; +uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1) { const auto exponent = Unary(state, spv::OpBitcast, TypeI32(state), arg1); return EmitExt(state, TypeF32(state), GLSLstd450Ldexp, {arg0, exponent}); } -uint32_t EmitFPRoundEven32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeF32(state), GLSLstd450RoundEven, {arg0}); -} - -uint32_t EmitFPFloor32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeF32(state), GLSLstd450Floor, {arg0}); -} - -uint32_t EmitFPCeil32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeF32(state), GLSLstd450Ceil, {arg0}); -} - -uint32_t EmitFPTrunc32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeF32(state), GLSLstd450Trunc, {arg0}); -} - -uint32_t EmitFPFract32(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - return EmitExt(state, TypeF32(state), GLSLstd450Fract, {arg0}); -} - -uint32_t EmitFPSin(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - auto source = EmitTrigCycleF32(state, arg0, true); +uint32_t EmitFPSin(EmitterState& state, uint32_t arg0) { + auto source = EmitTrigCycleF32(state, arg0, true); source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu)); return EmitExt(state, TypeF32(state), GLSLstd450Sin, {source}); } -uint32_t EmitFPCos(ValueEmitContext& ctx, uint32_t arg0) { - auto& state = ctx.state; - auto source = EmitTrigCycleF32(state, arg0, false); +uint32_t EmitFPCos(EmitterState& state, uint32_t arg0) { + auto source = EmitTrigCycleF32(state, arg0, false); source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu)); return EmitExt(state, TypeF32(state), GLSLstd450Cos, {source}); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp index b897c42a..330fd3b9 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp @@ -2,88 +2,6 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { -uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, - uint32_t false_value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, true_value, false_value}); - return ret; -} - -uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); - -void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, - uint32_t& out_low, uint32_t& out_high) { - const auto amount = EmitAndConstant(state, shift, 63u); - const auto dword_shift = EmitAndConstant(state, amount, 31u); - const auto less32 = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto carry_count_base = state.builder.AllocateId(); - const auto carry_count = state.builder.AllocateId(); - const auto low_part = state.builder.AllocateId(); - const auto high_part = state.builder.AllocateId(); - const auto carry_part = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - const auto high_less = state.builder.AllocateId(); - const auto high_ge = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)}); - state.builder.AddFunction( - {spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)}); - state.builder.AddFunction( - {spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift}); - state.builder.AddFunction( - {spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)}); - state.builder.AddFunction( - {spv::OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift}); - state.builder.AddFunction( - {spv::OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift}); - state.builder.AddFunction( - {spv::OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count}); - state.builder.AddFunction( - {spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)}); - state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), high_less, high_part, carry}); - state.builder.AddFunction({spv::OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift}); - out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0)); - out_high = EmitSelectValueU32(state, less32, high_less, high_ge); -} - -void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, - uint32_t shift, uint32_t& out_low, uint32_t& out_high) { - const auto amount = EmitAndConstant(state, shift, 63u); - const auto dword_shift = EmitAndConstant(state, amount, 31u); - const auto less32 = state.builder.AllocateId(); - const auto non_zero = state.builder.AllocateId(); - const auto carry_count_base = state.builder.AllocateId(); - const auto carry_count = state.builder.AllocateId(); - const auto low_part = state.builder.AllocateId(); - const auto high_part = state.builder.AllocateId(); - const auto carry_part = state.builder.AllocateId(); - const auto carry = state.builder.AllocateId(); - const auto low_less = state.builder.AllocateId(); - const auto low_ge = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)}); - state.builder.AddFunction( - {spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)}); - state.builder.AddFunction( - {spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift}); - state.builder.AddFunction( - {spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)}); - state.builder.AddFunction( - {spv::OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift}); - state.builder.AddFunction( - {spv::OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift}); - state.builder.AddFunction( - {spv::OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count}); - state.builder.AddFunction( - {spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)}); - state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), low_less, low_part, carry}); - state.builder.AddFunction( - {spv::OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift}); - out_low = EmitSelectValueU32(state, less32, low_less, low_ge); - out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0)); -} - uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction( @@ -98,12 +16,6 @@ uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t sh return ret; } -uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), ret, lhs, rhs}); - return ret; -} - uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t constant) { const auto ret = state.builder.AllocateId(); @@ -189,42 +101,6 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo return ret; } -uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpBitcast, TypeU32(state), ret, value}); - return ret; -} - -uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpBitcast, TypeF32(state), ret, value}); - return ret; -} - -uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpLogicalOr, TypeBool(state), ret, lhs, rhs}); - return ret; -} - -uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), ret, value}); - return ret; -} - F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) { F32Class cls; cls.bits = bits; @@ -324,13 +200,6 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo return EmitBitcastU32ToF32(state, result_bits); } -uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450Trunc, value}); - return ret; -} - uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) { const auto bits = state.builder.AllocateId(); const auto abs_bits = state.builder.AllocateId(); @@ -386,19 +255,6 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe return ret; } -uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpFNegate, TypeF32(state), ret, value}); - return ret; -} - -uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450FAbs, value}); - return ret; -} - uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) { const auto unpacked = state.builder.AllocateId(); const auto ret = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index b4451c06..e1ceb4d8 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -16,29 +16,7 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& return true; } -uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { - const auto shifted_one = state.builder.AllocateId(); - const auto merged_one = state.builder.AllocateId(); - const auto shifted_two = state.builder.AllocateId(); - const auto merged_two = state.builder.AllocateId(); - const auto quad_bits = state.builder.AllocateId(); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value, - ConstantU64(state, 0x0000000100000001ull)}); - state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one}); - state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one, - ConstantU64(state, 0x0000000200000002ull)}); - state.builder.AddFunction( - {spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two}); - state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two, - ConstantU64(state, 0x1111111111111111ull)}); - state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits, - ConstantU64(state, 0x0000000f0000000full)}); - return result; -} - -uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) { - auto& state = ctx.state; +uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) { if (kind == IR::StageInputKind::LocalInvocationIndex) { return EmitLocalInvocationIndex(state); } @@ -155,8 +133,7 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl return result; } -uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) { - auto& state = ctx.state; +uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) { const auto* input = InputBindingForParameter(state, attr); if (input == nullptr || input->variable_id == 0) { return ConstantU32(state, 0); @@ -217,7 +194,7 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32 auto& state = ctx.state; const auto* input = InputBindingForParameter(state, attr); if (!input->per_vertex) { - return EmitAttribute(ctx, attr, chan); + return EmitAttribute(ctx.state, attr, chan); } const auto load_vertex = [&](uint32_t vertex) { const auto pointer = state.builder.AllocateId(); @@ -427,6 +404,26 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) { } } // namespace +uint32_t EmitWqmU64(EmitterState& state, uint32_t value) { + const auto shifted_one = state.builder.AllocateId(); + const auto merged_one = state.builder.AllocateId(); + const auto shifted_two = state.builder.AllocateId(); + const auto merged_two = state.builder.AllocateId(); + const auto quad_bits = state.builder.AllocateId(); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value, + ConstantU64(state, 0x0000000100000001ull)}); + state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one}); + state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one, + ConstantU64(state, 0x0000000200000002ull)}); + state.builder.AddFunction( + {spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two}); + state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two, + ConstantU64(state, 0x1111111111111111ull)}); + state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits, + ConstantU64(state, 0x0000000f0000000full)}); + return result; +} void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; @@ -523,8 +520,7 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) { void EmitVoid(ValueEmitContext&) {} -void EmitBarrier(ValueEmitContext& ctx) { - auto& state = ctx.state; +void EmitBarrier(EmitterState& state) { const auto semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), @@ -547,8 +543,7 @@ uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) { return result; } -uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg) { - auto& state = ctx.state; +uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg) { uint32_t dword = 0; if (!UserDataDwordIndex(state, reg, dword)) { @@ -559,13 +554,12 @@ uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg) { } uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) { - return EmitBuiltinU32(ctx, static_cast(kind.U32()), index.U32()); + return EmitBuiltinU32(ctx.state, static_cast(kind.U32()), index.U32()); } -uint32_t EmitUndefU1(ValueEmitContext& ctx, const IR::Inst& inst) { - auto& state = ctx.state; +uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) { const auto result = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpUndef, ctx.TypeId(inst.GetType()), result}); + state.builder.AddFunction({spv::OpUndef, TypeId(state, inst.GetType()), result}); return result; } @@ -582,23 +576,15 @@ uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) { const auto can_fetch = state.builder.AllocateId(); state.builder.AddFunction( {spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active}); - return EmitNative(ctx, can_fetch, shuffled, + return EmitNative(ctx.state, can_fetch, shuffled, ConstantU32(state, 0)); } uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) { const auto flags = inst.Flags(); const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2)); - return EmitNative(ctx, write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)); -} - -uint32_t EmitWqmU64(ValueEmitContext& ctx, uint32_t value) { - return EmitWqmU64(ctx.state, value); -} - -uint32_t EmitLaneId(ValueEmitContext& ctx) { - auto& state = ctx.state; - return EmitSubgroupLocalInvocationId(state); + return EmitNative(ctx.state, write, ctx.Arg(inst, 0), + ctx.Arg(inst, 1)); } uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) { @@ -620,7 +606,8 @@ uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) { const auto hit = state.builder.AllocateId(); state.builder.AddFunction({spv::OpIEqual, TypeBool(state), hit, EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)}); - return EmitNative(ctx, hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)); + return EmitNative(ctx.state, hit, ctx.Arg(inst, 1), + ctx.Arg(inst, 0)); } uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) { @@ -671,7 +658,7 @@ uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) { } uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) { - return EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()); + return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32()); } uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) { @@ -684,7 +671,6 @@ uint32_t EmitGetShaderBase(ValueEmitContext& ctx) { return ctx.Def(IR::Value(uint64_t {0})); } - void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Fail(inst, "must be lowered before SPIR-V emission"); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index ed72601e..6bda42b6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -3,13 +3,8 @@ #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { -// Direct SPIR-V operations share emission as well as argument/result marshalling. -template -uint32_t EmitNative(ValueEmitContext& ctx, Args... args) { - const auto result = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction({opcode, ctx.TypeId(type), result, args...}); - return result; -} +// Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values +// (IR::Value), and instructions whose metadata or operand loads must stay lazy. #define EMIT_NATIVE(name, opcode, type, ...) \ inline constexpr auto Emit##name = EmitNative; EMIT_NATIVE(BitCastU16F16, OpBitcast, U32, uint32_t) @@ -18,13 +13,13 @@ inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16; inline constexpr auto EmitConvertU32U8 = EmitBitCastU16F16; inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16; EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t) -uint32_t EmitConvertU16U32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitConvertU8U32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitConvertF16F32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitConvertF32F16(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitConvertS32F32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitConvertU32F32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitConvertF32S32(ValueEmitContext& ctx, uint32_t arg0); +uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0); +uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0); +uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0); +inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32; +uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0); +uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0); +uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0); EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t) EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t) @@ -32,17 +27,19 @@ EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t) EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t, uint32_t) -uint32_t EmitCompositeExtractU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1); +uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1); inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64; inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64; inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64; -uint32_t EmitPackHalf2x16(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitPackSnorm2x16(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitPackUnorm2x16(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitPackFloat2x16Rtz(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitFPAbs32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPNeg32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPSaturate32(ValueEmitContext& ctx, uint32_t arg0); +inline constexpr auto EmitPackHalf2x16 = EmitGlsl; +inline constexpr auto EmitPackSnorm2x16 = + EmitGlsl; +inline constexpr auto EmitPackUnorm2x16 = + EmitGlsl; +uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1); +inline constexpr auto EmitFPAbs32 = EmitFAbsValue; +inline constexpr auto EmitFPNeg32 = EmitFNegateValue; +uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0); EMIT_NATIVE(BitFieldInsert, OpBitFieldInsert, U32, uint32_t, uint32_t, uint32_t, uint32_t) EMIT_NATIVE(BitFieldUExtract, OpBitFieldUExtract, U32, uint32_t, uint32_t, uint32_t) EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint32_t) @@ -53,13 +50,13 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t) EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t) EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t) EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t) -uint32_t EmitIAdd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitISub64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitIMul64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); +uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1); EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t) -uint32_t EmitSMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitUMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitIAbs32(ValueEmitContext& ctx, uint32_t arg0); +uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0); EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t) EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t) @@ -70,23 +67,23 @@ EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t) EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t) -uint32_t EmitBitwiseAnd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); +uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1); EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t) EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t) -uint32_t EmitBitCount64(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFindILsb32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFindUMsb32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFindUMsb64(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitSMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitSMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitUMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitUMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitSMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitSMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitUMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitUMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitSMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitUMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0); +uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0); +uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0); +uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0); +uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); EMIT_NATIVE(SLessThan32, OpSLessThan, U1, uint32_t, uint32_t) EMIT_NATIVE(ULessThan32, OpULessThan, U1, uint32_t, uint32_t) EMIT_NATIVE(IEqual32, OpIEqual, U1, uint32_t, uint32_t) @@ -97,11 +94,11 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t) EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t) -uint32_t EmitIEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitINotEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitULessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitSLessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitUGreaterThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); +uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1); EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t) EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t) @@ -118,31 +115,32 @@ EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t) EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t) -uint32_t EmitFPIsNan32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPCmpClass32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); +uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0); +inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32; EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t) EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t) EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t) -uint32_t EmitFPFma32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitFPMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitFPMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitFPMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitFPMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitFPMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2); -uint32_t EmitFPRecip32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPRecipIFlag32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPRecipSqrt32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPSqrt(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPExp2(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPLog2(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPLdexp(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1); -uint32_t EmitFPRoundEven32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPFloor32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPCeil32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPTrunc32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPFract32(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPSin(ValueEmitContext& ctx, uint32_t arg0); -uint32_t EmitFPCos(ValueEmitContext& ctx, uint32_t arg0); +inline constexpr auto EmitFPFma32 = + EmitGlsl; +uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1); +uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2); +uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0); +uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0); +uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0); +uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0); +uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0); +uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0); +uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1); +inline constexpr auto EmitFPRoundEven32 = EmitGlsl; +inline constexpr auto EmitFPFloor32 = EmitGlsl; +inline constexpr auto EmitFPCeil32 = EmitGlsl; +inline constexpr auto EmitFPTrunc32 = EmitGlsl; +inline constexpr auto EmitFPFract32 = EmitGlsl; +uint32_t EmitFPSin(EmitterState& state, uint32_t arg0); +uint32_t EmitFPCos(EmitterState& state, uint32_t arg0); #undef EMIT_NATIVE uint32_t EmitIdentity(ValueEmitContext& ctx, uint32_t value); @@ -154,20 +152,20 @@ inline constexpr auto EmitWaitcnt = EmitVoid; inline constexpr auto EmitSendmsg = EmitVoid; inline constexpr auto EmitTtraceData = EmitVoid; inline constexpr auto EmitInstPrefetch = EmitVoid; -void EmitBarrier(ValueEmitContext& ctx); +void EmitBarrier(EmitterState& state); void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst); -uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg); +uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg); uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index); -uint32_t EmitUndefU1(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst); inline constexpr auto EmitUndefU8 = EmitUndefU1; inline constexpr auto EmitUndefU16 = EmitUndefU1; inline constexpr auto EmitUndefU32 = EmitUndefU1; inline constexpr auto EmitUndefU64 = EmitUndefU1; uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst); -uint32_t EmitWqmU64(ValueEmitContext& ctx, uint32_t value); -uint32_t EmitLaneId(ValueEmitContext& ctx); +uint32_t EmitWqmU64(EmitterState& state, uint32_t value); +inline constexpr auto EmitLaneId = EmitSubgroupLocalInvocationId; uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate); uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst); @@ -184,72 +182,79 @@ inline constexpr auto EmitGetScratchResource = EmitGetSrtResource; inline constexpr auto EmitGetImageResource = EmitGetSrtResource; inline constexpr auto EmitGetSamplerResource = EmitGetSrtResource; inline constexpr auto EmitMakeImageAddress = EmitGetSrtResource; -void EmitMemory(ValueEmitContext& ctx, const IR::Inst& inst); -inline constexpr auto EmitBpermuteU32 = EmitMemory; -inline constexpr auto EmitReadConst = EmitMemory; -inline constexpr auto EmitReadConstBuffer = EmitMemory; -inline constexpr auto EmitLoadAddressU8 = EmitMemory; -inline constexpr auto EmitLoadAddressU16 = EmitMemory; -inline constexpr auto EmitLoadAddressU32 = EmitMemory; -inline constexpr auto EmitStoreAddressU8 = EmitMemory; -inline constexpr auto EmitStoreAddressU16 = EmitMemory; -inline constexpr auto EmitStoreAddressU32 = EmitMemory; -inline constexpr auto EmitLoadBufferU8 = EmitMemory; -inline constexpr auto EmitLoadBufferU16 = EmitMemory; -inline constexpr auto EmitLoadBufferU32 = EmitMemory; -inline constexpr auto EmitLoadBufferU32x2 = EmitMemory; -inline constexpr auto EmitLoadBufferU32x3 = EmitMemory; -inline constexpr auto EmitLoadBufferU32x4 = EmitMemory; -inline constexpr auto EmitStoreBufferU8 = EmitMemory; -inline constexpr auto EmitStoreBufferU16 = EmitMemory; -inline constexpr auto EmitStoreBufferU32 = EmitMemory; -inline constexpr auto EmitStoreBufferU32x2 = EmitMemory; -inline constexpr auto EmitStoreBufferU32x3 = EmitMemory; -inline constexpr auto EmitStoreBufferU32x4 = EmitMemory; -inline constexpr auto EmitBufferAtomicSwap32 = EmitMemory; -inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitMemory; -inline constexpr auto EmitBufferAtomicSwap64 = EmitMemory; -inline constexpr auto EmitBufferAtomicIAdd32 = EmitMemory; -inline constexpr auto EmitBufferAtomicISub32 = EmitMemory; -inline constexpr auto EmitBufferAtomicSMin32 = EmitMemory; -inline constexpr auto EmitBufferAtomicUMin32 = EmitMemory; -inline constexpr auto EmitBufferAtomicSMax32 = EmitMemory; -inline constexpr auto EmitBufferAtomicUMax32 = EmitMemory; -inline constexpr auto EmitBufferAtomicAnd32 = EmitMemory; -inline constexpr auto EmitBufferAtomicOr32 = EmitMemory; -inline constexpr auto EmitBufferAtomicOr64 = EmitMemory; -inline constexpr auto EmitBufferAtomicXor32 = EmitMemory; -inline constexpr auto EmitBufferAtomicFMin32 = EmitMemory; -inline constexpr auto EmitBufferAtomicFMax32 = EmitMemory; -inline constexpr auto EmitLoadSharedU8 = EmitMemory; -inline constexpr auto EmitLoadSharedU16 = EmitMemory; -inline constexpr auto EmitLoadSharedU32 = EmitMemory; -inline constexpr auto EmitLoadSharedU32x2 = EmitMemory; -inline constexpr auto EmitLoadSharedU32x3 = EmitMemory; -inline constexpr auto EmitLoadSharedU32x4 = EmitMemory; -inline constexpr auto EmitWriteSharedU8 = EmitMemory; -inline constexpr auto EmitWriteSharedU16 = EmitMemory; -inline constexpr auto EmitWriteSharedU32 = EmitMemory; -inline constexpr auto EmitWriteSharedU32x2 = EmitMemory; -inline constexpr auto EmitWriteSharedU32x3 = EmitMemory; -inline constexpr auto EmitWriteSharedU32x4 = EmitMemory; -inline constexpr auto EmitSharedAtomicFMin32 = EmitMemory; -inline constexpr auto EmitSharedAtomicFMax32 = EmitMemory; -inline constexpr auto EmitSharedAtomicSwap32 = EmitMemory; -inline constexpr auto EmitSharedAtomicIAdd32 = EmitMemory; -inline constexpr auto EmitSharedAtomicISub32 = EmitMemory; -inline constexpr auto EmitSharedAtomicInc32 = EmitMemory; -inline constexpr auto EmitSharedAtomicDec32 = EmitMemory; -inline constexpr auto EmitSharedAtomicSMin32 = EmitMemory; -inline constexpr auto EmitSharedAtomicUMin32 = EmitMemory; -inline constexpr auto EmitSharedAtomicSMax32 = EmitMemory; -inline constexpr auto EmitSharedAtomicUMax32 = EmitMemory; -inline constexpr auto EmitSharedAtomicAnd32 = EmitMemory; -inline constexpr auto EmitSharedAtomicOr32 = EmitMemory; -inline constexpr auto EmitSharedAtomicXor32 = EmitMemory; -inline constexpr auto EmitDataAppend = EmitMemory; -inline constexpr auto EmitDataConsume = EmitMemory; -inline constexpr auto EmitSwizzleU32 = EmitMemory; +void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst); +void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); +void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst); +uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst); +void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst); +inline constexpr auto EmitLoadAddressU8 = EmitLoadMemory; +inline constexpr auto EmitLoadAddressU16 = EmitLoadMemory; +inline constexpr auto EmitLoadAddressU32 = EmitLoadMemory; +inline constexpr auto EmitStoreAddressU8 = EmitStoreMemory; +inline constexpr auto EmitStoreAddressU16 = EmitStoreMemory; +inline constexpr auto EmitStoreAddressU32 = EmitStoreMemory; +inline constexpr auto EmitLoadBufferU8 = EmitLoadMemory; +inline constexpr auto EmitLoadBufferU16 = EmitLoadMemory; +inline constexpr auto EmitLoadBufferU32 = EmitLoadMemory; +inline constexpr auto EmitLoadBufferU32x2 = EmitLoadMemory; +inline constexpr auto EmitLoadBufferU32x3 = EmitLoadMemory; +inline constexpr auto EmitLoadBufferU32x4 = EmitLoadMemory; +inline constexpr auto EmitStoreBufferU8 = EmitStoreMemory; +inline constexpr auto EmitStoreBufferU16 = EmitStoreMemory; +inline constexpr auto EmitStoreBufferU32 = EmitStoreMemory; +inline constexpr auto EmitStoreBufferU32x2 = EmitStoreMemory; +inline constexpr auto EmitStoreBufferU32x3 = EmitStoreMemory; +inline constexpr auto EmitStoreBufferU32x4 = EmitStoreMemory; +inline constexpr auto EmitBufferAtomicSwap32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicSwap64 = EmitBufferAtomic64; +inline constexpr auto EmitBufferAtomicIAdd32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicISub32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicSMin32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicUMin32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicSMax32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicUMax32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicAnd32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicOr32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicOr64 = EmitBufferAtomic64; +inline constexpr auto EmitBufferAtomicXor32 = EmitAtomic32; +inline constexpr auto EmitBufferAtomicFMin32 = EmitBufferFloatAtomic; +inline constexpr auto EmitBufferAtomicFMax32 = EmitBufferFloatAtomic; +inline constexpr auto EmitLoadSharedU8 = EmitLoadMemory; +inline constexpr auto EmitLoadSharedU16 = EmitLoadMemory; +inline constexpr auto EmitLoadSharedU32 = EmitLoadMemory; +inline constexpr auto EmitLoadSharedU32x2 = EmitLoadMemory; +inline constexpr auto EmitLoadSharedU32x3 = EmitLoadMemory; +inline constexpr auto EmitLoadSharedU32x4 = EmitLoadMemory; +inline constexpr auto EmitWriteSharedU8 = EmitStoreMemory; +inline constexpr auto EmitWriteSharedU16 = EmitStoreMemory; +inline constexpr auto EmitWriteSharedU32 = EmitStoreMemory; +inline constexpr auto EmitWriteSharedU32x2 = EmitStoreMemory; +inline constexpr auto EmitWriteSharedU32x3 = EmitStoreMemory; +inline constexpr auto EmitWriteSharedU32x4 = EmitStoreMemory; +inline constexpr auto EmitSharedAtomicFMin32 = EmitSharedFloatAtomic; +inline constexpr auto EmitSharedAtomicFMax32 = EmitSharedFloatAtomic; +inline constexpr auto EmitSharedAtomicSwap32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicIAdd32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicISub32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicInc32 = EmitSharedIncDec; +inline constexpr auto EmitSharedAtomicDec32 = EmitSharedIncDec; +inline constexpr auto EmitSharedAtomicSMin32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicUMin32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicSMax32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicUMax32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicAnd32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicOr32 = EmitAtomic32; +inline constexpr auto EmitSharedAtomicXor32 = EmitAtomic32; +inline constexpr auto EmitDataAppend = EmitAppendConsume; +inline constexpr auto EmitDataConsume = EmitAppendConsume; +uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst); void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst); inline constexpr auto EmitImageQueryDimensions = EmitImage; inline constexpr auto EmitImageQueryLod = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 01a42014..ff887ead 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -165,6 +165,23 @@ inline void EmitLabel(EmitterState& state, uint32_t label) { state.builder.AddFunction({spv::OpLabel, label}); } +uint32_t TypeId(EmitterState& state, IR::Type type); + +// Shared instruction construction; typed aliases add no forwarding functions. +template +uint32_t EmitNative(EmitterState& state, Args... args) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, TypeId(state, type), result, args...}); + return result; +} + +uint32_t GlslStd450(EmitterState& state); + +template +uint32_t EmitGlsl(EmitterState& state, Args... args) { + return EmitNative(state, GlslStd450(state), opcode, args...); +} + inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { const auto result = state.builder.AllocateId(); state.builder.AddFunction({opcode, type, result, value}); @@ -172,14 +189,14 @@ inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint3 } inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, - uint32_t rhs) { + uint32_t rhs) { const auto result = state.builder.AllocateId(); state.builder.AddFunction({opcode, type, result, lhs, rhs}); return result; } -inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, - uint32_t true_value, uint32_t false_value) { +inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value, + uint32_t false_value) { const auto result = state.builder.AllocateId(); state.builder.AddFunction({spv::OpSelect, type, result, condition, true_value, false_value}); return result; @@ -195,7 +212,6 @@ struct ValueEmitContext { uint32_t FirstLane(uint32_t ballot); uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane); uint32_t Result(const IR::Inst& inst); - uint32_t TypeId(IR::Type type) const; uint32_t Define(const IR::Inst& inst, uint32_t value); uint32_t ResourceIndex(IR::Value value, IR::ValueOpcode opcode); const IR::Inst* ImageAddress(IR::Value value); @@ -313,8 +329,6 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value); uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components); -uint32_t GlslStd450(EmitterState& state); - void AllocateInputVariables(EmitterState& state); void AllocateOutputVariables(EmitterState& state); @@ -330,7 +344,7 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name void AddDescriptorAnnotationsAndNames(EmitterState& state); -void DefineModule(EmitterState& state); +void DefineModule(EmitterState& state); void DefineMeshOutputs(EmitterState& state); void EmitMeshEntryPoint(EmitterState& state); void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); @@ -365,7 +379,7 @@ uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane); -uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs); +inline constexpr auto EmitAddU32 = EmitNative; uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs); @@ -390,10 +404,9 @@ struct MemoryResourceAccess { MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); -MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, - const IR::MemoryInfo& mem, - uint32_t variable, - uint32_t pointer_type); +MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, + const IR::MemoryInfo& mem, + uint32_t variable, uint32_t pointer_type); uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access, uint32_t raw_index); @@ -404,14 +417,13 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access, uint32_t index); -uint32_t EmitStorageBufferElementPointer(EmitterState& state, - const MemoryResourceAccess& access, uint32_t index, - uint32_t pointer_type); +uint32_t EmitStorageBufferElementPointer(EmitterState& state, const MemoryResourceAccess& access, + uint32_t index, uint32_t pointer_type); uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value); -uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, - uint32_t false_value); +inline constexpr auto EmitTBufferSelectF32 = + EmitNative; bool IsSignedFormatComponent(Format::ComponentType type); @@ -427,20 +439,14 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control); -uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, - uint32_t false_value); - -void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift, - uint32_t& out_low, uint32_t& out_high); - -void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, - uint32_t shift, uint32_t& out_low, uint32_t& out_high); +inline constexpr auto EmitSelectValueU32 = + EmitNative; uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask); uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift); -uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs); +inline constexpr auto EmitOrU32 = EmitNative; uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t constant); @@ -453,17 +459,19 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); -uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value); +inline constexpr auto EmitBitcastF32ToU32 = EmitNative; -uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value); +inline constexpr auto EmitBitcastU32ToF32 = EmitNative; -uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs); +inline constexpr auto EmitAndU32 = EmitNative; -uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs); +inline constexpr auto EmitLogicalAndBool = + EmitNative; -uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs); +inline constexpr auto EmitLogicalOrBool = + EmitNative; -uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value); +inline constexpr auto EmitLogicalNotBool = EmitNative; F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits); @@ -476,15 +484,15 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask); uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value); -uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value); +inline constexpr auto EmitTruncF32Value = EmitGlsl; uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value); uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero); -uint32_t EmitFNegateValue(EmitterState& state, uint32_t value); +inline constexpr auto EmitFNegateValue = EmitNative; -uint32_t EmitFAbsValue(EmitterState& state, uint32_t value); +inline constexpr auto EmitFAbsValue = EmitGlsl; uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 0ba050a5..9e0dfa1c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -599,23 +599,6 @@ uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst, }); } -uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { - return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) { - const auto scope = - mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; - const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); - if (mem.kind == IR::ResourceKind::Lds) { - const auto semantics = - spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; - ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope), - ConstantU32(ctx.state, semantics)}); - } else { - EmitDeviceAtomicMemoryBarrier(ctx.state); - } - return old; - }); -} - template uint32_t EmitAtomicUpdate(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, Fn&& replacement) { @@ -643,137 +626,6 @@ uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) { return Select(state, TypeU32(state), wrap, limit, next); } -uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst, - const IR::MemoryInfo& mem) { - auto& state = ctx.state; - return EmitValueOrDefaultIfCondition( - state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() { - const auto resource = PrepareStorageBufferResourceAccess( - state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state)); - const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state), - ByteAddress(ctx, inst, mem), resource.byte_offset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address, - ConstantU32(state, 3u)); - return EmitValueOrDefaultIfCondition( - state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), - ConstantU64(state, 0), [&]() { - const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), - ctx.Arg(inst, inst.NumArgs() - 2)); - const auto old = state.builder.AllocateId(); - state.builder.AddFunction( - {SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old, - EmitStorageBufferElementPointer(state, resource, index, - TypeStorageBufferU64ElementPointer(state)), - ConstantU32(state, spv::ScopeDevice), - ConstantU32(state, spv::MemorySemanticsMaskNone), value}); - EmitDeviceAtomicMemoryBarrier(state); - return Unary(state, spv::OpBitcast, TypeU64(state), old); - }); - }); -} - -uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, - bool max_value) { - return EmitAtomicUpdate(ctx, inst, mem, [max_value](EmitterState& state, uint32_t old, - uint32_t value) { - return EmitFloatAtomicReplacement(state, old, value, max_value); - }); -} - -uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, - bool max_value) { - EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - EmitIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - ctx.state.builder.AddFunction( - {spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)}); - const auto data = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable}); - AtomicUpdate( - ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index), - mem.kind, [&](uint32_t old) { - const auto old_f = - Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old); - const auto compare_f = - Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2)); - const auto data_f = - Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data); - const auto compare = Binary( - ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan, - TypeBool(ctx.state), max_value ? old_f : compare_f, - max_value ? compare_f : old_f); - return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), - Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f)); - }); - }); - }); - return 0; -} - -uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) { - auto& state = ctx.state; - if (ctx.half == 1) { - return ctx.other_half->Def(IR::Value(const_cast(&inst))); - } - const auto m0 = ctx.Arg(inst, 0); - const auto base = - Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); - const auto size = - Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); - const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base, - ConstantU32(state, ctx.Memory(inst).offset)); - const auto raw_index = - Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - const auto mem = ctx.Memory(inst); - const auto access = PrepareMemoryResourceAccess(state, mem); - const auto index = EmitMemoryElementIndex(state, access, raw_index); - const auto exec = ctx.Arg(inst, 1); - const auto ballot = ctx.Ballot(inst.Arg(1)); - const auto low = state.builder.AllocateId(); - const auto high = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0}); - state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1}); - const auto count = Binary(state, spv::OpIAdd, TypeU32(state), - Unary(state, spv::OpBitCount, TypeU32(state), low), - Unary(state, spv::OpBitCount, TypeU32(state), high)); - const auto first = ctx.FirstLane(ballot); - const auto source_lane = - state.lane_count == 2 - ? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31)) - : first; - const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state), - EmitSubgroupLocalInvocationId(state), source_lane); - const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); - const auto m0_bounds = - mem.kind == IR::ResourceKind::Gds - ? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) - : Binary(state, spv::OpULessThan, TypeBool(state), - ConstantU32(state, ctx.Memory(inst).offset + 3u), size); - const auto condition = AndCondition( - state, is_first, - AndCondition(state, - state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count, - ConstantU32(state, 0)) - : exec, - AndCondition(state, storage_bounds, m0_bounds))); - const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { - const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value, - EmitMemoryElementPointer(state, access, index), - ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice - : spv::ScopeWorkgroup), - ConstantU32(state, spv::MemorySemanticsMaskNone), count}); - return value; - }); - const auto result = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result, - ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane}); - return result; -} - struct PreparedFormattedMemory { Format::BufferFormatInfo info; MemoryResourceAccess resource; @@ -1044,162 +896,259 @@ void DefineGetBdaPointer(EmitterState& state) { state.builder.AddFunction({spv::OpFunctionEnd}); } -void EmitMemory(ValueEmitContext& ctx, const IR::Inst& inst) { - auto& state = ctx.state; - const auto op = inst.GetOpcode(); +uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto& mem = ctx.Memory(inst); + return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) { + const auto scope = + mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; + const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); + if (mem.kind == IR::ResourceKind::Lds) { + const auto semantics = + spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope), + ConstantU32(ctx.state, semantics)}); + } else { + EmitDeviceAtomicMemoryBarrier(ctx.state); + } + return old; + }); +} + +uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto& mem = ctx.Memory(inst); + auto& state = ctx.state; + return EmitValueOrDefaultIfCondition( + state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() { + const auto resource = PrepareStorageBufferResourceAccess( + state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state)); + const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state), + ByteAddress(ctx, inst, mem), resource.byte_offset); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address, + ConstantU32(state, 3u)); + return EmitValueOrDefaultIfCondition( + state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), + ConstantU64(state, 0), [&]() { + const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), + ctx.Arg(inst, inst.NumArgs() - 2)); + const auto old = state.builder.AllocateId(); + state.builder.AddFunction( + {SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old, + EmitStorageBufferElementPointer(state, resource, index, + TypeStorageBufferU64ElementPointer(state)), + ConstantU32(state, spv::ScopeDevice), + ConstantU32(state, spv::MemorySemanticsMaskNone), value}); + EmitDeviceAtomicMemoryBarrier(state); + return Unary(state, spv::OpBitcast, TypeU64(state), old); + }); + }); +} + +uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto& mem = ctx.Memory(inst); + const bool max_value = inst.GetOpcode() == IR::ValueOpcode::BufferAtomicFMax32; + return EmitAtomicUpdate(ctx, inst, mem, + [max_value](EmitterState& state, uint32_t old, uint32_t value) { + return EmitFloatAtomicReplacement(state, old, value, max_value); + }); +} + +void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto& mem = ctx.Memory(inst); + const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32; + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + EmitIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + ctx.state.builder.AddFunction( + {spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)}); + const auto data = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable}); + AtomicUpdate( + ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index), + mem.kind, [&](uint32_t old) { + const auto old_f = + Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old); + const auto compare_f = + Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2)); + const auto data_f = + Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data); + const auto compare = Binary( + ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan, + TypeBool(ctx.state), max_value ? old_f : compare_f, + max_value ? compare_f : old_f); + return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), + Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f)); + }); + }); + }); +} + +uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { + const bool append = inst.GetOpcode() == IR::ValueOpcode::DataAppend; + auto& state = ctx.state; + if (ctx.half == 1) { + return ctx.other_half->Def(IR::Value(const_cast(&inst))); + } + const auto m0 = ctx.Arg(inst, 0); + const auto base = + Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); + const auto size = + Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); + const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base, + ConstantU32(state, ctx.Memory(inst).offset)); + const auto raw_index = + Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); + const auto mem = ctx.Memory(inst); + const auto access = PrepareMemoryResourceAccess(state, mem); + const auto index = EmitMemoryElementIndex(state, access, raw_index); + const auto exec = ctx.Arg(inst, 1); + const auto ballot = ctx.Ballot(inst.Arg(1)); + const auto low = state.builder.AllocateId(); + const auto high = state.builder.AllocateId(); + state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0}); + state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1}); + const auto count = Binary(state, spv::OpIAdd, TypeU32(state), + Unary(state, spv::OpBitCount, TypeU32(state), low), + Unary(state, spv::OpBitCount, TypeU32(state), high)); + const auto first = ctx.FirstLane(ballot); + const auto source_lane = + state.lane_count == 2 + ? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31)) + : first; + const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state), + EmitSubgroupLocalInvocationId(state), source_lane); + const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); + const auto m0_bounds = + mem.kind == IR::ResourceKind::Gds + ? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) + : Binary(state, spv::OpULessThan, TypeBool(state), + ConstantU32(state, ctx.Memory(inst).offset + 3u), size); + const auto condition = AndCondition( + state, is_first, + AndCondition(state, + state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count, + ConstantU32(state, 0)) + : exec, + AndCondition(state, storage_bounds, m0_bounds))); + const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value, + EmitMemoryElementPointer(state, access, index), + ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice + : spv::ScopeWorkgroup), + ConstantU32(state, spv::MemorySemanticsMaskNone), count}); + return value; + }); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result, + ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane}); + return result; +} + +uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + if (state.flattened_srt_variable == 0) { + ctx.Fail(inst, "requires the flattened SRT descriptor"); + } + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, + state.flattened_srt_variable, ConstantU32(state, 0), + ctx.Arg(inst, 1)}); + return EmitNative(state, pointer); +} + +void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { + auto mem = ctx.Memory(inst); + if (mem.planning_only) return; + auto& state = ctx.state; + mem.kind = IR::ResourceKind::ScalarBuffer; + const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), + ConstantU32(state, mem.offset)); + const auto index = + Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); + const auto access = PrepareMemoryResourceAccess(state, mem); + const auto element = EmitMemoryElementIndex(state, access, index); + const auto condition = EmitMemoryElementInBounds(state, access, element); + ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { + return EmitNative( + state, EmitMemoryElementPointer(state, access, element)); + })); +} + +void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto op = inst.GetOpcode(); + const auto& mem = ctx.Memory(inst); + if (op == IR::ValueOpcode::LoadAddressU32 && mem.planning_only) return; const auto buffer_components = IR::BufferComponentCount(op); - if (buffer_components > 1u) { - const auto access = IR::BufferAccessOf(op); - if (access == IR::BufferAccess::Read) { - ctx.Define(inst, LoadWideBuffer(ctx, inst, buffer_components)); - return; - } - if (access == IR::BufferAccess::Write) { - StoreWideBuffer(ctx, inst, buffer_components); - return; - } - } const auto shared_components = IR::SharedComponentCount(op); - if (shared_components > 1u) { - if (IR::SharedAccessOf(op) == IR::SharedAccess::Read) { - ctx.Define(inst, LoadWideShared(ctx, inst, shared_components)); - } else { - StoreWideShared(ctx, inst, shared_components); - } - return; - } - if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) && - ctx.Memory(inst).planning_only) { - return; - } - if (op == IR::ValueOpcode::ReadConst) { - if (state.flattened_srt_variable == 0) { - ctx.Fail(inst, "requires the flattened SRT descriptor"); - return; - } - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state), - pointer, state.flattened_srt_variable, ConstantU32(state, 0), - ctx.Arg(inst, 1)}); - ctx.Define(inst, EmitNative(ctx, pointer)); - return; - } - if (op == IR::ValueOpcode::ReadConstBuffer) { - auto mem = ctx.Memory(inst); - mem.kind = IR::ResourceKind::ScalarBuffer; - const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), - ConstantU32(state, mem.offset)); - const auto index = - Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - const auto access = PrepareMemoryResourceAccess(state, mem); - const auto element = EmitMemoryElementIndex(state, access, index); - const auto condition = EmitMemoryElementInBounds(state, access, element); - ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { - const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpLoad, TypeU32(state), value, - EmitMemoryElementPointer(state, access, element)}); - return value; - })); - return; - } - const auto address_info = IR::AddressOpcodeInfoOf(op); - const bool load_address = address_info.access == IR::AddressAccess::Read; - if (load_address && ctx.Memory(inst).kind != IR::ResourceKind::Scratch) { - ctx.Define(inst, LoadBda(ctx, inst, ctx.Memory(inst), address_info.data_bits)); - return; - } - const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 || - op == IR::ValueOpcode::LoadBufferU16 || - op == IR::ValueOpcode::LoadBufferU32; - const bool load_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Read; - if (load_address || load_buffer || load_shared) { - const auto mem = ctx.Memory(inst); - uint32_t value = 0; - if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) - value = FormattedLoad(ctx, inst, mem); - else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 || - op == IR::ValueOpcode::LoadSharedU8) - value = LoadSubword(ctx, inst, mem, 8, false); - else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 || - op == IR::ValueOpcode::LoadSharedU16) - value = LoadSubword(ctx, inst, mem, 16, false); - else - value = LoadWord(ctx, inst, mem); - ctx.Define(inst, value); - return; - } - const bool store_address = address_info.access == IR::AddressAccess::Write; - const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 || - op == IR::ValueOpcode::StoreBufferU16 || - op == IR::ValueOpcode::StoreBufferU32; - const bool store_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Write; - if (store_address || store_buffer || store_shared) { - const auto mem = ctx.Memory(inst); - if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted) - FormattedStore(ctx, inst, mem); - else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 || - op == IR::ValueOpcode::WriteSharedU8) - StoreSubword(ctx, inst, mem, 8); - else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 || - op == IR::ValueOpcode::WriteSharedU16) - StoreSubword(ctx, inst, mem, 16); - else - StoreWord(ctx, inst, mem); - return; - } - const auto atomic_opcode = SpirvAtomicOpcode(op); - if (atomic_opcode != 0 && inst.GetType() == IR::Type::U64) { - ctx.Define(inst, EmitBufferAtomic64(ctx, inst, ctx.Memory(inst))); - return; - } - if (atomic_opcode != 0) { - ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst))); - return; - } - if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) { - ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst), - op == IR::ValueOpcode::BufferAtomicFMax32)); - return; - } - if (op == IR::ValueOpcode::SharedAtomicInc32 || op == IR::ValueOpcode::SharedAtomicDec32) { - const auto replacement = op == IR::ValueOpcode::SharedAtomicInc32 - ? AtomicIncrement : AtomicDecrement; - ctx.Define(inst, EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement)); - return; - } - if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) { - SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32); - return; - } - if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume) { - ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend)); - return; - } - if (op == IR::ValueOpcode::SwizzleU32 || op == IR::ValueOpcode::BpermuteU32) { - uint32_t source = ctx.Arg(inst, 0); - uint32_t target = 0; - if (op == IR::ValueOpcode::SwizzleU32) { - state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, source}); - source = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpLoad, TypeU32(state), source, ctx.scratch_u32_variable}); - target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state), - inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0); - } else { - const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state), - Binary(state, spv::OpShiftRightLogical, TypeU32(state), - ctx.Arg(inst, 1), ConstantU32(state, 2)), - ConstantU32(state, 31)); - const auto base = - Binary(state, spv::OpBitwiseAnd, TypeU32(state), - EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u)); - target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index); - } - ctx.Define(inst, EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2))); - return; - } - ctx.Fail(inst, "has no memory SPIR-V emitter"); + const auto address_info = IR::AddressOpcodeInfoOf(op); + uint32_t value; + if (buffer_components > 1u) + value = LoadWideBuffer(ctx, inst, buffer_components); + else if (shared_components > 1u) + value = LoadWideShared(ctx, inst, shared_components); + else if (address_info.access == IR::AddressAccess::Read && + mem.kind != IR::ResourceKind::Scratch) + value = LoadBda(ctx, inst, mem, address_info.data_bits); + else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) + value = FormattedLoad(ctx, inst, mem); + else if (inst.GetType() == IR::Type::U8) + value = LoadSubword(ctx, inst, mem, 8, false); + else if (inst.GetType() == IR::Type::U16) + value = LoadSubword(ctx, inst, mem, 16, false); + else + value = LoadWord(ctx, inst, mem); + ctx.Define(inst, value); +} + +void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto op = inst.GetOpcode(); + const auto& mem = ctx.Memory(inst); + const auto buffer_components = IR::BufferComponentCount(op); + const auto shared_components = IR::SharedComponentCount(op); + const auto type = inst.Arg(inst.NumArgs() - 2).GetType(); + if (buffer_components > 1u) + StoreWideBuffer(ctx, inst, buffer_components); + else if (shared_components > 1u) + StoreWideShared(ctx, inst, shared_components); + else if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted) + FormattedStore(ctx, inst, mem); + else if (type == IR::Type::U8) + StoreSubword(ctx, inst, mem, 8); + else if (type == IR::Type::U16) + StoreSubword(ctx, inst, mem, 16); + else + StoreWord(ctx, inst, mem); +} + +uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto replacement = + inst.GetOpcode() == IR::ValueOpcode::SharedAtomicInc32 ? AtomicIncrement : AtomicDecrement; + return EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement); +} + +uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)}); + const auto source = EmitNative(state, ctx.scratch_u32_variable); + const auto target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state), + inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0); + return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2)); +} + +uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto source = ctx.Arg(inst, 0); + const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state), + Binary(state, spv::OpShiftRightLogical, TypeU32(state), + ctx.Arg(inst, 1), ConstantU32(state, 2)), + ConstantU32(state, 31)); + const auto base = Binary(state, spv::OpBitwiseAnd, TypeU32(state), + EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u)); + const auto target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index); + return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2)); } } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 09852f1c..19749874 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -25,12 +25,6 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { return ConstantU32(state, 0); } -uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpIAdd, TypeU32(state), ret, lhs, rhs}); - return ret; -} - uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) { const auto ret = state.builder.AllocateId(); state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs}); @@ -194,14 +188,6 @@ uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) { return ret; } -uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, - uint32_t false_value) { - const auto ret = state.builder.AllocateId(); - state.builder.AddFunction( - {spv::OpSelect, TypeF32(state), ret, condition, true_value, false_value}); - return ret; -} - bool IsSignedFormatComponent(Format::ComponentType type) { return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm || type == Format::ComponentType::Sscaled; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 0d3f7de5..fc8b77f3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -40,7 +40,7 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) { } uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) { - const auto value_type = ctx.TypeId(type); + const auto value_type = TypeId(ctx.state, type); return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type); } @@ -243,14 +243,22 @@ decltype(auto) Arg(ValueEmitContext& ctx, const IR::Inst& inst, size_t index) { } } -template -void Invoke(Return (*emit)(ValueEmitContext&, Args...), ValueEmitContext& ctx, - const IR::Inst& inst) { +template +void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::Inst& inst) { // A full instruction keeps metadata and predicated/lane operand loads lazy. + static_assert(std::is_same_v || + std::is_same_v); + auto& context = [&]() -> Context& { + if constexpr (std::is_same_v) + return ctx.state; + else + return ctx; + }(); constexpr bool has_inst = (std::is_same_v || ...); [&](std::index_sequence) { + static_assert(((!std::is_same_v || I == 0) && ...)); const auto call = [&] { - return emit(ctx, Arg(ctx, inst, I - (has_inst && I != 0))...); + return emit(context, Arg(ctx, inst, I - (has_inst && I != 0))...); }; if constexpr (std::is_void_v) { call(); @@ -274,7 +282,7 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured, const IR::Inst& inst) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) { - const auto type = ctx.TypeId(inst.GetType()); + const auto type = TypeId(ctx.state, inst.GetType()); if (type == 0 || inst.NumArgs() == 0) { ctx.Fail(inst, "has no native SPIR-V representation"); } @@ -295,7 +303,7 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher, const IR::Inst& inst) { if (inst.GetOpcode() == IR::ValueOpcode::Phi) { - const auto type = ctx.TypeId(inst.GetType()); + const auto type = TypeId(ctx.state, inst.GetType()); if (type == 0) { ctx.Fail(inst, "cannot be loaded by the dispatcher"); } @@ -425,7 +433,7 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState } // namespace -uint32_t ValueEmitContext::TypeId(IR::Type type) const { +uint32_t TypeId(EmitterState& state, IR::Type type) { switch (type) { case IR::Type::U1: return TypeBool(state); case IR::Type::U8: @@ -470,7 +478,7 @@ uint32_t ValueEmitContext::Def(IR::Value value) { return loaded->second.second; } const auto id = state.builder.AllocateId(); - state.builder.AddFunction({spv::OpLoad, TypeId(inst->GetType()), id, found->second}); + state.builder.AddFunction({spv::OpLoad, TypeId(state, inst->GetType()), id, found->second}); dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id}); return id; } @@ -536,7 +544,7 @@ uint32_t ValueEmitContext::FirstLane(uint32_t ballot) { } uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) { - const auto type = TypeId(inst.Arg(index).GetType()); + const auto type = TypeId(state, inst.Arg(index).GetType()); const auto scope = ConstantU32(state, spv::ScopeSubgroup); const auto low = state.builder.AllocateId(); if (other_half == nullptr) { @@ -574,7 +582,7 @@ uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) { if (const auto found = definitions.find(&inst); found != definitions.end()) { if (found->second != value) { state.builder.AddFunction( - {spv::OpCopyObject, TypeId(inst.GetType()), found->second, value}); + {spv::OpCopyObject, TypeId(state, inst.GetType()), found->second, value}); } return found->second; } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 54a32c1e..a3b3e006 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -16687,6 +16687,44 @@ TestCase VectorVop3LshlrevB64Captured() { return test; } +TestCase VectorDynamicU64ShiftEdges() { + using O = ShaderOpcode; + TestCase test; + test.name = "VectorDynamicU64ShiftEdges"; + test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u}; + test.expected = test.initial; + auto& code = test.code; + for (const uint64_t source : {uint64_t{0}, uint64_t{1}, + uint64_t{0x8000000000000000ull}, + uint64_t{0xffffffffffffffffull}, + uint64_t{0x0123456789abcdefull}}) { + AppendVMovLiteral(&code, 4, static_cast(source)); + AppendVMovLiteral(&code, 5, static_cast(source >> 32u)); + for (u32 index = 0; index < test.initial.size(); ++index) { + // A buffer load prevents the constant-shift specialization. + AppendVMovU32(&code, 30, index * sizeof(u32)); + AppendBufferLoadDword(&code, 3, 30); + for (const bool left : {true, false}) { + AppendVop3(&code, left ? 0x2ff : 0x300, 33, Vgpr(3), Vgpr(4)); + const auto amount = test.initial[index] & 63u; + const auto expected = left ? source << amount : source >> amount; + const auto output = static_cast(test.expected.size()); + AppendStoreVgpr(&code, 33, output); + AppendStoreVgpr(&code, 34, output + 1u); + test.expected.push_back(static_cast(expected)); + test.expected.push_back(static_cast(expected >> 32u)); + } + } + } + AppendEnd(&code); + test.initial.resize(test.expected.size()); + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64, + O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical", "OpSelect"}; + test.forbidden_spirv = {"OpTypeInt 64"}; + return test; +} + TestCase VectorVop3IntegerOps() { using O = ShaderOpcode; @@ -24745,6 +24783,7 @@ std::vector MakeCases() { AddCase(ScalarBfeI32CapturedRawSignExtends); AddCase(BitfieldExtractWidthPastEndEdges); AddCase(Scalar64BitOps); + AddCase(VectorDynamicU64ShiftEdges); AddCase(ScalarConditionalMoveB64); AddCase(ScalarConditionalMoveB64PreservesMasks); AddCase(ScalarAndn2B64SccUsesMaskShadow); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index f4c18c7f..d009ba15 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -8708,6 +8708,38 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { "dispatcher did not use one canonical U32x3 spill slot"); } +void TestNewShaderRecompilerPlanningOnlyLoads() { + using namespace ShaderRecompiler; + const uint32_t shader[] = {EncodeSopp(0x01)}; + auto options = MakeCompileOptions(ShaderType::Compute); + auto result = RecompileForTest(shader, options); + auto& program = result.program; + IR::IREmitter ir(program.blocks.front()); + IR::MemoryInfo memory; + memory.kind = IR::ResourceKind::ScalarAddress; + memory.planning_only = true; + program.memory_info.push_back(memory); + const auto address = ir.Emit(IR::ValueOpcode::GetAddressResource, + {IR::Value(0x1000u), IR::Value(0u)}); + const auto load = ir.Emit(IR::ValueOpcode::LoadAddressU32, + {address, IR::Value(0u), IR::Value(0u), IR::Value(true)}); + load.Instruction()->SetFlags(IR::MemoryFlags{.index = 0}); + memory.kind = IR::ResourceKind::ScalarBuffer; + program.memory_info.push_back(memory); + const auto buffer = ir.Emit(IR::ValueOpcode::GetBufferResource, + {IR::Value(0x2000u), IR::Value(0u), + IR::Value(16u), IR::Value(0u)}); + const auto read = ir.Emit(IR::ValueOpcode::ReadConstBuffer, + {buffer, IR::Value(0u)}); + read.Instruction()->SetFlags(IR::MemoryFlags{.index = 1}); + ir.Emit(IR::ValueOpcode::ReferenceU32, {load}); + ir.Emit(IR::ValueOpcode::ReferenceU32, {read}); + const auto spirv = Spirv::EmitProgram(program, options.input_info); + CheckSpirvBinaryValidates(spirv); + Check(spirv == result.spirv, + "planning-only loads or references emitted GPU operands/instructions"); +} + void TestNewShaderRecompilerU64PairTranslation() { using namespace ShaderRecompiler; @@ -13017,6 +13049,7 @@ int main() { TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerDispatcherSpillsU32x3(); + TestNewShaderRecompilerPlanningOnlyLoads(); TestNewShaderRecompilerU64PairTranslation(); TestComputeDispatchWaveSize(); TestNewShaderRecompilerBufferLoadsGuardedByExec();