shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

shader: reuse native emitters and simplify memory dispatch

This commit is contained in:
nmzik 2026-09-11 19:10:48 +04:00
parent 59ef88c959
commit 20e57994cf
10 changed files with 721 additions and 993 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

454
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp
View File

@ -41,32 +41,6 @@ uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_
return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
}
uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high);
}
uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
const auto pair_type = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
@ -84,58 +58,42 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
}
uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
return MakePair(state, low,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
}
uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value);
uint32_t low = 0;
uint32_t high = 0;
uint32_t EmitShift64(EmitterState& state, spv::Op opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value);
const auto amount = EmitAndConstant(state, shift, 63u);
const auto word_shift = EmitAndConstant(state, amount, 31u);
const auto at_least_32 =
Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
const auto nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
const auto carry_count = EmitAndConstant(
state, Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), 31u);
if (opcode == spv::OpShiftLeftLogical) {
EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else if (opcode == spv::OpShiftRightLogical) {
EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else {
const auto amount =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
const auto word_shift =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
const auto at_least_32 = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount,
ConstantU32(state, 32));
const auto nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
const auto high_shifted =
Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
const auto low_shifted =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
const auto carry_count =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
ConstantU32(state, 31));
const auto low = Binary(state, opcode, TypeU32(state), pair.low, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, carry_count),
ConstantU32(state, 0));
const auto low_below_32 =
Binary(state, spv::OpBitwiseOr, TypeU32(state), low_shifted, carry);
const auto negative =
Binary(state, spv::OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
const auto sign_fill = Select(state, TypeU32(state), negative,
ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
const auto high =
Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, opcode, TypeU32(state), pair.high, word_shift), carry);
return MakePair(state,
Select(state, TypeU32(state), at_least_32, high_shifted, low_below_32),
Select(state, TypeU32(state), at_least_32, sign_fill, high_shifted));
Select(state, TypeU32(state), at_least_32, ConstantU32(state, 0), low),
Select(state, TypeU32(state), at_least_32, low, high));
}
return MakePair(state, low, high);
const auto high = Binary(state, opcode, TypeU32(state), pair.high, word_shift);
const auto carry =
Select(state, TypeU32(state), nonzero,
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
ConstantU32(state, 0));
const auto low = Binary(
state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift), carry);
const auto fill = opcode == spv::OpShiftRightArithmetic
? Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, 31))
: ConstantU32(state, 0);
return MakePair(state, Select(state, TypeU32(state), at_least_32, high, low),
Select(state, TypeU32(state), at_least_32, fill, high));
}
uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
@ -176,26 +134,10 @@ uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t valu
: ConstantU32(state, 0);
const auto low = shift == 32u ? pair.high
: Binary(state, opcode, TypeU32(state), pair.high,
ConstantU32(state, shift - 32u));
ConstantU32(state, shift - 32u));
return MakePair(state, low, high);
}
uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
const auto pair = ExtractPair(state, value);
const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high});
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low});
const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero,
Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
}
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
bool max_value) {
const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
@ -217,19 +159,6 @@ uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, b
return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
}
uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
const auto min_ab = EmitMinMaxF32Value(state, a, b, false);
const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false);
const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state),
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
const auto any_nan =
Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
return Select(state, TypeF32(state), any_nan, min3, median);
}
uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
std::initializer_list<uint32_t> args) {
const auto result = state.builder.AllocateId();
@ -271,121 +200,127 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high);
}
uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {src});
}
} // namespace
uint32_t EmitConvertU16U32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitNative<spv::OpBitwiseAnd, IR::Type::U16>(ctx, arg0, ConstantU32(state, 0xffffu));
uint32_t EmitFPMedTri32(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
const auto min_ab = EmitMinMaxF32Value(state, a, b, false);
const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false);
const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state),
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
const auto any_nan =
Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
return Select(state, TypeF32(state), any_nan, min3, median);
}
uint32_t EmitConvertU8U32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitNative<spv::OpBitwiseAnd, IR::Type::U8>(ctx, arg0, ConstantU32(state, 0xffu));
uint32_t EmitFindUMsb64(EmitterState& state, uint32_t value) {
const auto pair = ExtractPair(state, value);
const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high});
state.builder.AddFunction(
{spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low});
const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero =
Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero,
Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
}
uint32_t EmitConvertF16F32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
const auto pair = state.builder.AllocateId();
uint32_t EmitIMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
return MakePair(state, low,
Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
}
uint32_t EmitISub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high);
}
uint32_t EmitIAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value);
const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high);
}
uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpBitwiseAnd, IR::Type::U16>(state, arg0, ConstantU32(state, 0xffffu));
}
uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpBitwiseAnd, IR::Type::U8>(state, arg0, ConstantU32(state, 0xffu));
}
uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0) {
const auto pair = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpCompositeConstruct, TypeF32Vector(state, 2), pair, arg0, ConstantF32(state, 0)});
return EmitPackHalf(state, pair);
return EmitPackHalf2x16(state, pair);
}
uint32_t EmitConvertF32F16(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitF16BitsToF32(state, arg0);
}
uint32_t EmitConvertS32F32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0) {
return EmitF32ToU32(state, arg0, true);
}
uint32_t EmitConvertU32F32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
return EmitF32ToU32(state, arg0, false);
}
uint32_t EmitConvertF32S32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0) {
const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0);
return EmitNative<spv::OpConvertSToF, IR::Type::F32>(ctx, signed_value);
return EmitNative<spv::OpConvertSToF, IR::Type::F32>(state, signed_value);
}
uint32_t EmitCompositeExtractU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1) {
return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(ctx, arg0, arg1.U32());
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
return EmitNative<spv::OpCompositeExtract, IR::Type::U32>(state, arg0, arg1.U32());
}
uint32_t EmitPackHalf2x16(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {arg0});
}
uint32_t EmitPackSnorm2x16(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeU32(state), GLSLstd450PackSnorm2x16, {arg0});
}
uint32_t EmitPackUnorm2x16(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeU32(state), GLSLstd450PackUnorm2x16, {arg0});
}
uint32_t EmitPackFloat2x16Rtz(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
const auto low = EmitF32ToF16RtzBits(state, arg0);
const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16));
uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1) {
const auto low = EmitF32ToF16RtzBits(state, arg0);
const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
EmitF32ToF16RtzBits(state, arg1), ConstantU32(state, 16));
return Binary(state, spv::OpBitwiseOr, TypeU32(state), low, high);
}
uint32_t EmitFPAbs32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitFAbsValue(state, arg0);
}
uint32_t EmitFPNeg32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitFNegateValue(state, arg0);
}
uint32_t EmitFPSaturate32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0) {
return EmitExt(state, TypeF32(state), GLSLstd450FClamp,
{arg0, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
}
uint32_t EmitIAdd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitAdd64(state, arg0, arg1);
}
uint32_t EmitISub64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitSub64(state, arg0, arg1);
}
uint32_t EmitIMul64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitMul64(state, arg0, arg1);
}
uint32_t EmitSMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMulHigh(state, arg0, arg1, true);
}
uint32_t EmitUMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMulHigh(state, arg0, arg1, false);
}
uint32_t EmitIAbs32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0) {
const auto value = arg0;
const auto neg = Unary(state, spv::OpSNegate, TypeU32(state), value);
const auto negative =
@ -414,225 +349,148 @@ uint32_t EmitShiftRightArithmetic64(ValueEmitContext& ctx, uint32_t arg0, IR::Va
: EmitShift64(state, spv::OpShiftRightArithmetic, arg0, ctx.Def(arg1));
}
uint32_t EmitBitwiseAnd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return Binary(state, spv::OpBitwiseAnd, TypeU64(state), arg0, arg1);
}
uint32_t EmitBitCount64(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0) {
const auto pair = ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), arg0));
return Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high);
}
uint32_t EmitFindILsb32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0) {
const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindILsb, {arg0});
return Unary(state, spv::OpBitcast, TypeU32(state), value);
}
uint32_t EmitFindUMsb32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0) {
const auto value = EmitExt(state, TypeI32(state), GLSLstd450FindUMsb, {arg0});
return Unary(state, spv::OpBitcast, TypeU32(state), value);
}
uint32_t EmitFindUMsb64(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitFindMsb64(state, arg0);
}
uint32_t EmitSMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMinMaxI32Value(state, arg0, arg1, false);
}
uint32_t EmitSMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMinMaxI32Value(state, arg0, arg1, true);
}
uint32_t EmitUMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMinMaxU32Value(state, arg0, arg1, false);
}
uint32_t EmitUMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMinMaxU32Value(state, arg0, arg1, true);
}
uint32_t EmitSMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitMinMax3(state, arg0, arg1, arg2, true, false);
}
uint32_t EmitSMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitMinMax3(state, arg0, arg1, arg2, true, true);
}
uint32_t EmitUMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitMinMax3(state, arg0, arg1, arg2, false, false);
}
uint32_t EmitUMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitMinMax3(state, arg0, arg1, arg2, false, true);
}
uint32_t EmitSMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitMed3(state, arg0, arg1, arg2, true);
}
uint32_t EmitUMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitMed3(state, arg0, arg1, arg2, false);
}
uint32_t EmitIEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareEqual64(state, arg0, arg1, false);
}
uint32_t EmitINotEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareEqual64(state, arg0, arg1, true);
}
uint32_t EmitULessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpULessThan, spv::OpULessThan);
}
uint32_t EmitSLessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpSLessThan, spv::OpULessThan);
}
uint32_t EmitUGreaterThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return CompareOrdered64(state, arg0, arg1, spv::OpUGreaterThan, spv::OpUGreaterThan);
}
uint32_t EmitFPIsNan32(ValueEmitContext& ctx, uint32_t arg0) {
return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(ctx, arg0, arg0);
uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0) {
return EmitNative<spv::OpFUnordNotEqual, IR::Type::U1>(state, arg0, arg0);
}
uint32_t EmitFPCmpClass32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
return EmitClassMaskF32(state, arg0, arg1);
}
uint32_t EmitFPFma32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Fma, {arg0, arg1, arg2});
}
uint32_t EmitFPMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMinMaxF32Value(state, arg0, arg1, false);
}
uint32_t EmitFPMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1) {
return EmitMinMaxF32Value(state, arg0, arg1, true);
}
uint32_t EmitFPMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitFMinMax3(state, arg0, arg1, arg2, false);
}
uint32_t EmitFPMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
return EmitFMinMax3(state, arg0, arg1, arg2, true);
}
uint32_t EmitFPMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2) {
auto& state = ctx.state;
return EmitFMed3(state, arg0, arg1, arg2);
}
uint32_t EmitFPRecip32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0) {
const auto source = EmitFlushF32DenormToSignedZero(state, arg0);
return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), source);
}
uint32_t EmitFPRecipIFlag32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0) {
// Integer-to-float inputs used by IFLAG cannot be denormal.
return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), arg0);
}
uint32_t EmitFPRecipSqrt32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0) {
return EmitExt(state, TypeF32(state), GLSLstd450InverseSqrt,
{EmitFlushF32DenormToSignedZero(state, arg0)});
}
uint32_t EmitFPSqrt(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0) {
return EmitExt(state, TypeF32(state), GLSLstd450Sqrt,
{EmitFlushF32DenormToSignedZero(state, arg0)});
}
uint32_t EmitFPExp2(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0) {
return EmitExt(state, TypeF32(state), GLSLstd450Exp2,
{EmitFlushF32DenormToSignedZero(state, arg0)});
}
uint32_t EmitFPLog2(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0) {
return EmitExt(state, TypeF32(state), GLSLstd450Log2,
{EmitFlushF32DenormToSignedZero(state, arg0)});
}
uint32_t EmitFPLdexp(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1) {
auto& state = ctx.state;
uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1) {
const auto exponent = Unary(state, spv::OpBitcast, TypeI32(state), arg1);
return EmitExt(state, TypeF32(state), GLSLstd450Ldexp, {arg0, exponent});
}
uint32_t EmitFPRoundEven32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450RoundEven, {arg0});
}
uint32_t EmitFPFloor32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Floor, {arg0});
}
uint32_t EmitFPCeil32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Ceil, {arg0});
}
uint32_t EmitFPTrunc32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Trunc, {arg0});
}
uint32_t EmitFPFract32(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
return EmitExt(state, TypeF32(state), GLSLstd450Fract, {arg0});
}
uint32_t EmitFPSin(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
auto source = EmitTrigCycleF32(state, arg0, true);
uint32_t EmitFPSin(EmitterState& state, uint32_t arg0) {
auto source = EmitTrigCycleF32(state, arg0, true);
source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
return EmitExt(state, TypeF32(state), GLSLstd450Sin, {source});
}
uint32_t EmitFPCos(ValueEmitContext& ctx, uint32_t arg0) {
auto& state = ctx.state;
auto source = EmitTrigCycleF32(state, arg0, false);
uint32_t EmitFPCos(EmitterState& state, uint32_t arg0) {
auto source = EmitTrigCycleF32(state, arg0, false);
source = Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
return EmitExt(state, TypeF32(state), GLSLstd450Cos, {source});
}

144
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp
View File

@ -2,88 +2,6 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
uint32_t false_value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, true_value, false_value});
return ret;
}
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
uint32_t& out_low, uint32_t& out_high) {
const auto amount = EmitAndConstant(state, shift, 63u);
const auto dword_shift = EmitAndConstant(state, amount, 31u);
const auto less32 = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto carry_count_base = state.builder.AllocateId();
const auto carry_count = state.builder.AllocateId();
const auto low_part = state.builder.AllocateId();
const auto high_part = state.builder.AllocateId();
const auto carry_part = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
const auto high_less = state.builder.AllocateId();
const auto high_ge = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
state.builder.AddFunction(
{spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
state.builder.AddFunction(
{spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
state.builder.AddFunction({spv::OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
}
void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
const auto amount = EmitAndConstant(state, shift, 63u);
const auto dword_shift = EmitAndConstant(state, amount, 31u);
const auto less32 = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto carry_count_base = state.builder.AllocateId();
const auto carry_count = state.builder.AllocateId();
const auto low_part = state.builder.AllocateId();
const auto high_part = state.builder.AllocateId();
const auto carry_part = state.builder.AllocateId();
const auto carry = state.builder.AllocateId();
const auto low_less = state.builder.AllocateId();
const auto low_ge = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
state.builder.AddFunction(
{spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
state.builder.AddFunction(
{spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
out_low = EmitSelectValueU32(state, less32, low_less, low_ge);
out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
}
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
@ -98,12 +16,6 @@ uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t sh
return ret;
}
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
uint32_t constant) {
const auto ret = state.builder.AllocateId();
@ -189,42 +101,6 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
return ret;
}
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitcast, TypeU32(state), ret, value});
return ret;
}
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitcast, TypeF32(state), ret, value});
return ret;
}
uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLogicalOr, TypeBool(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), ret, value});
return ret;
}
F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
F32Class cls;
cls.bits = bits;
@ -324,13 +200,6 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
return EmitBitcastU32ToF32(state, result_bits);
}
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450Trunc, value});
return ret;
}
uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
const auto bits = state.builder.AllocateId();
const auto abs_bits = state.builder.AllocateId();
@ -386,19 +255,6 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
return ret;
}
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpFNegate, TypeF32(state), ret, value});
return ret;
}
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450FAbs, value});
return ret;
}
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
const auto unpacked = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();

82
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
View File

@ -16,29 +16,7 @@ bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t&
return true;
}
uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
const auto shifted_two = state.builder.AllocateId();
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull)});
state.builder.AddFunction(
{spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull)});
state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full)});
return result;
}
uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
auto& state = ctx.state;
uint32_t EmitBuiltinU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
if (kind == IR::StageInputKind::LocalInvocationIndex) {
return EmitLocalInvocationIndex(state);
}
@ -155,8 +133,7 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
return result;
}
uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
auto& state = ctx.state;
uint32_t EmitAttribute(EmitterState& state, uint32_t attr, uint32_t chan) {
const auto* input = InputBindingForParameter(state, attr);
if (input == nullptr || input->variable_id == 0) {
return ConstantU32(state, 0);
@ -217,7 +194,7 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
auto& state = ctx.state;
const auto* input = InputBindingForParameter(state, attr);
if (!input->per_vertex) {
return EmitAttribute(ctx, attr, chan);
return EmitAttribute(ctx.state, attr, chan);
}
const auto load_vertex = [&](uint32_t vertex) {
const auto pointer = state.builder.AllocateId();
@ -427,6 +404,26 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) {
}
} // namespace
uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto shifted_one = state.builder.AllocateId();
const auto merged_one = state.builder.AllocateId();
const auto shifted_two = state.builder.AllocateId();
const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull)});
state.builder.AddFunction(
{spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull)});
state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full)});
return result;
}
void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
@ -523,8 +520,7 @@ uint32_t EmitIdentity(ValueEmitContext&, uint32_t value) {
void EmitVoid(ValueEmitContext&) {}
void EmitBarrier(ValueEmitContext& ctx) {
auto& state = ctx.state;
void EmitBarrier(EmitterState& state) {
const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
@ -547,8 +543,7 @@ uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
return result;
}
uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg) {
auto& state = ctx.state;
uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg) {
uint32_t dword = 0;
if (!UserDataDwordIndex(state, reg, dword)) {
@ -559,13 +554,12 @@ uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg) {
}
uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index) {
return EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
return EmitBuiltinU32(ctx.state, static_cast<IR::StageInputKind>(kind.U32()), index.U32());
}
uint32_t EmitUndefU1(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpUndef, ctx.TypeId(inst.GetType()), result});
state.builder.AddFunction({spv::OpUndef, TypeId(state, inst.GetType()), result});
return result;
}
@ -582,23 +576,15 @@ uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto can_fetch = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx, can_fetch, shuffled,
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, can_fetch, shuffled,
ConstantU32(state, 0));
}
uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto flags = inst.Flags<IR::DppMoveFlags>();
const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx, write, ctx.Arg(inst, 0), ctx.Arg(inst, 1));
}
uint32_t EmitWqmU64(ValueEmitContext& ctx, uint32_t value) {
return EmitWqmU64(ctx.state, value);
}
uint32_t EmitLaneId(ValueEmitContext& ctx) {
auto& state = ctx.state;
return EmitSubgroupLocalInvocationId(state);
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, write, ctx.Arg(inst, 0),
ctx.Arg(inst, 1));
}
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
@ -620,7 +606,8 @@ uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto hit = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIEqual, TypeBool(state), hit,
EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx, hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0));
return EmitNative<spv::OpSelect, IR::Type::U32>(ctx.state, hit, ctx.Arg(inst, 1),
ctx.Arg(inst, 0));
}
uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
@ -671,7 +658,7 @@ uint32_t EmitPermlane16U32(ValueEmitContext& ctx, const IR::Inst& inst) {
}
uint32_t EmitGetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
return EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32());
return EmitAttribute(ctx.state, inst.Arg(0).U32(), inst.Arg(1).U32());
}
uint32_t EmitGetInterpolationParameter(ValueEmitContext& ctx, const IR::Inst& inst) {
@ -684,7 +671,6 @@ uint32_t EmitGetShaderBase(ValueEmitContext& ctx) {
return ctx.Def(IR::Value(uint64_t {0}));
}
void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Fail(inst, "must be lowered before SPIR-V emission");
}

287
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h
View File

@ -3,13 +3,8 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
// Direct SPIR-V operations share emission as well as argument/result marshalling.
template <spv::Op opcode, IR::Type type, typename... Args>
uint32_t EmitNative(ValueEmitContext& ctx, Args... args) {
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({opcode, ctx.TypeId(type), result, args...});
return result;
}
// Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
// (IR::Value), and instructions whose metadata or operand loads must stay lazy.
#define EMIT_NATIVE(name, opcode, type, ...) \
inline constexpr auto Emit##name = EmitNative<spv::opcode, IR::Type::type, __VA_ARGS__>;
EMIT_NATIVE(BitCastU16F16, OpBitcast, U32, uint32_t)
@ -18,13 +13,13 @@ inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16;
inline constexpr auto EmitConvertU32U8 = EmitBitCastU16F16;
inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16;
EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t)
uint32_t EmitConvertU16U32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertU8U32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertF16F32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertF32F16(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertS32F32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertU32F32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertF32S32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0);
inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32;
uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0);
uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
@ -32,17 +27,19 @@ EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint
EMIT_NATIVE(CompositeConstructF32x2, OpCompositeConstruct, F32x2, uint32_t, uint32_t)
EMIT_NATIVE(CompositeConstructU32x4, OpCompositeConstruct, U32x4, uint32_t, uint32_t, uint32_t,
uint32_t)
uint32_t EmitCompositeExtractU64(ValueEmitContext& ctx, uint32_t arg0, IR::Value arg1);
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1);
inline constexpr auto EmitCompositeExtractU32x2 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x3 = EmitCompositeExtractU64;
inline constexpr auto EmitCompositeExtractU32x4 = EmitCompositeExtractU64;
uint32_t EmitPackHalf2x16(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitPackSnorm2x16(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitPackUnorm2x16(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitPackFloat2x16Rtz(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPAbs32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPNeg32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPSaturate32(ValueEmitContext& ctx, uint32_t arg0);
inline constexpr auto EmitPackHalf2x16 = EmitGlsl<GLSLstd450PackHalf2x16, IR::Type::U32, uint32_t>;
inline constexpr auto EmitPackSnorm2x16 =
EmitGlsl<GLSLstd450PackSnorm2x16, IR::Type::U32, uint32_t>;
inline constexpr auto EmitPackUnorm2x16 =
EmitGlsl<GLSLstd450PackUnorm2x16, IR::Type::U32, uint32_t>;
uint32_t EmitPackFloat2x16Rtz(EmitterState& state, uint32_t arg0, uint32_t arg1);
inline constexpr auto EmitFPAbs32 = EmitFAbsValue;
inline constexpr auto EmitFPNeg32 = EmitFNegateValue;
uint32_t EmitFPSaturate32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(BitFieldInsert, OpBitFieldInsert, U32, uint32_t, uint32_t, uint32_t, uint32_t)
EMIT_NATIVE(BitFieldUExtract, OpBitFieldUExtract, U32, uint32_t, uint32_t, uint32_t)
EMIT_NATIVE(BitFieldSExtract, OpBitFieldSExtract, U32, uint32_t, uint32_t, uint32_t)
@ -53,13 +50,13 @@ EMIT_NATIVE(IAdd32, OpIAdd, U32, uint32_t, uint32_t)
EMIT_NATIVE(ISub32, OpISub, U32, uint32_t, uint32_t)
EMIT_NATIVE(IMul32, OpIMul, U32, uint32_t, uint32_t)
EMIT_NATIVE(UDiv32, OpUDiv, U32, uint32_t, uint32_t)
uint32_t EmitIAdd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitISub64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitIMul64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitIAdd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitISub64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitIMul64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(IAddCarry32, OpIAddCarry, U32x2, uint32_t, uint32_t)
uint32_t EmitSMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMulHi(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitIAbs32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitSMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMulHi(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitIAbs32(EmitterState& state, uint32_t arg0);
EMIT_NATIVE(ShiftLeftLogical32, OpShiftLeftLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightLogical32, OpShiftRightLogical, U32, uint32_t, uint32_t)
EMIT_NATIVE(ShiftRightArithmetic32, OpShiftRightArithmetic, U32, uint32_t, uint32_t)
@ -70,23 +67,23 @@ EMIT_NATIVE(BitwiseAnd32, OpBitwiseAnd, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseOr32, OpBitwiseOr, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseXor32, OpBitwiseXor, U32, uint32_t, uint32_t)
EMIT_NATIVE(BitwiseNot32, OpNot, U32, uint32_t)
uint32_t EmitBitwiseAnd64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitBitwiseAnd64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(BitReverse32, OpBitReverse, U32, uint32_t)
EMIT_NATIVE(BitCount32, OpBitCount, U32, uint32_t)
uint32_t EmitBitCount64(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFindILsb32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFindUMsb32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFindUMsb64(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitSMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitSMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitSMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitSMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitSMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitBitCount64(EmitterState& state, uint32_t arg0);
uint32_t EmitFindILsb32(EmitterState& state, uint32_t arg0);
uint32_t EmitFindUMsb32(EmitterState& state, uint32_t arg0);
uint32_t EmitFindUMsb64(EmitterState& state, uint32_t arg0);
uint32_t EmitSMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitSMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitSMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitUMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
EMIT_NATIVE(SLessThan32, OpSLessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(ULessThan32, OpULessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(IEqual32, OpIEqual, U1, uint32_t, uint32_t)
@ -97,11 +94,11 @@ EMIT_NATIVE(UGreaterThan32, OpUGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(INotEqual32, OpINotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(SGreaterThanEqual32, OpSGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(UGreaterThanEqual32, OpUGreaterThanEqual, U1, uint32_t, uint32_t)
uint32_t EmitIEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitINotEqual64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThan64(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitIEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitINotEqual64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitULessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitSLessThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitUGreaterThan64(EmitterState& state, uint32_t arg0, uint32_t arg1);
EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
@ -118,31 +115,32 @@ EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
uint32_t EmitFPIsNan32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPCmpClass32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0);
inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
uint32_t EmitFPFma32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPMin32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPMax32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPMinTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPMaxTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPMedTri32(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPRecip32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPRecipIFlag32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPRecipSqrt32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPSqrt(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPExp2(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPLog2(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPLdexp(ValueEmitContext& ctx, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPRoundEven32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPFloor32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPCeil32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPTrunc32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPFract32(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPSin(ValueEmitContext& ctx, uint32_t arg0);
uint32_t EmitFPCos(ValueEmitContext& ctx, uint32_t arg0);
inline constexpr auto EmitFPFma32 =
EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPMax32(EmitterState& state, uint32_t arg0, uint32_t arg1);
uint32_t EmitFPMinTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPMaxTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPMedTri32(EmitterState& state, uint32_t arg0, uint32_t arg1, uint32_t arg2);
uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPRecipSqrt32(EmitterState& state, uint32_t arg0);
uint32_t EmitFPSqrt(EmitterState& state, uint32_t arg0);
uint32_t EmitFPExp2(EmitterState& state, uint32_t arg0);
uint32_t EmitFPLog2(EmitterState& state, uint32_t arg0);
uint32_t EmitFPLdexp(EmitterState& state, uint32_t arg0, uint32_t arg1);
inline constexpr auto EmitFPRoundEven32 = EmitGlsl<GLSLstd450RoundEven, IR::Type::F32, uint32_t>;
inline constexpr auto EmitFPFloor32 = EmitGlsl<GLSLstd450Floor, IR::Type::F32, uint32_t>;
inline constexpr auto EmitFPCeil32 = EmitGlsl<GLSLstd450Ceil, IR::Type::F32, uint32_t>;
inline constexpr auto EmitFPTrunc32 = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
inline constexpr auto EmitFPFract32 = EmitGlsl<GLSLstd450Fract, IR::Type::F32, uint32_t>;
uint32_t EmitFPSin(EmitterState& state, uint32_t arg0);
uint32_t EmitFPCos(EmitterState& state, uint32_t arg0);
#undef EMIT_NATIVE
uint32_t EmitIdentity(ValueEmitContext& ctx, uint32_t value);
@ -154,20 +152,20 @@ inline constexpr auto EmitWaitcnt = EmitVoid;
inline constexpr auto EmitSendmsg = EmitVoid;
inline constexpr auto EmitTtraceData = EmitVoid;
inline constexpr auto EmitInstPrefetch = EmitVoid;
void EmitBarrier(ValueEmitContext& ctx);
void EmitBarrier(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitGetUserData(ValueEmitContext& ctx, IR::ScalarReg reg);
uint32_t EmitGetUserData(EmitterState& state, IR::ScalarReg reg);
uint32_t EmitGetBuiltin(ValueEmitContext& ctx, IR::Value kind, IR::Value index);
uint32_t EmitUndefU1(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitUndefU1(EmitterState& state, const IR::Inst& inst);
inline constexpr auto EmitUndefU8 = EmitUndefU1;
inline constexpr auto EmitUndefU16 = EmitUndefU1;
inline constexpr auto EmitUndefU32 = EmitUndefU1;
inline constexpr auto EmitUndefU64 = EmitUndefU1;
uint32_t EmitDppMoveU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitWqmU64(ValueEmitContext& ctx, uint32_t value);
uint32_t EmitLaneId(ValueEmitContext& ctx);
uint32_t EmitWqmU64(EmitterState& state, uint32_t value);
inline constexpr auto EmitLaneId = EmitSubgroupLocalInvocationId;
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
@ -184,72 +182,79 @@ inline constexpr auto EmitGetScratchResource = EmitGetSrtResource;
inline constexpr auto EmitGetImageResource = EmitGetSrtResource;
inline constexpr auto EmitGetSamplerResource = EmitGetSrtResource;
inline constexpr auto EmitMakeImageAddress = EmitGetSrtResource;
void EmitMemory(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitBpermuteU32 = EmitMemory;
inline constexpr auto EmitReadConst = EmitMemory;
inline constexpr auto EmitReadConstBuffer = EmitMemory;
inline constexpr auto EmitLoadAddressU8 = EmitMemory;
inline constexpr auto EmitLoadAddressU16 = EmitMemory;
inline constexpr auto EmitLoadAddressU32 = EmitMemory;
inline constexpr auto EmitStoreAddressU8 = EmitMemory;
inline constexpr auto EmitStoreAddressU16 = EmitMemory;
inline constexpr auto EmitStoreAddressU32 = EmitMemory;
inline constexpr auto EmitLoadBufferU8 = EmitMemory;
inline constexpr auto EmitLoadBufferU16 = EmitMemory;
inline constexpr auto EmitLoadBufferU32 = EmitMemory;
inline constexpr auto EmitLoadBufferU32x2 = EmitMemory;
inline constexpr auto EmitLoadBufferU32x3 = EmitMemory;
inline constexpr auto EmitLoadBufferU32x4 = EmitMemory;
inline constexpr auto EmitStoreBufferU8 = EmitMemory;
inline constexpr auto EmitStoreBufferU16 = EmitMemory;
inline constexpr auto EmitStoreBufferU32 = EmitMemory;
inline constexpr auto EmitStoreBufferU32x2 = EmitMemory;
inline constexpr auto EmitStoreBufferU32x3 = EmitMemory;
inline constexpr auto EmitStoreBufferU32x4 = EmitMemory;
inline constexpr auto EmitBufferAtomicSwap32 = EmitMemory;
inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitMemory;
inline constexpr auto EmitBufferAtomicSwap64 = EmitMemory;
inline constexpr auto EmitBufferAtomicIAdd32 = EmitMemory;
inline constexpr auto EmitBufferAtomicISub32 = EmitMemory;
inline constexpr auto EmitBufferAtomicSMin32 = EmitMemory;
inline constexpr auto EmitBufferAtomicUMin32 = EmitMemory;
inline constexpr auto EmitBufferAtomicSMax32 = EmitMemory;
inline constexpr auto EmitBufferAtomicUMax32 = EmitMemory;
inline constexpr auto EmitBufferAtomicAnd32 = EmitMemory;
inline constexpr auto EmitBufferAtomicOr32 = EmitMemory;
inline constexpr auto EmitBufferAtomicOr64 = EmitMemory;
inline constexpr auto EmitBufferAtomicXor32 = EmitMemory;
inline constexpr auto EmitBufferAtomicFMin32 = EmitMemory;
inline constexpr auto EmitBufferAtomicFMax32 = EmitMemory;
inline constexpr auto EmitLoadSharedU8 = EmitMemory;
inline constexpr auto EmitLoadSharedU16 = EmitMemory;
inline constexpr auto EmitLoadSharedU32 = EmitMemory;
inline constexpr auto EmitLoadSharedU32x2 = EmitMemory;
inline constexpr auto EmitLoadSharedU32x3 = EmitMemory;
inline constexpr auto EmitLoadSharedU32x4 = EmitMemory;
inline constexpr auto EmitWriteSharedU8 = EmitMemory;
inline constexpr auto EmitWriteSharedU16 = EmitMemory;
inline constexpr auto EmitWriteSharedU32 = EmitMemory;
inline constexpr auto EmitWriteSharedU32x2 = EmitMemory;
inline constexpr auto EmitWriteSharedU32x3 = EmitMemory;
inline constexpr auto EmitWriteSharedU32x4 = EmitMemory;
inline constexpr auto EmitSharedAtomicFMin32 = EmitMemory;
inline constexpr auto EmitSharedAtomicFMax32 = EmitMemory;
inline constexpr auto EmitSharedAtomicSwap32 = EmitMemory;
inline constexpr auto EmitSharedAtomicIAdd32 = EmitMemory;
inline constexpr auto EmitSharedAtomicISub32 = EmitMemory;
inline constexpr auto EmitSharedAtomicInc32 = EmitMemory;
inline constexpr auto EmitSharedAtomicDec32 = EmitMemory;
inline constexpr auto EmitSharedAtomicSMin32 = EmitMemory;
inline constexpr auto EmitSharedAtomicUMin32 = EmitMemory;
inline constexpr auto EmitSharedAtomicSMax32 = EmitMemory;
inline constexpr auto EmitSharedAtomicUMax32 = EmitMemory;
inline constexpr auto EmitSharedAtomicAnd32 = EmitMemory;
inline constexpr auto EmitSharedAtomicOr32 = EmitMemory;
inline constexpr auto EmitSharedAtomicXor32 = EmitMemory;
inline constexpr auto EmitDataAppend = EmitMemory;
inline constexpr auto EmitDataConsume = EmitMemory;
inline constexpr auto EmitSwizzleU32 = EmitMemory;
void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitLoadAddressU8 = EmitLoadMemory;
inline constexpr auto EmitLoadAddressU16 = EmitLoadMemory;
inline constexpr auto EmitLoadAddressU32 = EmitLoadMemory;
inline constexpr auto EmitStoreAddressU8 = EmitStoreMemory;
inline constexpr auto EmitStoreAddressU16 = EmitStoreMemory;
inline constexpr auto EmitStoreAddressU32 = EmitStoreMemory;
inline constexpr auto EmitLoadBufferU8 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU16 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32x2 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32x3 = EmitLoadMemory;
inline constexpr auto EmitLoadBufferU32x4 = EmitLoadMemory;
inline constexpr auto EmitStoreBufferU8 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU16 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU32 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU32x2 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU32x3 = EmitStoreMemory;
inline constexpr auto EmitStoreBufferU32x4 = EmitStoreMemory;
inline constexpr auto EmitBufferAtomicSwap32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicCmpSwap32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicSwap64 = EmitBufferAtomic64;
inline constexpr auto EmitBufferAtomicIAdd32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicISub32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicSMin32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicUMin32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicSMax32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicUMax32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicAnd32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicOr32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicOr64 = EmitBufferAtomic64;
inline constexpr auto EmitBufferAtomicXor32 = EmitAtomic32;
inline constexpr auto EmitBufferAtomicFMin32 = EmitBufferFloatAtomic;
inline constexpr auto EmitBufferAtomicFMax32 = EmitBufferFloatAtomic;
inline constexpr auto EmitLoadSharedU8 = EmitLoadMemory;
inline constexpr auto EmitLoadSharedU16 = EmitLoadMemory;
inline constexpr auto EmitLoadSharedU32 = EmitLoadMemory;
inline constexpr auto EmitLoadSharedU32x2 = EmitLoadMemory;
inline constexpr auto EmitLoadSharedU32x3 = EmitLoadMemory;
inline constexpr auto EmitLoadSharedU32x4 = EmitLoadMemory;
inline constexpr auto EmitWriteSharedU8 = EmitStoreMemory;
inline constexpr auto EmitWriteSharedU16 = EmitStoreMemory;
inline constexpr auto EmitWriteSharedU32 = EmitStoreMemory;
inline constexpr auto EmitWriteSharedU32x2 = EmitStoreMemory;
inline constexpr auto EmitWriteSharedU32x3 = EmitStoreMemory;
inline constexpr auto EmitWriteSharedU32x4 = EmitStoreMemory;
inline constexpr auto EmitSharedAtomicFMin32 = EmitSharedFloatAtomic;
inline constexpr auto EmitSharedAtomicFMax32 = EmitSharedFloatAtomic;
inline constexpr auto EmitSharedAtomicSwap32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicIAdd32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicISub32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicInc32 = EmitSharedIncDec;
inline constexpr auto EmitSharedAtomicDec32 = EmitSharedIncDec;
inline constexpr auto EmitSharedAtomicSMin32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicUMin32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicSMax32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicUMax32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicAnd32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicOr32 = EmitAtomic32;
inline constexpr auto EmitSharedAtomicXor32 = EmitAtomic32;
inline constexpr auto EmitDataAppend = EmitAppendConsume;
inline constexpr auto EmitDataConsume = EmitAppendConsume;
uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst);
inline constexpr auto EmitImageQueryDimensions = EmitImage;
inline constexpr auto EmitImageQueryLod = EmitImage;

78
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
View File

@ -165,6 +165,23 @@ inline void EmitLabel(EmitterState& state, uint32_t label) {
state.builder.AddFunction({spv::OpLabel, label});
}
uint32_t TypeId(EmitterState& state, IR::Type type);
// Shared instruction construction; typed aliases add no forwarding functions.
template <spv::Op opcode, IR::Type type, typename... Args>
uint32_t EmitNative(EmitterState& state, Args... args) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, TypeId(state, type), result, args...});
return result;
}
uint32_t GlslStd450(EmitterState& state);
template <GLSLstd450 opcode, IR::Type type, typename... Args>
uint32_t EmitGlsl(EmitterState& state, Args... args) {
return EmitNative<spv::OpExtInst, type>(state, GlslStd450(state), opcode, args...);
}
inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, value});
@ -172,14 +189,14 @@ inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint3
}
inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
uint32_t rhs) {
uint32_t rhs) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({opcode, type, result, lhs, rhs});
return result;
}
inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition,
uint32_t true_value, uint32_t false_value) {
inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpSelect, type, result, condition, true_value, false_value});
return result;
@ -195,7 +212,6 @@ struct ValueEmitContext {
uint32_t FirstLane(uint32_t ballot);
uint32_t Shuffle(const IR::Inst& inst, size_t index, uint32_t lane);
uint32_t Result(const IR::Inst& inst);
uint32_t TypeId(IR::Type type) const;
uint32_t Define(const IR::Inst& inst, uint32_t value);
uint32_t ResourceIndex(IR::Value value, IR::ValueOpcode opcode);
const IR::Inst* ImageAddress(IR::Value value);
@ -313,8 +329,6 @@ uint32_t ConstantU64(EmitterState& state, uint64_t value);
uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
uint32_t GlslStd450(EmitterState& state);
void AllocateInputVariables(EmitterState& state);
void AllocateOutputVariables(EmitterState& state);
@ -330,7 +344,7 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name
void AddDescriptorAnnotationsAndNames(EmitterState& state);
void DefineModule(EmitterState& state);
void DefineModule(EmitterState& state);
void DefineMeshOutputs(EmitterState& state);
void EmitMeshEntryPoint(EmitterState& state);
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst);
@ -365,7 +379,7 @@ uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane);
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
inline constexpr auto EmitAddU32 = EmitNative<spv::OpIAdd, IR::Type::U32, uint32_t, uint32_t>;
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
@ -390,10 +404,9 @@ struct MemoryResourceAccess {
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
uint32_t variable,
uint32_t pointer_type);
MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const IR::MemoryInfo& mem,
uint32_t variable, uint32_t pointer_type);
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
uint32_t raw_index);
@ -404,14 +417,13 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index);
uint32_t EmitStorageBufferElementPointer(EmitterState& state,
const MemoryResourceAccess& access, uint32_t index,
uint32_t pointer_type);
uint32_t EmitStorageBufferElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index, uint32_t pointer_type);
uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value);
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
uint32_t false_value);
inline constexpr auto EmitTBufferSelectF32 =
EmitNative<spv::OpSelect, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
bool IsSignedFormatComponent(Format::ComponentType type);
@ -427,20 +439,14 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
uint32_t false_value);
void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
uint32_t& out_low, uint32_t& out_high);
void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
uint32_t shift, uint32_t& out_low, uint32_t& out_high);
inline constexpr auto EmitSelectValueU32 =
EmitNative<spv::OpSelect, IR::Type::U32, uint32_t, uint32_t, uint32_t>;
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
inline constexpr auto EmitOrU32 = EmitNative<spv::OpBitwiseOr, IR::Type::U32, uint32_t, uint32_t>;
uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
uint32_t constant);
@ -453,17 +459,19 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value);
inline constexpr auto EmitBitcastF32ToU32 = EmitNative<spv::OpBitcast, IR::Type::U32, uint32_t>;
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value);
inline constexpr auto EmitBitcastU32ToF32 = EmitNative<spv::OpBitcast, IR::Type::F32, uint32_t>;
uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
inline constexpr auto EmitAndU32 = EmitNative<spv::OpBitwiseAnd, IR::Type::U32, uint32_t, uint32_t>;
uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
inline constexpr auto EmitLogicalAndBool =
EmitNative<spv::OpLogicalAnd, IR::Type::U1, uint32_t, uint32_t>;
uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
inline constexpr auto EmitLogicalOrBool =
EmitNative<spv::OpLogicalOr, IR::Type::U1, uint32_t, uint32_t>;
uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value);
inline constexpr auto EmitLogicalNotBool = EmitNative<spv::OpLogicalNot, IR::Type::U1, uint32_t>;
F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits);
@ -476,15 +484,15 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value);
inline constexpr auto EmitTruncF32Value = EmitGlsl<GLSLstd450Trunc, IR::Type::F32, uint32_t>;
uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value);
inline constexpr auto EmitFNegateValue = EmitNative<spv::OpFNegate, IR::Type::F32, uint32_t>;
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
inline constexpr auto EmitFAbsValue = EmitGlsl<GLSLstd450FAbs, IR::Type::F32, uint32_t>;
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);

553
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
View File

@ -599,23 +599,6 @@ uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
});
}
uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) {
const auto scope =
mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
}
template <typename Fn>
uint32_t EmitAtomicUpdate(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem, Fn&& replacement) {
@ -643,137 +626,6 @@ uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) {
return Select(state, TypeU32(state), wrap, limit, next);
}
uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem) {
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
const auto resource = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
ByteAddress(ctx, inst, mem), resource.byte_offset);
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
ConstantU32(state, 3u));
return EmitValueOrDefaultIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
ConstantU64(state, 0), [&]() {
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state),
ctx.Arg(inst, inst.NumArgs() - 2));
const auto old = state.builder.AllocateId();
state.builder.AddFunction(
{SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old,
EmitStorageBufferElementPointer(state, resource, index,
TypeStorageBufferU64ElementPointer(state)),
ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, spv::MemorySemanticsMaskNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return Unary(state, spv::OpBitcast, TypeU64(state), old);
});
});
}
uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
return EmitAtomicUpdate(ctx, inst, mem, [max_value](EmitterState& state, uint32_t old,
uint32_t value) {
return EmitFloatAtomicReplacement(state, old, value, max_value);
});
}
uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
bool max_value) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
mem.kind, [&](uint32_t old) {
const auto old_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data);
const auto compare = Binary(
ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f);
return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
});
return 0;
}
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append) {
auto& state = ctx.state;
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, ctx.Memory(inst).offset));
const auto raw_index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count = Binary(state, spv::OpIAdd, TypeU32(state),
Unary(state, spv::OpBitCount, TypeU32(state), low),
Unary(state, spv::OpBitCount, TypeU32(state), high));
const auto first = ctx.FirstLane(ballot);
const auto source_lane =
state.lane_count == 2
? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31))
: first;
const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first,
AndCondition(state,
state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice
: spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsMaskNone), count});
return value;
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane});
return result;
}
struct PreparedFormattedMemory {
Format::BufferFormatInfo info;
MemoryResourceAccess resource;
@ -1044,162 +896,259 @@ void DefineGetBdaPointer(EmitterState& state) {
state.builder.AddFunction({spv::OpFunctionEnd});
}
void EmitMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto op = inst.GetOpcode();
uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) {
const auto scope =
mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics =
spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, semantics)});
} else {
EmitDeviceAtomicMemoryBarrier(ctx.state);
}
return old;
});
}
uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
auto& state = ctx.state;
return EmitValueOrDefaultIfCondition(
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
const auto resource = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
ByteAddress(ctx, inst, mem), resource.byte_offset);
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
ConstantU32(state, 3u));
return EmitValueOrDefaultIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
ConstantU64(state, 0), [&]() {
const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state),
ctx.Arg(inst, inst.NumArgs() - 2));
const auto old = state.builder.AllocateId();
state.builder.AddFunction(
{SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old,
EmitStorageBufferElementPointer(state, resource, index,
TypeStorageBufferU64ElementPointer(state)),
ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, spv::MemorySemanticsMaskNone), value});
EmitDeviceAtomicMemoryBarrier(state);
return Unary(state, spv::OpBitcast, TypeU64(state), old);
});
});
}
uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::BufferAtomicFMax32;
return EmitAtomicUpdate(ctx, inst, mem,
[max_value](EmitterState& state, uint32_t old, uint32_t value) {
return EmitFloatAtomicReplacement(state, old, value, max_value);
});
}
void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction(
{spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
{spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
mem.kind, [&](uint32_t old) {
const auto old_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data);
const auto compare = Binary(
ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f);
return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
});
}
uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
const bool append = inst.GetOpcode() == IR::ValueOpcode::DataAppend;
auto& state = ctx.state;
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, ctx.Memory(inst).offset));
const auto raw_index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count = Binary(state, spv::OpIAdd, TypeU32(state),
Unary(state, spv::OpBitCount, TypeU32(state), low),
Unary(state, spv::OpBitCount, TypeU32(state), high));
const auto first = ctx.FirstLane(ballot);
const auto source_lane =
state.lane_count == 2
? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31))
: first;
const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition(
state, is_first,
AndCondition(state,
state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index),
ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice
: spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsMaskNone), count});
return value;
});
const auto result = state.builder.AllocateId();
state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane});
return result;
}
uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
if (state.flattened_srt_variable == 0) {
ctx.Fail(inst, "requires the flattened SRT descriptor");
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.flattened_srt_variable, ConstantU32(state, 0),
ctx.Arg(inst, 1)});
return EmitNative<spv::OpLoad, IR::Type::U32>(state, pointer);
}
void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
auto mem = ctx.Memory(inst);
if (mem.planning_only) return;
auto& state = ctx.state;
mem.kind = IR::ResourceKind::ScalarBuffer;
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, mem.offset));
const auto index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto element = EmitMemoryElementIndex(state, access, index);
const auto condition = EmitMemoryElementInBounds(state, access, element);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
return EmitNative<spv::OpLoad, IR::Type::U32>(
state, EmitMemoryElementPointer(state, access, element));
}));
}
void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
const auto& mem = ctx.Memory(inst);
if (op == IR::ValueOpcode::LoadAddressU32 && mem.planning_only) return;
const auto buffer_components = IR::BufferComponentCount(op);
if (buffer_components > 1u) {
const auto access = IR::BufferAccessOf(op);
if (access == IR::BufferAccess::Read) {
ctx.Define(inst, LoadWideBuffer(ctx, inst, buffer_components));
return;
}
if (access == IR::BufferAccess::Write) {
StoreWideBuffer(ctx, inst, buffer_components);
return;
}
}
const auto shared_components = IR::SharedComponentCount(op);
if (shared_components > 1u) {
if (IR::SharedAccessOf(op) == IR::SharedAccess::Read) {
ctx.Define(inst, LoadWideShared(ctx, inst, shared_components));
} else {
StoreWideShared(ctx, inst, shared_components);
}
return;
}
if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
ctx.Memory(inst).planning_only) {
return;
}
if (op == IR::ValueOpcode::ReadConst) {
if (state.flattened_srt_variable == 0) {
ctx.Fail(inst, "requires the flattened SRT descriptor");
return;
}
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state),
pointer, state.flattened_srt_variable, ConstantU32(state, 0),
ctx.Arg(inst, 1)});
ctx.Define(inst, EmitNative<spv::OpLoad, IR::Type::U32>(ctx, pointer));
return;
}
if (op == IR::ValueOpcode::ReadConstBuffer) {
auto mem = ctx.Memory(inst);
mem.kind = IR::ResourceKind::ScalarBuffer;
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, mem.offset));
const auto index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto element = EmitMemoryElementIndex(state, access, index);
const auto condition = EmitMemoryElementInBounds(state, access, element);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpLoad, TypeU32(state), value,
EmitMemoryElementPointer(state, access, element)});
return value;
}));
return;
}
const auto address_info = IR::AddressOpcodeInfoOf(op);
const bool load_address = address_info.access == IR::AddressAccess::Read;
if (load_address && ctx.Memory(inst).kind != IR::ResourceKind::Scratch) {
ctx.Define(inst, LoadBda(ctx, inst, ctx.Memory(inst), address_info.data_bits));
return;
}
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadBufferU32;
const bool load_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Read;
if (load_address || load_buffer || load_shared) {
const auto mem = ctx.Memory(inst);
uint32_t value = 0;
if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
value = FormattedLoad(ctx, inst, mem);
else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadSharedU8)
value = LoadSubword(ctx, inst, mem, 8, false);
else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadSharedU16)
value = LoadSubword(ctx, inst, mem, 16, false);
else
value = LoadWord(ctx, inst, mem);
ctx.Define(inst, value);
return;
}
const bool store_address = address_info.access == IR::AddressAccess::Write;
const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::StoreBufferU32;
const bool store_shared = IR::SharedAccessOf(op) == IR::SharedAccess::Write;
if (store_address || store_buffer || store_shared) {
const auto mem = ctx.Memory(inst);
if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
FormattedStore(ctx, inst, mem);
else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
op == IR::ValueOpcode::WriteSharedU8)
StoreSubword(ctx, inst, mem, 8);
else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
op == IR::ValueOpcode::WriteSharedU16)
StoreSubword(ctx, inst, mem, 16);
else
StoreWord(ctx, inst, mem);
return;
}
const auto atomic_opcode = SpirvAtomicOpcode(op);
if (atomic_opcode != 0 && inst.GetType() == IR::Type::U64) {
ctx.Define(inst, EmitBufferAtomic64(ctx, inst, ctx.Memory(inst)));
return;
}
if (atomic_opcode != 0) {
ctx.Define(inst, EmitAtomic(ctx, inst, ctx.Memory(inst)));
return;
}
if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst),
op == IR::ValueOpcode::BufferAtomicFMax32));
return;
}
if (op == IR::ValueOpcode::SharedAtomicInc32 || op == IR::ValueOpcode::SharedAtomicDec32) {
const auto replacement = op == IR::ValueOpcode::SharedAtomicInc32
? AtomicIncrement : AtomicDecrement;
ctx.Define(inst, EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement));
return;
}
if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32) {
SharedFloatAtomic(ctx, inst, ctx.Memory(inst), op == IR::ValueOpcode::SharedAtomicFMax32);
return;
}
if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume) {
ctx.Define(inst, AppendConsume(ctx, inst, op == IR::ValueOpcode::DataAppend));
return;
}
if (op == IR::ValueOpcode::SwizzleU32 || op == IR::ValueOpcode::BpermuteU32) {
uint32_t source = ctx.Arg(inst, 0);
uint32_t target = 0;
if (op == IR::ValueOpcode::SwizzleU32) {
state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, source});
source = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpLoad, TypeU32(state), source, ctx.scratch_u32_variable});
target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
} else {
const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state),
ctx.Arg(inst, 1), ConstantU32(state, 2)),
ConstantU32(state, 31));
const auto base =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u));
target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index);
}
ctx.Define(inst, EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2)));
return;
}
ctx.Fail(inst, "has no memory SPIR-V emitter");
const auto address_info = IR::AddressOpcodeInfoOf(op);
uint32_t value;
if (buffer_components > 1u)
value = LoadWideBuffer(ctx, inst, buffer_components);
else if (shared_components > 1u)
value = LoadWideShared(ctx, inst, shared_components);
else if (address_info.access == IR::AddressAccess::Read &&
mem.kind != IR::ResourceKind::Scratch)
value = LoadBda(ctx, inst, mem, address_info.data_bits);
else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
value = FormattedLoad(ctx, inst, mem);
else if (inst.GetType() == IR::Type::U8)
value = LoadSubword(ctx, inst, mem, 8, false);
else if (inst.GetType() == IR::Type::U16)
value = LoadSubword(ctx, inst, mem, 16, false);
else
value = LoadWord(ctx, inst, mem);
ctx.Define(inst, value);
}
void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto op = inst.GetOpcode();
const auto& mem = ctx.Memory(inst);
const auto buffer_components = IR::BufferComponentCount(op);
const auto shared_components = IR::SharedComponentCount(op);
const auto type = inst.Arg(inst.NumArgs() - 2).GetType();
if (buffer_components > 1u)
StoreWideBuffer(ctx, inst, buffer_components);
else if (shared_components > 1u)
StoreWideShared(ctx, inst, shared_components);
else if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
FormattedStore(ctx, inst, mem);
else if (type == IR::Type::U8)
StoreSubword(ctx, inst, mem, 8);
else if (type == IR::Type::U16)
StoreSubword(ctx, inst, mem, 16);
else
StoreWord(ctx, inst, mem);
}
uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto replacement =
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicInc32 ? AtomicIncrement : AtomicDecrement;
return EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement);
}
uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)});
const auto source = EmitNative<spv::OpLoad, IR::Type::U32>(state, ctx.scratch_u32_variable);
const auto target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2));
}
uint32_t EmitBpermuteU32(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto source = ctx.Arg(inst, 0);
const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, spv::OpShiftRightLogical, TypeU32(state),
ctx.Arg(inst, 1), ConstantU32(state, 2)),
ConstantU32(state, 31));
const auto base = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u));
const auto target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index);
return EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2));
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

14
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
View File

@ -25,12 +25,6 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
return ConstantU32(state, 0);
}
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({spv::OpIAdd, TypeU32(state), ret, lhs, rhs});
return ret;
}
uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
@ -194,14 +188,6 @@ uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
return ret;
}
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
uint32_t false_value) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{spv::OpSelect, TypeF32(state), ret, condition, true_value, false_value});
return ret;
}
bool IsSignedFormatComponent(Format::ComponentType type) {
return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm ||
type == Format::ComponentType::Sscaled;

30
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
View File

@ -40,7 +40,7 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
}
uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
const auto value_type = ctx.TypeId(type);
const auto value_type = TypeId(ctx.state, type);
return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type);
}
@ -243,14 +243,22 @@ decltype(auto) Arg(ValueEmitContext& ctx, const IR::Inst& inst, size_t index) {
}
}
template <typename Return, typename... Args>
void Invoke(Return (*emit)(ValueEmitContext&, Args...), ValueEmitContext& ctx,
const IR::Inst& inst) {
template <typename Context, typename Return, typename... Args>
void Invoke(Return (*emit)(Context&, Args...), ValueEmitContext& ctx, const IR::Inst& inst) {
// A full instruction keeps metadata and predicated/lane operand loads lazy.
static_assert(std::is_same_v<Context, ValueEmitContext> ||
std::is_same_v<Context, EmitterState>);
auto& context = [&]() -> Context& {
if constexpr (std::is_same_v<Context, EmitterState>)
return ctx.state;
else
return ctx;
}();
constexpr bool has_inst = (std::is_same_v<Args, const IR::Inst&> || ...);
[&]<size_t... I>(std::index_sequence<I...>) {
static_assert(((!std::is_same_v<Args, const IR::Inst&> || I == 0) && ...));
const auto call = [&] {
return emit(ctx, Arg<Args>(ctx, inst, I - (has_inst && I != 0))...);
return emit(context, Arg<Args>(ctx, inst, I - (has_inst && I != 0))...);
};
if constexpr (std::is_void_v<Return>) {
call();
@ -274,7 +282,7 @@ void EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
const IR::Inst& inst) {
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
const auto type = ctx.TypeId(inst.GetType());
const auto type = TypeId(ctx.state, inst.GetType());
if (type == 0 || inst.NumArgs() == 0) {
ctx.Fail(inst, "has no native SPIR-V representation");
}
@ -295,7 +303,7 @@ void EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& s
void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
const IR::Inst& inst) {
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
const auto type = ctx.TypeId(inst.GetType());
const auto type = TypeId(ctx.state, inst.GetType());
if (type == 0) {
ctx.Fail(inst, "cannot be loaded by the dispatcher");
}
@ -425,7 +433,7 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
} // namespace
uint32_t ValueEmitContext::TypeId(IR::Type type) const {
uint32_t TypeId(EmitterState& state, IR::Type type) {
switch (type) {
case IR::Type::U1: return TypeBool(state);
case IR::Type::U8:
@ -470,7 +478,7 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
return loaded->second.second;
}
const auto id = state.builder.AllocateId();
state.builder.AddFunction({spv::OpLoad, TypeId(inst->GetType()), id, found->second});
state.builder.AddFunction({spv::OpLoad, TypeId(state, inst->GetType()), id, found->second});
dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
return id;
}
@ -536,7 +544,7 @@ uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
}
uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
const auto type = TypeId(inst.Arg(index).GetType());
const auto type = TypeId(state, inst.Arg(index).GetType());
const auto scope = ConstantU32(state, spv::ScopeSubgroup);
const auto low = state.builder.AllocateId();
if (other_half == nullptr) {
@ -574,7 +582,7 @@ uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
if (const auto found = definitions.find(&inst); found != definitions.end()) {
if (found->second != value) {
state.builder.AddFunction(
{spv::OpCopyObject, TypeId(inst.GetType()), found->second, value});
{spv::OpCopyObject, TypeId(state, inst.GetType()), found->second, value});
}
return found->second;
}

39
tests/ShaderRecompilerComputeTests.cpp
View File

@ -16687,6 +16687,44 @@ TestCase VectorVop3LshlrevB64Captured() {
return test;
}
TestCase VectorDynamicU64ShiftEdges() {
using O = ShaderOpcode;
TestCase test;
test.name = "VectorDynamicU64ShiftEdges";
test.initial = {0u, 1u, 31u, 32u, 33u, 63u, 64u, 65u};
test.expected = test.initial;
auto& code = test.code;
for (const uint64_t source : {uint64_t{0}, uint64_t{1},
uint64_t{0x8000000000000000ull},
uint64_t{0xffffffffffffffffull},
uint64_t{0x0123456789abcdefull}}) {
AppendVMovLiteral(&code, 4, static_cast<u32>(source));
AppendVMovLiteral(&code, 5, static_cast<u32>(source >> 32u));
for (u32 index = 0; index < test.initial.size(); ++index) {
// A buffer load prevents the constant-shift specialization.
AppendVMovU32(&code, 30, index * sizeof(u32));
AppendBufferLoadDword(&code, 3, 30);
for (const bool left : {true, false}) {
AppendVop3(&code, left ? 0x2ff : 0x300, 33, Vgpr(3), Vgpr(4));
const auto amount = test.initial[index] & 63u;
const auto expected = left ? source << amount : source >> amount;
const auto output = static_cast<u32>(test.expected.size());
AppendStoreVgpr(&code, 33, output);
AppendStoreVgpr(&code, 34, output + 1u);
test.expected.push_back(static_cast<u32>(expected));
test.expected.push_back(static_cast<u32>(expected >> 32u));
}
}
}
AppendEnd(&code);
test.initial.resize(test.expected.size());
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORD, O::V_LSHLREV_B64,
O::V_LSHRREV_B64, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.required_spirv = {"OpShiftLeftLogical", "OpShiftRightLogical", "OpSelect"};
test.forbidden_spirv = {"OpTypeInt 64"};
return test;
}
TestCase VectorVop3IntegerOps() {
using O = ShaderOpcode;
@ -24745,6 +24783,7 @@ std::vector<TestCase> MakeCases() {
AddCase(ScalarBfeI32CapturedRawSignExtends);
AddCase(BitfieldExtractWidthPastEndEdges);
AddCase(Scalar64BitOps);
AddCase(VectorDynamicU64ShiftEdges);
AddCase(ScalarConditionalMoveB64);
AddCase(ScalarConditionalMoveB64PreservesMasks);
AddCase(ScalarAndn2B64SccUsesMaskShadow);

33
tests/shaderCfgTests.cpp
View File

@ -8708,6 +8708,38 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
"dispatcher did not use one canonical U32x3 spill slot");
}
void TestNewShaderRecompilerPlanningOnlyLoads() {
using namespace ShaderRecompiler;
const uint32_t shader[] = {EncodeSopp(0x01)};
auto options = MakeCompileOptions(ShaderType::Compute);
auto result = RecompileForTest(shader, options);
auto& program = result.program;
IR::IREmitter ir(program.blocks.front());
IR::MemoryInfo memory;
memory.kind = IR::ResourceKind::ScalarAddress;
memory.planning_only = true;
program.memory_info.push_back(memory);
const auto address = ir.Emit(IR::ValueOpcode::GetAddressResource,
{IR::Value(0x1000u), IR::Value(0u)});
const auto load = ir.Emit(IR::ValueOpcode::LoadAddressU32,
{address, IR::Value(0u), IR::Value(0u), IR::Value(true)});
load.Instruction()->SetFlags(IR::MemoryFlags{.index = 0});
memory.kind = IR::ResourceKind::ScalarBuffer;
program.memory_info.push_back(memory);
const auto buffer = ir.Emit(IR::ValueOpcode::GetBufferResource,
{IR::Value(0x2000u), IR::Value(0u),
IR::Value(16u), IR::Value(0u)});
const auto read = ir.Emit(IR::ValueOpcode::ReadConstBuffer,
{buffer, IR::Value(0u)});
read.Instruction()->SetFlags(IR::MemoryFlags{.index = 1});
ir.Emit(IR::ValueOpcode::ReferenceU32, {load});
ir.Emit(IR::ValueOpcode::ReferenceU32, {read});
const auto spirv = Spirv::EmitProgram(program, options.input_info);
CheckSpirvBinaryValidates(spirv);
Check(spirv == result.spirv,
"planning-only loads or references emitted GPU operands/instructions");
}
void TestNewShaderRecompilerU64PairTranslation() {
using namespace ShaderRecompiler;
@ -13017,6 +13049,7 @@ int main() {
TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry();
TestNewShaderRecompilerCfgIrreducibleDispatcher();
TestNewShaderRecompilerDispatcherSpillsU32x3();
TestNewShaderRecompilerPlanningOnlyLoads();
TestNewShaderRecompilerU64PairTranslation();
TestComputeDispatchWaveSize();
TestNewShaderRecompilerBufferLoadsGuardedByExec();