shader: implement RDNA2 DS masked OR through shared atomic updates

Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>
This commit is contained in:
nmzik
2026-10-04 17:18:35 +04:00
parent c3f4962715
commit 98557a83b6
11 changed files with 144 additions and 28 deletions
@@ -232,6 +232,7 @@ void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& ins
uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitSharedAtomic64(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -259,8 +259,14 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
}
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
auto address = ByteAddress(ctx, inst, mem);
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
// RDNA2 DS region addresses retain bits [15:2] after adding the byte offset.
address = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 0xffffu));
}
return Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state),
ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2));
address, ConstantU32(ctx.state, 2));
}
uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
@@ -1415,6 +1421,15 @@ uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst) {
return EmitAtomicUpdate(ctx, inst, ctx.Memory(inst), replacement);
}
void EmitSharedAtomicMaskedOr32(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto keep = Unary(ctx.state, spv::OpNot, TypeU32(ctx.state), ctx.Arg(inst, 1));
EmitAtomicUpdate(ctx, inst, ctx.Memory(inst),
[keep](EmitterState& state, uint32_t old, uint32_t value) {
return Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), old, keep), value);
});
}
uint32_t EmitSwizzleU32(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0));
@@ -92,7 +92,8 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
{0x05u, Opcode::DS_MIN_I32, 1, 32}, {0x06u, Opcode::DS_MAX_I32, 1, 32},
{0x07u, Opcode::DS_MIN_U32, 1, 32}, {0x08u, Opcode::DS_MAX_U32, 1, 32},
{0x09u, Opcode::DS_AND_B32, 1, 32}, {0x0au, Opcode::DS_OR_B32, 1, 32},
{0x0bu, Opcode::DS_XOR_B32, 1, 32}, {0x0du, Opcode::DS_WRITE_B32, 1, 32},
{0x0bu, Opcode::DS_XOR_B32, 1, 32}, {0x0cu, Opcode::DS_MSKOR_B32, 1, 32},
{0x0du, Opcode::DS_WRITE_B32, 1, 32},
{0x0eu, Opcode::DS_WRITE2_B32, 2, 32}, {0x0fu, Opcode::DS_WRITE2ST64_B32, 2, 32},
{0x12u, Opcode::DS_MIN_F32, 1, 32}, {0x13u, Opcode::DS_MAX_F32, 1, 32},
{0x1eu, Opcode::DS_WRITE_B8, 1, 8}, {0x1fu, Opcode::DS_WRITE_B16, 1, 16},
@@ -192,6 +193,7 @@ bool IsDsAtomicOpcode(Opcode opcode) {
case Opcode::DS_OR_RTN_B32:
case Opcode::DS_XOR_B32:
case Opcode::DS_XOR_RTN_B32:
case Opcode::DS_MSKOR_B32:
case Opcode::DS_WRXCHG_RTN_B32: return true;
default: return false;
}
@@ -203,6 +205,7 @@ uint32_t DsSourceCount(Opcode opcode) {
case Opcode::DS_WRITE2ST64_B32:
case Opcode::DS_WRITE2_B64:
case Opcode::DS_WRITE2ST64_B64:
case Opcode::DS_MSKOR_B32:
case Opcode::DS_MIN_F32:
case Opcode::DS_MAX_F32: return 3u;
case Opcode::DS_PERMUTE_B32:
@@ -697,6 +697,7 @@ std::string InstructionToString(const Instruction& inst) {
case Opcode::DS_OR_RTN_B32:
case Opcode::DS_XOR_B32:
case Opcode::DS_XOR_RTN_B32:
case Opcode::DS_MSKOR_B32:
case Opcode::DS_WRXCHG_RTN_B32:
case Opcode::DS_MIN_F32:
case Opcode::DS_MAX_F32:
@@ -558,6 +558,7 @@ enum class Opcode {
DS_OR_RTN_B32,
DS_XOR_B32,
DS_XOR_RTN_B32,
DS_MSKOR_B32,
DS_WRXCHG_RTN_B32,
DS_MIN_F32,
DS_MAX_F32,
@@ -239,9 +239,6 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
case Decoder::Opcode::DS_WRITE_ADDTID_B32:
return index == 0u ? decoded.src1 : MakeM0Operand();
case Decoder::Opcode::DS_MIN_F32:
case Decoder::Opcode::DS_MAX_F32:
return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
case Decoder::Opcode::DS_WRITE_B8:
case Decoder::Opcode::DS_WRITE_B16:
case Decoder::Opcode::DS_WRITE_B8_D16_HI:
@@ -255,12 +252,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
case Decoder::Opcode::DS_WRITE2_B64:
case Decoder::Opcode::DS_WRITE2ST64_B64:
return index == 0u ? decoded.src1 : index == 1u ? decoded.src0 : decoded.src2;
default:
if (decoded.opcode >= Decoder::Opcode::DS_ADD_U32 &&
decoded.opcode <= Decoder::Opcode::DS_WRXCHG_RTN_B32) {
return index == 0u ? decoded.src1 : decoded.src0;
}
return decoded.src0;
default: return decoded.src0;
}
}
return DecodedSourceAt(decoded, index);
@@ -580,12 +572,13 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
bool returns_value) {
const auto memory = MemoryInfoFromDecoded(inst);
const auto address = ReadU32(MemorySourceAt(inst, 1));
const auto data_src = MemorySourceAt(inst, 0);
const auto address = ReadU32(inst.src0);
const IR::Value value =
memory.data_dwords == 2u ? IR::Value(ReadU64(data_src)) : IR::Value(ReadU32(data_src));
const auto result = ir.Emit(opcode, {address, value, ir.GetExec()},
AddMemoryInfo(memory, inst.pc));
memory.data_dwords == 2u ? IR::Value(ReadU64(inst.src1)) : IR::Value(ReadU32(inst.src1));
const auto flags = AddMemoryInfo(memory, inst.pc);
const auto result = inst.src_count == 3u
? ir.Emit(opcode, {address, value, ReadU32(inst.src2), ir.GetExec()}, flags)
: ir.Emit(opcode, {address, value, ir.GetExec()}, flags);
if (returns_value) {
WriteOperand(inst.dst, result);
}
@@ -857,14 +850,6 @@ void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
}
}
void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
const auto memory = MemoryInfoFromDecoded(inst);
ir.Emit(opcode,
{ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)),
ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()},
AddMemoryInfo(memory, inst.pc));
}
void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
const auto memory = MemoryInfoFromDecoded(inst);
WriteOperand(inst.dst, ir.Emit(opcode,
@@ -1048,6 +1033,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, false);
case Decoder::Opcode::DS_XOR_RTN_B32:
return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicXor32, true);
case Decoder::Opcode::DS_MSKOR_B32:
return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicMaskedOr32, false);
case Decoder::Opcode::DS_WRXCHG_RTN_B32:
return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicSwap32, true);
@@ -1110,9 +1097,9 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
case Decoder::Opcode::IMAGE_GATHER4H: return IMAGE_GATHER(inst);
case Decoder::Opcode::DS_MIN_F32:
return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMin32);
return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMin32, false);
case Decoder::Opcode::DS_MAX_F32:
return DS_MINMAX_F32(inst, IR::ValueOpcode::SharedAtomicFMax32);
return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicFMax32, false);
case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst);
case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false);
case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true);
@@ -105,7 +105,6 @@ private:
void DS_READ2(const Decoder::Instruction& inst);
void DS_WRITE(const Decoder::Instruction& inst);
void DS_WRITE2(const Decoder::Instruction& inst);
void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
void DS_ADDTID(const Decoder::Instruction& inst, bool write);
void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
@@ -178,7 +178,8 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) {
case ValueOpcode::SharedAtomicAnd32:
case ValueOpcode::SharedAtomicOr32:
case ValueOpcode::SharedAtomicOr64:
case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic;
case ValueOpcode::SharedAtomicXor32:
case ValueOpcode::SharedAtomicMaskedOr32: return SharedAccess::Atomic;
case ValueOpcode::DataAppend: return SharedAccess::Append;
case ValueOpcode::DataConsume: return SharedAccess::Consume;
default: return SharedAccess::None;
@@ -290,6 +290,7 @@ VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1)
VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicMaskedOr32, Void, U32, U32, U32, U1)
VALUE_OPCODE(DataAppend, U32, U32, U1)
VALUE_OPCODE(DataConsume, U32, U32, U1)
VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)