shader: preserve RDNA2 DLC coherence for flat address loads

This commit is contained in:
Claxten
2026-10-06 03:28:30 +04:00
committed by nmzik
parent cdb64bfd4f
commit 9caec9f2c8
5 changed files with 75 additions and 14 deletions
@@ -205,7 +205,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
if (memory_index >= program.memory_info.size()) {
Fail(program, "address operation has invalid memory metadata");
}
const auto kind = program.memory_info[memory_index].kind;
const auto& memory = program.memory_info[memory_index];
const auto kind = memory.kind;
if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
if (program.scratch_dwords == 0) {
Fail(program, "scratch operation has no per-thread storage");
@@ -217,6 +218,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
}
} else if (address_access == IR::AddressAccess::Write) {
Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
} else {
// Descriptor stores can alias coherent physical-address loads.
requirements.coherent_buffers |= memory.coherent;
}
}
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
@@ -180,29 +180,46 @@ void RecordBdaFault(EmitterState& state, uint32_t page) {
Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit));
}
uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address, bool coherent = false) {
auto& state = ctx.state;
const auto bda = GetBdaPointer(state, address);
const auto present =
Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantU64(state, 0));
return EmitValueOrZeroIfCondition(state, present, [&]() {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer,
bda);
auto pointer = state.builder.AllocateId();
if (coherent) {
const auto block = state.builder.DecoratedType(
spv::OpTypeStruct, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
{spv::OpMemberDecorate, {0, spv::DecorationCoherent}},
{spv::OpDecorate, {spv::DecorationBlock}}}, TypeU32(state));
state.builder.AddFunction(
spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, block),
pointer, bda);
const auto base = pointer;
pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypePhysicalU32Pointer(state), pointer,
base, ConstantU32(state, 0));
} else {
state.builder.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer,
bda);
}
const auto value = state.builder.AllocateId();
constexpr uint32_t alignment = sizeof(uint32_t);
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer,
spv::MemoryAccessAlignedMask, alignment);
spv::MemoryAccessAlignedMask |
(coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone),
alignment);
return value;
});
}
uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits) {
uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint32_t bits,
bool coherent = false) {
auto& state = ctx.state;
return EmitValueOrZeroIfCondition(state, active, [&]() {
const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeU64(state), address,
ConstantU64(state, ~uint64_t {3}));
const auto first = LoadBdaDword(ctx, aligned);
const auto first = LoadBdaDword(ctx, aligned, coherent);
const auto byte =
Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3));
@@ -212,7 +229,7 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
TypeBool(state), byte, ConstantU32(state, bits == 16u ? 2u : 0u));
const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() {
return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeU64(state), aligned,
ConstantU64(state, sizeof(uint32_t))));
ConstantU64(state, sizeof(uint32_t))), coherent);
});
const auto shift =
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, ConstantU32(state, 3));
@@ -1359,7 +1376,7 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
else if (address_info.access == IR::AddressAccess::Read &&
mem.kind != IR::ResourceKind::Scratch)
value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),
address_info.data_bits);
address_info.data_bits, mem.coherent);
else if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
value = mem.kind == IR::ResourceKind::IndirectBuffer ? LoadIndirectFormattedX(ctx, inst)
: FormattedLoad(ctx, inst, mem);