diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 96b20cb89..a6c413992 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -655,6 +655,14 @@ void EmitStoreCompletion(EmitterState& state) { spv::MemorySemanticsImageMemoryMask)); } +void EmitDataCacheInvalidate(EmitterState& state) { + // RDNA2 GL0/GL1 invalidation makes vector memory visible without a workgroup rendezvous. + state.builder.AddFunction( + spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice), + ConstantU32(state, spv::MemorySemanticsAcquireMask | spv::MemorySemanticsUniformMemoryMask | + spv::MemorySemanticsImageMemoryMask)); +} + void EmitBarrier(EmitterState& state) { const auto tessellation = state.program.stage == ShaderType::TessellationControl; if (!tessellation && ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 9feea9d3f..d28a14414 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -218,6 +218,7 @@ inline constexpr auto EmitTtraceData = EmitVoid; inline constexpr auto EmitInstPrefetch = EmitVoid; void EmitBarrier(EmitterState& state); void EmitStoreCompletion(EmitterState& state); +void EmitDataCacheInvalidate(EmitterState& state); void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitMeshDrawParameter(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitGetTessellationAttribute(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index 63d5519c6..dda13c833 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -307,6 +307,7 @@ void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& ctx.state.lane_half = half; if (half == 0 || (inst.GetOpcode() != IR::ValueOpcode::Barrier && inst.GetOpcode() != IR::ValueOpcode::StoreCompletion && + inst.GetOpcode() != IR::ValueOpcode::DataCacheInvalidate && inst.GetOpcode() != IR::ValueOpcode::MeshAllocate)) { emit_instruction(lane, inst); } diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 9b177ec88..072692091 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -70,6 +70,8 @@ constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = { {0x52u, Opcode::BUFFER_ATOMIC_ADD_X2, 2, 32}, {0x59u, Opcode::BUFFER_ATOMIC_AND_X2, 2, 32}, {0x5au, Opcode::BUFFER_ATOMIC_OR_X2, 2, 32}, + {0x71u, Opcode::BUFFER_GL0_INV, 0, 0}, + {0x72u, Opcode::BUFFER_GL1_INV, 0, 0}, {0x80u, Opcode::BUFFER_LOAD_FORMAT_D16_X, 1, 16, false, false, true}, {0x83u, Opcode::BUFFER_LOAD_FORMAT_D16_XYZW, 2, 16, false, false, true}, {0x84u, Opcode::BUFFER_STORE_FORMAT_D16_X, 1, 16, false, false, true}, @@ -312,6 +314,9 @@ void DecodeMubuf(uint32_t pc, std::span code, uint32_t word_inde if (inst.opcode == Opcode::UNSUPPORTED) { SetUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented"); } + if (inst.opcode == Opcode::BUFFER_GL0_INV || inst.opcode == Opcode::BUFFER_GL1_INV) { + return; + } DecodeVectorGpr(vdata, inst.dst); if (inst.opcode == Opcode::BUFFER_LOAD_FORMAT_D16_XYZW || diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 3bbe6239c..77ac23085 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -566,6 +566,9 @@ std::string InstructionToString(const Instruction& inst) { OperandToString(inst.src0).c_str())); case Opcode::S_BARRIER: return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_barrier", inst.pc)); + case Opcode::BUFFER_GL0_INV: + case Opcode::BUFFER_GL1_INV: + return fmt::format("0x{:08x}: {}", inst.pc, magic_enum::enum_name(inst.opcode)); case Opcode::V_NOP: return WithUnsupportedReason(inst, fmt::format("0x{:08x}: v_nop", inst.pc)); case Opcode::S_ENDPGM: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 434dd7fee..4fb6c7b3e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -599,6 +599,8 @@ enum class Opcode { BUFFER_ATOMIC_XOR, BUFFER_ATOMIC_FMIN, BUFFER_ATOMIC_FMAX, + BUFFER_GL0_INV, + BUFFER_GL1_INV, FLAT_LOAD_UBYTE, FLAT_LOAD_SBYTE, FLAT_LOAD_USHORT, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 11067c81c..beaef6372 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -940,6 +940,10 @@ void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) { void Translator::EmitMemory(const Decoder::Instruction& inst) { switch (inst.opcode) { + case Decoder::Opcode::BUFFER_GL0_INV: + case Decoder::Opcode::BUFFER_GL1_INV: + ir.Emit(IR::ValueOpcode::DataCacheInvalidate); + return; case Decoder::Opcode::IMAGE_BVH_INTERSECT_RAY: { const auto result = ir.Emit(IR::ValueOpcode::BvhIntersect, {ConstructU32x4(inst.src1, 4), MakeImageAddress(inst, inst.src0), ir.GetExec()}, diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index a2561c85d..10fb4b79a 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -94,6 +94,7 @@ bool HasSideEffects(ValueOpcode opcode) { case ValueOpcode::SetTessellationAttribute: case ValueOpcode::MeshAllocate: case ValueOpcode::StoreCompletion: + case ValueOpcode::DataCacheInvalidate: case ValueOpcode::Barrier: return true; default: return false; } diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index adcb24841..9e7dacb2e 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -350,6 +350,7 @@ VALUE_OPCODE(SetAttribute, Void, U32x4, U1) VALUE_OPCODE(ControlNop, Void) VALUE_OPCODE(StoreCompletion, Void) +VALUE_OPCODE(DataCacheInvalidate, Void) VALUE_OPCODE(Barrier, Void) VALUE_OPCODE(Sendmsg, Void) VALUE_OPCODE(TtraceData, Void) diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 7a7b32e43..b4529d640 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -509,6 +509,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs case ValueOpcode::ReadConst: info.uses_flattened_srt = true; break; case ValueOpcode::BitwiseXor32: info.has_bitwise_xor = true; break; case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break; + case ValueOpcode::DataCacheInvalidate: info.coherent_buffers = true; break; case ValueOpcode::BvhIntersect: info.bvh = true; break; case ValueOpcode::ConditionRef: info.subgroup_vote |= diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 50d326b80..bd6aa3b2f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -1967,6 +1967,59 @@ void TestNggVertexEntryState() { } } +void TestNativeDataCacheInvalidation() { + using namespace ShaderRecompiler; + // Captured f5edf70dbb1f6057:0x194 and 0x1a4, each followed by S_WAITCNT. + const std::array cache_ops = {0xe1c40000u, 0u, 0xbf8c0000u, + 0xe1c80000u, 0u, 0xbf8c0000u, EncodeSopp(0x01)}; + Decoder::Program decoded; + Decoder::DecodeProgram(cache_ops, decoded); + for (const uint32_t index : {0u, 2u}) { + const auto &inst = decoded.instructions[index]; + Check(inst.opcode == (index == 0 ? Decoder::Opcode::BUFFER_GL0_INV + : Decoder::Opcode::BUFFER_GL1_INV) && + inst.word_count == 2 && inst.src_count == 0 && inst.data_dwords == 0 && + inst.dst.kind == Decoder::OperandKind::Unknown, + "RDNA2 cache invalidation acquired a buffer operand or lost its encoding"); + } + + auto options = MakeCompileOptions(ShaderType::Compute); + options.dump_ir = true; + ShaderComputeInputInfo compute{}; + compute.threads_num[1] = compute.threads_num[2] = 1; + options.input_info.compute = &compute; + for (const uint32_t wave_size : {32u, 64u}) { + compute.threads_num[0] = compute.wave_size = options.wave_size = wave_size; + const auto minimal = RecompileForTest(cache_ops, options); + CheckSpirvBinaryValidates(minimal.spirv); + Check(minimal.program.info.buffers.empty(), + "cache invalidation introduced a phantom buffer binding"); + const auto source = DisassembleSpirvBinary(minimal.spirv); + const auto first = source.find("OpMemoryBarrier %uint_1 %uint_2114"); + const auto second = source.find("OpMemoryBarrier %uint_1 %uint_2114", first + 1); + Check(first != std::string::npos && second != std::string::npos && + source.find("OpMemoryBarrier", second + 1) == std::string::npos && + source.find("OpControlBarrier") == std::string::npos, + "cache invalidation must fence device memory once per wave without convergence"); + + const std::array reload = { + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(0, 0, 0), + 0xe1c40000u, 0u, 0xbf8c0000u, + EncodeMubuf0(0x0c, 0, false), EncodeMubuf1(1, 0, 0), + EncodeMubuf0(0x1c, 4, false), EncodeMubuf1(1, 0, 0), EncodeSopp(0x01), + }; + const auto result = RecompileForTest(reload, options); + CheckSpirvBinaryValidates(result.spirv); + const auto store = result.ir_dump.find("StoreBufferU32"); + const auto fence = result.ir_dump.find("DataCacheInvalidate"); + const auto load = result.ir_dump.find("LoadBufferU32"); + Check(store < fence && fence < load && load != std::string::npos && + result.program.info.coherent_buffers && + DisassembleSpirvBinary(result.spirv).find(" Coherent") != std::string::npos, + "plain buffer reload lost its cache fence order or coherent visibility"); + } +} + void TestNativeStoreCompletion() { using namespace ShaderRecompiler; const std::array register_wait = {EncodeSopk(0x17, 4, static_cast(0x9234)), EncodeSopp(0x01)}; @@ -16523,6 +16576,7 @@ int main() { TestNewShaderDecoderArchitecture(); TestImageAddressOperands(); TestSopkCompareImmediateExtension(); + TestNativeDataCacheInvalidation(); TestNativeStoreCompletion(); TestNewShaderRecompilerCapturedVopcSdwaCmpxClass(); TestVopcCmpxClassF16Decoder();