diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 4a2f942f5..d66d1299a 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -420,6 +420,7 @@ void EnsureLdsStorage(EmitterState& state); struct MemoryResourceAccess { IR::ResourceKind kind = IR::ResourceKind::None; uint32_t object_pointer = 0; + uint32_t descriptor_index = 0; uint32_t length = 0; uint32_t byte_offset = 0; uint32_t element_bits = 32; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index d8055185b..6159e56ed 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1075,6 +1075,63 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) { }); } +uint32_t LoadRawBuffer(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + const MemoryResourceAccess& resource, uint32_t components) { + auto& state = ctx.state; + const auto scalar = [&] { + std::array values {}; + for (uint32_t component = 0; component < components; component++) { + values[component] = + LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); + } + return ConstructU32Composite(state, components, values); + }; + if (!mem.IsCoherentBufferPairLoad(inst.GetOpcode())) return scalar(); + + const auto first = ByteAddress(ctx, inst, mem, resource.byte_offset); + const auto second = ByteAddress(ctx, inst, RebaseRawComponent(mem, 1), resource.byte_offset); + const auto wide = + PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[3], + resource.non_uniform ? resource.descriptor_index : 0u); + const auto index = + Binary(state, spv::OpShiftRightLogical, TypeU32(state), first, ConstantU32(state, 3)); + auto atomic = + Binary(state, spv::OpIEqual, TypeBool(state), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 7)), + ConstantU32(state, 0)); + atomic = AndCondition(state, atomic, + Binary(state, spv::OpIEqual, TypeBool(state), second, + EmitAddU32(state, first, ConstantU32(state, 4)))); + // The 64-bit alias requires the entire pair; partial and swizzled reads retain DWORD bounds. + atomic = AndCondition(state, atomic, EmitMemoryElementInBounds(state, wide, index)); + const auto atomic_label = state.builder.AllocateId(), scalar_label = state.builder.AllocateId(); + const auto merge = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.builder.AddFunction(spv::OpBranchConditional, atomic, atomic_label, scalar_label); + EmitLabel(state, atomic_label); + const auto pointer = EmitStorageBufferElementPointer( + state, wide, index, TypeStorageBufferElementPointer(state, 64)); + const auto snapshot = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpAtomicLoad, TypeU64(state), snapshot, pointer, + ConstantU32(state, spv::ScopeDevice), + ConstantU32(state, spv::MemorySemanticsAcquireMask | + spv::MemorySemanticsUniformMemoryMask)); + const std::array words { + EmitCompositeExtractU64(state, snapshot, IR::Value(0u)), + EmitCompositeExtractU64(state, snapshot, IR::Value(1u))}; + const auto pair = ConstructU32Composite(state, 2, words); + state.builder.AddFunction(spv::OpBranch, merge); + EmitLabel(state, scalar_label); + const auto fallback = scalar(); + const auto scalar_exit = state.current_label; + state.builder.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpPhi, TypeU32Composite(state, 2), result, pair, atomic_label, + fallback, scalar_exit); + return result; +} + uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { auto& state = ctx.state; const auto mem = ctx.Memory(inst); @@ -1100,12 +1157,7 @@ uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compon return LoadFormattedPrepared(ctx, inst, selected, resource, info, components); } if (resource.object_pointer == 0u) return ConstantU32CompositeZero(state, components); - std::array values {}; - for (uint32_t component = 0; component < components; component++) { - values[component] = - LoadWordPrepared(ctx, inst, RebaseRawComponent(selected, component), resource); - } - return ConstructU32Composite(state, components, values); + return LoadRawBuffer(ctx, inst, selected, resource, components); }); }); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 033a93059..043aa6b7d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -207,10 +207,11 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, const bool non_uniform = array_index != 0u; const auto static_index = state.program.info.buffers.at(mem.resource).descriptor_index; if (!non_uniform) array_index = ConstantU32(state, static_index); - MemoryResourceAccess access { - .kind = mem.kind, - .non_uniform = non_uniform, - .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone}; + MemoryResourceAccess access {.kind = mem.kind, + .descriptor_index = array_index, + .non_uniform = non_uniform, + .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask + : spv::MemoryAccessMaskNone}; access.object_pointer = state.builder.AllocateId(); state.builder.AddFunction(spv::OpAccessChain, buffer.pointer_type, access.object_pointer, buffer.variable, array_index); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index c739482e0..b1b0edbb1 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -82,6 +82,11 @@ struct MemoryInfo { bool coherent = false; bool planning_only = false; + [[nodiscard]] bool IsCoherentBufferPairLoad(ValueOpcode opcode) const { + return kind == ResourceKind::Buffer && opcode == ValueOpcode::LoadBufferU32x2 && coherent && + !typed && !formatted && data_bits == 32u; + } + [[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const { if (typed) return false; if (data_bits == 8u) return !formatted && opcode == ValueOpcode::LoadBufferU8; diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index b2df912f8..7a7b32e43 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -340,6 +340,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs } if (kind == ResourceKind::Buffer) { info.coherent_buffers |= memory.coherent; + info.buffer_int64_atomics |= memory.IsCoherentBufferPairLoad(op); if (memory.resource >= info.buffers.size()) { return Fail("buffer operation has invalid resource metadata"); } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 03a4d874f..c42b29bc1 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -31086,6 +31086,69 @@ TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() { return test; } +TestCase CoherentBufferLoadDwordx2Bounds(u32 wave, u32 prefix) { + using O = ShaderOpcode; + TestCase test; + test.name = wave == 64 ? (prefix ? "CoherentBufferPairBoundsOffsetWave64" + : "CoherentBufferPairBoundsWave64") + : (prefix ? "CoherentBufferPairBoundsOffsetWave32" + : "CoherentBufferPairBoundsWave32"); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.initial.resize(12); + test.expected = test.initial; + const u32 addresses[] = {0u, 4u, 8u, 0xfffffffcu}; + const u32 base = prefix / 4; + const u32 values[][2] = {{test.initial[base], test.initial[base + 1]}, + {test.initial[base + 1], test.initial[base + 2]}, + {test.initial[base + 2], 0u}, + {0u, test.initial[base]}}; + for (u32 i = 0; i < std::size(addresses); ++i) { + AppendVMovLiteral(&test.code, 20, addresses[i]); + test.code.push_back(EncodeMubuf0(0x0d, 0, false, true, true) | (1u << 15u)); + test.code.push_back(EncodeMubuf1(2, 0, 20)); + for (u32 component = 0; component < 2; ++component) { + AppendStoreVgpr(&test.code, 2 + component, 4 + i * 2 + component); + test.expected[4 + i * 2 + component] = values[i][component]; + } + } + AppendEnd(&test.code); + test.user_data = MakeStructuredStorageBufferData(0, 12); + test.has_user_data = true; + test.storage_buffer_offsets = {prefix}; + test.storage_buffer_ranges = {{0u, 12u + prefix}}; + test.required_spirv = {"OpAtomicLoad %ulong", "Int64Atomics", + "ArrayStride 8"}; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + test.compute_info.wave_size = wave; + test.has_compute_info = true; + return test; +} + +TestCase CoherentBufferLoadDwordx2Swizzled() { + using O = ShaderOpcode; + TestCase test; + test.name = "CoherentBufferLoadDwordx2Swizzled"; + AppendVMovU32(&test.code, 20, 0); + test.code.push_back(EncodeMubuf0(0x0d, 0, true, false, true) | (1u << 15u)); + test.code.push_back(EncodeMubuf1(2, 0, 20)); + AppendStoreVgpr(&test.code, 2, 9); + AppendStoreVgpr(&test.code, 3, 10); + AppendEnd(&test.code); + test.initial = {0x11111111u, 0x22222222u, 0, 0, 0, 0, + 0, 0, 0x33333333u, 0, 0}; + test.expected = test.initial; + test.expected[9] = test.initial[0]; + test.expected[10] = + test.initial[8]; // Four-byte elements span eight-index groups. + test.user_data = MakeStructuredStorageBufferData(8, 8); + test.user_data[1] |= 1u << 31u; + test.has_user_data = true; + test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD, + O::S_ENDPGM}; + return test; +} + TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() { using O = ShaderOpcode; @@ -35635,6 +35698,65 @@ TestCase Wave64AppendConsumeHighHalf() { return test; } +TestCase CoherentBufferPairContention(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = wave_size == 64 ? "CoherentBufferPairContentionWave64" + : "CoherentBufferPairContentionWave32"; + auto &code = test.code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 6, 0); + AppendSMovLiteral(&code, 20, 128); + const auto loop = code.size(); + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); + code.push_back( + EncodeVop1(0x37, 3, Vgpr(2))); // v_not_b32: correlated high word. + AppendBufferStoreOpcode(&code, 0x50, 2, 20); + code.push_back(EncodeMubuf0(0x0d, 0, false, true, true) | (1u << 15u)); + code.push_back(EncodeMubuf1(4, 12, 20)); + code.push_back(EncodeVop2(0x1d, 4, Vgpr(4), 5)); + code.push_back(EncodeVop1(0x37, 4, Vgpr(4))); + code.push_back(EncodeVop2(0x1c, 6, Vgpr(4), 6)); + code.push_back(EncodeSop2(0x01, 20, 20, InlineU32(1))); + code.push_back(EncodeSopc(0x07, 20, InlineU32(0))); + code.push_back(EncodeSopp(0x05, static_cast(loop - code.size() - 1))); + AppendStoreVgprAtLaneDwordOffset(&code, 6, 0, 2); + code.push_back(EncodeSopp(0x0a, 0)); + // Restore a deterministic mailbox after all readers finish. + code.push_back(EncodeSop1(0x04, 126, InlineU32(1))); + AppendVMovU32(&code, 2, 0); + AppendVMovLiteral(&code, 3, 0xffffffffu); + AppendBufferStoreOpcode(&code, 0x50, 2, 20); + AppendEnd(&code); + test.initial.assign(wave_size * 2 + 2, 0); + test.initial[1] = 0xffffffffu; + test.expected = test.initial; + test.required_spirv = {"OpAtomicExchange %ulong", "OpAtomicLoad %ulong", + "Int64Atomics", "OpLoopMerge"}; + test.opcodes = {O::V_MOV_B32, + O::S_MOV_B32, + O::S_MOV_B64, + O::V_ADD_NC_U32, + O::V_NOT_B32, + O::V_XOR_B32, + O::V_OR_B32, + O::BUFFER_ATOMIC_SWAP_X2, + O::BUFFER_LOAD_DWORDX2, + O::S_SUB_U32, + O::S_CMP_LG_U32, + O::S_CBRANCH_SCC1, + O::V_LSHLREV_B32, + O::BUFFER_STORE_DWORD, + O::S_BARRIER, + O::S_ENDPGM}; + test.compute_info.threads_num[0] = wave_size * 2; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.has_compute_info = true; + return test; +} + TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false, bool store_completion = false) { using O = ShaderOpcode; @@ -42221,6 +42343,12 @@ std::vector MakeCases() { cases.push_back(BufferSubwordStoresAtHostOffset(2)); cases.push_back(BufferSubwordStoresAtHostOffset(1)); AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress); + for (const u32 wave : {32u, 64u}) { + for (const u32 prefix : {0u, 4u}) + cases.push_back(CoherentBufferLoadDwordx2Bounds(wave, prefix)); + cases.push_back(CoherentBufferPairContention(wave)); + } + AddCase(CoherentBufferLoadDwordx2Swizzled); AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress); AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress); AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail); @@ -48414,6 +48542,12 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) { VulkanHarness vulkan; + for (const u32 wave : {32u, 64u}) { + for (const u32 prefix : {0u, 4u}) + RunCase(&vulkan, CoherentBufferLoadDwordx2Bounds(wave, prefix)); + RunCase(&vulkan, CoherentBufferPairContention(wave)); + } + RunCase(&vulkan, CoherentBufferLoadDwordx2Swizzled()); RunCase(&vulkan, BufferWorkgroupPublication(32)); RunCase(&vulkan, BufferWorkgroupPublication(64)); RunCase(&vulkan, BufferWorkgroupPublication(32, true));