raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: keep coherent 64bit atomic

Browse Source
This commit is contained in:
nmzik
2026-10-17 07:52:20 +02:00
parent b485dd24be
commit d1fe1324c8
6 changed files with 204 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+1
View File
@@ -420,6 +420,7 @@ void EnsureLdsStorage(EmitterState& state);
struct MemoryResourceAccess {
IR::ResourceKind kind = IR::ResourceKind::None;
uint32_t object_pointer = 0;
uint32_t descriptor_index = 0;
uint32_t length = 0;
uint32_t byte_offset = 0;
uint32_t element_bits = 32;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+58 -6
View File
@@ -1075,6 +1075,63 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) {
});
}
uint32_t LoadRawBuffer(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t components) {
auto& state = ctx.state;
const auto scalar = [&] {
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
values[component] =
LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource);
}
return ConstructU32Composite(state, components, values);
};
if (!mem.IsCoherentBufferPairLoad(inst.GetOpcode())) return scalar();
const auto first = ByteAddress(ctx, inst, mem, resource.byte_offset);
const auto second = ByteAddress(ctx, inst, RebaseRawComponent(mem, 1), resource.byte_offset);
const auto wide =
PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[3],
resource.non_uniform ? resource.descriptor_index : 0u);
const auto index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), first, ConstantU32(state, 3));
auto atomic =
Binary(state, spv::OpIEqual, TypeBool(state),
Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 7)),
ConstantU32(state, 0));
atomic = AndCondition(state, atomic,
Binary(state, spv::OpIEqual, TypeBool(state), second,
EmitAddU32(state, first, ConstantU32(state, 4))));
// The 64-bit alias requires the entire pair; partial and swizzled reads retain DWORD bounds.
atomic = AndCondition(state, atomic, EmitMemoryElementInBounds(state, wide, index));
const auto atomic_label = state.builder.AllocateId(), scalar_label = state.builder.AllocateId();
const auto merge = state.builder.AllocateId();
state.builder.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
state.builder.AddFunction(spv::OpBranchConditional, atomic, atomic_label, scalar_label);
EmitLabel(state, atomic_label);
const auto pointer = EmitStorageBufferElementPointer(
state, wide, index, TypeStorageBufferElementPointer(state, 64));
const auto snapshot = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAtomicLoad, TypeU64(state), snapshot, pointer,
ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, spv::MemorySemanticsAcquireMask |
spv::MemorySemanticsUniformMemoryMask));
const std::array<uint32_t, 4> words {
EmitCompositeExtractU64(state, snapshot, IR::Value(0u)),
EmitCompositeExtractU64(state, snapshot, IR::Value(1u))};
const auto pair = ConstructU32Composite(state, 2, words);
state.builder.AddFunction(spv::OpBranch, merge);
EmitLabel(state, scalar_label);
const auto fallback = scalar();
const auto scalar_exit = state.current_label;
state.builder.AddFunction(spv::OpBranch, merge);
EmitLabel(state, merge);
const auto result = state.builder.AllocateId();
state.builder.AddFunction(spv::OpPhi, TypeU32Composite(state, 2), result, pair, atomic_label,
fallback, scalar_exit);
return result;
}
uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) {
auto& state = ctx.state;
const auto mem = ctx.Memory(inst);
@@ -1100,12 +1157,7 @@ uint32_t LoadBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compon
return LoadFormattedPrepared(ctx, inst, selected, resource, info, components);
}
if (resource.object_pointer == 0u) return ConstantU32CompositeZero(state, components);
std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) {
values[component] =
LoadWordPrepared(ctx, inst, RebaseRawComponent(selected, component), resource);
}
return ConstructU32Composite(state, components, values);
return LoadRawBuffer(ctx, inst, selected, resource, components);
});
});
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+5 -4
View File
@@ -207,10 +207,11 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
const bool non_uniform = array_index != 0u;
const auto static_index = state.program.info.buffers.at(mem.resource).descriptor_index;
if (!non_uniform) array_index = ConstantU32(state, static_index);
MemoryResourceAccess access {
.kind = mem.kind,
.non_uniform = non_uniform,
.memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone};
MemoryResourceAccess access {.kind = mem.kind,
.descriptor_index = array_index,
.non_uniform = non_uniform,
.memory_access = mem.coherent ? spv::MemoryAccessVolatileMask
: spv::MemoryAccessMaskNone};
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, buffer.pointer_type, access.object_pointer,
buffer.variable, array_index);
src/graphics/shader/recompiler/ir/ShaderIR.h
+5
View File
@@ -82,6 +82,11 @@ struct MemoryInfo {
bool coherent = false;
bool planning_only = false;
[[nodiscard]] bool IsCoherentBufferPairLoad(ValueOpcode opcode) const {
return kind == ResourceKind::Buffer && opcode == ValueOpcode::LoadBufferU32x2 && coherent &&
!typed && !formatted && data_bits == 32u;
}
[[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const {
if (typed) return false;
if (data_bits == 8u) return !formatted && opcode == ValueOpcode::LoadBufferU8;
src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp
+1
View File
@@ -340,6 +340,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
}
if (kind == ResourceKind::Buffer) {
info.coherent_buffers |= memory.coherent;
info.buffer_int64_atomics |= memory.IsCoherentBufferPairLoad(op);
if (memory.resource >= info.buffers.size()) {
return Fail("buffer operation has invalid resource metadata");
}
tests/ShaderRecompilerComputeTests.cpp
+134
View File
@@ -31086,6 +31086,69 @@ TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
return test;
}
TestCase CoherentBufferLoadDwordx2Bounds(u32 wave, u32 prefix) {
using O = ShaderOpcode;
TestCase test;
test.name = wave == 64 ? (prefix ? "CoherentBufferPairBoundsOffsetWave64"
: "CoherentBufferPairBoundsWave64")
: (prefix ? "CoherentBufferPairBoundsOffsetWave32"
: "CoherentBufferPairBoundsWave32");
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.initial.resize(12);
test.expected = test.initial;
const u32 addresses[] = {0u, 4u, 8u, 0xfffffffcu};
const u32 base = prefix / 4;
const u32 values[][2] = {{test.initial[base], test.initial[base + 1]},
{test.initial[base + 1], test.initial[base + 2]},
{test.initial[base + 2], 0u},
{0u, test.initial[base]}};
for (u32 i = 0; i < std::size(addresses); ++i) {
AppendVMovLiteral(&test.code, 20, addresses[i]);
test.code.push_back(EncodeMubuf0(0x0d, 0, false, true, true) | (1u << 15u));
test.code.push_back(EncodeMubuf1(2, 0, 20));
for (u32 component = 0; component < 2; ++component) {
AppendStoreVgpr(&test.code, 2 + component, 4 + i * 2 + component);
test.expected[4 + i * 2 + component] = values[i][component];
}
}
AppendEnd(&test.code);
test.user_data = MakeStructuredStorageBufferData(0, 12);
test.has_user_data = true;
test.storage_buffer_offsets = {prefix};
test.storage_buffer_ranges = {{0u, 12u + prefix}};
test.required_spirv = {"OpAtomicLoad %ulong", "Int64Atomics",
"ArrayStride 8"};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.wave_size = wave;
test.has_compute_info = true;
return test;
}
TestCase CoherentBufferLoadDwordx2Swizzled() {
using O = ShaderOpcode;
TestCase test;
test.name = "CoherentBufferLoadDwordx2Swizzled";
AppendVMovU32(&test.code, 20, 0);
test.code.push_back(EncodeMubuf0(0x0d, 0, true, false, true) | (1u << 15u));
test.code.push_back(EncodeMubuf1(2, 0, 20));
AppendStoreVgpr(&test.code, 2, 9);
AppendStoreVgpr(&test.code, 3, 10);
AppendEnd(&test.code);
test.initial = {0x11111111u, 0x22222222u, 0, 0, 0, 0,
0, 0, 0x33333333u, 0, 0};
test.expected = test.initial;
test.expected[9] = test.initial[0];
test.expected[10] =
test.initial[8]; // Four-byte elements span eight-index groups.
test.user_data = MakeStructuredStorageBufferData(8, 8);
test.user_data[1] |= 1u << 31u;
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
@@ -35635,6 +35698,65 @@ TestCase Wave64AppendConsumeHighHalf() {
return test;
}
TestCase CoherentBufferPairContention(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "CoherentBufferPairContentionWave64"
: "CoherentBufferPairContentionWave32";
auto &code = test.code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 6, 0);
AppendSMovLiteral(&code, 20, 128);
const auto loop = code.size();
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
code.push_back(
EncodeVop1(0x37, 3, Vgpr(2))); // v_not_b32: correlated high word.
AppendBufferStoreOpcode(&code, 0x50, 2, 20);
code.push_back(EncodeMubuf0(0x0d, 0, false, true, true) | (1u << 15u));
code.push_back(EncodeMubuf1(4, 12, 20));
code.push_back(EncodeVop2(0x1d, 4, Vgpr(4), 5));
code.push_back(EncodeVop1(0x37, 4, Vgpr(4)));
code.push_back(EncodeVop2(0x1c, 6, Vgpr(4), 6));
code.push_back(EncodeSop2(0x01, 20, 20, InlineU32(1)));
code.push_back(EncodeSopc(0x07, 20, InlineU32(0)));
code.push_back(EncodeSopp(0x05, static_cast<u32>(loop - code.size() - 1)));
AppendStoreVgprAtLaneDwordOffset(&code, 6, 0, 2);
code.push_back(EncodeSopp(0x0a, 0));
// Restore a deterministic mailbox after all readers finish.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendVMovU32(&code, 2, 0);
AppendVMovLiteral(&code, 3, 0xffffffffu);
AppendBufferStoreOpcode(&code, 0x50, 2, 20);
AppendEnd(&code);
test.initial.assign(wave_size * 2 + 2, 0);
test.initial[1] = 0xffffffffu;
test.expected = test.initial;
test.required_spirv = {"OpAtomicExchange %ulong", "OpAtomicLoad %ulong",
"Int64Atomics", "OpLoopMerge"};
test.opcodes = {O::V_MOV_B32,
O::S_MOV_B32,
O::S_MOV_B64,
O::V_ADD_NC_U32,
O::V_NOT_B32,
O::V_XOR_B32,
O::V_OR_B32,
O::BUFFER_ATOMIC_SWAP_X2,
O::BUFFER_LOAD_DWORDX2,
O::S_SUB_U32,
O::S_CMP_LG_U32,
O::S_CBRANCH_SCC1,
O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD,
O::S_BARRIER,
O::S_ENDPGM};
test.compute_info.threads_num[0] = wave_size * 2;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false,
bool store_completion = false) {
using O = ShaderOpcode;
@@ -42221,6 +42343,12 @@ std::vector<TestCase> MakeCases() {
cases.push_back(BufferSubwordStoresAtHostOffset(2));
cases.push_back(BufferSubwordStoresAtHostOffset(1));
AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
for (const u32 wave : {32u, 64u}) {
for (const u32 prefix : {0u, 4u})
cases.push_back(CoherentBufferLoadDwordx2Bounds(wave, prefix));
cases.push_back(CoherentBufferPairContention(wave));
}
AddCase(CoherentBufferLoadDwordx2Swizzled);
AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail);
@@ -48414,6 +48542,12 @@ int main(int argc, char **argv) {
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) {
VulkanHarness vulkan;
for (const u32 wave : {32u, 64u}) {
for (const u32 prefix : {0u, 4u})
RunCase(&vulkan, CoherentBufferLoadDwordx2Bounds(wave, prefix));
RunCase(&vulkan, CoherentBufferPairContention(wave));
}
RunCase(&vulkan, CoherentBufferLoadDwordx2Swizzled());
RunCase(&vulkan, BufferWorkgroupPublication(32));
RunCase(&vulkan, BufferWorkgroupPublication(64));
RunCase(&vulkan, BufferWorkgroupPublication(32, true));