shader: keep coherent 64bit atomic

This commit is contained in:
nmzik
2026-10-17 07:52:20 +02:00
parent b485dd24be
commit d1fe1324c8
6 changed files with 204 additions and 10 deletions
+134
View File
@@ -31086,6 +31086,69 @@ TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
return test;
}
TestCase CoherentBufferLoadDwordx2Bounds(u32 wave, u32 prefix) {
using O = ShaderOpcode;
TestCase test;
test.name = wave == 64 ? (prefix ? "CoherentBufferPairBoundsOffsetWave64"
: "CoherentBufferPairBoundsWave64")
: (prefix ? "CoherentBufferPairBoundsOffsetWave32"
: "CoherentBufferPairBoundsWave32");
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
test.initial.resize(12);
test.expected = test.initial;
const u32 addresses[] = {0u, 4u, 8u, 0xfffffffcu};
const u32 base = prefix / 4;
const u32 values[][2] = {{test.initial[base], test.initial[base + 1]},
{test.initial[base + 1], test.initial[base + 2]},
{test.initial[base + 2], 0u},
{0u, test.initial[base]}};
for (u32 i = 0; i < std::size(addresses); ++i) {
AppendVMovLiteral(&test.code, 20, addresses[i]);
test.code.push_back(EncodeMubuf0(0x0d, 0, false, true, true) | (1u << 15u));
test.code.push_back(EncodeMubuf1(2, 0, 20));
for (u32 component = 0; component < 2; ++component) {
AppendStoreVgpr(&test.code, 2 + component, 4 + i * 2 + component);
test.expected[4 + i * 2 + component] = values[i][component];
}
}
AppendEnd(&test.code);
test.user_data = MakeStructuredStorageBufferData(0, 12);
test.has_user_data = true;
test.storage_buffer_offsets = {prefix};
test.storage_buffer_ranges = {{0u, 12u + prefix}};
test.required_spirv = {"OpAtomicLoad %ulong", "Int64Atomics",
"ArrayStride 8"};
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
test.compute_info.wave_size = wave;
test.has_compute_info = true;
return test;
}
TestCase CoherentBufferLoadDwordx2Swizzled() {
using O = ShaderOpcode;
TestCase test;
test.name = "CoherentBufferLoadDwordx2Swizzled";
AppendVMovU32(&test.code, 20, 0);
test.code.push_back(EncodeMubuf0(0x0d, 0, true, false, true) | (1u << 15u));
test.code.push_back(EncodeMubuf1(2, 0, 20));
AppendStoreVgpr(&test.code, 2, 9);
AppendStoreVgpr(&test.code, 3, 10);
AppendEnd(&test.code);
test.initial = {0x11111111u, 0x22222222u, 0, 0, 0, 0,
0, 0, 0x33333333u, 0, 0};
test.expected = test.initial;
test.expected[9] = test.initial[0];
test.expected[10] =
test.initial[8]; // Four-byte elements span eight-index groups.
test.user_data = MakeStructuredStorageBufferData(8, 8);
test.user_data[1] |= 1u << 31u;
test.has_user_data = true;
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX2, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
}
TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
using O = ShaderOpcode;
@@ -35635,6 +35698,65 @@ TestCase Wave64AppendConsumeHighHalf() {
return test;
}
TestCase CoherentBufferPairContention(u32 wave_size) {
using O = ShaderOpcode;
TestCase test;
test.name = wave_size == 64 ? "CoherentBufferPairContentionWave64"
: "CoherentBufferPairContentionWave32";
auto &code = test.code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 6, 0);
AppendSMovLiteral(&code, 20, 128);
const auto loop = code.size();
code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0));
code.push_back(
EncodeVop1(0x37, 3, Vgpr(2))); // v_not_b32: correlated high word.
AppendBufferStoreOpcode(&code, 0x50, 2, 20);
code.push_back(EncodeMubuf0(0x0d, 0, false, true, true) | (1u << 15u));
code.push_back(EncodeMubuf1(4, 12, 20));
code.push_back(EncodeVop2(0x1d, 4, Vgpr(4), 5));
code.push_back(EncodeVop1(0x37, 4, Vgpr(4)));
code.push_back(EncodeVop2(0x1c, 6, Vgpr(4), 6));
code.push_back(EncodeSop2(0x01, 20, 20, InlineU32(1)));
code.push_back(EncodeSopc(0x07, 20, InlineU32(0)));
code.push_back(EncodeSopp(0x05, static_cast<u32>(loop - code.size() - 1)));
AppendStoreVgprAtLaneDwordOffset(&code, 6, 0, 2);
code.push_back(EncodeSopp(0x0a, 0));
// Restore a deterministic mailbox after all readers finish.
code.push_back(EncodeSop1(0x04, 126, InlineU32(1)));
AppendVMovU32(&code, 2, 0);
AppendVMovLiteral(&code, 3, 0xffffffffu);
AppendBufferStoreOpcode(&code, 0x50, 2, 20);
AppendEnd(&code);
test.initial.assign(wave_size * 2 + 2, 0);
test.initial[1] = 0xffffffffu;
test.expected = test.initial;
test.required_spirv = {"OpAtomicExchange %ulong", "OpAtomicLoad %ulong",
"Int64Atomics", "OpLoopMerge"};
test.opcodes = {O::V_MOV_B32,
O::S_MOV_B32,
O::S_MOV_B64,
O::V_ADD_NC_U32,
O::V_NOT_B32,
O::V_XOR_B32,
O::V_OR_B32,
O::BUFFER_ATOMIC_SWAP_X2,
O::BUFFER_LOAD_DWORDX2,
O::S_SUB_U32,
O::S_CMP_LG_U32,
O::S_CBRANCH_SCC1,
O::V_LSHLREV_B32,
O::BUFFER_STORE_DWORD,
O::S_BARRIER,
O::S_ENDPGM};
test.compute_info.threads_num[0] = wave_size * 2;
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.wave_size = wave_size;
test.has_compute_info = true;
return test;
}
TestCase BufferWorkgroupPublication(u32 wave_size, bool dlc_only = false,
bool store_completion = false) {
using O = ShaderOpcode;
@@ -42221,6 +42343,12 @@ std::vector<TestCase> MakeCases() {
cases.push_back(BufferSubwordStoresAtHostOffset(2));
cases.push_back(BufferSubwordStoresAtHostOffset(1));
AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
for (const u32 wave : {32u, 64u}) {
for (const u32 prefix : {0u, 4u})
cases.push_back(CoherentBufferLoadDwordx2Bounds(wave, prefix));
cases.push_back(CoherentBufferPairContention(wave));
}
AddCase(CoherentBufferLoadDwordx2Swizzled);
AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
AddCase(BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail);
@@ -48414,6 +48542,12 @@ int main(int argc, char **argv) {
}
if (argc == 2 && std::strcmp(argv[1], "--buffer-publication-only") == 0) {
VulkanHarness vulkan;
for (const u32 wave : {32u, 64u}) {
for (const u32 prefix : {0u, 4u})
RunCase(&vulkan, CoherentBufferLoadDwordx2Bounds(wave, prefix));
RunCase(&vulkan, CoherentBufferPairContention(wave));
}
RunCase(&vulkan, CoherentBufferLoadDwordx2Swizzled());
RunCase(&vulkan, BufferWorkgroupPublication(32));
RunCase(&vulkan, BufferWorkgroupPublication(64));
RunCase(&vulkan, BufferWorkgroupPublication(32, true));