From 5313181ecf8aea5011ed005b44a73b2914cea989 Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 14 Oct 2026 20:39:25 +0200 Subject: [PATCH] shader: bound workgroup selector prefixes from runtime starting indices --- .../ir/passes/ResourceMaterialization.cpp | 5 +- .../recompiler/ir/passes/ResourceTracking.cpp | 17 +- tests/ResourceTrackingTests.cpp | 108 +++++++- tests/ShaderRecompilerComputeTests.cpp | 236 ++++++++++-------- 4 files changed, 252 insertions(+), 114 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 7cced8885..a3de48d00 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -577,8 +577,9 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p (material.Type() != 0u || material.SwizzleEnabled() || material.AddTid() || material.OutOfBounds() != 0u || uint64_t {selector->offset} + 4u > step || uint64_t {first} + count > material.NumRecords())) return false; - const auto end = count != 0u - ? (uint64_t {first} + count - 1u) * step + selector->offset + 4u : 0u; + const auto last = count != 0u ? uint64_t {first} + count - 1u : 0u; + if (last > UINT32_MAX) return false; + const auto end = count != 0u ? last * step + selector->offset + 4u : 0u; if (end > uint64_t {UINT32_MAX} + 1u || (indirect.workgroup_axis == UINT32_MAX && end > size)) return false; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 9481ba62f..28ff2182f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -2204,6 +2204,7 @@ private: const auto* memory = ScalarReadMemory(*read, memory_index); Value index; Value count; + Value first(0u); uint32_t workgroup_axis = UINT32_MAX; DescriptorSource count_source; uint32_t row_stride = 0; @@ -2214,6 +2215,20 @@ private: ((offset | stride | memory->offset) & 3u) == 0u && uint64_t {offset} + memory->offset <= UINT32_MAX) { workgroup_axis = WorkgroupAxis(index); + if (workgroup_axis == UINT32_MAX) { + const auto* sum = index.Resolve().TryInstruction(); + if (sum != nullptr && sum->GetOpcode() == ValueOpcode::IAdd32) { + for (uint32_t arm = 0; arm < 2u; ++arm) { + const auto axis = WorkgroupAxis(sum->Arg(arm)); + const auto base = sum->Arg(arm ^ 1u); + if (axis == UINT32_MAX || + !ValidateRuntimeValue(m_program, base, RuntimeValueType::Integer)) continue; + workgroup_axis = axis; + first = base; + break; + } + } + } const auto* bound = BoundedLoop(index, read->Parent()); if (bound != nullptr && bound->GetOpcode() == ValueOpcode::ULessThan32) { count = bound->Arg(1); @@ -2269,7 +2284,7 @@ private: if (rows) indirect.selector->rows.emplace( DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows { .source = InternSource(count_source), .stride = row_stride}); - if (!count.IsEmpty() || workgroup_axis != UINT32_MAX) indirect.selector_first = Value(0u); + if (!count.IsEmpty() || workgroup_axis != UINT32_MAX) indirect.selector_first = first; return true; } diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 0cc349afe..8ebdb3928 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -5236,8 +5236,9 @@ void TestWorkgroupSelectedBufferRecords() { void TestGpuSelectedFormattedRecordStore() { namespace Prospero = Libs::Graphics::Prospero; + enum class Prefix { None, Forward, Reversed, Lane, TwoAxes }; const auto make = [](uint32_t stride = 592u, uint32_t shift = 16u, bool split = false, - bool wide = false) { + bool wide = false, Prefix prefix = Prefix::None) { auto fixture = std::make_unique(); auto& f = *fixture; std::array table_words, selector_words; @@ -5247,8 +5248,20 @@ void TestGpuSelectedFormattedRecordStore() { } const auto table = wide ? f.Address(table_words[0], table_words[1]) : f.Buffer(table_words); const auto lane = f.Emit(ValueOpcode::LaneId); + auto selector_index = lane; + if (prefix != Prefix::None) { + const auto group = f.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::WorkgroupId)), Value(0u)}); + const auto first = prefix == Prefix::Lane ? lane : prefix == Prefix::TwoAxes + ? f.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::WorkgroupId)), Value(1u)}) + : f.Emit(ValueOpcode::LoadAddressU32, {table, Value(0u), Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::ScalarAddress, .offset = 0x234u}, 0x1088u)); + selector_index = f.Emit(ValueOpcode::IAdd32, + {prefix == Prefix::Reversed ? group : first, prefix == Prefix::Reversed ? first : group}); + } const auto packed = wide ? f.Emit(ValueOpcode::ReadConstBuffer, - {f.Buffer(selector_words), f.Emit(ValueOpcode::IMul32, {lane, Value(8u)})}, + {f.Buffer(selector_words), f.Emit(ValueOpcode::IMul32, {selector_index, Value(8u)})}, f.AddMemory({.kind = ResourceKind::ScalarBuffer, .offset = 4u}, 0x1c4cu)) : f.Emit(ValueOpcode::LoadBufferU32, {f.Buffer(selector_words), lane, Value(0u), Value(0u), Value(true)}, @@ -5514,6 +5527,97 @@ void TestGpuSelectedFormattedRecordStore() { for (const auto [shift, split] : {std::pair{0u, false}, std::pair{16u, true}}) CheckFatal([&] { make(16u, shift, split, true); }, "not a valid runtime value", "finite RGBA16 store accepted an unbounded key or independently wrapping DWORDs"); + + struct PrefixMemory : LinearTestMemory { uint32_t read_attempts = 0u; } prefix_memory; + const auto read_prefix = [](void* userdata, uint64_t address, std::span words) { + auto* memory = static_cast(userdata); + ++memory->read_attempts; + return ReadLinearTestMemory(memory, address, words); + }; + std::ranges::fill(prefix_memory.words, 0xdeadbeefu); + constexpr uint32_t selector_offset = 0x1000u; + constexpr uint32_t first_offset = table_offset - 48u + 0x234u; + for (uint32_t row = 0; row < 2u; ++row) { + const std::array descriptor{0x100000u + row * 256u, 8u << 16u, 20u, rgba}; + std::copy(descriptor.begin(), descriptor.end(), + prefix_memory.words.begin() + table_offset / 4u + row * 4u); + } + const std::array numeric_descriptor{0xbf8d214eu, 0xbf78904bu, 0x3e930d39u, 0x33d6bf95u}; + std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), + prefix_memory.words.begin() + table_offset / 4u + 4u * 4u); + for (uint32_t index = 0; index < 12u; ++index) + prefix_memory.words[selector_offset / 4u + index * 2u + 1u] = + ((index < 4u ? index & 1u : index < 6u ? 1u : 4u) << 16u) | 0xbeefu; + std::array prefix_data{uint32_t(prefix_memory.base) + table_offset - 48u, 0u, 0u, 0u, + uint32_t(prefix_memory.base) + selector_offset, 8u << 16u, 12u, 0x5204u}; + std::array counts{2u, 1u, 1u}; + SrtRuntime prefix_runtime{.user_data = prefix_data, .read_memory = read_prefix, + .userdata = &prefix_memory, .read_specialization_memory = read_prefix, + .workgroup_counts = counts}; + prefix_memory.watched_address = prefix_memory.base + selector_offset + 4u; + for (const auto prefix : {Prefix::Forward, Prefix::Reversed}) { + const auto selected = make(16u, 16u, false, true, prefix); + auto selected_plan = ExtractResourcePlan(selected->program); + auto& indirect = *selected_plan.descriptor_sources[ + selected_plan.info.buffers[wide_root].source].indirect_descriptor; + Check(indirect.workgroup_axis == 0u && !indirect.selector_first.IsImmediate() && + indirect.key_count.IsEmpty() && indirect.selector && indirect.selector->stride == 8u && + indirect.selector->offset == 4u && indirect.table_immediate == 48u, + "runtime-first selector lost its workgroup prefix or captured table geometry"); + const auto refresh = [&] { + prefix_memory.watched_reads = 0u; + prefix_memory.read_attempts = 0u; + return MaterializeResources(selected_plan, prefix_runtime, snapshot, specialization); + }; + for (const uint32_t first : {0u, 2u, 4u, 0u}) { + prefix_memory.words[first_offset / 4u] = first; + Check(refresh() && snapshot.buffers.size() == selected_plan.info.buffers.size() + (first == 4u ? 0u : 1u) && + snapshot.buffers[wide_root].dwords[0] == 0x100000u + (first == 4u ? 256u : 0u), + "runtime-first refresh froze a prior selector range or enumerated unrelated numeric rows"); + for (const auto read : {std::pair{prefix_memory.base + first_offset, uint64_t{4u}}, + std::pair{prefix_memory.base + selector_offset + first * 8u + 4u, uint64_t{12u}}}) + Check(std::ranges::find(snapshot.specialization_reads, read) != snapshot.specialization_reads.end(), + "runtime first or selected prefix escaped specialization read capture"); + } + prefix_memory.fail_address = prefix_memory.base + first_offset; + Check(!refresh() && prefix_memory.read_attempts == 1u, + "unreadable runtime first reached the selector payload"); + prefix_memory.fail_address = UINT64_MAX; + prefix_memory.words[first_offset / 4u] = 6u; + Check(!refresh(), "an actually selected invalid numeric descriptor was silently omitted"); + for (const uint32_t first : {UINT32_MAX, 0x20000000u}) { + prefix_memory.words[first_offset / 4u] = first; + Check(!refresh() && prefix_memory.read_attempts == 1u, + "wrapped first-plus-group or scaled prefix read selector payload"); + } + prefix_memory.words[first_offset / 4u] = UINT32_MAX; + counts[0] = 7u; + indirect.selector->stride = 0xfffffffcu; + Check(!refresh() && prefix_memory.read_attempts == 1u, + "wrapped first-plus-group overflowed the host prefix product before rejection"); + indirect.selector->stride = 8u; + counts[0] = 1u; + prefix_memory.words[first_offset / 4u] = 0x1fffffffu; + Check(refresh() && prefix_memory.watched_reads == 0u && snapshot.buffers[wide_root].dwords[0] == 0x100000u, + "last nonwrapping selector DWORD lost scalar OOB key zero"); + prefix_memory.words[first_offset / 4u] = UINT32_MAX; + for (const auto empty : {std::array{0u, 1u, 1u}, std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u}}) { + counts = empty; + Check(refresh() && prefix_memory.watched_reads == 0u && + snapshot.buffers.size() == selected_plan.info.buffers.size() && + snapshot.buffers[wide_root].dwords[0] == 0u, + "empty dispatch evaluated an unreachable wrapped selector prefix"); + } + counts = {2u, 1u, 1u}; + } + for (const auto prefix : {Prefix::Lane, Prefix::TwoAxes}) { + const auto rejected = make(16u, 16u, false, true, prefix); + const auto& indirect = rejected->program.descriptor_sources[ + rejected->program.info.buffers[wide_root].source].indirect_descriptor; + Check(indirect && indirect->workgroup_axis == UINT32_MAX && !indirect->selector && + indirect->key_count == Value(32u), + "lane-varying or multiple-axis first incorrectly narrowed the finite descriptor domain"); + } } void TestFiniteBufferSelection() { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index d576b5844..a327acaee 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -10968,118 +10968,136 @@ public: constexpr auto record_keys = base + 0xd1000; constexpr auto record_table = base + 0xd2000; constexpr auto record_output = base + 0xd7000; - std::vector record_shader{ - EncodeSop2(0x26, 16, 13, InlineU32(12)), - EncodeSmem0(0x08, 16, 0), EncodeSmem1(0, 16), EncodeSopp(0x0c, 0), - EncodeSop2(0x26, 17, 16, 255), 192u, - EncodeSmem0(0x0a, 20, 2), EncodeSmem1(160, 17), - EncodeSmem0(0x0a, 24, 2), EncodeSmem1(176, 17), EncodeSopp(0x0c, 0), - EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 5, 0), - EncodeSopp(0x0c, 0)}; - AppendVop3(&record_shader, 0x346, 8, 13, 12, Vgpr(0)); - record_shader.insert(record_shader.end(), { - EncodeMubuf0(0x1d, 0, true, false), EncodeMubuf1(4, 2, 8)}); - AppendVMovLiteral(&record_shader, 10, 0x5au); - AppendSMovLiteral(&record_shader, 126, 0x55555555u); - AppendSMovLiteral(&record_shader, 127, 0x55555555u); - record_shader.insert(record_shader.end(), { - EncodeMubuf0(0x18, 0, true, false, true), EncodeMubuf1(10, 6, 0)}); - AppendEnd(&record_shader); - ShaderMapUserData(reinterpret_cast(record_shader.data()), - {.type = Prospero::ShaderBinaryType::kCs, - .code_size_bytes = static_cast(record_shader.size() * sizeof(u32))}); - std::array records; - records.fill(0xdeadbeefu); - const std::array numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au}; - std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), records.begin() + 76u); - for (u32 row = 0; row < 2u; ++row) { - const auto payload = base + 0xd3000 + row * 0x1000; - const auto target = base + 0xd5000 + row * 0x1000; - const std::array descriptors{ - static_cast(payload), static_cast(payload >> 32u) | (8u << 16u), 4u, - (u32(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | DstSel(4, 5, 6, 7), - static_cast(target), static_cast(target >> 32u) | (1u << 16u), 4u, 0x5204u}; - std::copy(descriptors.begin(), descriptors.end(), records.begin() + row * 48u + 40u); - const std::array data{0x40003c00u + row, 0xc0003800u + row, - 0x44004200u + row, 0xc400bc00u + row, 0x48004600u + row, 0xc800c600u + row, - 0x4c004a00u + row, 0xcc00ca00u + row}; - LibKernel::Memory::WriteBacking(payload, data.data(), sizeof(data)); - } - LibKernel::Memory::WriteBacking(record_table, records.data(), sizeof(records)); - const auto produce = [&](uint64_t address, u32 stride, std::array values) { - shaders.SetCsShader({.data_addr = reinterpret_cast(writer.data()), - .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1, - .wave_size = 64, .user_sgpr = 15}); - for (u32 word = 0; word < 3u; ++word) { - set_buffer(word * 4u, address + word * stride, 4u); - shaders.SetCsUserSgpr(12u + word, values[word], HW::UserSgprType::Unknown); + for (const bool based : {false, true}) { + std::vector record_shader; + if (based) { + // The later captured shader starts its prefix at a clean SRT DWORD. + record_shader = {EncodeSmem0(0x00, 18, 2), EncodeSmem1(0x234, 125), EncodeSopp(0x0c, 0), + EncodeSop2(0x02, 18, 18, 13), + EncodeSop2(0x1e, 16, 18, InlineU32(3)), + EncodeSop2(0x02, 16, 16, InlineU32(4))}; + } else record_shader = {EncodeSop2(0x26, 16, 13, InlineU32(12))}; + record_shader.insert(record_shader.end(), { + EncodeSmem0(0x08, 16, 0), EncodeSmem1(0, 16), EncodeSopp(0x0c, 0)}); + if (based) record_shader.insert(record_shader.end(), { + EncodeSop2(0x27, 16, 16, 255), (5u << 16u) | 16u}); + record_shader.insert(record_shader.end(), { + EncodeSop2(0x26, 17, 16, 255), 192u, + EncodeSmem0(based ? 0x02 : 0x0a, 20, 2), EncodeSmem1(160, 17), + EncodeSmem0(based ? 0x02 : 0x0a, 24, 2), EncodeSmem1(176, 17), EncodeSopp(0x0c, 0), + EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 5, 0), + EncodeSopp(0x0c, 0)}); + AppendVop3(&record_shader, 0x346, 8, 13, 12, Vgpr(0)); + record_shader.insert(record_shader.end(), { + EncodeMubuf0(0x1d, 0, true, false), EncodeMubuf1(4, 2, 8)}); + AppendVMovLiteral(&record_shader, 10, 0x5au); + AppendSMovLiteral(&record_shader, 126, 0x55555555u); + AppendSMovLiteral(&record_shader, 127, 0x55555555u); + record_shader.insert(record_shader.end(), { + EncodeMubuf0(0x18, 0, true, false, true), EncodeMubuf1(10, 6, 0)}); + AppendEnd(&record_shader); + ShaderMapUserData(reinterpret_cast(record_shader.data()), + {.type = Prospero::ShaderBinaryType::kCs, + .code_size_bytes = static_cast(record_shader.size() * sizeof(u32))}); + std::array records; + records.fill(0xdeadbeefu); + const std::array numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au}; + std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), records.begin() + 76u); + for (u32 row = 0; row < 2u; ++row) { + const auto payload = base + 0xd3000 + row * 0x1000; + const auto target = base + 0xd5000 + row * 0x1000; + const std::array descriptors{ + static_cast(payload), static_cast(payload >> 32u) | (8u << 16u), 4u, + (u32(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | DstSel(4, 5, 6, 7), + static_cast(target), static_cast(target >> 32u) | (1u << 16u), 4u, 0x5204u}; + std::copy(descriptors.begin(), descriptors.end(), records.begin() + row * 48u + 40u); + const std::array data{0x40003c00u + row, 0xc0003800u + row, + 0x44004200u + row, 0xc400bc00u + row, 0x48004600u + row, 0xc800c600u + row, + 0x4c004a00u + row, 0xcc00ca00u + row}; + LibKernel::Memory::WriteBacking(payload, data.data(), sizeof(data)); } - processor.DispatchDirect(1, 1, 1, 0x41u); - }; - for (const u32 wave : {32u, 64u}) { - for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u}, - std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u}, - std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) { - const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end(); - const std::array stale_args{}; - std::array stale_keys; - stale_keys.fill(sentinel); - Require(name, "record fixture reset", context.InvalidateMemory(record_args, 0x8000), - "record fixture escaped its mapped allocation"); - LibKernel::Memory::WriteBacking(record_args, stale_args.data(), sizeof(stale_args)); - LibKernel::Memory::WriteBacking(record_keys, stale_keys.data(), sizeof(stale_keys)); - std::vector expected_output(wave * 4u, sentinel); - LibKernel::Memory::WriteBacking(record_output, expected_output.data(), expected_output.size() * 4u); - const std::array untouched{sentinel, sentinel, sentinel, sentinel}; - for (u32 row = 0; row < 2u; ++row) - LibKernel::Memory::WriteBacking(base + 0xd5000 + row * 0x1000, untouched.data(), sizeof(untouched)); - produce(record_args, 4u, dimensions); - produce(record_keys, 12u, {1u, 0u, 22369622u}); - std::array cpu_args{}; - std::array cpu_keys{}; - Require(name, "GPU record dependencies remain pending", - cache.HasGpuDirtyBytes(record_args, 12u) && cache.HasGpuDirtyBytes(record_keys, 16u) && - LibKernel::Memory::TryReadBacking(record_args, cpu_args.data(), sizeof(cpu_args)) && - LibKernel::Memory::TryReadBacking(record_keys, cpu_keys.data(), sizeof(cpu_keys)) && - cpu_args == stale_args && cpu_keys == stale_keys, - "the producer did not leave dispatch counts and selectors GPU-owned"); - shaders.SetCsShader({.data_addr = reinterpret_cast(record_shader.data()), - .num_thread_x = wave, .num_thread_y = 1, .num_thread_z = 1, - .user_sgpr = 13, .tgid_x_en = true}); - set_buffer(0, record_keys, 3u, 12u); - set_buffer(4, record_table, 2u, 192u); - set_buffer(8, record_output, wave * 2u, 8u); - shaders.SetCsUserSgpr(12, std::countr_zero(wave), HW::UserSgprType::Unknown); - processor.DispatchIndirect(record_args, 0x41u | - (wave == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u)); - cache.ReadMemory(record_output, expected_output.size() * 4u); - std::vector actual_output(expected_output.size()); - Require(name, "record output readback", LibKernel::Memory::TryReadBacking( - record_output, actual_output.data(), actual_output.size() * 4u), - "indirect record output could not be read back"); - if (!empty) - for (u32 group = 0; group < dimensions[0]; ++group) { - const auto row = 1u - group; - for (u32 lane = 0; lane < wave; ++lane) { - const auto out = (group * wave + lane) * 2u; - expected_output[out] = expected_output[out + 1u] = 0u; - if (lane < 4u) - Require(name, "unchanged formatted input", LibKernel::Memory::TryReadBacking( - base + 0xd3000 + row * 0x1000 + lane * 8u, expected_output.data() + out, 8u), - "the formatted source escaped its mapped allocation"); + LibKernel::Memory::WriteBacking(record_table, records.data(), sizeof(records)); + const auto produce = [&](uint64_t address, u32 stride, std::array values) { + shaders.SetCsShader({.data_addr = reinterpret_cast(writer.data()), + .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = 64, .user_sgpr = 15}); + for (u32 word = 0; word < 3u; ++word) { + set_buffer(word * 4u, address + word * stride, 4u); + shaders.SetCsUserSgpr(12u + word, values[word], HW::UserSgprType::Unknown); + } + processor.DispatchDirect(1, 1, 1, 0x41u); + }; + for (const u32 wave : {32u, 64u}) { + u32 first = based ? 2u : 0u; + for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u}, + std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u}, + std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) { + const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end(); + const std::array stale_args{}; + std::array stale_keys; + stale_keys.fill(sentinel); + Require(name, "record fixture reset", context.InvalidateMemory(record_args, 0x8000), + "record fixture escaped its mapped allocation"); + LibKernel::Memory::WriteBacking(record_args, stale_args.data(), sizeof(stale_args)); + LibKernel::Memory::WriteBacking(record_keys, stale_keys.data(), sizeof(stale_keys)); + LibKernel::Memory::WriteBacking(record_table + 0x234, &first, sizeof(first)); + std::vector expected_output(wave * 4u, sentinel); + LibKernel::Memory::WriteBacking(record_output, expected_output.data(), expected_output.size() * 4u); + const std::array untouched{sentinel, sentinel, sentinel, sentinel}; + for (u32 row = 0; row < 2u; ++row) + LibKernel::Memory::WriteBacking(base + 0xd5000 + row * 0x1000, untouched.data(), sizeof(untouched)); + produce(record_args, 4u, dimensions); + produce(record_keys + (based ? first * 8u + 4u : 0u), based ? 8u : 12u, + based ? std::array{0x1beefu, 0xbeefu, 0x1fbeefu} + : std::array{1u, 0u, 22369622u}); + std::array cpu_args{}; + std::array cpu_keys{}; + Require(name, "GPU record dependencies remain pending", + cache.HasGpuDirtyBytes(record_args, 12u) && cache.HasGpuDirtyBytes(record_keys, sizeof(stale_keys)) && + LibKernel::Memory::TryReadBacking(record_args, cpu_args.data(), sizeof(cpu_args)) && + LibKernel::Memory::TryReadBacking(record_keys, cpu_keys.data(), sizeof(cpu_keys)) && + cpu_args == stale_args && cpu_keys == stale_keys, + "the producer did not leave dispatch counts and selectors GPU-owned"); + shaders.SetCsShader({.data_addr = reinterpret_cast(record_shader.data()), + .num_thread_x = wave, .num_thread_y = 1, .num_thread_z = 1, + .user_sgpr = 13, .tgid_x_en = true}); + set_buffer(0, record_keys, based ? 5u : 3u, based ? 8u : 12u); + set_buffer(4, record_table, 2u, 192u); + if (based) shaders.SetCsUserSgpr(5, record_table >> 32u, HW::UserSgprType::Unknown); + set_buffer(8, record_output, wave * 2u, 8u); + shaders.SetCsUserSgpr(12, std::countr_zero(wave), HW::UserSgprType::Unknown); + processor.DispatchIndirect(record_args, 0x41u | + (wave == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u)); + cache.ReadMemory(record_output, expected_output.size() * 4u); + std::vector actual_output(expected_output.size()); + Require(name, "record output readback", LibKernel::Memory::TryReadBacking( + record_output, actual_output.data(), actual_output.size() * 4u), + "indirect record output could not be read back"); + if (!empty) + for (u32 group = 0; group < dimensions[0]; ++group) { + const auto row = 1u - group; + for (u32 lane = 0; lane < wave; ++lane) { + const auto out = (group * wave + lane) * 2u; + expected_output[out] = expected_output[out + 1u] = 0u; + if (lane < 4u) + Require(name, "unchanged formatted input", LibKernel::Memory::TryReadBacking( + base + 0xd3000 + row * 0x1000 + lane * 8u, expected_output.data() + out, 8u), + "the formatted source escaped its mapped allocation"); + } } + Require(name, "coherent exact workgroup selector prefix", actual_output == expected_output, + "GPU counts or selectors were stale, over-enumerated, or selected the wrong record"); + for (u32 row = 0; row < 2u; ++row) { + const auto target = base + 0xd5000 + row * 0x1000; + cache.ReadMemory(target, sizeof(untouched)); + auto expected = untouched; + if (!empty && (dimensions[0] == 2u || row == 1u)) expected[0] = 0xa55aa55au; + std::array actual{}; + Require(name, "selected byte store and canaries", LibKernel::Memory::TryReadBacking( + target, actual.data(), sizeof(actual)) && actual == expected, + "masked/OOB record stores changed an inactive row or adjacent bytes"); } - Require(name, "coherent exact workgroup selector prefix", actual_output == expected_output, - "GPU counts or selectors were stale, over-enumerated, or selected the wrong record"); - for (u32 row = 0; row < 2u; ++row) { - const auto target = base + 0xd5000 + row * 0x1000; - cache.ReadMemory(target, sizeof(untouched)); - auto expected = untouched; - if (!empty && (dimensions[0] == 2u || row == 1u)) expected[0] = 0xa55aa55au; - std::array actual{}; - Require(name, "selected byte store and canaries", LibKernel::Memory::TryReadBacking( - target, actual.data(), sizeof(actual)) && actual == expected, - "masked/OOB record stores changed an inactive row or adjacent bytes"); + if (based) first ^= 2u; } } }