diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index e066e0ac5..0bdabcc8f 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -347,6 +347,7 @@ struct PipelineCache::ProgramCache { }; if constexpr (std::is_same_v) { runtime.workgroup_counts = input_info.workgroup_counts; + runtime.indirect_args_address = input_info.indirect_args_address; } bool source_changed = false; if (entry != programs.end()) { diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 08f19478e..d357e3f3c 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -457,6 +457,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, } ShaderComputeInputInfo input_info {}; input_info.async_compute = async_compute; + input_info.indirect_args_address = args_addr; const auto compute_program = m_context.GetPipelineCache().GetComputeProgram( cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info); buffer.EndRendering(); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index fe7604154..7cced8885 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -307,9 +307,10 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p const auto root_resource = specializations[resource_index]; const auto evaluate_count = [&](uint32_t& count) { if (indirect.workgroup_axis != UINT32_MAX) { - if (indirect.workgroup_axis >= runtime.workgroup_counts.size()) return false; - count = runtime.workgroup_counts[indirect.workgroup_axis]; - return count != 0u; + if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false; + count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end() + ? 0u : runtime.workgroup_counts[indirect.workgroup_axis]; + return true; } if (!clean.Evaluate(indirect.key_count, count)) return false; if (indirect.signed_count && std::bit_cast(count) < 0) count = 0; @@ -337,14 +338,28 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p const auto read_keys = [&](uint64_t base, uint64_t size, uint64_t first, uint64_t step, uint64_t count) { keys.resize(count); + if (count == 0u) return true; if (step == 4u) { return read_selector(base, size, first, keys); } - for (auto& key: keys) { - if (!read_selector(base, size, first, {&key, 1})) + if (step < 4u) return false; + for (uint64_t copied = 0; copied < count;) { + const auto offset = first + copied * step; + // Scalar reads align each strided offset independently. + const auto bias = selector->global ? 0u : offset & 3u; + const auto length = std::min(count - copied, + 1u + (uint64_t {MaxIndirectDescriptorProbes - 1u} * 4u - bias) / step); + keys.resize(count + (bias + (length - 1u) * step) / 4u + 1u); + const auto batch = std::span(keys).subspan(count); + if (!read_selector(base, size, offset, batch)) { + keys.resize(count); return false; - first += step; + } + for (uint64_t index = 0; index < length; ++index) + keys[copied + index] = batch[(bias + index * step) / 4u]; + copied += length; } + keys.resize(count); return true; }; uint64_t table_base = 0; @@ -564,7 +579,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p uint64_t {first} + count > material.NumRecords())) return false; const auto end = count != 0u ? (uint64_t {first} + count - 1u) * step + selector->offset + 4u : 0u; - if (end > uint64_t {UINT32_MAX} + 1u || end > size) + if (end > uint64_t {UINT32_MAX} + 1u || + (indirect.workgroup_axis == UINT32_MAX && end > size)) return false; if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count)) return false; @@ -1331,11 +1347,16 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime runtime.read_specialization_memory == nullptr)) { return false; } - const bool capture_reads = program.capture_specialization_reads || !program.source_reads.empty(); + const auto uses_workgroup_count = [](const DescriptorSource& source) { + return source.indirect_descriptor && source.indirect_descriptor->workgroup_axis != UINT32_MAX; + }; + const bool capture_reads = program.capture_specialization_reads || !program.source_reads.empty() || + (runtime.indirect_args_address != 0u && + std::ranges::any_of(program.descriptor_sources, uses_workgroup_count)); auto& reads = snapshot.specialization_reads; reads = program.source_reads; SrtReadCapture capture {runtime, reads}; - const auto observed = capture_reads ? capture.ObservedRuntime() : runtime; + auto observed = capture_reads ? capture.ObservedRuntime() : runtime; SrtWalker clean(program, CleanRuntime(observed)); SrtWalker walker(program, observed, capture_reads || program.requires_specialization_memory ? &clean : nullptr); @@ -1343,6 +1364,18 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime return false; } const auto active = std::span(program.active_sources); + std::array workgroup_counts {}; + if (runtime.indirect_args_address != 0u) { + for (uint32_t source = 0; source < program.descriptor_sources.size(); ++source) { + if ((!active.empty() && !active[source]) || + !uses_workgroup_count(program.descriptor_sources[source])) continue; + if ((runtime.indirect_args_address & 3u) != 0u || + !ReadTable(runtime.indirect_args_address, sizeof(workgroup_counts), 0u, + observed, workgroup_counts)) return false; + observed.workgroup_counts = workgroup_counts; + break; + } + } snapshot.uniform_fill = {}; const auto& fill = program.uniform_fill; const auto words = fill.fill.words; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index fc48fe5c5..9481ba62f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -2204,6 +2204,7 @@ private: const auto* memory = ScalarReadMemory(*read, memory_index); Value index; Value count; + uint32_t workgroup_axis = UINT32_MAX; DescriptorSource count_source; uint32_t row_stride = 0; uint32_t offset = 0, stride = 0; @@ -2212,19 +2213,21 @@ private: if (MatchTableOffset(read->Arg(1), index, offset, stride) && ((offset | stride | memory->offset) & 3u) == 0u && uint64_t {offset} + memory->offset <= UINT32_MAX) { + workgroup_axis = WorkgroupAxis(index); const auto* bound = BoundedLoop(index, read->Parent()); if (bound != nullptr && bound->GetOpcode() == ValueOpcode::ULessThan32) { count = bound->Arg(1); } } // Raw pointers have no V# bounds for enumerating an unbounded selector domain. - if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty()) return false; - const bool rows = count.IsEmpty() && memory->offset == 0u && + if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty() && + workgroup_axis == UINT32_MAX) return false; + const bool rows = count.IsEmpty() && workgroup_axis == UINT32_MAX && memory->offset == 0u && MatchCountedRows(*read, count_source, row_stride); if (rows) { stride = 4u; offset = 0u; - } else if (count.IsEmpty()) { + } else if (count.IsEmpty() && workgroup_axis == UINT32_MAX) { auto address = read->Arg(1); auto active = Value(true); const auto* lane = address.Resolve().TryInstruction(); @@ -2255,17 +2258,18 @@ private: } // A bounded image heap already supports GPU selection. Only enumerate a // narrower proven selector domain instead of scanning unrelated material data. - if (require_bound && count.IsEmpty() && !rows) return false; + if (require_bound && count.IsEmpty() && workgroup_axis == UINT32_MAX && !rows) return false; if (!MakeRuntimeTableSource(*read, material_source)) return false; selector.source = InternSource(material_source); selector.stride = stride; selector.offset = offset + (memory->offset & ~3u); indirect.selector = std::move(selector); indirect.key_count = count; + indirect.workgroup_axis = workgroup_axis; if (rows) indirect.selector->rows.emplace( DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows { .source = InternSource(count_source), .stride = row_stride}); - if (!count.IsEmpty()) indirect.selector_first = Value(0u); + if (!count.IsEmpty() || workgroup_axis != UINT32_MAX) indirect.selector_first = Value(0u); return true; } @@ -3025,6 +3029,11 @@ private: indirect.table_scalar = true; return finish(); } + DescriptorSource material_source; + if (!plan.split_offsets && MatchScalarMaterialKey(key, indirect, material_source, true)) { + plan.retain_reads = true; + return finish(); + } if (!legacy_store) { if (table_source.dword_count == 2u && !plan.split_offsets) { const auto maximum = SelectorMaximum(key); @@ -3082,7 +3091,6 @@ private: !ImmediateU32(read->Arg(2), offset) || offset != 0u || !ImmediateU32(read->Arg(3), scalar) || scalar != 0u || !EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false; - DescriptorSource material_source; if (!MakeRuntimeTableSource(*read, material_source) || !BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false; indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source), diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 9b9192625..e530b770f 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -19,6 +19,7 @@ struct SrtRuntime { void* userdata = nullptr; SrtMemoryReader read_specialization_memory = nullptr; std::span workgroup_counts; + uint64_t indirect_args_address = 0; }; class SrtReadCapture { diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index deff9fc60..9f2544109 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -578,11 +578,13 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions; const auto host_subgroup_size = info.host_subgroup_size; const auto workgroup_counts = std::to_array(info.workgroup_counts); + const auto indirect_args_address = info.indirect_args_address; info = {}; info.async_compute = async_compute; info.dispatch_thread_dimensions = dispatch_thread_dimensions; info.host_subgroup_size = host_subgroup_size; std::ranges::copy(workgroup_counts, info.workgroup_counts); + info.indirect_args_address = indirect_args_address; info.threads_num[0] = regs.cs_regs.num_thread_x; info.threads_num[1] = regs.cs_regs.num_thread_y; info.threads_num[2] = regs.cs_regs.num_thread_z; diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 39a4aa40b..e5b8285bc 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -150,6 +150,7 @@ struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo { uint8_t float_mode = 0xc0; uint32_t dispatch_threads_num[3] = {0, 0, 0}; uint32_t workgroup_counts[3] = {0, 0, 0}; + uint64_t indirect_args_address = 0; bool group_id[3] = {false, false, false}; bool dispatch_thread_dimensions = false; bool lds_storage = false; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index e098213e8..0cc349afe 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -222,7 +222,8 @@ MakeIndirectImageFixture(bool malformed, uint32_t table_stride = 32, uint32_t ta ValueOpcode::GetBuiltin, {Value(static_cast(finite_key ? StageInputKind::WorkgroupId : StageInputKind::GlobalInvocationId)), Value(0u)}); - const auto selector = finite_key ? invocation + const auto selector = finite_key + ? fixture->Emit(ValueOpcode::BitwiseXor32, {invocation, Value(1u)}) : fixture->Emit(ValueOpcode::ReadFirstLane, {invocation, Value(true)}); Value member; if (nested) { @@ -2699,7 +2700,7 @@ void TestUniformizedMaterialImageKeys() { memory.fail_address = base + 16u * 16u + 8u; memory.reads = memory.descriptor_reads = 0u; Check(MaterializeResources(packed, runtime, snapshot, specialization) && - memory.reads == 19u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u && + memory.reads == 4u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u && snapshot.images[0].dwords[0] == 0x200u && snapshot.images[1].dwords[0] == 0x400u && snapshot.images[2].dwords[0] == 0x600u, "packed selector did not extract before clamping or read outside its sixteen records"); @@ -2719,7 +2720,7 @@ void TestUniformizedMaterialImageKeys() { snapshot.images.size() == 3u && snapshot.images[0].dwords[0] == 0x200u && snapshot.images[1].dwords[0] == 0x400u && snapshot.images[2].dwords[0] == 0x600u && std::ranges::find(snapshot.specialization_reads, - std::pair{base + base_offset + immediate, 4u}) != + std::pair{base + base_offset + immediate, 244u}) != snapshot.specialization_reads.end(), "global selector did not preserve exact base-plus-offset while scalar descriptor reads aligned the base"); } @@ -2731,7 +2732,7 @@ void TestUniformizedMaterialImageKeys() { memory.fail_address = base + 16u * 16u + 8u; memory.reads = memory.descriptor_reads = 0u; Check(MaterializeResources(merged, runtime, snapshot, specialization) && - memory.reads == 19u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u && + memory.reads == 4u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u && snapshot.images[0].dwords[0] == 0x200u && snapshot.images[1].dwords[0] == 0x400u && snapshot.images[2].dwords[0] == 0x600u, "merged nibble selector did not preserve its sixteen-record domain and lower-half projection"); @@ -2978,12 +2979,14 @@ void TestScalarBufferSelectorDomain() { ResourceSnapshot snapshot; ResourceSpecialization specialization; memory.watched_address = 0x1000u + 144u; - memory.fail_address = 0x1000u + 160u; Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 3u && memory.watched_reads == 1u && - memory.watched_dwords == 1u && snapshot.images[0].dwords[0] == 0x100u && + memory.watched_dwords == 9u && snapshot.images[0].dwords[0] == 0x100u && snapshot.images[2].dwords[0] == 0x102u, - "scalar selector read outside its wrapped domain or omitted the OOB zero key"); + "scalar selector interpreted batched padding or omitted the OOB zero key"); + Check(std::ranges::find(snapshot.specialization_reads, + std::pair{0x1090u, 36u}) != snapshot.specialization_reads.end(), + "scalar selector batch escaped specialization overlap tracking"); const auto scaled_lane = make_plan(true, false, true); memory.words[176u / 4u] = UINT32_MAX; Check(MaterializeResources(scaled_lane, runtime, snapshot, specialization) && @@ -3723,7 +3726,7 @@ void TestSharedAppendSelectorBounds() { Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 1u && memory.descriptor_reads == 0u && memory.watched_reads == 0u, "empty shared append lists probed material keys or image descriptors"); - memory.fail_address = 0x10b0u; + memory.fail_address = UINT64_MAX; for (const auto counts : {std::array{1u, 2u}, std::array{3u, 1u}, std::array{1u, 1u}, std::array{0u, 0u}}) { memory.words[0x800u / 4u] = counts[0]; memory.words[0x804u / 4u] = counts[1]; @@ -3778,10 +3781,13 @@ void TestSharedAppendSelectorBounds() { words[0] = memory.counts[(address - 0x40000000u) / 4u]; return true; } - if (words.size() == 1u && address >= 0x1090u && - (address - 0x1090u) % 160u == 0u && (address - 0x1090u) / 160u < 65536u) { - words[0] = 1u; - ++memory.keys_read; + if (address >= 0x1090u && (address - 0x1090u) % 160u == 0u && + address - 0x1090u + words.size_bytes() <= 65535u * 160u + 4u) { + std::ranges::fill(words, 0xdeadbeefu); + for (size_t word = 0; word < words.size(); word += 40u) { + words[word] = 1u; + ++memory.keys_read; + } return true; } if (words.size() != 8u || address != 0x50000020u) return false; @@ -4784,7 +4790,8 @@ void TestBoundedFormattedBufferStore() { std::unique_ptr MakeBufferRecordFixture(uint32_t step = 1u, bool bypass = false, bool gpu_selected = false, uint32_t bias = 0u, - bool split_offsets = false) { + bool split_offsets = false, + bool workgroup_selected = false) { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; auto fixture = std::make_unique(); auto& f = *fixture; @@ -4796,8 +4803,8 @@ std::unique_ptr MakeBufferRecordFixture(uint32_t step = 1u, bool bypass if (gpu_selected) { std::array selector_words; for (uint32_t word = 0; word < 4u; ++word) selector_words[word] = f.UserData(4u + word); - const auto group = f.Emit(ValueOpcode::GetBuiltin, - {Value(static_cast(StageInputKind::WorkgroupId)), Value(0u)}); + const auto group = workgroup_selected ? f.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::WorkgroupId)), Value(0u)}) : lane; key = f.Emit(ValueOpcode::ReadConstBuffer, {f.Buffer(selector_words), f.Emit(ValueOpcode::IMul32, {group, Value(12u)})}, f.AddMemory({.kind = ResourceKind::ScalarBuffer}, 0x1bcu)); @@ -5090,6 +5097,143 @@ void TestGpuSelectedBufferRecords() { "not a valid runtime value", "split dynamic descriptor offsets lost independent U32 wrapping"); } +void TestWorkgroupSelectedBufferRecords() { + namespace Prospero = Libs::Graphics::Prospero; + auto fixture = MakeBufferRecordFixture(1u, false, true, 0u, false, true); + auto plan = ExtractResourcePlan(fixture->program); + std::vector roots; + for (uint32_t root = 0; root < plan.info.buffers.size(); ++root) { + const auto& source = plan.descriptor_sources[plan.info.buffers[root].source]; + if (!source.indirect_descriptor) continue; + const auto& indirect = *source.indirect_descriptor; + Check(indirect.workgroup_axis == 0u && indirect.selector_first == Value(0u) && + indirect.selector && indirect.selector->stride == 12u && + indirect.table_stride == 192u && !indirect.table_scalar, + "workgroup selector lost its exact prefix or descriptor geometry"); + roots.push_back(root); + } + Check(roots.size() == 2u, "workgroup selector did not retain both buffer operations"); + LinearTestMemory memory; + constexpr uint32_t table = 256u, selector = 0x1000u; + memory.words.resize((selector + MaxIndirectDescriptorProbes * 12u) / 4u); + std::fill_n(memory.words.begin() + table / 4u, 384u / 4u, 0xdeadbeefu); + for (uint32_t row = 0; row < 2u; ++row) { + for (uint32_t member = 0; member < 2u; ++member) { + const auto first = (table + row * 192u + 160u + member * 16u) / 4u; + memory.words[first] = 0x100000u + row * 0x1000u + member * 0x10000u; + memory.words[first + 1u] = (member == 0u ? 8u : 1u) << 16u; + memory.words[first + 2u] = 16u; + memory.words[first + 3u] = member == 0u + ? (uint32_t(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | Libs::Graphics::DstSel(4, 5, 6, 7) + : 0x5204u; + } + } + const std::array numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au}; + std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), memory.words.begin() + (table + 304u) / 4u); + std::array user_data{uint32_t(memory.base) + table, 192u << 16u, 2u, 0x5204u, + uint32_t(memory.base) + selector, 12u << 16u, MaxIndirectDescriptorProbes, 0x5204u}; + memory.watched_address = memory.base; + SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, + .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory, + .indirect_args_address = memory.base}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const auto materialize = [&] { + memory.reads = memory.watched_reads = 0u; + return MaterializeResources(plan, runtime, snapshot, specialization); + }; + for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u}, + std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u}, + std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) { + std::copy(dimensions.begin(), dimensions.end(), memory.words.begin()); + memory.words[selector / 4u] = dimensions[0] == 1u ? 0u : 1u; + memory.words[selector / 4u + 3u] = 0u; + memory.words[selector / 4u + 6u] = 22369622u; // Would select the numeric fields at +288/+304. + const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end(); + Check(materialize() && memory.watched_reads == 1u && memory.watched_dwords == 3u && + snapshot.buffers.size() == plan.info.buffers.size() + (empty ? 0u : (dimensions[0] - 1u) * 2u) && + memory.reads == (empty ? 1u : 3u + dimensions[0] * 2u), + "workgroup prefix reread arguments, visited an inactive key or retained stale candidates"); + Check(std::ranges::find(snapshot.specialization_reads, std::pair{memory.base, uint64_t{12u}}) != + snapshot.specialization_reads.end(), + "indirect dispatch arguments escaped writable-alias tracking"); + if (!empty) + Check(std::ranges::find(snapshot.specialization_reads, + std::pair{memory.base + selector, uint64_t{(dimensions[0] - 1u) * 12u + 4u}}) != + snapshot.specialization_reads.end(), + "batched selector prefix escaped writable-alias tracking"); + } + const auto valid = specialization; + memory.words[selector / 4u] = 0x40000001u; // key*192 wraps to the valid second row. + Check(materialize() && specialization == valid, + "GPU descriptor key lost its native U32 multiplication wrap"); + memory.words[selector / 4u] = 22369622u; + Check(!materialize(), "an actually selected numeric descriptor was silently filtered"); + memory.words[selector / 4u] = 1u; + user_data[5] = 0u; + user_data[6] = 14u; // Second key has fewer than four in-bounds bytes: SMEM returns zero. + Check(materialize() && snapshot.buffers.size() == plan.info.buffers.size() + 2u, + "partial scalar selector did not zero-fill its OOB DWORD"); + user_data[6] = 0u; + Check(materialize() && snapshot.buffers.size() == plan.info.buffers.size() && + memory.reads == 3u, + "empty selector V# did not select key zero without reading payload"); + // Existing vector selectors use the V# stride; each scalar snapshot read + // independently rounds its resulting byte offset down to a DWORD. + for (const uint32_t stride : {5u, 6u}) { + for (const auto root : roots) { + auto& indirect = *plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor; + indirect.selector->stride = 0u; + indirect.selector_first = Value(1u); + } + user_data[5] = stride << 16u; + user_data[6] = 5u; + memory.words[0] = 4u; + std::fill_n(memory.words.begin() + selector / 4u, 8u, 0xdeadbeefu); + for (uint32_t index = 0; index < 4u; ++index) + memory.words[selector / 4u + ((index + 1u) * stride) / 4u] = index & 1u; + Check(materialize() && memory.reads == 7u && + snapshot.buffers.size() == plan.info.buffers.size() + 2u, + "batched vector selector changed independent scalar DWORD alignment"); + } + for (const auto root : roots) { + auto& indirect = *plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor; + indirect.selector->stride = 12u; + indirect.selector_first = Value(0u); + } + user_data[5] = 12u << 16u; + user_data[6] = MaxIndirectDescriptorProbes; + std::fill(memory.words.begin() + selector / 4u, memory.words.end(), 0u); + memory.words[0] = MaxIndirectDescriptorProbes; + Check(materialize() && memory.reads == 9u, + "strided selector prefix was not read in bounded contiguous batches"); + memory.words[0] = MaxIndirectDescriptorProbes + 1u; + Check(!materialize() && memory.reads == 1u, + "selector cap was checked after reading its GPU payload"); + memory.words[0] = 2u; + for (const auto root : roots) + plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor->selector->offset = UINT32_MAX - 3u; + Check(!materialize() && memory.reads == 1u, + "selector offset arithmetic accepted a wrapped prefix span"); + for (const auto root : roots) + plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor->selector->offset = 0u; + runtime.indirect_args_address = 0u; + const std::array direct_counts{1u, 1u, 1u}; + runtime.workgroup_counts = std::span(direct_counts).first(1u); + Check(!materialize() && memory.reads == 0u, + "workgroup prefix accepted an incomplete dispatch extent"); + runtime.workgroup_counts = direct_counts; + memory.fail_address = memory.base; + Check(materialize() && memory.watched_reads == 0u, + "direct workgroup counts unnecessarily read indirect arguments"); + const auto ordinary = MakeBufferRecordFixture(1u, false, true); + const auto ordinary_plan = ExtractResourcePlan(ordinary->program); + runtime.indirect_args_address = memory.base; + user_data[2] = 0u; + Check(MaterializeResources(ordinary_plan, runtime, snapshot, specialization), + "a plan without workgroup counts read the indirect argument pointer"); +} + void TestGpuSelectedFormattedRecordStore() { namespace Prospero = Libs::Graphics::Prospero; const auto make = [](uint32_t stride = 592u, uint32_t shift = 16u, bool split = false, @@ -7024,6 +7168,7 @@ int main() { Run("bounded formatted buffer store", TestBoundedFormattedBufferStore); Run("bounded buffer records", TestBoundedBufferRecords); Run("GPU-selected buffer records", TestGpuSelectedBufferRecords); + Run("workgroup-selected buffer records", TestWorkgroupSelectedBufferRecords); Run("GPU-selected formatted record store", TestGpuSelectedFormattedRecordStore); Run("finite buffer selection", TestFiniteBufferSelection); Run("packed scalar buffer selection", TestPackedScalarBufferSelection); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 2b7d66992..d576b5844 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1684,12 +1684,14 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) { ShaderRecompiler::IR::ExtractResourcePlan(translated.program); ShaderRecompiler::IR::ResourceSnapshot resources; ShaderRecompiler::IR::ResourceSpecialization specialization; + const std::array workgroup_counts{test.dispatch_x, test.dispatch_y, test.dispatch_z}; const ShaderRecompiler::IR::SrtRuntime runtime{ .user_data = options.user_data, .shader_base = reinterpret_cast(test.code.data()), .read_memory = ReadTestMemory, .userdata = const_cast *>(&test.initial), .read_specialization_memory = ReadTestMemory, + .workgroup_counts = workgroup_counts, }; Require(test.name, "resource materialization", ShaderRecompiler::IR::MaterializeResources( @@ -10960,6 +10962,128 @@ public: } } + // GPU-written dispatch counts bound the stride12 selector prefix; only + // those keys select the two real descriptor members in each stride192 row. + constexpr auto record_args = base + 0xd0000; + constexpr auto record_keys = base + 0xd1000; + constexpr auto record_table = base + 0xd2000; + constexpr auto record_output = base + 0xd7000; + std::vector record_shader{ + EncodeSop2(0x26, 16, 13, InlineU32(12)), + EncodeSmem0(0x08, 16, 0), EncodeSmem1(0, 16), EncodeSopp(0x0c, 0), + EncodeSop2(0x26, 17, 16, 255), 192u, + EncodeSmem0(0x0a, 20, 2), EncodeSmem1(160, 17), + EncodeSmem0(0x0a, 24, 2), EncodeSmem1(176, 17), EncodeSopp(0x0c, 0), + EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 5, 0), + EncodeSopp(0x0c, 0)}; + AppendVop3(&record_shader, 0x346, 8, 13, 12, Vgpr(0)); + record_shader.insert(record_shader.end(), { + EncodeMubuf0(0x1d, 0, true, false), EncodeMubuf1(4, 2, 8)}); + AppendVMovLiteral(&record_shader, 10, 0x5au); + AppendSMovLiteral(&record_shader, 126, 0x55555555u); + AppendSMovLiteral(&record_shader, 127, 0x55555555u); + record_shader.insert(record_shader.end(), { + EncodeMubuf0(0x18, 0, true, false, true), EncodeMubuf1(10, 6, 0)}); + AppendEnd(&record_shader); + ShaderMapUserData(reinterpret_cast(record_shader.data()), + {.type = Prospero::ShaderBinaryType::kCs, + .code_size_bytes = static_cast(record_shader.size() * sizeof(u32))}); + std::array records; + records.fill(0xdeadbeefu); + const std::array numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au}; + std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), records.begin() + 76u); + for (u32 row = 0; row < 2u; ++row) { + const auto payload = base + 0xd3000 + row * 0x1000; + const auto target = base + 0xd5000 + row * 0x1000; + const std::array descriptors{ + static_cast(payload), static_cast(payload >> 32u) | (8u << 16u), 4u, + (u32(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | DstSel(4, 5, 6, 7), + static_cast(target), static_cast(target >> 32u) | (1u << 16u), 4u, 0x5204u}; + std::copy(descriptors.begin(), descriptors.end(), records.begin() + row * 48u + 40u); + const std::array data{0x40003c00u + row, 0xc0003800u + row, + 0x44004200u + row, 0xc400bc00u + row, 0x48004600u + row, 0xc800c600u + row, + 0x4c004a00u + row, 0xcc00ca00u + row}; + LibKernel::Memory::WriteBacking(payload, data.data(), sizeof(data)); + } + LibKernel::Memory::WriteBacking(record_table, records.data(), sizeof(records)); + const auto produce = [&](uint64_t address, u32 stride, std::array values) { + shaders.SetCsShader({.data_addr = reinterpret_cast(writer.data()), + .num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1, + .wave_size = 64, .user_sgpr = 15}); + for (u32 word = 0; word < 3u; ++word) { + set_buffer(word * 4u, address + word * stride, 4u); + shaders.SetCsUserSgpr(12u + word, values[word], HW::UserSgprType::Unknown); + } + processor.DispatchDirect(1, 1, 1, 0x41u); + }; + for (const u32 wave : {32u, 64u}) { + for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u}, + std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u}, + std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) { + const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end(); + const std::array stale_args{}; + std::array stale_keys; + stale_keys.fill(sentinel); + Require(name, "record fixture reset", context.InvalidateMemory(record_args, 0x8000), + "record fixture escaped its mapped allocation"); + LibKernel::Memory::WriteBacking(record_args, stale_args.data(), sizeof(stale_args)); + LibKernel::Memory::WriteBacking(record_keys, stale_keys.data(), sizeof(stale_keys)); + std::vector expected_output(wave * 4u, sentinel); + LibKernel::Memory::WriteBacking(record_output, expected_output.data(), expected_output.size() * 4u); + const std::array untouched{sentinel, sentinel, sentinel, sentinel}; + for (u32 row = 0; row < 2u; ++row) + LibKernel::Memory::WriteBacking(base + 0xd5000 + row * 0x1000, untouched.data(), sizeof(untouched)); + produce(record_args, 4u, dimensions); + produce(record_keys, 12u, {1u, 0u, 22369622u}); + std::array cpu_args{}; + std::array cpu_keys{}; + Require(name, "GPU record dependencies remain pending", + cache.HasGpuDirtyBytes(record_args, 12u) && cache.HasGpuDirtyBytes(record_keys, 16u) && + LibKernel::Memory::TryReadBacking(record_args, cpu_args.data(), sizeof(cpu_args)) && + LibKernel::Memory::TryReadBacking(record_keys, cpu_keys.data(), sizeof(cpu_keys)) && + cpu_args == stale_args && cpu_keys == stale_keys, + "the producer did not leave dispatch counts and selectors GPU-owned"); + shaders.SetCsShader({.data_addr = reinterpret_cast(record_shader.data()), + .num_thread_x = wave, .num_thread_y = 1, .num_thread_z = 1, + .user_sgpr = 13, .tgid_x_en = true}); + set_buffer(0, record_keys, 3u, 12u); + set_buffer(4, record_table, 2u, 192u); + set_buffer(8, record_output, wave * 2u, 8u); + shaders.SetCsUserSgpr(12, std::countr_zero(wave), HW::UserSgprType::Unknown); + processor.DispatchIndirect(record_args, 0x41u | + (wave == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u)); + cache.ReadMemory(record_output, expected_output.size() * 4u); + std::vector actual_output(expected_output.size()); + Require(name, "record output readback", LibKernel::Memory::TryReadBacking( + record_output, actual_output.data(), actual_output.size() * 4u), + "indirect record output could not be read back"); + if (!empty) + for (u32 group = 0; group < dimensions[0]; ++group) { + const auto row = 1u - group; + for (u32 lane = 0; lane < wave; ++lane) { + const auto out = (group * wave + lane) * 2u; + expected_output[out] = expected_output[out + 1u] = 0u; + if (lane < 4u) + Require(name, "unchanged formatted input", LibKernel::Memory::TryReadBacking( + base + 0xd3000 + row * 0x1000 + lane * 8u, expected_output.data() + out, 8u), + "the formatted source escaped its mapped allocation"); + } + } + Require(name, "coherent exact workgroup selector prefix", actual_output == expected_output, + "GPU counts or selectors were stale, over-enumerated, or selected the wrong record"); + for (u32 row = 0; row < 2u; ++row) { + const auto target = base + 0xd5000 + row * 0x1000; + cache.ReadMemory(target, sizeof(untouched)); + auto expected = untouched; + if (!empty && (dimensions[0] == 2u || row == 1u)) expected[0] = 0xa55aa55au; + std::array actual{}; + Require(name, "selected byte store and canaries", LibKernel::Memory::TryReadBacking( + target, actual.data(), sizeof(actual)) && actual == expected, + "masked/OOB record stores changed an inactive row or adjacent bytes"); + } + } + } + // Exchange values across two waves in the last 512 bytes of each // workgroup's 64-KiB LDS, on a host restricted to 48-KiB native shared memory. constexpr auto lds_output = base + 0xb0000; @@ -37009,8 +37133,10 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { test.dispatch_x = keys.size(); test.bda_mappings = {{0u, 0u}}; auto &code = test.code; - code.insert(code.end(), {EncodeSop2(0x26, 32, 24, InlineU32(12)), - EncodeSmem0(0x08, 32, 0), EncodeSmem1(0u, 32), EncodeSopp(0x0c, 0), + // A vector load keeps this regression on the unbounded affine-table path. + code.insert(code.end(), {EncodeVop1(0x01, 2, 24), + EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(3, 0, 2), + EncodeSopp(0x0c, 0), EncodeVop1(0x02, 32, Vgpr(3)), EncodeSop2(0x26, 33, 32, 255), 192u, EncodeSop2(0x02, 33, 33, 255), 96u, EncodeSmem0(0x0a, 36, 2), EncodeSmem1(144u, 33), @@ -37113,6 +37239,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) { for (const bool wide : {false, true}) for (const u32 wave : {32u, 64u}) { const std::array keys{wide ? 31u : 5u, 0u, 3u, 1u, 4u, 2u, 6u}; + const auto selector_index = [wide](u32 group) { return wide ? group ^ 1u : group; }; TestCase test; test.name = wide ? "GpuSelectedFormattedRgba16Store" : "GpuSelectedFormattedRecordStore"; test.initial.assign((slot * 7u + keys.size() * wave * 4u) / 4u, 0xdeadbeefu); @@ -37136,7 +37263,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) { records[0] = table; std::copy_n(records.begin(), 4, test.user_data.begin() + 32u); if (wide) { test.user_data[32] = table - 48u; test.user_data[33] = 0u; } - auto selector_buffer = MakeStructuredStorageBufferData(0u, keys.size() * 4u); + auto selector_buffer = MakeStructuredStorageBufferData(0u, (keys.size() + u32(wide)) * 4u); selector_buffer[0] = selectors; std::copy_n(selector_buffer.begin(), 4, test.user_data.begin() + 4u); auto output = MakeStructuredStorageBufferData(0u, keys.size() * wave * 4u); @@ -37152,7 +37279,10 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) { test.compute_info.workgroup_register = 44u; test.dispatch_x = keys.size(); auto &code = test.code; - code.push_back(EncodeSop2(0x1e, 23, 44, InlineU32(2))); + // Keep the finite32-key proof independent of workgroup-prefix materialization: + // this fixture changes GPU keys after compiling its complete candidate domain. + if (wide) code.push_back(EncodeSop2(0x12, 23, 44, InlineU32(1))); + code.push_back(EncodeSop2(0x1e, 23, wide ? 23u : 44u, InlineU32(2))); if (wide) { code.insert(code.end(), {EncodeSmem0(0x08, 22, 2), EncodeSmem1(0, 23), EncodeSop2(0x27, 22, 22, 255), (5u << 16u) | 16u, @@ -37219,7 +37349,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) { producer.compute_info.wave_size = 32u; for (u32 group = 0; group < keys.size(); ++group) { AppendVMovLiteral(&producer.code, 1, (keys[group] << 16u) | 0xbeefu); - AppendVMovLiteral(&producer.code, 2, selectors + group * 4u); + AppendVMovLiteral(&producer.code, 2, selectors + selector_index(group) * 4u); AppendBufferStoreDword(&producer.code, 1, 2); } AppendEnd(&producer.code); @@ -37228,7 +37358,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) { auto expected_bytes = std::as_writable_bytes(std::span(test.expected)); for (u32 group = 0; group < keys.size(); ++group) { const auto row = keys[group]; - test.expected[selectors / 4u + group] = (row << 16u) | 0xbeefu; + test.expected[selectors / 4u + selector_index(group)] = (row << 16u) | 0xbeefu; for (u32 lane = 0; lane < wave; ++lane) { const bool valid = wide ? row == 0u || row == 3u : row < 6u; test.expected[slot * 7u / 4u + group * wave + lane] =