diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 88c1ff17c..40103eb01 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -813,6 +813,7 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, const Format::BufferFormatInfo& info, uint32_t data, uint32_t components) { + if (resource.object_pointer == 0u) return; // RDNA2 formatted stores transfer the entire format, zero-filling missing VGPRs. const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info); EmitIfCondition(ctx.state, plan.in_bounds, [&]() { @@ -852,14 +853,15 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst, void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - const auto data = ctx.Arg(inst, inst.NumArgs() - 2); - const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem)); - if (info.type == Format::ComponentType::Unknown) { - StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data); - return; - } - StoreFormattedPrepared(ctx, inst, mem, resource, info, data, 1u); + const auto data = ctx.Arg(inst, inst.NumArgs() - 2); + EmitBufferAccess(ctx, inst, mem, 0u, [&](const auto& selected, const auto& resource) { + const auto info = Format::GetFormatInfo(BufferFormat(ctx, selected)); + if (info.type == Format::ComponentType::Unknown) { + StoreWordPrepared(ctx, inst, RebaseRawComponent(selected, 0u), resource, data); + return; + } + StoreFormattedPrepared(ctx, inst, selected, resource, info, data, 1u); + }); }); } diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 81f69588b..737399ea6 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -249,11 +249,18 @@ bool NormalizeIndirectBuffer(DescriptorValue& value, bool indexed, const BufferR descriptor.OutOfBounds() != (indexed ? 0u : 3u)) { return SpecializationFail("indirect buffers require exact linear index or byte bounds"); } - if (formatted_read) { + if (resource.formatted) { const auto format = Format::GetFormatInfo(descriptor.Format()); if (!indexed || format.type == Format::ComponentType::Unknown) - return SpecializationFail("indirect formatted loads require a supported indexed format"); - if (descriptor.Base48() == 0u || descriptor.Stride() == 0u || descriptor.NumRecords() == 0u) { + return SpecializationFail("indirect formatted buffers require a supported indexed format"); + if (!formatted_read && + ((descriptor.Format() != Prospero::BufferFormat::k32UInt && + descriptor.Format() != Prospero::BufferFormat::k8SInt) || + descriptor.DstSelXYZW() != DstSel(4, 0, 0, 0) || + descriptor.Stride() != format.byte_size)) + return SpecializationFail("indirect formatted X stores require tight SInt8 or UInt32 identity records"); + if (formatted_read && + (descriptor.Base48() == 0u || descriptor.Stride() == 0u || descriptor.NumRecords() == 0u)) { // Every index is out of bounds. Preserve the format and swizzle constants. value.dwords[1] &= ~(0x3fffu << 16u); value.dwords[2] = 0u; @@ -263,14 +270,10 @@ bool NormalizeIndirectBuffer(DescriptorValue& value, bool indexed, const BufferR if (descriptor.Stride() < format.byte_size || (descriptor.Stride() & (alignment - 1u)) != 0u || (descriptor.Base48() & (alignment - 1u)) != 0u) - return SpecializationFail("indirect formatted loads require aligned full-format records"); + return SpecializationFail("indirect formatted buffers require aligned full-format records"); } else if ((!indexed && descriptor.Stride() != 1u) || (indexed && resource.max_byte_extent != 1u && descriptor.Stride() != 4u)) { return SpecializationFail("indirect buffer stores require exact linear DWORD or byte bounds"); - } else if (resource.formatted && (descriptor.Format() != Prospero::BufferFormat::k32UInt || - descriptor.DstSelXYZW() != DstSel(4, 0, 0, 0) || - (descriptor.Base48() & 3u) != 0u)) { - return SpecializationFail("indirect formatted X stores require an aligned 32-bit UInt identity mapping"); } return true; } @@ -1614,16 +1617,6 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& inst.SetFlags(image_remap[inst.Flags()]); continue; } - if (inst.GetOpcode() == ValueOpcode::StoreBufferU32) { - auto& memory = memory_info[inst.Flags().index]; - if (memory.kind == ResourceKind::Buffer && memory.formatted && - program.descriptor_sources[buffers[memory.resource].source].indirect_descriptor) { - // Candidate validation proved an index-only k32UInt identity store. - // Reuse finite raw-store dispatch and its original-index bounds check. - memory.formatted = false; - } - continue; - } if (BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read) { const auto& memory = memory_info[inst.Flags().index]; const bool native_oob = memory.kind == ResourceKind::Buffer && diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 118162ccc..51e349b72 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -400,7 +400,8 @@ public: std::vector> buffer_selections; for (const auto& plan: m_indirect_descriptors) { auto key = plan.key; - if (m_sources[plan.source].indirect_descriptor->table_maximum) { + if (m_sources[plan.source].indirect_descriptor->table_maximum && + plan.handle->GetOpcode() != ValueOpcode::GetBufferResource) { auto* block = plan.handle->Parent(); const auto where = std::ranges::find_if(block->Instructions(), [&](const Inst& inst) { return &inst == plan.handle; }); @@ -481,7 +482,10 @@ public: const auto where = std::ranges::find_if(block->Instructions(), [&](const Inst& inst) { return &inst == anchor; }); const auto& indirect = *m_sources[plan.source].indirect_descriptor; - if (indirect.table_scalar) { + if (indirect.table_maximum) { + key = Value(&*block->PrependNewInst( + where, ValueOpcode::ShiftLeftLogical32, {key, Value(2u)})); + } else if (indirect.table_scalar) { // Wrapped affine offsets span one power-of-two congruence class. // Keep the existing mapping's byte key while compressing its empty gaps. key = plan.reads[0]->Arg(1); @@ -508,7 +512,7 @@ public: plan.handle->SetArg(dword, plan.key); } } - if (plan.reads[0] != nullptr) { + if (plan.reads[0] != nullptr && !plan.retain_reads) { for (uint32_t word = 0; word < plan.handle->NumArgs(); ++word) m_program.memory_info[plan.memory[word]].planning_only = true; } @@ -2713,7 +2717,7 @@ private: return !IndirectBufferAccess(*consumer.user); }); })) { - return false; + plan.retain_reads = true; } plan.memory[dword] = memory_index; plan.reads[dword] = read; @@ -2725,6 +2729,15 @@ private: return true; } + bool SetExactTableMaximum(Value key, DescriptorSource::IndirectDescriptor& indirect) { + const auto maximum = SelectorMaximum(key); + if (maximum > UINT32_MAX / 4u || + uint64_t {maximum} * indirect.table_stride + indirect.table_offset > UINT32_MAX) + return false; + indirect.table_maximum = maximum; + return true; + } + bool TryMakeNestedDescriptor(Inst& handle, const DescriptorSource& descriptor, IndirectDescriptorPlan& plan) { if (handle.GetOpcode() != ValueOpcode::GetImageResource && @@ -2769,9 +2782,7 @@ private: DescriptorSource::IndirectDescriptor indirect; if (!MatchTableOffset(low->Arg(1), key, indirect.table_offset, indirect.table_stride) || indirect.table_stride < 8u || (indirect.table_stride & 3u) != 0u) return false; - const auto maximum = SelectorMaximum(key); - if (maximum > UINT32_MAX / 4u || - uint64_t {maximum} * indirect.table_stride + indirect.table_offset > UINT32_MAX || + if (!SetExactTableMaximum(key, indirect) || (key.TryInstruction() != nullptr && !AvailableBefore(*key.Instruction(), handle))) return false; DescriptorSource table_source; @@ -2780,7 +2791,6 @@ private: indirect.table_source = InternSource(table_source); indirect.table_immediate = low_memory->offset & ~3u; indirect.table_scalar = true; - indirect.table_maximum = maximum; indirect.pointer_offset = descriptor_offset; DescriptorSource source; source.dword_count = handle.NumArgs(); @@ -3007,6 +3017,11 @@ private: indirect.signed_count = bound->GetOpcode() == ValueOpcode::SLessThan32; return finish(); } + if (table_source.dword_count == 4u && !plan.split_offsets && + plan.selection_anchor != nullptr && SetExactTableMaximum(key, indirect)) { + indirect.table_scalar = true; + return finish(); + } if (!legacy_store) { // A bounded V# supplies every reachable descriptor without reading the GPU key. // Separate U32 offsets could wrap each DWORD to a different table position. @@ -3300,7 +3315,7 @@ private: bool IsIndirectPlanningMemory(uint32_t index) const { return std::any_of(m_indirect_descriptors.begin(), m_indirect_descriptors.end(), [&](const IndirectDescriptorPlan& plan) { - return plan.reads[0] != nullptr && + return plan.reads[0] != nullptr && !plan.retain_reads && std::find(plan.memory.begin(), plan.memory.begin() + plan.handle->NumArgs(), index) != plan.memory.begin() + plan.handle->NumArgs(); }); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 8658d0834..eb4fa75b7 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -4640,14 +4640,14 @@ void TestBoundedFormattedBufferStore() { memory.fail_address = UINT64_MAX; Check(materialize(), "valid uniform descriptor table did not recover after a failed refresh"); ApplyResourceSpecialization(fixture->program, specialization); - bool lowered = false; + bool formatted = false; for (auto* block : fixture->program.blocks) for (auto& inst : *block) if (inst.GetOpcode() == ValueOpcode::StoreBufferU32) { const auto& info = fixture->program.memory_info[inst.Flags().index]; - lowered = !info.formatted && info.idxen; + formatted = info.formatted && info.idxen; } - Check(lowered, "proven UInt32 formatted store did not reuse the raw DWORD store"); + Check(formatted, "UInt32 candidate store lost the shared formatted store path"); for (const auto [step, bypass, offset] : {std::tuple{2u, false, false}, std::tuple{1u, true, false}, std::tuple{1u, false, true}}) CheckFatal([&] { make(step, bypass, offset); }, "not a valid runtime value", @@ -4962,6 +4962,168 @@ void TestGpuSelectedBufferRecords() { "not a valid runtime value", "split dynamic descriptor offsets lost independent U32 wrapping"); } +void TestGpuSelectedFormattedRecordStore() { + namespace Prospero = Libs::Graphics::Prospero; + const auto make = [](uint32_t stride = 592u, uint32_t shift = 16u, bool split = false) { + auto fixture = std::make_unique(); + auto& f = *fixture; + std::array table_words, selector_words; + for (uint32_t word = 0; word < 4u; ++word) { + table_words[word] = f.UserData(word); + selector_words[word] = f.UserData(4u + word); + } + const auto table = f.Buffer(table_words); + const auto lane = f.Emit(ValueOpcode::LaneId); + const auto packed = f.Emit(ValueOpcode::LoadBufferU32, + {f.Buffer(selector_words), lane, Value(0u), Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .idxen = true}, 0x1bcu)); + const auto key = f.Emit(ValueOpcode::ReadLane, + {f.Emit(ValueOpcode::ShiftRightLogical32, {packed, Value(shift)}), Value(0u)}); + const auto offset = f.Emit(ValueOpcode::IMul32, {key, Value(stride)}); + std::array words; + for (uint32_t word = 0; word < 4u; ++word) { + const auto dynamic = split ? f.Emit(ValueOpcode::IAdd32, {offset, Value(word * 4u)}) : offset; + words[word] = f.Emit(ValueOpcode::ReadConstBuffer, {table, dynamic}, + f.AddMemory({.kind = ResourceKind::ScalarBuffer, + .offset = 448u + (split ? 0u : word * 4u), + .component_index = word, .component_count = 4u}, 0x2e8u)); + } + f.Emit(ValueOpcode::LoadBufferU32, + {f.Buffer(words), lane, Value(0u), Value(0u), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .formatted = true, .idxen = true}, 0x2f8u)); + for (uint32_t use = 0; use < 2u; ++use) + f.Emit(ValueOpcode::StoreBufferU32, + {f.Buffer(words), lane, Value(0u), Value(0u), Value(uint32_t(-127)), Value(true)}, + f.AddMemory({.kind = ResourceKind::Buffer, .formatted = true, + .idxen = true, .coherent = true}, 0x1618u + use * 8u)); + f.PlanAndTrack(); + return fixture; + }; + auto fixture = make(); + const auto plan = ExtractResourcePlan(fixture->program); + uint32_t root = UINT32_MAX; + for (uint32_t resource = 0; resource < plan.info.buffers.size(); ++resource) { + const auto& indirect = plan.descriptor_sources[plan.info.buffers[resource].source].indirect_descriptor; + if (!indirect) continue; + Check(root == UINT32_MAX && indirect->table_scalar && indirect->table_maximum == 65535u && + !indirect->pointer_offset && !indirect->selector && indirect->key_count.IsEmpty() && + indirect->table_stride == 592u && indirect->table_immediate == 448u, + "formatted record store lost its exact rows or captured the GPU selector"); + root = resource; + } + Check(root != UINT32_MAX, "formatted record store has no native candidate root"); + const Inst* selection = nullptr; + uint32_t scalar_reads = 0u, bda_reads = 0u, stores = 0u; + for (const auto* block : fixture->program.blocks) + for (const auto& inst : *block) { + if (inst.GetOpcode() == ValueOpcode::GetBufferResourceIndex) { + Check(selection == nullptr && inst.Flags() == root, + "formatted record stores duplicated selection searches"); + selection = &inst; + const auto* encoded = inst.Arg(0).Resolve().TryInstruction(); + Check(encoded && encoded->GetOpcode() == ValueOpcode::ShiftLeftLogical32 && + encoded->Arg(1) == Value(2u) && + encoded->Arg(0).ResolveInstruction()->GetOpcode() == ValueOpcode::ReadLane, + "exact record selection used an affine wrapped-byte ordinal"); + bool defined = false; + for (const auto& prior : *block) { + if (&prior == &inst) break; + defined |= &prior == encoded; + } + Check(defined, "exact row key was defined after its resource selection"); + } + if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { + const auto& memory = fixture->program.memory_info[inst.Flags().index]; + Check(memory.kind == ResourceKind::ScalarBuffer && !memory.planning_only, + "store planning consumed descriptor words still used by the BDA load"); + ++scalar_reads; + } + if (inst.GetOpcode() == ValueOpcode::LoadBufferU32 && inst.Flags().pc == 0x2f8u) { + const auto& memory = fixture->program.memory_info[inst.Flags().index]; + Check(memory.kind == ResourceKind::IndirectBuffer && memory.formatted, + "store planning replaced the existing unrestricted BDA formatted load"); + ++bda_reads; + } + if (inst.GetOpcode() == ValueOpcode::StoreBufferU32) { + Check(selection && inst.Arg(0).ResolveInstruction()->Arg(0).ResolveInstruction() == selection, + "formatted store lost its dominating shared selection"); + ++stores; + } + } + Check(selection && scalar_reads == 4u && bda_reads == 1u && stores == 2u, + "shared formatted record fixture lost an exercised descriptor consumer"); + LinearTestMemory memory; + std::ranges::fill(memory.words, 0xdeadbeefu); + constexpr uint32_t table_offset = 256u; + constexpr uint32_t first = (table_offset + 448u) / 4u; + for (uint32_t row = 0; row < 6u; ++row) { + const auto word = first + row * 592u / 4u; + memory.words[word] = 0x100000u + row * 32u; + memory.words[word + 1u] = 1u << 16u; + memory.words[word + 2u] = 20u; + memory.words[word + 3u] = (uint32_t(Prospero::BufferFormat::k8SInt) << 12u) | 4u; + } + memory.fail_address = memory.base + 0x2000u; + std::array user_data{uint32_t(memory.base) + table_offset, 592u << 16u, 6u, 0x5204u, + uint32_t(memory.fail_address), 4u << 16u, 64u, 0x5204u}; + SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory, + .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const auto materialize = [&] { return MaterializeResources(plan, runtime, snapshot, specialization); }; + for (const uint32_t count : {0u, 6u, 0u, 6u}) { + user_data[2] = count; + memory.reads = 0u; + Check(materialize() && memory.reads == count && + snapshot.buffers.size() == plan.info.buffers.size() + count, + "exact formatted records read GPU selector bytes, row padding or stale candidates"); + } + const auto mapping = specialization.buffers[root].indirect_mapping_offset; + Check(snapshot.flattened_srt[mapping] == 6u, "exact formatted records lost the dense row map"); + for (uint32_t row = 0; row < 6u; ++row) + Check(snapshot.flattened_srt[mapping + 1u + row] == row + 1u && + std::ranges::find(snapshot.specialization_reads, + std::pair{memory.base + table_offset + row * 592u + 448u, uint64_t{16u}}) != + snapshot.specialization_reads.end(), + "exact row descriptor escaped selection mapping or writable-alias capture"); + const auto original = specialization; + memory.words[first] += 1u; + Check(materialize() && specialization == original, + "signed byte destination refresh required alignment or needless specialization"); + for (const auto [word, value] : std::array, 5>{{ + {first + 1u, 2u << 16u}, {first + 1u, (1u << 16u) | (1u << 31u)}, + {first + 3u, (uint32_t(Prospero::BufferFormat::k8UInt) << 12u) | 4u}, + {first + 3u, memory.words[first + 3u] | (1u << 28u)}, + {first + 3u, memory.words[first + 3u] | (4u << 3u)}}}) { + const auto saved = memory.words[word]; + memory.words[word] = value; + Check(!materialize(), "signed byte candidate accepted an unproved format, stride or bounds"); + memory.words[word] = saved; + } + user_data[1] = 0u; + user_data[2] = 5u * 592u + 448u + 12u; + memory.watched_address = memory.base + table_offset + 5u * 592u + 448u; + Check(materialize() && memory.watched_dwords == 3u && + snapshot.flattened_srt[specialization.buffers[root].indirect_mapping_offset] == 5u && + snapshot.buffers.size() == plan.info.buffers.size() + 5u, + "partial direct V# did not zero-fill its missing format DWORD"); + user_data[1] = 592u << 16u; + user_data[2] = 6u; + Check(materialize(), "valid signed byte records did not recover after a partial tail"); + ApplyResourceSpecialization(fixture->program, specialization); + for (const auto* block : fixture->program.blocks) + for (const auto& inst : *block) + if (inst.GetOpcode() == ValueOpcode::StoreBufferU32) { + const auto& info = fixture->program.memory_info[inst.Flags().index]; + Check(info.kind == ResourceKind::Buffer && info.formatted && info.coherent, + "signed byte store bypassed shared formatted conversion or GLC"); + } + for (const auto [stride, shift, split] : + {std::tuple{592u, 0u, false}, std::tuple{65540u, 16u, false}, std::tuple{592u, 16u, true}}) + CheckFatal([&] { make(stride, shift, split); }, "not a valid runtime value", + "exact record store accepted an unbounded, wrapping or split descriptor offset"); +} + void TestFiniteBufferSelection() { enum class Variant { Valid, WiderExec, Unbounded, Cycle, TooMany }; const auto make = [](Variant variant) { @@ -6460,6 +6622,7 @@ int main() { Run("bounded formatted buffer store", TestBoundedFormattedBufferStore); Run("bounded buffer records", TestBoundedBufferRecords); Run("GPU-selected buffer records", TestGpuSelectedBufferRecords); + Run("GPU-selected formatted record store", TestGpuSelectedFormattedRecordStore); Run("finite buffer selection", TestFiniteBufferSelection); Run("packed scalar buffer selection", TestPackedScalarBufferSelection); Run("conditional sampler phi", TestConditionalSamplerPhi); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 8eb09faa8..5ab7c7db3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -36925,6 +36925,122 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { std::printf("[compute] %-32s ok\n", "GpuSelectedRecordBufferAccess"); } +void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) { + constexpr u32 selectors = 128, table = 256, stride = 592, member = 448; + constexpr std::array keys{5u, 0u, 3u, 1u, 4u, 2u, 6u}; + const u32 alignment = vulkan.RuntimeContext().GetPhysicalDeviceProperties() + .limits.minStorageBufferOffsetAlignment; + const u32 slot = (4096u + alignment - 1u) / alignment * alignment; + for (const u32 wave : {32u, 64u}) { + TestCase test; + test.name = "GpuSelectedFormattedRecordStore"; + test.initial.assign((slot * 7u + keys.size() * wave * 4u) / 4u, 0xdeadbeefu); + auto bytes = std::as_writable_bytes(std::span(test.initial)); + for (u32 row = 0; row < 6u; ++row) { + const std::array descriptor{slot * (row + 1u), 1u << 16u, 20u, + (u32(Prospero::BufferFormat::k8SInt) << 12u) | DstSel(4, 0, 0, 0)}; + std::copy(descriptor.begin(), descriptor.end(), + test.initial.begin() + (table + row * stride + member) / 4u); + for (u32 element = 0; element < 20u; ++element) + bytes[slot * (row + 1u) + element] = std::byte(0x80u + row + element); + } + auto records = MakeStructuredStorageBufferData(stride, 6u); + records[0] = table; + std::copy_n(records.begin(), 4, test.user_data.begin() + 32u); + auto selector_buffer = MakeStructuredStorageBufferData(0u, keys.size() * 4u); + selector_buffer[0] = selectors; + std::copy_n(selector_buffer.begin(), 4, test.user_data.begin() + 4u); + auto output = MakeStructuredStorageBufferData(0u, keys.size() * wave * 4u); + output[0] = slot * 7u; + std::copy_n(output.begin(), 4, test.user_data.begin() + 48u); + test.has_user_data = test.has_compute_info = true; + test.bda_mappings = {{0u, 0u}}; + test.compute_info.wave_size = wave; + test.compute_info.threads_num[0] = wave; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1u; + test.compute_info.thread_ids_num = 1u; + test.compute_info.group_id[0] = true; + test.compute_info.workgroup_register = 44u; + test.dispatch_x = keys.size(); + auto &code = test.code; + code.insert(code.end(), {EncodeSop2(0x1e, 23, 44, InlineU32(2)), + EncodeVop1(0x01, 30, 23), EncodeMubuf0(0x0c), EncodeMubuf1(4, 1, 30), + EncodeVop2(0x16, 6, InlineU32(16), 4)}); + AppendVop3(&code, 0x360, 22, Vgpr(6), InlineU32(0)); + code.insert(code.end(), {EncodeSop2(0x26, 23, 22, 255), stride, + EncodeSmem0(0x0a, 8, 16), EncodeSmem1(member, 23), + EncodeMubuf0(0x00, 0, true, false, true), EncodeMubuf1(4, 2, 0), + EncodeSop2(0x1e, 34, 44, InlineU32(std::countr_zero(wave))), + EncodeVop2(0x25, 30, 34, 0), EncodeVop2(0x1a, 30, InlineU32(2), 30)}); + AppendBufferStoreDword(&code, 4, 30); + code.push_back(EncodeVop2(0x1a, 10, InlineU32(5), 0)); + code.insert(code.end(), {EncodeVop2(0x25, 10, 255, 10), u32(-200)}); + AppendSMovLiteral(&code, 126, 0x55555555u); + AppendSMovLiteral(&code, 127, wave == 64u ? 0x55555555u : 0u); + code.insert(code.end(), {EncodeMubuf0(0x04, 0, true, false, true), + EncodeMubuf1(10, 2, 0)}); + AppendEnd(&code); + test.decoded_counts = {{"V_LSHLREV_B32", 2u}, {"BUFFER_STORE_FORMAT_X", 1u}}; + test.required_spirv = {"get_bda_pointer", "convert_indirect_buffer_format"}; + test.forbidden_spirv = {"OpAtomicCompareExchange"}; + const auto compiled = CompileCase(test, vulkan.SubgroupSize()); + u32 bda_loads = 0, native_stores = 0; + for (const auto *block : compiled.program.blocks) + for (const auto &inst : *block) { + using namespace ShaderRecompiler::IR; + if (inst.GetOpcode() != ValueOpcode::LoadBufferU32 && + inst.GetOpcode() != ValueOpcode::StoreBufferU32) continue; + const auto &memory = compiled.program.memory_info[inst.Flags().index]; + if (inst.GetOpcode() == ValueOpcode::LoadBufferU32) + bda_loads += memory.kind == ResourceKind::IndirectBuffer && memory.formatted; + else + native_stores += memory.kind == ResourceKind::Buffer && memory.formatted; + } + Require(test.name, "shared scalar descriptor reads", bda_loads == 1u && native_stores == 1u, + "native store planning replaced the BDA read or bypassed formatted conversion"); + for (const auto &value : compiled.resources.buffers) { + ShaderBufferResource descriptor{}; + std::copy_n(value.dwords.begin(), 4u, descriptor.fields); + test.storage_buffer_ranges.emplace_back(descriptor.Base48(), + std::max(4u, u32(descriptor.GetSize()))); + } + TestCase producer; + producer.name = "FormattedRecordSelectorProducer"; + producer.initial = test.initial; + producer.has_compute_info = true; + producer.compute_info.threads_num[0] = producer.compute_info.threads_num[1] = + producer.compute_info.threads_num[2] = 1u; + producer.compute_info.wave_size = 32u; + for (u32 group = 0; group < keys.size(); ++group) { + AppendVMovLiteral(&producer.code, 1, (keys[group] << 16u) | 0xbeefu); + AppendVMovLiteral(&producer.code, 2, selectors + group * 4u); + AppendBufferStoreDword(&producer.code, 1, 2); + } + AppendEnd(&producer.code); + const auto producer_shader = CompileCase(producer, vulkan.SubgroupSize()); + test.expected = test.initial; + auto expected_bytes = std::as_writable_bytes(std::span(test.expected)); + for (u32 group = 0; group < keys.size(); ++group) { + const auto row = keys[group]; + test.expected[selectors / 4u + group] = (row << 16u) | 0xbeefu; + for (u32 lane = 0; lane < wave; ++lane) { + test.expected[slot * 7u / 4u + group * wave + lane] = + row < 6u && lane < 20u ? u32(int32_t(0x80u + row + lane) - 256) : 0u; + if (row < 6u && lane < 20u && (lane & 1u) == 0u) + expected_bytes[slot * (row + 1u) + lane] = + std::byte(u32(std::clamp(int32_t(lane * 32u) - 200, -128, 127)) & 255u); + } + } + auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true); + vulkan.Dispatch(producer, producer_shader, buffer); + vulkan.Dispatch(test, compiled, buffer); + CompareWords(test, "shared BDA reads, signed byte saturation, EXEC, OOB and neighboring canaries", + test.expected, vulkan.ReadBuffer(test.name, buffer, test.expected.size())); + vulkan.DestroyBuffer(&buffer); + std::printf("[compute] %s wave=%u ok\n", test.name, wave); + } +} + void CheckFiniteBufferSelection(VulkanHarness &vulkan) { for (const u32 wave : {32u, 64u}) { TestCase test; @@ -45934,6 +46050,7 @@ int main(int argc, char **argv) { CheckIndirectBufferStore(vulkan); CheckBoundedRecordBufferAccess(vulkan); CheckGpuSelectedRecordBufferAccess(vulkan); + CheckGpuSelectedFormattedRecordStore(vulkan); CheckRuntimeBufferRecords(vulkan); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors()); @@ -46641,6 +46758,7 @@ int main(int argc, char **argv) { CheckIndirectBufferStore(vulkan); CheckBoundedRecordBufferAccess(vulkan); CheckGpuSelectedRecordBufferAccess(vulkan); + CheckGpuSelectedFormattedRecordStore(vulkan); CheckFiniteBufferSelection(vulkan); CheckRuntimeBufferRecords(vulkan); CheckComputeThreadDimensions(vulkan);