From 6455546f268ab1ed883f8dcf7c333fa7dc3b0e48 Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 14 Oct 2026 22:04:55 +0200 Subject: [PATCH] shader: admit bounded multi-component formatted buffer loads Co-authored-by: fxpw --- .../recompiler/ir/passes/ResourceTracking.cpp | 10 ++- tests/ResourceTrackingTests.cpp | 37 +++++++--- tests/ShaderRecompilerComputeTests.cpp | 72 ++++++++++++++----- 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index dd23cff9f..381d3a107 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -2637,7 +2637,8 @@ private: bool IndirectBufferAccess(const Inst& inst) const { const auto opcode = inst.GetOpcode(); if (opcode != ValueOpcode::StoreBufferU32 && opcode != ValueOpcode::StoreBufferU8 && - opcode != ValueOpcode::StoreBufferU32x4 && opcode != ValueOpcode::LoadBufferU32x2) return false; + opcode != ValueOpcode::StoreBufferU32x4 && opcode != ValueOpcode::LoadBufferU32x2 && + opcode != ValueOpcode::LoadBufferU32x3 && opcode != ValueOpcode::LoadBufferU32x4) return false; const auto index = inst.Flags().index; if (index >= m_program.memory_info.size()) return false; const auto& memory = m_program.memory_info[index]; @@ -2652,8 +2653,11 @@ private: if (opcode == ValueOpcode::StoreBufferU32x4) return memory.formatted && memory.data_bits == 32u && memory.data_dwords == 4u && memory.component_count == 4u; - return memory.formatted && memory.data_bits == 16u && memory.data_dwords == 2u && - memory.component_count == 4u; + const auto components = BufferComponentCount(opcode); + return memory.formatted && memory.data_dwords == components && + ((memory.data_bits == 32u && memory.component_count == components) || + (opcode == ValueOpcode::LoadBufferU32x2 && memory.data_bits == 16u && + memory.component_count == 4u)); } bool MatchDescriptorTable(Inst& handle, const DescriptorSource& descriptor, diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 9f8cb1f45..8d57b3a0d 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -4791,7 +4791,8 @@ void TestBoundedFormattedBufferStore() { std::unique_ptr MakeBufferRecordFixture(uint32_t step = 1u, bool bypass = false, bool gpu_selected = false, uint32_t bias = 0u, bool split_offsets = false, - bool workgroup_selected = false) { + bool workgroup_selected = false, + uint32_t d32_components = 0u) { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; auto fixture = std::make_unique(); auto& f = *fixture; @@ -4851,12 +4852,21 @@ std::unique_ptr MakeBufferRecordFixture(uint32_t step = 1u, bool bypass for (uint32_t use = 0; use < (gpu_selected ? 2u : formatted ? 112u : 16u); ++use) { const auto handle = f.Buffer(words); if (formatted) { - const auto loaded = f.Emit(ValueOpcode::LoadBufferU32x2, + const auto opcode = d32_components == 3u ? ValueOpcode::LoadBufferU32x3 + : d32_components == 4u ? ValueOpcode::LoadBufferU32x4 + : ValueOpcode::LoadBufferU32x2; + const auto extract = d32_components == 3u ? ValueOpcode::CompositeExtractU32x3 + : d32_components == 4u ? ValueOpcode::CompositeExtractU32x4 + : ValueOpcode::CompositeExtractU32x2; + const auto loaded = f.Emit(opcode, {handle, lane, Value(0u), Value(0u), Value(true)}, - f.AddMemory({.kind = ResourceKind::Buffer, .data_dwords = 2u, .data_bits = 16u, - .component_count = 4u, .formatted = true, .idxen = true}, 0x1914u)); + f.AddMemory({.kind = ResourceKind::Buffer, + .data_dwords = d32_components == 0u ? 2u : d32_components, + .data_bits = d32_components == 0u ? 16u : 32u, + .component_count = d32_components == 0u ? 4u : d32_components, + .formatted = true, .idxen = true}, 0x1914u)); f.Emit(ValueOpcode::ReferenceU32, - {f.Emit(ValueOpcode::CompositeExtractU32x2, {loaded, Value(0u)})}); + {f.Emit(extract, {loaded, Value(0u)})}); } else { f.Emit(ValueOpcode::StoreBufferU8, {handle, lane, Value(0u), Value(0u), Value(uint8_t{42u}), Value(true)}, @@ -4868,10 +4878,13 @@ std::unique_ptr MakeBufferRecordFixture(uint32_t step = 1u, bool bypass return fixture; } -void TestBoundedBufferRecords() { +void TestBoundedBufferRecords(uint32_t d32_components) { namespace Prospero = Libs::Graphics::Prospero; using Libs::Graphics::DstSel; - auto fixture = MakeBufferRecordFixture(); + auto fixture = MakeBufferRecordFixture(1u, false, false, 0u, false, false, d32_components); + const auto load_opcode = d32_components == 3u ? ValueOpcode::LoadBufferU32x3 + : d32_components == 4u ? ValueOpcode::LoadBufferU32x4 + : ValueOpcode::LoadBufferU32x2; const auto plan = ExtractResourcePlan(fixture->program); Check(plan.info.buffers.size() == 2u, "record descriptors did not share their resource roots"); uint32_t selections = 0u; @@ -4884,7 +4897,7 @@ void TestBoundedBufferRecords() { "record selection moved outside its loop or lost its buffer root"); selected[inst.Flags()] = &inst; } - if (inst.GetOpcode() == ValueOpcode::LoadBufferU32x2 || inst.GetOpcode() == ValueOpcode::StoreBufferU8) { + if (inst.GetOpcode() == load_opcode || inst.GetOpcode() == ValueOpcode::StoreBufferU8) { const auto root = fixture->program.memory_info[inst.Flags().index].resource; Check(inst.Arg(0).Instruction()->Arg(0).Instruction() == selected[root], "record operation did not use its dominating shared selection"); @@ -4957,7 +4970,7 @@ void TestBoundedBufferRecords() { ApplyResourceSpecialization(fixture->program, specialization); uint32_t loads = 0u; for (const auto* block : fixture->program.blocks) - for (const auto& inst : *block) loads += inst.GetOpcode() == ValueOpcode::LoadBufferU32x2; + for (const auto& inst : *block) loads += inst.GetOpcode() == load_opcode; Check(loads == 112u, "empty root incorrectly folded nonempty formatted candidates"); memory.words[first] = first_address; memory.words[first + 2u] = 16u; @@ -4973,7 +4986,7 @@ void TestBoundedBufferRecords() { memory.words[0] = MaxIndirectDescriptorProbes + 1u; Check(!materialize(), "record table accepted an unbounded candidate count"); for (const auto [step, bypass] : {std::pair{2u, false}, std::pair{1u, true}}) { - const auto unbounded = MakeBufferRecordFixture(step, bypass); + const auto unbounded = MakeBufferRecordFixture(step, bypass, false, 0u, false, false, d32_components); for (const auto& buffer : unbounded->program.info.buffers) { const auto& source = unbounded->program.descriptor_sources[buffer.source]; if (source.indirect_descriptor) @@ -7276,7 +7289,9 @@ int main() { Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); Run("writable descriptor phi", TestWritableDescriptorPhi); Run("bounded formatted buffer store", TestBoundedFormattedBufferStore); - Run("bounded buffer records", TestBoundedBufferRecords); + Run("bounded buffer records", [] { + for (const uint32_t components : {0u, 2u, 3u, 4u}) TestBoundedBufferRecords(components); + }); Run("GPU-selected buffer records", TestGpuSelectedBufferRecords); Run("workgroup-selected buffer records", TestWorkgroupSelectedBufferRecords); Run("GPU-selected formatted record store", TestGpuSelectedFormattedRecordStore); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 0e9284736..4257565fc 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -37380,6 +37380,7 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { // Multiplication by 192 wraps: these keys reach congruence slots 1,4,0,3,2,1,OOB,7,5. constexpr std::array keys{0u, 1u, 0x15555555u, 0x15555556u, 0x2aaaaaabu, 0x40000000u, UINT32_MAX, 2u, 0x2aaaaaacu}; + for (const u32 components : {0u, 2u, 3u, 4u}) for (const u32 wave : {32u, 64u}) { for (const u32 table_bytes : {0u, 536u, 544u}) { TestCase test; @@ -37406,13 +37407,24 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(3, 0, 2), EncodeSopp(0x0c, 0), EncodeVop1(0x02, 32, Vgpr(3)), EncodeSop2(0x26, 33, 32, 255), 192u, - EncodeSop2(0x02, 33, 33, 255), 96u, - EncodeSmem0(0x0a, 36, 2), EncodeSmem1(144u, 33), + EncodeSop2(0x02, 33, 33, 255), 96u}); + if (components == 0u) + code.insert(code.end(), {EncodeSmem0(0x0a, 36, 2), EncodeSmem1(144u, 33)}); + code.insert(code.end(), { EncodeSmem0(0x0a, 8, 2), EncodeSmem1(160u, 33), - EncodeSmem0(0x0a, 12, 2), EncodeSmem1(176u, 33), EncodeSopp(0x0c, 0), + EncodeSmem0(0x0a, 12, 2), EncodeSmem1(176u, 33), EncodeSopp(0x0c, 0)}); + if (components == 0u) { + code.insert(code.end(), { EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 2, 0), EncodeMubuf0(0x0a, 0, true, false), EncodeMubuf1(6, 9, 0), - EncodeMubuf0(0x00, 0, true, false), EncodeMubuf1(7, 9, 0), + EncodeMubuf0(0x00, 0, true, false), EncodeMubuf1(7, 9, 0)}); + } else { + for (u32 component = components; component < 4u; ++component) + AppendVMovLiteral(&code, 4u + component, 0x13579bdfu); + code.insert(code.end(), {EncodeMubuf0(components - 1u, 0, true, false, true), + EncodeMubuf1(4, 2, 0)}); + } + code.insert(code.end(), { EncodeSopp(0x0c, 0), EncodeSop2(0x1e, 34, 24, InlineU32(std::countr_zero(wave))), EncodeVop2(0x25, 20, 34, 0), EncodeMubuf0(0x1e, 0, true, false), EncodeMubuf1(4, 4, 20)}); @@ -37422,18 +37434,32 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { code.insert(code.end(), {EncodeMubuf0(0x19, 0, true, false, true), EncodeMubuf1(10, 3, 20)}); AppendEnd(&code); for (u32 row = 0; row < 6u; ++row) { - const std::array record{slot * (row + 1u), 8u << 16u, 1u, + std::array record{slot * (row + 1u), 8u << 16u, 1u, static_cast(Prospero::BufferFormat::k16_16_16_16Float) << 12u | DstSel(4, 5, 6, 7), slot * (row + 7u), 1u << 16u, u32(keys.size()) * wave, 0x5204u}; + if (components != 0u) { + constexpr std::array formats{Prospero::BufferFormat::k16_16Float, + Prospero::BufferFormat::k16_16_16_16SNorm, Prospero::BufferFormat::k10_10_10_2SNorm}; + const std::array swizzles{DstSel(4, 5, 0, 1), DstSel(7, 4, 1, 0), DstSel(5, 4, 7, 6)}; + record[3] = (u32(formats[row % 3u]) << 12u) | swizzles[row % 3u]; + if (row == 3u) std::fill_n(record.begin(), 4u, 0u); + } std::copy(record.begin(), record.end(), test.initial.begin() + (table + 192u + row * 64u) / 4u); // BDA loads use a separate tuple, as the captured shader's record members do. - std::copy_n(record.begin(), 4u, test.initial.begin() + (table + 176u + row * 64u) / 4u); - test.initial[slot * (row + 1u) / 4u] = ((0x3800u + row) << 16u) | (0x3c00u + row); - test.initial[slot * (row + 1u) / 4u + 1u] = ((0xc000u + row) << 16u) | (0x4000u + row); + if (components == 0u) { + std::copy_n(record.begin(), 4u, test.initial.begin() + (table + 176u + row * 64u) / 4u); + test.initial[slot * (row + 1u) / 4u] = ((0x3800u + row) << 16u) | (0x3c00u + row); + test.initial[slot * (row + 1u) / 4u + 1u] = ((0xc000u + row) << 16u) | (0x4000u + row); + } else { + constexpr std::array, 3> payload{{ + {0x3800bc00u, 0u}, {0x7fff8000u, 0x80010000u}, {0xc007fe00u, 0u}}}; + std::copy(payload[row % 3u].begin(), payload[row % 3u].end(), + test.initial.begin() + slot * (row + 1u) / 4u); + } } // Compile against poison selectors. A separate GPU dispatch writes the actual keys. auto compiled = CompileCase(test, vulkan.SubgroupSize()); - if (table_bytes == 0u) + if (table_bytes == 0u || components != 0u) test.forbidden_spirv = {"convert_indirect_buffer_format"}; else test.required_spirv = {"get_bda_pointer", "convert_indirect_buffer_format"}; @@ -37448,12 +37474,13 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { if (opcode == spv::OpFunctionCall && words[3] == bda_function) ++bda_calls; offset += words.size(); } - Require(test.name, "empty descriptor table specialization", (bda_calls == 0u) == (table_bytes == 0u), + Require(test.name, "empty descriptor table specialization", (bda_calls == 0u) == (table_bytes == 0u || components != 0u), "empty table retained BDA reads or nonempty table lost GPU descriptor loads"); for (const auto &descriptor : compiled.resources.buffers) { ShaderBufferResource resource{}; std::copy_n(descriptor.dwords.begin(), 4u, resource.fields); - test.storage_buffer_ranges.emplace_back(resource.Base48(), std::max(4u, u32(resource.GetSize()))); + test.storage_buffer_ranges.emplace_back(resource.Base48(), + components != 0u && resource.Base48() == slot * 5u ? 6u : std::max(4u, u32(resource.GetSize()))); } TestCase producer; producer.name = "GpuRecordSelectorProducer"; @@ -37479,11 +37506,24 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) { for (u32 lane = 0; lane < wave; ++lane) { const u32 out = slot * 13u / 4u + (group * wave + lane) * 4u; const bool source = uint64_t{dynamic} + 160u + 16u <= table_bytes && lane == 0u; - test.expected[out] = source ? test.initial[slot * (row + 1u) / 4u] : 0u; - test.expected[out + 1u] = source ? test.initial[slot * (row + 1u) / 4u + 1u] : 0u; - const bool bda_source = uint64_t{dynamic} + 144u + 16u <= table_bytes && lane == 0u; - test.expected[out + 2u] = bda_source ? 0x3c00u + row : 0u; - test.expected[out + 3u] = bda_source ? 0x3f800000u + (row << 13u) : 0u; + if (components == 0u) { + test.expected[out] = source ? test.initial[slot * (row + 1u) / 4u] : 0u; + test.expected[out + 1u] = source ? test.initial[slot * (row + 1u) / 4u + 1u] : 0u; + const bool bda_source = uint64_t{dynamic} + 144u + 16u <= table_bytes && lane == 0u; + test.expected[out + 2u] = bda_source ? 0x3c00u + row : 0u; + test.expected[out + 3u] = bda_source ? 0x3f800000u + (row << 13u) : 0u; + } else { + constexpr std::array, 3> values{{ + {0xbf800000u, 0x3f000000u, 0u, 0x3f800000u}, + {0xbf800000u, 0xbf800000u, 0x3f800000u, 0u}, + {0x3f800000u, 0xbf800000u, 0xbf800000u, 0u}}}; + constexpr std::array, 3> defaults{{ + {0u, 0u, 0u, 0x3f800000u}, {0u, 0u, 0x3f800000u, 0u}, {0u, 0u, 0u, 0u}}}; + const bool descriptor = uint64_t{dynamic} + 160u + 16u <= table_bytes && row != 3u; + for (u32 component = 0; component < 4u; ++component) + test.expected[out + component] = component >= components ? 0x13579bdfu : + !descriptor ? 0u : source && row != 4u ? values[row % 3u][component] : defaults[row % 3u][component]; + } if (uint64_t{dynamic} + 176u + 16u <= table_bytes && (lane & 1u) == 0u) bytes[slot * (row + 7u) + group * wave + lane] = std::byte{0xc3}; }