From b4d32394b6227c50ea12266e8dd90c99fe456bc2 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 4 Oct 2026 03:18:09 +0400 Subject: [PATCH] shader: keep GPU-derived scalar resource reads on the GPU Co-authored-by: Ali Almohaya Co-authored-by: TheCruZ --- .../recompiler/backend/spirv/SpirvEmitter.cpp | 3 +- .../backend/spirv/spirvEmitterMemory.cpp | 33 ++++++++ src/graphics/shader/recompiler/ir/Program.cpp | 5 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 4 +- .../recompiler/ir/passes/ResourceTracking.cpp | 10 ++- tests/ShaderRecompilerComputeTests.cpp | 78 +++++++++++++++++++ tests/shaderCfgTests.cpp | 57 ++++++++++---- 7 files changed, 169 insertions(+), 21 deletions(-) diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 67f9d6cb8..d9aa57bf5 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -222,7 +222,8 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program Fail(program, "buffer operation has invalid memory metadata"); } const auto& memory = program.memory_info[memory_index]; - if (memory.kind == IR::ResourceKind::IndirectBuffer) { + if (memory.kind == IR::ResourceKind::IndirectBuffer && + inst.GetOpcode() != IR::ValueOpcode::ReadConstBuffer) { requirements.subgroup_local_invocation_id = true; } if (memory.kind == IR::ResourceKind::Buffer) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 10a7f4921..ad10b2bb4 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -809,6 +809,35 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor }); } +uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto& handle = *inst.Arg(0).ResolveInstruction(); + const auto word1 = ctx.Arg(handle, 1); + const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16), + ConstantU32(state, 14)); + const auto size = Binary( + state, spv::OpIMul, TypeScalarU64(state), + Unary(state, spv::OpUConvert, TypeScalarU64(state), + EmitUMax32(state, stride, ConstantU32(state, 1))), + Unary(state, spv::OpUConvert, TypeScalarU64(state), ctx.Arg(handle, 2))); + const auto offset = Binary( + state, spv::OpIAdd, TypeScalarU64(state), + Unary(state, spv::OpUConvert, TypeScalarU64(state), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1), + ConstantU32(state, ~3u))), + ConstantDeviceAddress(state, ctx.Memory(inst).offset & ~3u)); + const auto end = Binary(state, spv::OpIAdd, TypeScalarU64(state), offset, + ConstantDeviceAddress(state, sizeof(uint32_t))); + return EmitValueOrZeroIfCondition( + state, Binary(state, spv::OpULessThanEqual, TypeBool(state), end, size), [&]() { + const auto base = DeviceAddressFromWords( + state, Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(handle, 0), + ConstantU32(state, ~3u)), + Binary(state, spv::OpBitwiseAnd, TypeU32(state), word1, ConstantU32(state, 0xffffu))); + return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), base, offset)); + }); +} + uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t components) { auto& state = ctx.state; const auto& handle = *inst.Arg(0).ResolveInstruction(); @@ -1249,6 +1278,10 @@ uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { auto mem = ctx.Memory(inst); if (mem.planning_only) return; + if (mem.kind == IR::ResourceKind::IndirectBuffer) { + ctx.Define(inst, LoadIndirectScalarBuffer(ctx, inst)); + return; + } auto& state = ctx.state; mem.kind = IR::ResourceKind::ScalarBuffer; auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1), diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index 7c9f16805..ffaa2355e 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -399,7 +399,8 @@ void ValidateProgram(const Program& program, bool require_ssa) { ValueOpcodeName(inst.GetOpcode()))); } const auto& memory = program.memory_info[memory_index]; - if (memory.kind != ResourceKind::ScalarBuffer) { + if (memory.kind != ResourceKind::ScalarBuffer && + memory.kind != ResourceKind::IndirectBuffer) { return Fail(fmt::format("{} has an invalid scalar-memory resource kind", ValueOpcodeName(inst.GetOpcode()))); } @@ -457,7 +458,7 @@ void ValidateProgram(const Program& program, bool require_ssa) { } if (memory.kind == ResourceKind::IndirectBuffer && !memory.SupportsIndirectBufferLoad(inst.GetOpcode())) { - return Fail("indirect buffer requires a raw DWORD x2/x3/x4 load"); + return Fail("indirect buffer requires a scalar or raw DWORD x2/x3/x4 load"); } if (buffer_components > 1u && (!vector_buffer || memory.data_bits != 32u || diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 5df80941d..82ac5153c 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -74,8 +74,8 @@ struct MemoryInfo { [[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const { return !formatted && !typed && data_bits == 32u && - (opcode == ValueOpcode::LoadBufferU32x2 || opcode == ValueOpcode::LoadBufferU32x3 || - opcode == ValueOpcode::LoadBufferU32x4); + (opcode == ValueOpcode::ReadConstBuffer || opcode == ValueOpcode::LoadBufferU32x2 || + opcode == ValueOpcode::LoadBufferU32x3 || opcode == ValueOpcode::LoadBufferU32x4); } bool operator==(const MemoryInfo& other) const = default; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index b3f8e02be..d200e5121 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -736,12 +736,14 @@ private: for (auto* read: m_scalar_reads) { const auto flags = read->Flags(); auto& memory = m_program.memory_info[flags.index]; - memory.planning_only = true; const auto* handle = read->Arg(0).Resolve().TryInstruction(); auto resolved = std::ranges::find_if(m_resolved_handles, [&](const auto& entry) { return entry.handle == handle && entry.pc == flags.pc; }); EXIT_IF(resolved == m_resolved_handles.end()); + uint32_t bad_dword = 0; + if (!ValidateSource(resolved->source, bad_dword)) continue; + memory.planning_only = true; uint32_t slot = 0; for (; slot < m_program.srt_reads.size(); ++slot) { const auto* other = m_program.srt_reads[slot].value.Resolve().TryInstruction(); @@ -2071,10 +2073,12 @@ private: source = indirect->source; } else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, memory.resource * 4u, handle, source)) { - if (memory.kind != ResourceKind::Buffer || !memory.SupportsIndirectBufferLoad(op)) { + if (memory.kind != (op == ValueOpcode::ReadConstBuffer ? ResourceKind::ScalarBuffer + : ResourceKind::Buffer) || + !memory.SupportsIndirectBufferLoad(op)) { Fail(flags.pc, "buffer descriptor is not a valid runtime value; GPU-selected access " - "requires a raw DWORD x2/x3/x4 load"); + "requires a scalar or raw DWORD x2/x3/x4 load"); } m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer; m_info.uses_dma = true; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index d995fb415..c9dcdc0e0 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -27852,6 +27852,80 @@ TestCase ScalarLoadAlignsDynamicBase() { return test; } +TestCase ScalarBufferFromLoopReadlane(u32 wave_size) { + using O = ShaderOpcode; + constexpr u32 GuestBase = 0x10000; + constexpr u32 DescriptorCount = 5; + constexpr u32 PayloadBase = GuestBase + DescriptorCount * 32; + std::vector code; + code.push_back(EncodeVop2(0x1a, 1, InlineU32(2), 0)); + AppendBufferLoadDword(&code, 2, 1); + AppendSMovLiteral(&code, 8, GuestBase + 3); + AppendSMovLiteral(&code, 9, 0); + AppendSMovLiteral(&code, 28, 0); + const auto loop = static_cast(code.size()); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); // SMEM ignores EXEC. + code.push_back(EncodeSmem0(0x03, 16, 4)); // descriptor and runtime soffset + code.push_back(EncodeSmem1(3, 125)); // null keeps this a flattening candidate. + code.push_back(EncodeSmem0(0x0a, 32, 8)); + code.push_back(EncodeSmem1(7, 20)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(1))); + code.push_back(EncodeVop1(0x01, 3, 28)); + code.push_back(EncodeVop2(0x1a, 31, InlineU32(4), 3)); + for (u32 component = 0; component < 4; ++component) { + code.push_back(EncodeVop1(0x01, 4, 32 + component)); + AppendBufferStoreDword(&code, 4, 31); + code.push_back(EncodeVop2(0x25, 31, InlineU32(4), 31)); + } + code.push_back(EncodeSop1(0x04, 126, 193)); + code.push_back(EncodeSop2(0x00, 29, 28, InlineU32(wave_size == 64 ? 32 : 0))); + AppendVop3(&code, 0x360, 8, Vgpr(2), 29); + code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1))); + code.push_back(EncodeSopc(0x0a, 28, InlineU32(DescriptorCount))); + code.push_back(EncodeSopp(0x05, loop - static_cast(code.size()) - 1u)); + AppendEnd(&code); + + TestCase test; + test.name = wave_size == 32 ? "ScalarBufferFromLoopReadlane32" + : "ScalarBufferFromLoopReadlane64"; + test.code = std::move(code); + test.initial.resize(128 + DescriptorCount * 8 + 8); + constexpr u32 records[] = {20, 5, 3, 0x40000005u, 20}; + for (u32 i = 0; i < DescriptorCount; ++i) { + test.initial[(wave_size == 64 ? 32 : 0) + i] = + GuestBase + ((i + 1) % DescriptorCount) * 32 + 3; + const auto row = 128 + i * 8; + test.initial[row] = PayloadBase + 3; + test.initial[row + 1] = (i == 1 || i == 3 ? 4u << 16u : 0u) | + (i == 1 ? 1u << 31u : 0u); + test.initial[row + 2] = records[i]; + // Scalar addressing ignores FORMAT, swizzle, ADD_TID, INDEX_STRIDE and OOB_SELECT. + test.initial[row + 3] = i == 1 ? (3u << 21u) | (1u << 23u) | (3u << 28u) : 0u; + test.initial[row + 4] = i == 4 ? 0xffffffffu : 11u; + } + for (u32 i = 0; i < 8; ++i) + test.initial[128 + DescriptorCount * 8 + i] = 0x11111111u * (i + 1u); + test.expected = {0x44444444u, 0x55555555u, 0, 0, + 0x44444444u, 0x55555555u, 0, 0, + 0, 0, 0, 0, + 0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u, + 0, 0, 0, 0}; + test.bda_mappings = {{GuestBase, 512}}; + test.required_spirv = {"get_bda_pointer", "OpLoopMerge", "OpGroupNonUniformShuffle"}; + test.forbidden_spirv = {"flattened_srt"}; + test.opcodes = {O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::S_MOV_B32, + O::S_MOV_B64, O::S_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX4, + O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::V_ADD_NC_U32, + O::S_ADD_U32, O::V_READLANE_B32, O::S_CMP_LT_U32, + O::S_CBRANCH_SCC1, O::S_ENDPGM}; + test.compute_info.wave_size = wave_size; + test.compute_info.threads_num[0] = wave_size; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; +} + TestCase FlatVirtualAddressRebasesGuestAllocation() { using O = ShaderOpcode; @@ -32613,6 +32687,8 @@ std::vector MakeCases() { AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); AddCase(ScalarLoadAlignsComponentsAndMasksAddress); AddCase(ScalarLoadAlignsDynamicBase); + cases.push_back(ScalarBufferFromLoopReadlane(32)); + cases.push_back(ScalarBufferFromLoopReadlane(64)); AddCase(BufferLoadStore); AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset); AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset); @@ -37730,6 +37806,8 @@ int main(int argc, char **argv) { } if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) { VulkanHarness vulkan; + RunCase(&vulkan, ScalarBufferFromLoopReadlane(32)); + RunCase(&vulkan, ScalarBufferFromLoopReadlane(64)); CheckIndirectBufferStore(vulkan); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors()); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index d80967a54..c8b0bff92 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -7894,12 +7894,11 @@ void TestNewShaderRecompilerCfgLoopBreakContinue() { CheckSpirvBinaryValidates(result.spirv); } -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() { const uint32_t shader[] = { EncodeSMovB32(0, 128), // preheader: s0 = 0 EncodeSmem0(0x08, 8, 4), - 0u, // loop: s_buffer_load_dword s8, s[4:7] + 0u, // loop: s_buffer_load_dword s8, s[8:11], s0 EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop @@ -7909,13 +7908,25 @@ void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() { 0xbf810000u, }; + std::array user_data{}; + for (const uint32_t base : {8u, 48u}) { + user_data[base] = base == 8u ? 0x1000u : 0x2000u; + user_data[base + 2] = 64u; + user_data[base + 3] = 0x30005204u; + } auto options = MakeCompileOptions(ShaderType::Compute); - options.dump_ir = true; - ExpectFatal([&] { (void)RecompileForTest(shader, options); }, - "self-modifying scalar-buffer descriptor did not terminate " - "compilation"); + options.user_data = user_data; + const auto result = RecompileForTest(shader, options); + Check(result.program.info.uses_dma && !result.program.dispatcher_fallback && + SpirvContainsOpcode(result.spirv, 246), + "self-modifying scalar descriptor lost its structured GPU loop"); + Check(result.program.info.buffers.size() == 1u && result.program.info.buffers[0].written && + result.resources.flattened_srt.empty(), + "self-modifying scalar descriptor was flattened or bound on the host"); + CheckSpirvBinaryValidates(result.spirv); } +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS void TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher() { const uint32_t shader[] = { EncodeSMovB32(0, 128), // preheader: s0 = 0 @@ -8976,8 +8987,7 @@ void TestBoundedMaterialBufferStores(bool scalar_key) { #endif } -void TestNativeDescriptorProvenanceRejectsGpuSelection() { -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS +void TestNativeDescriptorProvenanceKeepsGpuSelection() { const std::array selected{ EncodeSmem0(0x02, 8, 6), 125u << 25u, EncodeVop1(0x01, 1, 128), @@ -9003,7 +9013,15 @@ void TestNativeDescriptorProvenanceRejectsGpuSelection() { EncodeSopp(0x07, 0xfff6u), EncodeSopp(0x01), }; + const std::array table{0x1000u, 0u, 4u, 3u << 28u, + 0x2000u, 0u, 4u, 3u << 28u}; std::array user_data{}; + user_data[0] = 0x3000u; + user_data[2] = 64u; + user_data[3] = 0x30005204u; + const auto address = reinterpret_cast(table.data()); + user_data[12] = static_cast(address); + user_data[13] = static_cast(address >> 32u); ShaderComputeInputInfo input_info{}; input_info.thread_ids_num = 1; input_info.threads_num[0] = 64; @@ -9012,10 +9030,23 @@ void TestNativeDescriptorProvenanceRejectsGpuSelection() { options.input_info.compute = &input_info; options.user_data = user_data; for (const auto &shader : {selected, partial, loop}) { - ExpectFatal([&] { ShaderRecompiler::TranslateProgram(shader, options); }, - "native descriptor provenance accepted a GPU-selected or loop-carried value"); + const auto result = RecompileForTest(shader, options, ReadHostTestMemory); + Check(result.program.info.uses_dma && result.program.info.buffers.size() == 1u && + result.program.info.buffers[0].written, + "GPU-selected scalar descriptor was materialized as a host binding"); + uint32_t indirect_reads = 0; + for (const auto *block : result.program.blocks) { + for (const auto &inst : *block) { + if (inst.GetOpcode() != ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer) continue; + const auto &memory = result.program.memory_info[ + inst.Flags().index]; + indirect_reads += memory.kind == ShaderRecompiler::IR::ResourceKind::IndirectBuffer && + !memory.planning_only; + } + } + Check(indirect_reads == 1u, "GPU-selected scalar read was flattened or removed"); + CheckSpirvBinaryValidates(result.spirv); } -#endif } void TestCfgSiblingSharedExit() { @@ -14618,8 +14649,8 @@ int main() { TestNewShaderRecompilerCfgTerminalExitMergePS(); TestNewShaderRecompilerCfgPostEndTargetMergePS(); TestNewShaderRecompilerCfgLoopBreakContinue(); -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured(); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher(); #endif TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured(); @@ -14647,7 +14678,7 @@ int main() { TestNativeScalarReadDescriptorPlanning(); TestNativeGuardedSamplerSource(); for (const bool scalar_key : {false, true}) TestBoundedMaterialBufferStores(scalar_key); - TestNativeDescriptorProvenanceRejectsGpuSelection(); + TestNativeDescriptorProvenanceKeepsGpuSelection(); TestCfgSiblingSharedExit(); TestNewShaderRecompilerCfgNestedTailEarlyExit(); TestNewShaderRecompilerCfgSharedReturnAfterNestedSelections();