diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 3d2ad7154..f9333b431 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -972,6 +972,10 @@ uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryIn uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index); +uint32_t LdsDwordCount(const EmitterState& state); + +uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index); + uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index); uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index dbd956348..6b240aa12 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -495,11 +495,14 @@ uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) { return pointer; } +uint32_t LdsDwordCount(const EmitterState& state) { + return state.needs_function_lds ? 8192u : state.compute_input_info->lds_size_dwords; +} + uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) { const auto in_bounds = state.builder.AllocateId(); - const auto dwords = state.needs_function_lds ? 8192u : 1024u; state.builder.AddFunction( - {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, dwords)}); + {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))}); return in_bounds; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 5314793c7..b083623fa 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -1,5 +1,7 @@ #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include + namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state, @@ -736,7 +738,8 @@ void EmitHeaderAndTypes(EmitterState& state) { if (state.stage == ShaderType::Compute || state.needs_function_lds) { const auto storage_class = state.needs_function_lds ? StorageClassFunction : StorageClassWorkgroup; - const auto lds_size = ConstantU32(state, state.needs_function_lds ? 8192u : 1024u); + const auto lds_dwords = std::max(LdsDwordCount(state), 1u); + const auto lds_size = ConstantU32(state, lds_dwords); state.builder.AddType({OpTypeArray, state.lds_array_type, state.uint_type, lds_size}); state.builder.AddType( {OpTypePointer, state.ptr_workgroup_array, storage_class, state.lds_array_type}); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp index 07b763dc1..f17c18d45 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp @@ -158,7 +158,7 @@ uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) { if (mem.kind == IR::ResourceKind::Lds) { - return 0; + return EmitLdsElementInBounds(ctx.state, index); } if (mem.kind == IR::ResourceKind::Gds) { return EmitGdsElementInBounds(ctx.state, index); @@ -546,7 +546,8 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo ConstantU32(state, ScopeSubgroup), ballot}); const auto is_first = Binary(state, OpIEqual, state.bool_type, EmitSubgroupLocalInvocationId(state), first); - const auto storage_bounds = gds ? EmitGdsElementInBounds(state, index) : 0u; + const auto storage_bounds = + gds ? EmitGdsElementInBounds(state, index) : EmitLdsElementInBounds(state, index); const auto m0_bounds = gds ? Binary(state, OpINotEqual, state.bool_type, size, ConstantU32(state, 0)) : Binary(state, OpULessThan, state.bool_type, diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index fbc7aa9e2..6643341a4 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -894,15 +894,16 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, ShaderComputeInputInfo& info) { info = {}; - info.threads_num[0] = regs.cs_regs.num_thread_x; - info.threads_num[1] = regs.cs_regs.num_thread_y; - info.threads_num[2] = regs.cs_regs.num_thread_z; - info.group_id[0] = regs.cs_regs.tgid_x_en != 0; - info.group_id[1] = regs.cs_regs.tgid_y_en != 0; - info.group_id[2] = regs.cs_regs.tgid_z_en != 0; - info.wave_size = regs.cs_regs.wave_size; - info.thread_ids_num = regs.cs_regs.tidig_comp_cnt + 1; - info.tg_size_en = regs.cs_regs.tg_size_en != 0; + info.threads_num[0] = regs.cs_regs.num_thread_x; + info.threads_num[1] = regs.cs_regs.num_thread_y; + info.threads_num[2] = regs.cs_regs.num_thread_z; + info.lds_size_dwords = static_cast(regs.cs_regs.lds_size) * 128u; + info.group_id[0] = regs.cs_regs.tgid_x_en != 0; + info.group_id[1] = regs.cs_regs.tgid_y_en != 0; + info.group_id[2] = regs.cs_regs.tgid_z_en != 0; + info.wave_size = regs.cs_regs.wave_size; + info.thread_ids_num = regs.cs_regs.tidig_comp_cnt + 1; + info.tg_size_en = regs.cs_regs.tg_size_en != 0; info.workgroup_register = regs.cs_regs.user_sgpr; } @@ -1330,10 +1331,12 @@ void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info) { LOGF("\t workgroup_register = %d\n" "\t thread_ids_num = %d\n" "\t wave_size = %u\n" + "\t lds_size_dwords = %u\n" "\t threads_num = {%u, %u, %u}\n" "\t tg_size_en = %s\n", - info.workgroup_register, info.thread_ids_num, info.wave_size, info.threads_num[0], - info.threads_num[1], info.threads_num[2], info.tg_size_en ? "true" : "false"); + info.workgroup_register, info.thread_ids_num, info.wave_size, info.lds_size_dwords, + info.threads_num[0], info.threads_num[1], info.threads_num[2], + info.tg_size_en ? "true" : "false"); LOGF("\t threadgroup_id = {%s, %s, %s}\n", info.group_id[0] ? "true" : "false", info.group_id[1] ? "true" : "false", info.group_id[2] ? "true" : "false"); } @@ -1723,6 +1726,7 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp ret.ids.push_back(input_info.workgroup_register); ret.ids.push_back(input_info.wave_size); ret.ids.push_back(input_info.thread_ids_num); + ret.ids.push_back(input_info.lds_size_dwords); for (int i = 0; i < 3; i++) { ret.ids.push_back(input_info.threads_num[i]); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 8e2e61597..7a0329c72 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -91,6 +91,7 @@ struct ShaderVertexInputInfo { struct ShaderComputeInputInfo { uint32_t threads_num[3] = {0, 0, 0}; uint32_t dispatch_threads_num[3] = {0, 0, 0}; + uint32_t lds_size_dwords = 0; bool group_id[3] = {false, false, false}; bool dispatch_thread_dimensions = false; uint32_t wave_size = 64; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 1db2d6f58..0d302302f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -846,7 +846,11 @@ struct TestCase { std::vector expected_storage_image_r32ui; std::vector required_spirv; std::vector forbidden_spirv; - ShaderComputeInputInfo compute_info{}; + ShaderComputeInputInfo compute_info = [] { + ShaderComputeInputInfo info{}; + info.lds_size_dwords = 1024; + return info; + }(); bool has_compute_info = false; u32 dispatch_x = 1; u32 dispatch_y = 1; @@ -1061,8 +1065,7 @@ CompiledShader CompileCase(const TestCase &test) { ShaderRecompiler::CompileOptions options; options.stage = ShaderType::Compute; options.dump_ir = true; - options.compute_input_info = - test.has_compute_info ? &test.compute_info : nullptr; + options.compute_input_info = &test.compute_info; options.user_data = user_data.data(); options.read_memory = ReadTestMemory; options.read_memory_data = const_cast *>(&test.initial); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 4bd99441c..c2329e76f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -162,6 +162,72 @@ uint32_t SpirvInstructionOpcodeCount(const std::vector &binary, return count; } +uint32_t SpirvArrayLengthCount(const std::vector &binary, + uint32_t requested_length) { + std::vector length_ids; + for (size_t i = 5; i < binary.size();) { + const uint32_t opcode = binary[i] & 0xffffu; + const uint32_t word_count = binary[i] >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return 0; + } + if (opcode == 43u && word_count == 4u && + binary[i + 3u] == requested_length) { // OpConstant + length_ids.push_back(binary[i + 2u]); + } + i += word_count; + } + + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t opcode = binary[i] & 0xffffu; + const uint32_t word_count = binary[i] >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (opcode == 28u && word_count == 4u && // OpTypeArray + std::find(length_ids.begin(), length_ids.end(), binary[i + 3u]) != + length_ids.end()) { + count++; + } + i += word_count; + } + return count; +} + +uint32_t SpirvUnsignedLessThanBoundCount(const std::vector &binary, + uint32_t requested_bound) { + std::vector bound_ids; + for (size_t i = 5; i < binary.size();) { + const uint32_t opcode = binary[i] & 0xffffu; + const uint32_t word_count = binary[i] >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return 0; + } + if (opcode == 43u && word_count == 4u && + binary[i + 3u] == requested_bound) { // OpConstant + bound_ids.push_back(binary[i + 2u]); + } + i += word_count; + } + + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t opcode = binary[i] & 0xffffu; + const uint32_t word_count = binary[i] >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (opcode == 176u && word_count == 5u && // OpULessThan + std::find(bound_ids.begin(), bound_ids.end(), binary[i + 4u]) != + bound_ids.end()) { + count++; + } + i += word_count; + } + return count; +} + bool SpirvSourceHasInstructionOperand(const std::string &source, const char *opcode, const char *operand) { std::istringstream lines(source); @@ -379,6 +445,7 @@ ShaderComputeInputInfo RegressionComputeInputInfo() { input_info.threads_num[0] = 1; input_info.threads_num[1] = 1; input_info.threads_num[2] = 1; + input_info.lds_size_dwords = 1024; input_info.workgroup_register = 40; return input_info; } @@ -9020,6 +9087,72 @@ void TestNewShaderRecompilerPixelPipelineEntry() { CheckSpirvBinaryValidates(vcc_spirv); } +void TestComputeLdsAllocationIdentity() { + const uint32_t shader[] = { + EncodeDs0(0x0d, 4288u), // ds_write_b32 v0, v1 offset:4288 + EncodeDs1(0, 1, 0), EncodeSopp(0x01), + }; + HW::ComputeShaderInfo regs{}; + regs.cs_regs.data_addr = reinterpret_cast(shader); + regs.cs_regs.num_thread_x = 64; + regs.cs_regs.num_thread_y = 1; + regs.cs_regs.num_thread_z = 1; + ShaderMappedData mapped{}; + mapped.code_size_bytes = sizeof(shader); + ShaderMapUserData(regs.cs_regs.data_addr, mapped); + HW::ShaderRegisters sh{}; + + const auto compile = [&](uint16_t encoded_lds_size, + uint32_t expected_dwords) { + regs.cs_regs.lds_size = encoded_lds_size; + ShaderComputeInputInfo input_info{}; + std::span spirv; + Check(ShaderCompileInfoCS(regs, sh, input_info, spirv), + "compute LDS allocation shader did not compile"); + Check(input_info.lds_size_dwords == expected_dwords, + "COMPUTE_PGM_RSRC2 LDS allocation units were not decoded"); + const std::vector binary(spirv.begin(), spirv.end()); + Check(SpirvArrayLengthCount(binary, expected_dwords) == 1u, + "SPIR-V workgroup array did not use the declared LDS allocation"); + Check(SpirvUnsignedLessThanBoundCount(binary, expected_dwords) == 1u, + "LDS bounds check did not use the declared allocation"); + CheckSpirvBinaryValidates(binary); + return input_info; + }; + + constexpr uint32_t lds_1152_rsrc2 = 0x00048188u; + constexpr uint32_t lds_896_rsrc2 = 0x00038188u; + constexpr auto decode_lds_field = [](uint32_t rsrc2) { + return static_cast( + (rsrc2 >> Pm4::COMPUTE_PGM_RSRC2_LDS_SIZE_SHIFT) & + Pm4::COMPUTE_PGM_RSRC2_LDS_SIZE_MASK); + }; + const auto lds_1152 = compile(decode_lds_field(lds_1152_rsrc2), 1152u); + const auto lds_896 = compile(decode_lds_field(lds_896_rsrc2), 896u); + Check(ShaderGetIdCS(regs, lds_1152, true) != + ShaderGetIdCS(regs, lds_896, true), + "compute pipeline identity omitted the LDS allocation"); + + const uint32_t append_shader[] = { + EncodeSMovB32(124, 132), // m0 = 4 bytes + EncodeDs0(0x3e), EncodeDs1(1, 0, 0), // ds_append v1 + EncodeSopp(0x01), + }; + ShaderComputeInputInfo append_info = RegressionComputeInputInfo(); + append_info.lds_size_dwords = 1152u; + ShaderRecompiler::CompileOptions append_options; + append_options.stage = ShaderType::Compute; + append_options.compute_input_info = &append_info; + ShaderRecompiler::CompileResult append_result; + std::string error; + Check(ShaderRecompiler::TryRecompile(append_shader, append_options, + append_result, &error), + error.c_str()); + Check(SpirvUnsignedLessThanBoundCount(append_result.spirv, 1152u) == 1u, + "typed LDS append omitted the declared allocation bound"); + CheckSpirvBinaryValidates(append_result.spirv); +} + void TestPixelProgramCacheDescriptorSetIdentity() { const uint32_t shader_01[] = {0xbf810000u}; const uint32_t shader_10[] = {0xbf810000u}; @@ -9250,6 +9383,7 @@ int main() { TestNewShaderRecompilerStageInputInfo(); TestGraphicsCreateInterpolantMapping(); TestNewShaderRecompilerPixelPipelineEntry(); + TestComputeLdsAllocationIdentity(); TestPixelProgramCacheDescriptorSetIdentity(); TestNewShaderRecompilerUnsupportedMemoryDecode();