From 03c55e57120ebfe34c48922a50400723653aec19 Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 23 Sep 2026 10:49:08 +0400 Subject: [PATCH] Keep shader user data inline for cached programs --- .../renderer/pipeline/pipelineCache.cpp | 7 ++-- src/graphics/shader/shader.cpp | 26 ++++++++------ src/graphics/shader/shaderCompiler.h | 4 ++- tests/shaderCfgTests.cpp | 36 +++++++++++++------ 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 26351a6e0..cdaf187b0 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -276,14 +276,15 @@ struct PipelineCache::ProgramCache { stage = ShaderType::Compute; } + const auto user_data = std::span(params.user_data).first(params.user_data_count); lookup_key.stage = stage; lookup_key.hash = params.hash; - lookup_key.user_data_count = static_cast(params.user_data.size()); + lookup_key.user_data_count = params.user_data_count; lookup_key.code_size = static_cast(params.code.size()); BuildStageStaticKey(input_info, lookup_key.static_state); auto entry = programs.find(lookup_key); const ShaderRecompiler::IR::SrtRuntime runtime { - .user_data = params.user_data, + .user_data = user_data, .shader_base = params.Base(), .read_specialization_memory = ReadShaderGuestMemory, }; @@ -329,7 +330,7 @@ struct PipelineCache::ProgramCache { ShaderRecompiler::CompileOptions options; options.stage = stage; options.shader_hash = params.hash; - options.user_data = params.user_data; + options.user_data = user_data; options.back_code = params.back_code; options.dump_ir = Config::GetShaderLogDirection() != Config::LogDirection::Silent; options.early_dump = options.dump_ir; diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 76c207ef1..8739984ec 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -108,7 +108,7 @@ static uint64_t GetDeclaredShaderHash(uint64_t shader_addr) { static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash, std::span user_data, - const ShaderMappedData& data) { + const ShaderMappedData& data, uint32_t user_data_base = 0) { if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) { EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64 " has invalid AGC shader_size=0x%08" PRIx32 "\n", @@ -116,12 +116,16 @@ static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uin } const auto code_words = data.code_size_bytes / sizeof(uint32_t); const auto code = std::span {reinterpret_cast(shader_addr), code_words}; - return { - .code = code, - .user_data = std::vector(user_data.begin(), user_data.end()), - .hash = declared_hash != 0 ? declared_hash - : XXH3_64bits(code.data(), code.size_bytes()), + ShaderParams params { + .code = code, + .user_data_count = static_cast(user_data.size()) + user_data_base, + .hash = declared_hash != 0 ? declared_hash + : XXH3_64bits(code.data(), code.size_bytes()), }; + EXIT_IF(user_data.size() > HW::UserSgprInfo::SGPRS_MAX || + params.user_data_count > params.user_data.size()); + std::copy(user_data.begin(), user_data.end(), params.user_data.begin() + user_data_base); + return params; } #if 0 @@ -756,11 +760,13 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& const HW::UserConfig& user_config, ShaderVertexInputInfo& info) { const auto& sh = context.GetShaderRegisters(); const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():"); + const bool merged = (context.GetShaderStages() & 0x20u) != 0; auto params = GetShaderParams( regs.es_regs.data_addr, "ShaderRecompiler VS", GetDeclaredShaderHash(regs.es_regs.data_addr), - std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data); - if ((context.GetShaderStages() & 0x20u) == 0) { + std::span(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data, + merged ? 8u : 0u); + if (!merged) { if (!ShaderGetStaticVertexInputInfo(regs.es_regs.data_addr, regs.gs_user_sgpr, regs.gs_regs.rsrc2.user_sgpr, sh, data, info)) { EXIT("failed to prepare vertex shader program\n"); @@ -770,7 +776,6 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context& } // NGG user SGPRs start at s8; a separately compiled GS back half also receives // its user-data pointer in s0:s1. - params.user_data.insert(params.user_data.begin(), 8u, 0u); info = {}; info.logical_stage = ShaderType::Mesh; info.pa_cl_vs_out_cntl = sh.m_paClVsOutCntl; @@ -841,14 +846,13 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context& GetShaderParams(regs.ls_regs.data_addr, "ShaderRecompiler LS", GetDeclaredShaderHash(regs.ls_regs.data_addr), local_users, local), GetShaderParams(regs.hs_regs.data_addr, "ShaderRecompiler HS", - GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control), + GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control, 8u), GetShaderParams(regs.es_regs.data_addr, "ShaderRecompiler TES", GetDeclaredShaderHash(regs.es_regs.data_addr), evaluation_users, evaluation), }; // The fused HS back half receives its separate user-data address in s0:s1. // RDNA2 reserves s0:s7 before the native HS user SGPRs. - params[1].user_data.insert(params[1].user_data.begin(), 8u, 0u); params[1].user_data[0] = static_cast(regs.hs_regs.user_data_addr); params[1].user_data[1] = static_cast(regs.hs_regs.user_data_addr >> 32u); diff --git a/src/graphics/shader/shaderCompiler.h b/src/graphics/shader/shaderCompiler.h index a53f82864..d4b05f5af 100644 --- a/src/graphics/shader/shaderCompiler.h +++ b/src/graphics/shader/shaderCompiler.h @@ -3,6 +3,7 @@ #include "graphics/shader/shader.h" +#include #include #include @@ -15,7 +16,8 @@ class UserConfig; struct ShaderParams { std::span code; - std::vector user_data; + std::array user_data {}; // 32 user SGPRs plus the merged-stage s0:s7 prefix. + uint32_t user_data_count = 0; uint64_t hash = 0; std::span back_code; diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 52ad6d1b8..969a1d89a 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -153,7 +153,7 @@ void CompilePixelRuntime(const ShaderParams ¶ms, ShaderPixelInputInfo &input_info) { auto options = MakeCompileOptions(ShaderType::Pixel); options.shader_hash = params.hash; - options.user_data = params.user_data; + options.user_data = std::span(params.user_data).first(params.user_data_count); options.input_info.pixel = &input_info; auto result = RecompileForTest(params.code, options); @@ -1735,7 +1735,7 @@ void TestNggVertexEntryState() { previous_key = key; auto options = MakeCompileOptions(ShaderType::Vertex); options.user_data_base = 8; - options.user_data = params.user_data; + options.user_data = std::span(params.user_data).first(params.user_data_count); options.input_info.vertex = &input; options.wave_size = input.wave_size; const auto result = RecompileForTest(params.code, options); @@ -9645,7 +9645,7 @@ void TestMergedShaderUserDataSnapshot() { regs.gs_user_sgpr.value[0]++; ShaderVertexInputInfo second_input{}; const auto second = PrepareProgram(regs, context, user_config, second_input); - Check(first.user_data.size() == 12 && second.user_data.size() == 12 && + Check(first.user_data_count == 12 && second.user_data_count == 12 && std::equal(front_data.begin(), front_data.end(), first.user_data.begin() + 8) && second.user_data[8] == front_data[0] + 1, "merged shader parameters did not snapshot ordinary user SGPRs at s8"); @@ -9656,7 +9656,7 @@ void TestMergedShaderUserDataSnapshot() { CompileOptions options{}; options.stage = ShaderType::Mesh; options.user_data_base = 0; - options.user_data = first.user_data; + options.user_data = std::span(first.user_data).first(first.user_data_count); options.input_info.vertex = &first_input; options.back_code = first.back_code; const auto translated = TranslateProgram(first.code, options); @@ -9664,13 +9664,14 @@ void TestMergedShaderUserDataSnapshot() { Check(program.info.buffers.size() == 2 && program.srt_reads.size() == 4, "merged shader lost front user SGPRs or the back-stage SRT load"); for (const auto *params : {&first, &second}) { - const IR::SrtRuntime runtime{.user_data = params->user_data, + const auto user_data = std::span(params->user_data).first(params->user_data_count); + const IR::SrtRuntime runtime{.user_data = user_data, .read_memory = ReadHostTestMemory}; IR::DescriptorValue front_descriptor, back_descriptor; const auto &table = params == &first ? first_table : second_table; Check(IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[0].source, front_descriptor) && IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[1].source, back_descriptor) && - std::equal(params->user_data.begin() + 8, params->user_data.end(), + std::equal(user_data.begin() + 8, user_data.end(), front_descriptor.dwords.begin()) && std::equal(table.begin(), table.end(), back_descriptor.dwords.begin()), "merged shader resource plan did not follow the current s0:s1 pointer and s8 data"); @@ -9688,12 +9689,27 @@ void TestMergedShaderUserDataSnapshot() { ShaderVertexInputInfo input{}; const auto params = PrepareProgram(regs, context, user_config, input); Check(params.back_code.empty() && params.hash == XXH3_64bits(monolithic, sizeof(monolithic)) && - input.mesh.scratch_size_dwords == 3 && params.user_data.size() == 12 && + input.mesh.scratch_size_dwords == 3 && params.user_data_count == 12 && params.user_data[0] == 0 && params.user_data[1] == 0 && - std::equal(second.user_data.begin() + 8, second.user_data.end(), + std::equal(second.user_data.begin() + 8, second.user_data.begin() + second.user_data_count, params.user_data.begin() + 8), "monolithic NGG shader used stale GS-back state or lost its s8 user data"); } + regs.gs_regs.rsrc2.user_sgpr = HW::UserSgprInfo::SGPRS_MAX; + for (uint32_t i = 4; i < HW::UserSgprInfo::SGPRS_MAX; ++i) { + regs.gs_user_sgpr.value[i] = 0x10001000u + i; + } + ShaderVertexInputInfo full_input{}; + const auto full = PrepareProgram(regs, context, user_config, full_input); + Check(full.user_data_count == 8u + HW::UserSgprInfo::SGPRS_MAX && + std::ranges::all_of(std::span(full.user_data).first(8), + [](uint32_t word) { return word == 0; }) && + std::equal(std::begin(regs.gs_user_sgpr.value), std::end(regs.gs_user_sgpr.value), + full.user_data.begin() + 8), + "merged shader did not preserve the full user-SGPR range after its reserved prefix"); + regs.gs_user_sgpr.value[HW::UserSgprInfo::SGPRS_MAX - 1]++; + Check(full.user_data.back() == 0x10001000u + HW::UserSgprInfo::SGPRS_MAX - 1, + "merged shader parameters borrowed mutable register storage"); context.SetMaxOutputPerSubgroup(256); context.SetGsMaxVertOut(8); user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriFan); @@ -12839,7 +12855,7 @@ void TestComputeLdsAllocationIdentity() { "COMPUTE_PGM_RSRC2 LDS allocation units were not decoded"); auto options = MakeCompileOptions(ShaderType::Compute); options.shader_hash = params.hash; - options.user_data = params.user_data; + options.user_data = std::span(params.user_data).first(params.user_data_count); options.input_info.compute = &input_info; options.wave_size = input_info.wave_size; @@ -12905,7 +12921,7 @@ void TestComputeLdsAllocationIdentity() { "AGC per-thread scratch size was not propagated"); auto scratch_options = MakeCompileOptions(ShaderType::Compute); scratch_options.shader_hash = scratch_params.hash; - scratch_options.user_data = scratch_params.user_data; + scratch_options.user_data = std::span(scratch_params.user_data).first(scratch_params.user_data_count); scratch_options.input_info.compute = &scratch_info; scratch_options.wave_size = scratch_info.wave_size;