Keep shader user data inline for cached programs

This commit is contained in:
nmzik
2026-09-23 10:49:08 +04:00
parent 5d1ad31c49
commit 03c55e5712
4 changed files with 48 additions and 25 deletions
@@ -276,14 +276,15 @@ struct PipelineCache::ProgramCache {
stage = ShaderType::Compute;
}
const auto user_data = std::span(params.user_data).first(params.user_data_count);
lookup_key.stage = stage;
lookup_key.hash = params.hash;
lookup_key.user_data_count = static_cast<uint32_t>(params.user_data.size());
lookup_key.user_data_count = params.user_data_count;
lookup_key.code_size = static_cast<uint32_t>(params.code.size());
BuildStageStaticKey(input_info, lookup_key.static_state);
auto entry = programs.find(lookup_key);
const ShaderRecompiler::IR::SrtRuntime runtime {
.user_data = params.user_data,
.user_data = user_data,
.shader_base = params.Base(),
.read_specialization_memory = ReadShaderGuestMemory,
};
@@ -329,7 +330,7 @@ struct PipelineCache::ProgramCache {
ShaderRecompiler::CompileOptions options;
options.stage = stage;
options.shader_hash = params.hash;
options.user_data = params.user_data;
options.user_data = user_data;
options.back_code = params.back_code;
options.dump_ir = Config::GetShaderLogDirection() != Config::LogDirection::Silent;
options.early_dump = options.dump_ir;
+15 -11
View File
@@ -108,7 +108,7 @@ static uint64_t GetDeclaredShaderHash(uint64_t shader_addr) {
static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uint64_t declared_hash,
std::span<const uint32_t> user_data,
const ShaderMappedData& data) {
const ShaderMappedData& data, uint32_t user_data_base = 0) {
if (data.code_size_bytes == 0 || data.code_size_bytes % sizeof(uint32_t) != 0) {
EXIT("%s hash=0x%016" PRIx64 " shader=0x%016" PRIx64
" has invalid AGC shader_size=0x%08" PRIx32 "\n",
@@ -116,12 +116,16 @@ static ShaderParams GetShaderParams(uint64_t shader_addr, const char* label, uin
}
const auto code_words = data.code_size_bytes / sizeof(uint32_t);
const auto code = std::span {reinterpret_cast<const uint32_t*>(shader_addr), code_words};
return {
.code = code,
.user_data = std::vector<uint32_t>(user_data.begin(), user_data.end()),
.hash = declared_hash != 0 ? declared_hash
: XXH3_64bits(code.data(), code.size_bytes()),
ShaderParams params {
.code = code,
.user_data_count = static_cast<uint32_t>(user_data.size()) + user_data_base,
.hash = declared_hash != 0 ? declared_hash
: XXH3_64bits(code.data(), code.size_bytes()),
};
EXIT_IF(user_data.size() > HW::UserSgprInfo::SGPRS_MAX ||
params.user_data_count > params.user_data.size());
std::copy(user_data.begin(), user_data.end(), params.user_data.begin() + user_data_base);
return params;
}
#if 0
@@ -756,11 +760,13 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
const HW::UserConfig& user_config, ShaderVertexInputInfo& info) {
const auto& sh = context.GetShaderRegisters();
const auto data = ShaderGetMappedData(regs.es_regs.data_addr, "ShaderGetInputInfoVS():");
const bool merged = (context.GetShaderStages() & 0x20u) != 0;
auto params = GetShaderParams(
regs.es_regs.data_addr, "ShaderRecompiler VS",
GetDeclaredShaderHash(regs.es_regs.data_addr),
std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data);
if ((context.GetShaderStages() & 0x20u) == 0) {
std::span<const uint32_t>(regs.gs_user_sgpr.value, regs.gs_regs.rsrc2.user_sgpr), data,
merged ? 8u : 0u);
if (!merged) {
if (!ShaderGetStaticVertexInputInfo(regs.es_regs.data_addr, regs.gs_user_sgpr,
regs.gs_regs.rsrc2.user_sgpr, sh, data, info)) {
EXIT("failed to prepare vertex shader program\n");
@@ -770,7 +776,6 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::Context&
}
// NGG user SGPRs start at s8; a separately compiled GS back half also receives
// its user-data pointer in s0:s1.
params.user_data.insert(params.user_data.begin(), 8u, 0u);
info = {};
info.logical_stage = ShaderType::Mesh;
info.pa_cl_vs_out_cntl = sh.m_paClVsOutCntl;
@@ -841,14 +846,13 @@ PrepareTessellationPrograms(const HW::VertexShaderInfo& regs, const HW::Context&
GetShaderParams(regs.ls_regs.data_addr, "ShaderRecompiler LS",
GetDeclaredShaderHash(regs.ls_regs.data_addr), local_users, local),
GetShaderParams(regs.hs_regs.data_addr, "ShaderRecompiler HS",
GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control),
GetDeclaredShaderHash(regs.hs_regs.data_addr), local_users, control, 8u),
GetShaderParams(regs.es_regs.data_addr, "ShaderRecompiler TES",
GetDeclaredShaderHash(regs.es_regs.data_addr), evaluation_users,
evaluation),
};
// The fused HS back half receives its separate user-data address in s0:s1.
// RDNA2 reserves s0:s7 before the native HS user SGPRs.
params[1].user_data.insert(params[1].user_data.begin(), 8u, 0u);
params[1].user_data[0] = static_cast<uint32_t>(regs.hs_regs.user_data_addr);
params[1].user_data[1] = static_cast<uint32_t>(regs.hs_regs.user_data_addr >> 32u);
+3 -1
View File
@@ -3,6 +3,7 @@
#include "graphics/shader/shader.h"
#include <array>
#include <span>
#include <vector>
@@ -15,7 +16,8 @@ class UserConfig;
struct ShaderParams {
std::span<const uint32_t> code;
std::vector<uint32_t> user_data;
std::array<uint32_t, 40> user_data {}; // 32 user SGPRs plus the merged-stage s0:s7 prefix.
uint32_t user_data_count = 0;
uint64_t hash = 0;
std::span<const uint32_t> back_code;
+26 -10
View File
@@ -153,7 +153,7 @@ void CompilePixelRuntime(const ShaderParams &params,
ShaderPixelInputInfo &input_info) {
auto options = MakeCompileOptions(ShaderType::Pixel);
options.shader_hash = params.hash;
options.user_data = params.user_data;
options.user_data = std::span(params.user_data).first(params.user_data_count);
options.input_info.pixel = &input_info;
auto result = RecompileForTest(params.code, options);
@@ -1735,7 +1735,7 @@ void TestNggVertexEntryState() {
previous_key = key;
auto options = MakeCompileOptions(ShaderType::Vertex);
options.user_data_base = 8;
options.user_data = params.user_data;
options.user_data = std::span(params.user_data).first(params.user_data_count);
options.input_info.vertex = &input;
options.wave_size = input.wave_size;
const auto result = RecompileForTest(params.code, options);
@@ -9645,7 +9645,7 @@ void TestMergedShaderUserDataSnapshot() {
regs.gs_user_sgpr.value[0]++;
ShaderVertexInputInfo second_input{};
const auto second = PrepareProgram(regs, context, user_config, second_input);
Check(first.user_data.size() == 12 && second.user_data.size() == 12 &&
Check(first.user_data_count == 12 && second.user_data_count == 12 &&
std::equal(front_data.begin(), front_data.end(), first.user_data.begin() + 8) &&
second.user_data[8] == front_data[0] + 1,
"merged shader parameters did not snapshot ordinary user SGPRs at s8");
@@ -9656,7 +9656,7 @@ void TestMergedShaderUserDataSnapshot() {
CompileOptions options{};
options.stage = ShaderType::Mesh;
options.user_data_base = 0;
options.user_data = first.user_data;
options.user_data = std::span(first.user_data).first(first.user_data_count);
options.input_info.vertex = &first_input;
options.back_code = first.back_code;
const auto translated = TranslateProgram(first.code, options);
@@ -9664,13 +9664,14 @@ void TestMergedShaderUserDataSnapshot() {
Check(program.info.buffers.size() == 2 && program.srt_reads.size() == 4,
"merged shader lost front user SGPRs or the back-stage SRT load");
for (const auto *params : {&first, &second}) {
const IR::SrtRuntime runtime{.user_data = params->user_data,
const auto user_data = std::span(params->user_data).first(params->user_data_count);
const IR::SrtRuntime runtime{.user_data = user_data,
.read_memory = ReadHostTestMemory};
IR::DescriptorValue front_descriptor, back_descriptor;
const auto &table = params == &first ? first_table : second_table;
Check(IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[0].source, front_descriptor) &&
IR::SrtWalker(program, runtime).EvaluateDescriptor(program.info.buffers[1].source, back_descriptor) &&
std::equal(params->user_data.begin() + 8, params->user_data.end(),
std::equal(user_data.begin() + 8, user_data.end(),
front_descriptor.dwords.begin()) &&
std::equal(table.begin(), table.end(), back_descriptor.dwords.begin()),
"merged shader resource plan did not follow the current s0:s1 pointer and s8 data");
@@ -9688,12 +9689,27 @@ void TestMergedShaderUserDataSnapshot() {
ShaderVertexInputInfo input{};
const auto params = PrepareProgram(regs, context, user_config, input);
Check(params.back_code.empty() && params.hash == XXH3_64bits(monolithic, sizeof(monolithic)) &&
input.mesh.scratch_size_dwords == 3 && params.user_data.size() == 12 &&
input.mesh.scratch_size_dwords == 3 && params.user_data_count == 12 &&
params.user_data[0] == 0 && params.user_data[1] == 0 &&
std::equal(second.user_data.begin() + 8, second.user_data.end(),
std::equal(second.user_data.begin() + 8, second.user_data.begin() + second.user_data_count,
params.user_data.begin() + 8),
"monolithic NGG shader used stale GS-back state or lost its s8 user data");
}
regs.gs_regs.rsrc2.user_sgpr = HW::UserSgprInfo::SGPRS_MAX;
for (uint32_t i = 4; i < HW::UserSgprInfo::SGPRS_MAX; ++i) {
regs.gs_user_sgpr.value[i] = 0x10001000u + i;
}
ShaderVertexInputInfo full_input{};
const auto full = PrepareProgram(regs, context, user_config, full_input);
Check(full.user_data_count == 8u + HW::UserSgprInfo::SGPRS_MAX &&
std::ranges::all_of(std::span(full.user_data).first(8),
[](uint32_t word) { return word == 0; }) &&
std::equal(std::begin(regs.gs_user_sgpr.value), std::end(regs.gs_user_sgpr.value),
full.user_data.begin() + 8),
"merged shader did not preserve the full user-SGPR range after its reserved prefix");
regs.gs_user_sgpr.value[HW::UserSgprInfo::SGPRS_MAX - 1]++;
Check(full.user_data.back() == 0x10001000u + HW::UserSgprInfo::SGPRS_MAX - 1,
"merged shader parameters borrowed mutable register storage");
context.SetMaxOutputPerSubgroup(256);
context.SetGsMaxVertOut(8);
user_config.SetPrimitiveType(Prospero::PrimitiveType::kTriFan);
@@ -12839,7 +12855,7 @@ void TestComputeLdsAllocationIdentity() {
"COMPUTE_PGM_RSRC2 LDS allocation units were not decoded");
auto options = MakeCompileOptions(ShaderType::Compute);
options.shader_hash = params.hash;
options.user_data = params.user_data;
options.user_data = std::span(params.user_data).first(params.user_data_count);
options.input_info.compute = &input_info;
options.wave_size = input_info.wave_size;
@@ -12905,7 +12921,7 @@ void TestComputeLdsAllocationIdentity() {
"AGC per-thread scratch size was not propagated");
auto scratch_options = MakeCompileOptions(ShaderType::Compute);
scratch_options.shader_hash = scratch_params.hash;
scratch_options.user_data = scratch_params.user_data;
scratch_options.user_data = std::span(scratch_params.user_data).first(scratch_params.user_data_count);
scratch_options.input_info.compute = &scratch_info;
scratch_options.wave_size = scratch_info.wave_size;