Implement RDNA2 per-invocation scratch memory

This commit is contained in:
Senaxx
2026-08-19 17:32:55 +02:00
committed by nmzik
parent 2db0b3cfa7
commit 56a15c2036
20 changed files with 184 additions and 53 deletions
@@ -46,7 +46,8 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
std::string dump = "CFG:\n";
dump += CFG::GraphToString(cfg);
dump += "\nIR:\n";
dump += fmt::format("mode={}\n", ir.dispatcher_fallback ? "dispatcher" : "structured");
dump += fmt::format("mode={} scratch_dwords={}\n",
ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords);
dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir);
return dump;
}
@@ -686,6 +687,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
output.shader_hash = options.shader_hash;
output.user_data_base = options.user_data_base;
output.user_data_count = options.user_data_count;
output.scratch_dwords = options.scratch_dwords;
return true;
};
if (!lower_program(ir)) {
@@ -18,6 +18,7 @@ struct CompileOptions {
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
uint32_t scratch_dwords = 0;
uint64_t shader_hash = 0;
uint64_t shader_base = 0;
std::optional<uint64_t> flat_memory_base;
@@ -158,6 +158,11 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
return Fail(error, "typed address handle has an invalid dense resource");
}
break;
case IR::ValueOpcode::GetScratchResource:
if (inst.NumArgs() != 0 || program.scratch_dwords == 0) {
return Fail(error, "typed scratch handle has invalid shader metadata");
}
break;
case IR::ValueOpcode::GetImageResource:
if (dense >= program.info.images.size()) {
return Fail(error, "typed image handle has an invalid dense resource");
@@ -198,6 +203,18 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.values->memory_info.size()) {
return Fail(error, "address operation has invalid memory metadata");
}
if (program.values->memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
if (program.scratch_dwords == 0) {
return Fail(error, "scratch operation has no per-thread storage");
}
requirements.function_scratch = true;
}
}
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.values->memory_info.size()) {
@@ -535,6 +535,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::GetSrtResource:
case IR::ValueOpcode::GetBufferResource:
case IR::ValueOpcode::GetAddressResource:
case IR::ValueOpcode::GetScratchResource:
case IR::ValueOpcode::GetImageResource:
case IR::ValueOpcode::GetSamplerResource:
case IR::ValueOpcode::MakeImageAddress: return true;
@@ -321,6 +321,7 @@ struct EmitterState {
uint32_t vsharp_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
std::array<uint32_t, 14> sampled_image_variables {};
std::array<uint32_t, 10> storage_image_variables {};
uint32_t sampler_variable = 0;
@@ -356,8 +357,8 @@ uint32_t TypeFunction(EmitterState& state);
uint32_t TypeStorageBufferPointer(EmitterState& state);
uint32_t TypeStorageBufferElementPointer(EmitterState& state);
uint32_t TypePushConstantElementPointer(EmitterState& state);
uint32_t TypeLdsArrayPointer(EmitterState& state, uint32_t storage_class);
uint32_t TypeLdsElementPointer(EmitterState& state, uint32_t storage_class);
uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
inline void EmitLabel(EmitterState& state, uint32_t label) {
state.current_label = label;
@@ -105,12 +105,12 @@ uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const I
}
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
const auto base = state.program.info.addresses[mem.resource].specialized_base;
if (mem.kind == IR::ResourceKind::Scratch) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid = Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
}
const auto base = state.program.info.addresses[mem.resource].specialized_base;
if (mem.kind == IR::ResourceKind::Flat ||
(mem.kind == IR::ResourceKind::Global &&
state.program.info.addresses[mem.resource].unbased)) {
@@ -75,7 +75,8 @@ static void EnsureLdsStorage(EmitterState& state) {
EXIT("function LDS was not prepared before SPIR-V function emission\n");
}
state.lds_variable = state.builder.DefineGlobalVariable(
TypeLdsArrayPointer(state, StorageClassWorkgroup), StorageClassWorkgroup);
TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)),
StorageClassWorkgroup);
state.builder.AddName(state.lds_variable, "lds_dwords");
}
@@ -98,10 +99,16 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
}
access.length = state.gds_length;
return access;
case IR::ResourceKind::Scratch:
if (state.scratch_variable == 0) {
EXIT("scratch storage was not prepared before SPIR-V function emission\n");
}
access.object_pointer = state.scratch_variable;
access.length = ConstantU32(state, state.program.scratch_dwords);
return access;
case IR::ResourceKind::ScalarAddress:
case IR::ResourceKind::Flat:
case IR::ResourceKind::Global:
case IR::ResourceKind::Scratch: {
case IR::ResourceKind::Global: {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource,
@@ -155,11 +162,11 @@ uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAcce
uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index) {
const auto pointer = state.builder.AllocateId();
if (access.kind == IR::ResourceKind::Lds) {
state.builder.AddFunction({OpAccessChain,
TypeLdsElementPointer(state, state.stage == ShaderType::Compute
? StorageClassWorkgroup
: StorageClassFunction),
if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
: state.stage == ShaderType::Compute ? StorageClassWorkgroup
: StorageClassFunction;
state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
pointer, access.object_pointer, index});
} else {
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
@@ -94,13 +94,13 @@ uint32_t TypePushConstantElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassPushConstant, TypeU32(state));
}
uint32_t TypeLdsArrayPointer(EmitterState& state, uint32_t storage_class) {
const auto count = ConstantU32(state, std::max(LdsDwordCount(state), 1u));
uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords) {
const auto count = ConstantU32(state, std::max(dwords, 1u));
const auto array = state.builder.Type(OpTypeArray, {TypeU32(state), count});
return TypePointer(state, storage_class, array);
}
uint32_t TypeLdsElementPointer(EmitterState& state, uint32_t storage_class) {
uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class) {
return TypePointer(state, storage_class, TypeU32(state));
}
@@ -601,6 +601,9 @@ void DefineModule(EmitterState& state) {
if (state.requirements.function_lds) {
state.lds_variable = state.builder.AllocateId();
}
if (state.requirements.function_scratch) {
state.scratch_variable = state.builder.AllocateId();
}
state.main_func = state.builder.AllocateId();
state.entry_label = state.builder.AllocateId();
@@ -659,6 +662,9 @@ void DefineModule(EmitterState& state) {
if (state.requirements.function_lds) {
state.builder.AddName(state.lds_variable, "lds_dwords");
}
if (state.requirements.function_scratch) {
state.builder.AddName(state.scratch_variable, "scratch_dwords");
}
AddInputAnnotationsAndNames(state);
AddOutputAnnotationsAndNames(state);
AddDescriptorAnnotationsAndNames(state);
@@ -580,8 +580,15 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std:
{OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
EmitLabel(state, state.entry_label);
if (state.requirements.function_lds) {
state.builder.AddFunction({OpVariable, TypeLdsArrayPointer(state, StorageClassFunction),
state.lds_variable, StorageClassFunction});
state.builder.AddFunction(
{OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)),
state.lds_variable, StorageClassFunction});
}
if (state.requirements.function_scratch) {
state.builder.AddFunction(
{OpVariable,
TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
state.scratch_variable, StorageClassFunction});
}
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable,
@@ -34,7 +34,7 @@ Translator::AddressOperands Translator::ReadAddressOperands(const IR::Instructio
high_or_base.reg.file != IR::RegisterFile::Vector
? ReadU32(high_or_base)
: low;
return {GetAddressResource(offset, IR::Value(0u)), offset, IR::Value(0u)};
return {ir.Emit(IR::ValueOpcode::GetScratchResource), offset, IR::Value(0u)};
}
if (inst.memory.kind == IR::ResourceKind::Global &&
high_or_base.kind == IR::OperandKind::Register &&
@@ -222,9 +222,9 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
mem.image_nsa_addr[i] = decoded.image_nsa_addr[i];
}
mem.memory_segment = decoded.memory_segment;
mem.address_is_full =
kind == ResourceKind::Flat || kind == ResourceKind::Scratch ||
(kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.address_is_full = kind == ResourceKind::Flat ||
(kind == ResourceKind::Global &&
decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.data_signed = decoded.data_signed;
mem.typed = decoded.typed;
mem.formatted = decoded.formatted;
@@ -236,7 +236,7 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
mem.offen = decoded.offen;
mem.resource = ResourceIndexFromOperand(decoded.src1);
mem.sampler = ResourceIndexFromOperand(decoded.src2);
if (kind == ResourceKind::Lds || kind == ResourceKind::Gds) {
if (kind == ResourceKind::Lds || kind == ResourceKind::Gds || kind == ResourceKind::Scratch) {
mem.resource = 0;
mem.sampler = 0;
}
@@ -430,6 +430,7 @@ struct SpirvRequirements {
bool compute_derivatives = false;
bool image_gather_extended = false;
bool function_lds = false;
bool function_scratch = false;
bool pixel_valid_mask = false;
};
@@ -440,6 +441,7 @@ struct Program {
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
uint32_t scratch_dwords = 0;
bool dispatcher_fallback = false;
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
std::string fallback_reason;
@@ -192,6 +192,7 @@ VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1)
VALUE_OPCODE(GetSrtResource, SrtResource)
VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32)
VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32)
VALUE_OPCODE(GetScratchResource, AddressResource)
VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32)
VALUE_OPCODE(GetSamplerResource, SamplerResource, U32, U32, U32, U32)
VALUE_OPCODE(MakeImageAddress, ImageAddress, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32,
@@ -679,11 +679,10 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
} else {
if (!runtime.flat_memory_base.has_value()) {
if (error != nullptr) {
*error =
fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
"guest-address translation",
address.kind == ResourceKind::Flat ? "FLAT" : "global/scratch",
address.first_use_pc);
*error = fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
"guest-address translation",
address.kind == ResourceKind::Flat ? "FLAT" : "global",
address.first_use_pc);
}
return false;
}
@@ -764,6 +764,18 @@ private:
if (!IsAddressResourceKind(memory.kind)) {
return Fail(flags.pc, error, "address operation has invalid resource kind");
}
if (memory.kind == ResourceKind::Scratch) {
handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetScratchResource ||
handle->NumArgs() != 0) {
return Fail(flags.pc, error, "scratch operation requires GetScratchResource");
}
if (m_program.scratch_dwords == 0) {
return Fail(flags.pc, error,
"scratch operation requires a nonzero AGC per-thread size");
}
return true;
}
bool unbased = false;
if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) {
return false;
+29 -10
View File
@@ -794,6 +794,7 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo& regs,
shader_addr);
return false;
}
info.scratch_size_dwords = data.scratch_size_dwords;
if (data.user_data == nullptr) {
LOGF("ShaderGetInputInfoVS(): no AGC user data for shader=0x%016" PRIx64 " es=0x%016" PRIx64
@@ -842,6 +843,12 @@ static void ShaderGetStaticInputInfoPS(
KYTY_PROFILER_FUNCTION();
ps_info = {};
ShaderMappedData data;
if (!ShaderGetMappedData(regs.ps_regs.data_addr, data)) {
EXIT("ShaderGetInputInfoPS(): shader=0x%016" PRIx64 " is missing from ShaderMap\n",
regs.ps_regs.data_addr);
}
ps_info.scratch_size_dwords = data.scratch_size_dwords;
// SPI_PS_IN_CONTROL.NUM_INTERP occupies bits 5:0. Keep the remaining control
// flags in the hardware state and extract only the input count here.
@@ -891,17 +898,23 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
const HW::ShaderRegisters& /*sh*/,
ShaderComputeInputInfo& info) {
info = {};
ShaderMappedData data;
if (!ShaderGetMappedData(regs.cs_regs.data_addr, data)) {
EXIT("ShaderGetInputInfoCS(): shader=0x%016" PRIx64 " is missing from ShaderMap\n",
regs.cs_regs.data_addr);
}
info.threads_num[0] = regs.cs_regs.num_thread_x;
info.threads_num[1] = regs.cs_regs.num_thread_y;
info.threads_num[2] = regs.cs_regs.num_thread_z;
info.lds_size_dwords = static_cast<uint32_t>(regs.cs_regs.lds_size) * 128u;
info.group_id[0] = regs.cs_regs.tgid_x_en != 0;
info.group_id[1] = regs.cs_regs.tgid_y_en != 0;
info.group_id[2] = regs.cs_regs.tgid_z_en != 0;
info.wave_size = regs.cs_regs.wave_size;
info.thread_ids_num = regs.cs_regs.tidig_comp_cnt + 1;
info.tg_size_en = regs.cs_regs.tg_size_en != 0;
info.threads_num[0] = regs.cs_regs.num_thread_x;
info.threads_num[1] = regs.cs_regs.num_thread_y;
info.threads_num[2] = regs.cs_regs.num_thread_z;
info.lds_size_dwords = static_cast<uint32_t>(regs.cs_regs.lds_size) * 128u;
info.scratch_size_dwords = data.scratch_size_dwords;
info.group_id[0] = regs.cs_regs.tgid_x_en != 0;
info.group_id[1] = regs.cs_regs.tgid_y_en != 0;
info.group_id[2] = regs.cs_regs.tgid_z_en != 0;
info.wave_size = regs.cs_regs.wave_size;
info.thread_ids_num = regs.cs_regs.tidig_comp_cnt + 1;
info.tg_size_en = regs.cs_regs.tg_size_en != 0;
info.workgroup_register = regs.cs_regs.user_sgpr;
}
@@ -1450,6 +1463,7 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi
options.shader_base = shader_addr;
options.user_data_base = 8;
options.user_data_count = regs.gs_regs.rsrc2.user_sgpr;
options.scratch_dwords = input_info.scratch_size_dwords;
options.user_data = regs.gs_user_sgpr.value;
options.read_specialization_memory = ReadShaderGuestMemory;
options.descriptor_set = 0;
@@ -1504,6 +1518,7 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis
options.shader_hash = shader_hash;
options.shader_base = shader_addr;
options.user_data_count = regs.ps_regs.rsrc2.user_sgpr;
options.scratch_dwords = input_info.scratch_size_dwords;
options.user_data = regs.ps_user_sgpr.value;
options.read_specialization_memory = ReadShaderGuestMemory;
options.descriptor_set = input_info.descriptor_set;
@@ -1554,6 +1569,7 @@ bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderReg
options.shader_hash = shader_addr;
options.shader_base = shader_addr;
options.user_data_count = regs.cs_regs.user_sgpr;
options.scratch_dwords = input_info.scratch_size_dwords;
options.user_data = regs.cs_user_sgpr.value;
options.read_specialization_memory = ReadShaderGuestMemory;
options.descriptor_set = 0;
@@ -1610,6 +1626,7 @@ ShaderId ShaderGetIdVS(const HW::VertexShaderInfo& regs, const ShaderVertexInput
ret.ids.push_back(static_cast<uint32_t>(input_info.fetch_embedded));
ret.ids.push_back(input_info.resources_num);
ret.ids.push_back(input_info.export_count);
ret.ids.push_back(input_info.scratch_size_dwords);
for (int i = 0; i < input_info.resources_num; i++) {
const auto& r = input_info.resources[i];
@@ -1663,6 +1680,7 @@ ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputIn
ret.ids.push_back(input_info.descriptor_set);
ret.ids.push_back(input_info.push_constant_offset);
ret.ids.push_back(input_info.scratch_size_dwords);
ret.ids.push_back(input_info.input_num);
ret.ids.push_back(input_info.ps_system_input_base);
ret.ids.push_back(static_cast<uint32_t>(input_info.ps_pos_x));
@@ -1726,6 +1744,7 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp
ret.ids.push_back(input_info.wave_size);
ret.ids.push_back(input_info.thread_ids_num);
ret.ids.push_back(input_info.lds_size_dwords);
ret.ids.push_back(input_info.scratch_size_dwords);
for (int i = 0; i < 3; i++) {
ret.ids.push_back(input_info.threads_num[i]);
+13 -9
View File
@@ -89,21 +89,23 @@ struct ShaderVertexInputInfo {
int resource_fetch_components[RES_MAX] = {};
ShaderVertexInputBuffer buffers[RES_MAX];
ShaderStageRuntime stage;
int resources_num = 0;
int fetch_shader_reg = 0;
int fetch_attrib_reg = 0;
int fetch_buffer_reg = 0;
int buffers_num = 0;
int export_count = 0;
uint32_t param_export_mask = 0;
bool fetch_external = false;
bool fetch_embedded = false;
int resources_num = 0;
int fetch_shader_reg = 0;
int fetch_attrib_reg = 0;
int fetch_buffer_reg = 0;
int buffers_num = 0;
int export_count = 0;
uint32_t scratch_size_dwords = 0;
uint32_t param_export_mask = 0;
bool fetch_external = false;
bool fetch_embedded = false;
};
struct ShaderComputeInputInfo {
uint32_t threads_num[3] = {0, 0, 0};
uint32_t dispatch_threads_num[3] = {0, 0, 0};
uint32_t lds_size_dwords = 0;
uint32_t scratch_size_dwords = 0;
bool group_id[3] = {false, false, false};
bool dispatch_thread_dimensions = false;
uint32_t wave_size = 64;
@@ -122,6 +124,7 @@ struct ShaderPixelInputInfo {
uint32_t mrt_output_mask = 0;
uint32_t descriptor_set = 0;
uint32_t push_constant_offset = 0;
uint32_t scratch_size_dwords = 0;
bool ps_pos_x = false;
bool ps_pos_y = false;
bool ps_pos_xy = false;
@@ -241,6 +244,7 @@ struct ShaderMappedData {
ShaderSemantic* input_semantics = nullptr;
uint32_t num_input_semantics = 0;
uint32_t code_size_bytes = 0;
uint32_t scratch_size_dwords = 0;
};
void ShaderInit();