shader: share formatted stores for GPU-selected signed byte records

This commit is contained in:
nmzik
2026-10-14 17:10:02 +02:00
parent 04cb4c6553
commit 6d6da57a95
5 changed files with 329 additions and 38 deletions
@@ -813,6 +813,7 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
const IR::MemoryInfo& mem, const MemoryResourceAccess& resource,
const Format::BufferFormatInfo& info, uint32_t data,
uint32_t components) {
if (resource.object_pointer == 0u) return;
// RDNA2 formatted stores transfer the entire format, zero-filling missing VGPRs.
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info);
EmitIfCondition(ctx.state, plan.in_bounds, [&]() {
@@ -852,14 +853,15 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
const auto data = ctx.Arg(inst, inst.NumArgs() - 2);
const auto info = Format::GetFormatInfo(BufferFormat(ctx, mem));
if (info.type == Format::ComponentType::Unknown) {
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data);
return;
}
StoreFormattedPrepared(ctx, inst, mem, resource, info, data, 1u);
const auto data = ctx.Arg(inst, inst.NumArgs() - 2);
EmitBufferAccess(ctx, inst, mem, 0u, [&](const auto& selected, const auto& resource) {
const auto info = Format::GetFormatInfo(BufferFormat(ctx, selected));
if (info.type == Format::ComponentType::Unknown) {
StoreWordPrepared(ctx, inst, RebaseRawComponent(selected, 0u), resource, data);
return;
}
StoreFormattedPrepared(ctx, inst, selected, resource, info, data, 1u);
});
});
}
@@ -249,11 +249,18 @@ bool NormalizeIndirectBuffer(DescriptorValue& value, bool indexed, const BufferR
descriptor.OutOfBounds() != (indexed ? 0u : 3u)) {
return SpecializationFail("indirect buffers require exact linear index or byte bounds");
}
if (formatted_read) {
if (resource.formatted) {
const auto format = Format::GetFormatInfo(descriptor.Format());
if (!indexed || format.type == Format::ComponentType::Unknown)
return SpecializationFail("indirect formatted loads require a supported indexed format");
if (descriptor.Base48() == 0u || descriptor.Stride() == 0u || descriptor.NumRecords() == 0u) {
return SpecializationFail("indirect formatted buffers require a supported indexed format");
if (!formatted_read &&
((descriptor.Format() != Prospero::BufferFormat::k32UInt &&
descriptor.Format() != Prospero::BufferFormat::k8SInt) ||
descriptor.DstSelXYZW() != DstSel(4, 0, 0, 0) ||
descriptor.Stride() != format.byte_size))
return SpecializationFail("indirect formatted X stores require tight SInt8 or UInt32 identity records");
if (formatted_read &&
(descriptor.Base48() == 0u || descriptor.Stride() == 0u || descriptor.NumRecords() == 0u)) {
// Every index is out of bounds. Preserve the format and swizzle constants.
value.dwords[1] &= ~(0x3fffu << 16u);
value.dwords[2] = 0u;
@@ -263,14 +270,10 @@ bool NormalizeIndirectBuffer(DescriptorValue& value, bool indexed, const BufferR
if (descriptor.Stride() < format.byte_size ||
(descriptor.Stride() & (alignment - 1u)) != 0u ||
(descriptor.Base48() & (alignment - 1u)) != 0u)
return SpecializationFail("indirect formatted loads require aligned full-format records");
return SpecializationFail("indirect formatted buffers require aligned full-format records");
} else if ((!indexed && descriptor.Stride() != 1u) ||
(indexed && resource.max_byte_extent != 1u && descriptor.Stride() != 4u)) {
return SpecializationFail("indirect buffer stores require exact linear DWORD or byte bounds");
} else if (resource.formatted && (descriptor.Format() != Prospero::BufferFormat::k32UInt ||
descriptor.DstSelXYZW() != DstSel(4, 0, 0, 0) ||
(descriptor.Base48() & 3u) != 0u)) {
return SpecializationFail("indirect formatted X stores require an aligned 32-bit UInt identity mapping");
}
return true;
}
@@ -1614,16 +1617,6 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
inst.SetFlags(image_remap[inst.Flags<uint32_t>()]);
continue;
}
if (inst.GetOpcode() == ValueOpcode::StoreBufferU32) {
auto& memory = memory_info[inst.Flags<MemoryFlags>().index];
if (memory.kind == ResourceKind::Buffer && memory.formatted &&
program.descriptor_sources[buffers[memory.resource].source].indirect_descriptor) {
// Candidate validation proved an index-only k32UInt identity store.
// Reuse finite raw-store dispatch and its original-index bounds check.
memory.formatted = false;
}
continue;
}
if (BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read) {
const auto& memory = memory_info[inst.Flags<MemoryFlags>().index];
const bool native_oob = memory.kind == ResourceKind::Buffer &&
@@ -400,7 +400,8 @@ public:
std::vector<std::pair<const IndirectDescriptorPlan*, Value>> buffer_selections;
for (const auto& plan: m_indirect_descriptors) {
auto key = plan.key;
if (m_sources[plan.source].indirect_descriptor->table_maximum) {
if (m_sources[plan.source].indirect_descriptor->table_maximum &&
plan.handle->GetOpcode() != ValueOpcode::GetBufferResource) {
auto* block = plan.handle->Parent();
const auto where = std::ranges::find_if(block->Instructions(),
[&](const Inst& inst) { return &inst == plan.handle; });
@@ -481,7 +482,10 @@ public:
const auto where = std::ranges::find_if(block->Instructions(),
[&](const Inst& inst) { return &inst == anchor; });
const auto& indirect = *m_sources[plan.source].indirect_descriptor;
if (indirect.table_scalar) {
if (indirect.table_maximum) {
key = Value(&*block->PrependNewInst(
where, ValueOpcode::ShiftLeftLogical32, {key, Value(2u)}));
} else if (indirect.table_scalar) {
// Wrapped affine offsets span one power-of-two congruence class.
// Keep the existing mapping's byte key while compressing its empty gaps.
key = plan.reads[0]->Arg(1);
@@ -508,7 +512,7 @@ public:
plan.handle->SetArg(dword, plan.key);
}
}
if (plan.reads[0] != nullptr) {
if (plan.reads[0] != nullptr && !plan.retain_reads) {
for (uint32_t word = 0; word < plan.handle->NumArgs(); ++word)
m_program.memory_info[plan.memory[word]].planning_only = true;
}
@@ -2713,7 +2717,7 @@ private:
return !IndirectBufferAccess(*consumer.user);
});
})) {
return false;
plan.retain_reads = true;
}
plan.memory[dword] = memory_index;
plan.reads[dword] = read;
@@ -2725,6 +2729,15 @@ private:
return true;
}
bool SetExactTableMaximum(Value key, DescriptorSource::IndirectDescriptor& indirect) {
const auto maximum = SelectorMaximum(key);
if (maximum > UINT32_MAX / 4u ||
uint64_t {maximum} * indirect.table_stride + indirect.table_offset > UINT32_MAX)
return false;
indirect.table_maximum = maximum;
return true;
}
bool TryMakeNestedDescriptor(Inst& handle, const DescriptorSource& descriptor,
IndirectDescriptorPlan& plan) {
if (handle.GetOpcode() != ValueOpcode::GetImageResource &&
@@ -2769,9 +2782,7 @@ private:
DescriptorSource::IndirectDescriptor indirect;
if (!MatchTableOffset(low->Arg(1), key, indirect.table_offset, indirect.table_stride) ||
indirect.table_stride < 8u || (indirect.table_stride & 3u) != 0u) return false;
const auto maximum = SelectorMaximum(key);
if (maximum > UINT32_MAX / 4u ||
uint64_t {maximum} * indirect.table_stride + indirect.table_offset > UINT32_MAX ||
if (!SetExactTableMaximum(key, indirect) ||
(key.TryInstruction() != nullptr && !AvailableBefore(*key.Instruction(), handle)))
return false;
DescriptorSource table_source;
@@ -2780,7 +2791,6 @@ private:
indirect.table_source = InternSource(table_source);
indirect.table_immediate = low_memory->offset & ~3u;
indirect.table_scalar = true;
indirect.table_maximum = maximum;
indirect.pointer_offset = descriptor_offset;
DescriptorSource source;
source.dword_count = handle.NumArgs();
@@ -3007,6 +3017,11 @@ private:
indirect.signed_count = bound->GetOpcode() == ValueOpcode::SLessThan32;
return finish();
}
if (table_source.dword_count == 4u && !plan.split_offsets &&
plan.selection_anchor != nullptr && SetExactTableMaximum(key, indirect)) {
indirect.table_scalar = true;
return finish();
}
if (!legacy_store) {
// A bounded V# supplies every reachable descriptor without reading the GPU key.
// Separate U32 offsets could wrap each DWORD to a different table position.
@@ -3300,7 +3315,7 @@ private:
bool IsIndirectPlanningMemory(uint32_t index) const {
return std::any_of(m_indirect_descriptors.begin(), m_indirect_descriptors.end(),
[&](const IndirectDescriptorPlan& plan) {
return plan.reads[0] != nullptr &&
return plan.reads[0] != nullptr && !plan.retain_reads &&
std::find(plan.memory.begin(), plan.memory.begin() + plan.handle->NumArgs(), index) !=
plan.memory.begin() + plan.handle->NumArgs();
});