mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-19 18:36:53 +02:00
shader: resolve workgroup-selected descriptor prefixes from dispatch arguments
Co-authored-by: fxpw <fxpw@outlook.com>
This commit is contained in:
nmzik
@@ -347,6 +347,7 @@ struct PipelineCache::ProgramCache {
|
||||
};
|
||||
if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
|
||||
runtime.workgroup_counts = input_info.workgroup_counts;
|
||||
runtime.indirect_args_address = input_info.indirect_args_address;
|
||||
}
|
||||
bool source_changed = false;
|
||||
if (entry != programs.end()) {
|
||||
|
||||
@@ -457,6 +457,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
}
|
||||
ShaderComputeInputInfo input_info {};
|
||||
input_info.async_compute = async_compute;
|
||||
input_info.indirect_args_address = args_addr;
|
||||
const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
|
||||
cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
|
||||
buffer.EndRendering();
|
||||
|
||||
@@ -307,9 +307,10 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
|
||||
const auto root_resource = specializations[resource_index];
|
||||
const auto evaluate_count = [&](uint32_t& count) {
|
||||
if (indirect.workgroup_axis != UINT32_MAX) {
|
||||
if (indirect.workgroup_axis >= runtime.workgroup_counts.size()) return false;
|
||||
count = runtime.workgroup_counts[indirect.workgroup_axis];
|
||||
return count != 0u;
|
||||
if (runtime.workgroup_counts.size() != 3u || indirect.workgroup_axis >= 3u) return false;
|
||||
count = std::ranges::find(runtime.workgroup_counts, 0u) != runtime.workgroup_counts.end()
|
||||
? 0u : runtime.workgroup_counts[indirect.workgroup_axis];
|
||||
return true;
|
||||
}
|
||||
if (!clean.Evaluate(indirect.key_count, count)) return false;
|
||||
if (indirect.signed_count && std::bit_cast<int32_t>(count) < 0) count = 0;
|
||||
@@ -337,14 +338,28 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
|
||||
const auto read_keys = [&](uint64_t base, uint64_t size, uint64_t first,
|
||||
uint64_t step, uint64_t count) {
|
||||
keys.resize(count);
|
||||
if (count == 0u) return true;
|
||||
if (step == 4u) {
|
||||
return read_selector(base, size, first, keys);
|
||||
}
|
||||
for (auto& key: keys) {
|
||||
if (!read_selector(base, size, first, {&key, 1}))
|
||||
if (step < 4u) return false;
|
||||
for (uint64_t copied = 0; copied < count;) {
|
||||
const auto offset = first + copied * step;
|
||||
// Scalar reads align each strided offset independently.
|
||||
const auto bias = selector->global ? 0u : offset & 3u;
|
||||
const auto length = std::min(count - copied,
|
||||
1u + (uint64_t {MaxIndirectDescriptorProbes - 1u} * 4u - bias) / step);
|
||||
keys.resize(count + (bias + (length - 1u) * step) / 4u + 1u);
|
||||
const auto batch = std::span(keys).subspan(count);
|
||||
if (!read_selector(base, size, offset, batch)) {
|
||||
keys.resize(count);
|
||||
return false;
|
||||
first += step;
|
||||
}
|
||||
for (uint64_t index = 0; index < length; ++index)
|
||||
keys[copied + index] = batch[(bias + index * step) / 4u];
|
||||
copied += length;
|
||||
}
|
||||
keys.resize(count);
|
||||
return true;
|
||||
};
|
||||
uint64_t table_base = 0;
|
||||
@@ -564,7 +579,8 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
|
||||
uint64_t {first} + count > material.NumRecords())) return false;
|
||||
const auto end = count != 0u
|
||||
? (uint64_t {first} + count - 1u) * step + selector->offset + 4u : 0u;
|
||||
if (end > uint64_t {UINT32_MAX} + 1u || end > size)
|
||||
if (end > uint64_t {UINT32_MAX} + 1u ||
|
||||
(indirect.workgroup_axis == UINT32_MAX && end > size))
|
||||
return false;
|
||||
if (!read_keys(base, size, uint64_t {first} * step + selector->offset, step, count))
|
||||
return false;
|
||||
@@ -1331,11 +1347,16 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
runtime.read_specialization_memory == nullptr)) {
|
||||
return false;
|
||||
}
|
||||
const bool capture_reads = program.capture_specialization_reads || !program.source_reads.empty();
|
||||
const auto uses_workgroup_count = [](const DescriptorSource& source) {
|
||||
return source.indirect_descriptor && source.indirect_descriptor->workgroup_axis != UINT32_MAX;
|
||||
};
|
||||
const bool capture_reads = program.capture_specialization_reads || !program.source_reads.empty() ||
|
||||
(runtime.indirect_args_address != 0u &&
|
||||
std::ranges::any_of(program.descriptor_sources, uses_workgroup_count));
|
||||
auto& reads = snapshot.specialization_reads;
|
||||
reads = program.source_reads;
|
||||
SrtReadCapture capture {runtime, reads};
|
||||
const auto observed = capture_reads ? capture.ObservedRuntime() : runtime;
|
||||
auto observed = capture_reads ? capture.ObservedRuntime() : runtime;
|
||||
SrtWalker clean(program, CleanRuntime(observed));
|
||||
SrtWalker walker(program, observed,
|
||||
capture_reads || program.requires_specialization_memory ? &clean : nullptr);
|
||||
@@ -1343,6 +1364,18 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
return false;
|
||||
}
|
||||
const auto active = std::span<const uint8_t>(program.active_sources);
|
||||
std::array<uint32_t, 3> workgroup_counts {};
|
||||
if (runtime.indirect_args_address != 0u) {
|
||||
for (uint32_t source = 0; source < program.descriptor_sources.size(); ++source) {
|
||||
if ((!active.empty() && !active[source]) ||
|
||||
!uses_workgroup_count(program.descriptor_sources[source])) continue;
|
||||
if ((runtime.indirect_args_address & 3u) != 0u ||
|
||||
!ReadTable(runtime.indirect_args_address, sizeof(workgroup_counts), 0u,
|
||||
observed, workgroup_counts)) return false;
|
||||
observed.workgroup_counts = workgroup_counts;
|
||||
break;
|
||||
}
|
||||
}
|
||||
snapshot.uniform_fill = {};
|
||||
const auto& fill = program.uniform_fill;
|
||||
const auto words = fill.fill.words;
|
||||
|
||||
@@ -2204,6 +2204,7 @@ private:
|
||||
const auto* memory = ScalarReadMemory(*read, memory_index);
|
||||
Value index;
|
||||
Value count;
|
||||
uint32_t workgroup_axis = UINT32_MAX;
|
||||
DescriptorSource count_source;
|
||||
uint32_t row_stride = 0;
|
||||
uint32_t offset = 0, stride = 0;
|
||||
@@ -2212,19 +2213,21 @@ private:
|
||||
if (MatchTableOffset(read->Arg(1), index, offset, stride) &&
|
||||
((offset | stride | memory->offset) & 3u) == 0u &&
|
||||
uint64_t {offset} + memory->offset <= UINT32_MAX) {
|
||||
workgroup_axis = WorkgroupAxis(index);
|
||||
const auto* bound = BoundedLoop(index, read->Parent());
|
||||
if (bound != nullptr && bound->GetOpcode() == ValueOpcode::ULessThan32) {
|
||||
count = bound->Arg(1);
|
||||
}
|
||||
}
|
||||
// Raw pointers have no V# bounds for enumerating an unbounded selector domain.
|
||||
if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty()) return false;
|
||||
const bool rows = count.IsEmpty() && memory->offset == 0u &&
|
||||
if (memory->kind == ResourceKind::ScalarAddress && count.IsEmpty() &&
|
||||
workgroup_axis == UINT32_MAX) return false;
|
||||
const bool rows = count.IsEmpty() && workgroup_axis == UINT32_MAX && memory->offset == 0u &&
|
||||
MatchCountedRows(*read, count_source, row_stride);
|
||||
if (rows) {
|
||||
stride = 4u;
|
||||
offset = 0u;
|
||||
} else if (count.IsEmpty()) {
|
||||
} else if (count.IsEmpty() && workgroup_axis == UINT32_MAX) {
|
||||
auto address = read->Arg(1);
|
||||
auto active = Value(true);
|
||||
const auto* lane = address.Resolve().TryInstruction();
|
||||
@@ -2255,17 +2258,18 @@ private:
|
||||
}
|
||||
// A bounded image heap already supports GPU selection. Only enumerate a
|
||||
// narrower proven selector domain instead of scanning unrelated material data.
|
||||
if (require_bound && count.IsEmpty() && !rows) return false;
|
||||
if (require_bound && count.IsEmpty() && workgroup_axis == UINT32_MAX && !rows) return false;
|
||||
if (!MakeRuntimeTableSource(*read, material_source)) return false;
|
||||
selector.source = InternSource(material_source);
|
||||
selector.stride = stride;
|
||||
selector.offset = offset + (memory->offset & ~3u);
|
||||
indirect.selector = std::move(selector);
|
||||
indirect.key_count = count;
|
||||
indirect.workgroup_axis = workgroup_axis;
|
||||
if (rows) indirect.selector->rows.emplace(
|
||||
DescriptorSource::IndirectDescriptor::SelectorRead::CountedRows {
|
||||
.source = InternSource(count_source), .stride = row_stride});
|
||||
if (!count.IsEmpty()) indirect.selector_first = Value(0u);
|
||||
if (!count.IsEmpty() || workgroup_axis != UINT32_MAX) indirect.selector_first = Value(0u);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -3025,6 +3029,11 @@ private:
|
||||
indirect.table_scalar = true;
|
||||
return finish();
|
||||
}
|
||||
DescriptorSource material_source;
|
||||
if (!plan.split_offsets && MatchScalarMaterialKey(key, indirect, material_source, true)) {
|
||||
plan.retain_reads = true;
|
||||
return finish();
|
||||
}
|
||||
if (!legacy_store) {
|
||||
if (table_source.dword_count == 2u && !plan.split_offsets) {
|
||||
const auto maximum = SelectorMaximum(key);
|
||||
@@ -3082,7 +3091,6 @@ private:
|
||||
!ImmediateU32(read->Arg(2), offset) || offset != 0u ||
|
||||
!ImmediateU32(read->Arg(3), scalar) || scalar != 0u ||
|
||||
!EquivalentValue(m_program, selected->Arg(0), read->Arg(4))) return false;
|
||||
DescriptorSource material_source;
|
||||
if (!MakeRuntimeTableSource(*read, material_source) ||
|
||||
!BoundedSelectedIndex(read->Arg(1), read->Arg(4), indirect)) return false;
|
||||
indirect.selector.emplace(DescriptorSource::IndirectDescriptor::SelectorRead{.source = InternSource(material_source),
|
||||
|
||||
@@ -19,6 +19,7 @@ struct SrtRuntime {
|
||||
void* userdata = nullptr;
|
||||
SrtMemoryReader read_specialization_memory = nullptr;
|
||||
std::span<const uint32_t> workgroup_counts;
|
||||
uint64_t indirect_args_address = 0;
|
||||
};
|
||||
|
||||
class SrtReadCapture {
|
||||
|
||||
@@ -578,11 +578,13 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
|
||||
const bool dispatch_thread_dimensions = info.dispatch_thread_dimensions;
|
||||
const auto host_subgroup_size = info.host_subgroup_size;
|
||||
const auto workgroup_counts = std::to_array(info.workgroup_counts);
|
||||
const auto indirect_args_address = info.indirect_args_address;
|
||||
info = {};
|
||||
info.async_compute = async_compute;
|
||||
info.dispatch_thread_dimensions = dispatch_thread_dimensions;
|
||||
info.host_subgroup_size = host_subgroup_size;
|
||||
std::ranges::copy(workgroup_counts, info.workgroup_counts);
|
||||
info.indirect_args_address = indirect_args_address;
|
||||
info.threads_num[0] = regs.cs_regs.num_thread_x;
|
||||
info.threads_num[1] = regs.cs_regs.num_thread_y;
|
||||
info.threads_num[2] = regs.cs_regs.num_thread_z;
|
||||
|
||||
@@ -150,6 +150,7 @@ struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo {
|
||||
uint8_t float_mode = 0xc0;
|
||||
uint32_t dispatch_threads_num[3] = {0, 0, 0};
|
||||
uint32_t workgroup_counts[3] = {0, 0, 0};
|
||||
uint64_t indirect_args_address = 0;
|
||||
bool group_id[3] = {false, false, false};
|
||||
bool dispatch_thread_dimensions = false;
|
||||
bool lds_storage = false;
|
||||
|
||||
+160
-15
@@ -222,7 +222,8 @@ MakeIndirectImageFixture(bool malformed, uint32_t table_stride = 32, uint32_t ta
|
||||
ValueOpcode::GetBuiltin,
|
||||
{Value(static_cast<uint32_t>(finite_key ? StageInputKind::WorkgroupId
|
||||
: StageInputKind::GlobalInvocationId)), Value(0u)});
|
||||
const auto selector = finite_key ? invocation
|
||||
const auto selector = finite_key
|
||||
? fixture->Emit(ValueOpcode::BitwiseXor32, {invocation, Value(1u)})
|
||||
: fixture->Emit(ValueOpcode::ReadFirstLane, {invocation, Value(true)});
|
||||
Value member;
|
||||
if (nested) {
|
||||
@@ -2699,7 +2700,7 @@ void TestUniformizedMaterialImageKeys() {
|
||||
memory.fail_address = base + 16u * 16u + 8u;
|
||||
memory.reads = memory.descriptor_reads = 0u;
|
||||
Check(MaterializeResources(packed, runtime, snapshot, specialization) &&
|
||||
memory.reads == 19u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u &&
|
||||
memory.reads == 4u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u &&
|
||||
snapshot.images[0].dwords[0] == 0x200u && snapshot.images[1].dwords[0] == 0x400u &&
|
||||
snapshot.images[2].dwords[0] == 0x600u,
|
||||
"packed selector did not extract before clamping or read outside its sixteen records");
|
||||
@@ -2719,7 +2720,7 @@ void TestUniformizedMaterialImageKeys() {
|
||||
snapshot.images.size() == 3u && snapshot.images[0].dwords[0] == 0x200u &&
|
||||
snapshot.images[1].dwords[0] == 0x400u && snapshot.images[2].dwords[0] == 0x600u &&
|
||||
std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair<uint64_t, uint64_t>{base + base_offset + immediate, 4u}) !=
|
||||
std::pair<uint64_t, uint64_t>{base + base_offset + immediate, 244u}) !=
|
||||
snapshot.specialization_reads.end(),
|
||||
"global selector did not preserve exact base-plus-offset while scalar descriptor reads aligned the base");
|
||||
}
|
||||
@@ -2731,7 +2732,7 @@ void TestUniformizedMaterialImageKeys() {
|
||||
memory.fail_address = base + 16u * 16u + 8u;
|
||||
memory.reads = memory.descriptor_reads = 0u;
|
||||
Check(MaterializeResources(merged, runtime, snapshot, specialization) &&
|
||||
memory.reads == 19u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u &&
|
||||
memory.reads == 4u && memory.descriptor_reads == 3u && snapshot.images.size() == 3u &&
|
||||
snapshot.images[0].dwords[0] == 0x200u && snapshot.images[1].dwords[0] == 0x400u &&
|
||||
snapshot.images[2].dwords[0] == 0x600u,
|
||||
"merged nibble selector did not preserve its sixteen-record domain and lower-half projection");
|
||||
@@ -2978,12 +2979,14 @@ void TestScalarBufferSelectorDomain() {
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
memory.watched_address = 0x1000u + 144u;
|
||||
memory.fail_address = 0x1000u + 160u;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.images.size() == 3u && memory.watched_reads == 1u &&
|
||||
memory.watched_dwords == 1u && snapshot.images[0].dwords[0] == 0x100u &&
|
||||
memory.watched_dwords == 9u && snapshot.images[0].dwords[0] == 0x100u &&
|
||||
snapshot.images[2].dwords[0] == 0x102u,
|
||||
"scalar selector read outside its wrapped domain or omitted the OOB zero key");
|
||||
"scalar selector interpreted batched padding or omitted the OOB zero key");
|
||||
Check(std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair<uint64_t, uint64_t>{0x1090u, 36u}) != snapshot.specialization_reads.end(),
|
||||
"scalar selector batch escaped specialization overlap tracking");
|
||||
const auto scaled_lane = make_plan(true, false, true);
|
||||
memory.words[176u / 4u] = UINT32_MAX;
|
||||
Check(MaterializeResources(scaled_lane, runtime, snapshot, specialization) &&
|
||||
@@ -3723,7 +3726,7 @@ void TestSharedAppendSelectorBounds() {
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.images.size() == 1u && memory.descriptor_reads == 0u && memory.watched_reads == 0u,
|
||||
"empty shared append lists probed material keys or image descriptors");
|
||||
memory.fail_address = 0x10b0u;
|
||||
memory.fail_address = UINT64_MAX;
|
||||
for (const auto counts : {std::array{1u, 2u}, std::array{3u, 1u}, std::array{1u, 1u}, std::array{0u, 0u}}) {
|
||||
memory.words[0x800u / 4u] = counts[0];
|
||||
memory.words[0x804u / 4u] = counts[1];
|
||||
@@ -3778,10 +3781,13 @@ void TestSharedAppendSelectorBounds() {
|
||||
words[0] = memory.counts[(address - 0x40000000u) / 4u];
|
||||
return true;
|
||||
}
|
||||
if (words.size() == 1u && address >= 0x1090u &&
|
||||
(address - 0x1090u) % 160u == 0u && (address - 0x1090u) / 160u < 65536u) {
|
||||
words[0] = 1u;
|
||||
++memory.keys_read;
|
||||
if (address >= 0x1090u && (address - 0x1090u) % 160u == 0u &&
|
||||
address - 0x1090u + words.size_bytes() <= 65535u * 160u + 4u) {
|
||||
std::ranges::fill(words, 0xdeadbeefu);
|
||||
for (size_t word = 0; word < words.size(); word += 40u) {
|
||||
words[word] = 1u;
|
||||
++memory.keys_read;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (words.size() != 8u || address != 0x50000020u) return false;
|
||||
@@ -4784,7 +4790,8 @@ void TestBoundedFormattedBufferStore() {
|
||||
|
||||
std::unique_ptr<Fixture> MakeBufferRecordFixture(uint32_t step = 1u, bool bypass = false,
|
||||
bool gpu_selected = false, uint32_t bias = 0u,
|
||||
bool split_offsets = false) {
|
||||
bool split_offsets = false,
|
||||
bool workgroup_selected = false) {
|
||||
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
|
||||
auto fixture = std::make_unique<Fixture>();
|
||||
auto& f = *fixture;
|
||||
@@ -4796,8 +4803,8 @@ std::unique_ptr<Fixture> MakeBufferRecordFixture(uint32_t step = 1u, bool bypass
|
||||
if (gpu_selected) {
|
||||
std::array<Value, 4> selector_words;
|
||||
for (uint32_t word = 0; word < 4u; ++word) selector_words[word] = f.UserData(4u + word);
|
||||
const auto group = f.Emit(ValueOpcode::GetBuiltin,
|
||||
{Value(static_cast<uint32_t>(StageInputKind::WorkgroupId)), Value(0u)});
|
||||
const auto group = workgroup_selected ? f.Emit(ValueOpcode::GetBuiltin,
|
||||
{Value(static_cast<uint32_t>(StageInputKind::WorkgroupId)), Value(0u)}) : lane;
|
||||
key = f.Emit(ValueOpcode::ReadConstBuffer,
|
||||
{f.Buffer(selector_words), f.Emit(ValueOpcode::IMul32, {group, Value(12u)})},
|
||||
f.AddMemory({.kind = ResourceKind::ScalarBuffer}, 0x1bcu));
|
||||
@@ -5090,6 +5097,143 @@ void TestGpuSelectedBufferRecords() {
|
||||
"not a valid runtime value", "split dynamic descriptor offsets lost independent U32 wrapping");
|
||||
}
|
||||
|
||||
void TestWorkgroupSelectedBufferRecords() {
|
||||
namespace Prospero = Libs::Graphics::Prospero;
|
||||
auto fixture = MakeBufferRecordFixture(1u, false, true, 0u, false, true);
|
||||
auto plan = ExtractResourcePlan(fixture->program);
|
||||
std::vector<uint32_t> roots;
|
||||
for (uint32_t root = 0; root < plan.info.buffers.size(); ++root) {
|
||||
const auto& source = plan.descriptor_sources[plan.info.buffers[root].source];
|
||||
if (!source.indirect_descriptor) continue;
|
||||
const auto& indirect = *source.indirect_descriptor;
|
||||
Check(indirect.workgroup_axis == 0u && indirect.selector_first == Value(0u) &&
|
||||
indirect.selector && indirect.selector->stride == 12u &&
|
||||
indirect.table_stride == 192u && !indirect.table_scalar,
|
||||
"workgroup selector lost its exact prefix or descriptor geometry");
|
||||
roots.push_back(root);
|
||||
}
|
||||
Check(roots.size() == 2u, "workgroup selector did not retain both buffer operations");
|
||||
LinearTestMemory memory;
|
||||
constexpr uint32_t table = 256u, selector = 0x1000u;
|
||||
memory.words.resize((selector + MaxIndirectDescriptorProbes * 12u) / 4u);
|
||||
std::fill_n(memory.words.begin() + table / 4u, 384u / 4u, 0xdeadbeefu);
|
||||
for (uint32_t row = 0; row < 2u; ++row) {
|
||||
for (uint32_t member = 0; member < 2u; ++member) {
|
||||
const auto first = (table + row * 192u + 160u + member * 16u) / 4u;
|
||||
memory.words[first] = 0x100000u + row * 0x1000u + member * 0x10000u;
|
||||
memory.words[first + 1u] = (member == 0u ? 8u : 1u) << 16u;
|
||||
memory.words[first + 2u] = 16u;
|
||||
memory.words[first + 3u] = member == 0u
|
||||
? (uint32_t(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | Libs::Graphics::DstSel(4, 5, 6, 7)
|
||||
: 0x5204u;
|
||||
}
|
||||
}
|
||||
const std::array<uint32_t, 4> numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au};
|
||||
std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), memory.words.begin() + (table + 304u) / 4u);
|
||||
std::array<uint32_t, 8> user_data{uint32_t(memory.base) + table, 192u << 16u, 2u, 0x5204u,
|
||||
uint32_t(memory.base) + selector, 12u << 16u, MaxIndirectDescriptorProbes, 0x5204u};
|
||||
memory.watched_address = memory.base;
|
||||
SrtRuntime runtime{.user_data = user_data, .read_memory = ReadLinearTestMemory,
|
||||
.userdata = &memory, .read_specialization_memory = ReadLinearTestMemory,
|
||||
.indirect_args_address = memory.base};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
const auto materialize = [&] {
|
||||
memory.reads = memory.watched_reads = 0u;
|
||||
return MaterializeResources(plan, runtime, snapshot, specialization);
|
||||
};
|
||||
for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u},
|
||||
std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u},
|
||||
std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) {
|
||||
std::copy(dimensions.begin(), dimensions.end(), memory.words.begin());
|
||||
memory.words[selector / 4u] = dimensions[0] == 1u ? 0u : 1u;
|
||||
memory.words[selector / 4u + 3u] = 0u;
|
||||
memory.words[selector / 4u + 6u] = 22369622u; // Would select the numeric fields at +288/+304.
|
||||
const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end();
|
||||
Check(materialize() && memory.watched_reads == 1u && memory.watched_dwords == 3u &&
|
||||
snapshot.buffers.size() == plan.info.buffers.size() + (empty ? 0u : (dimensions[0] - 1u) * 2u) &&
|
||||
memory.reads == (empty ? 1u : 3u + dimensions[0] * 2u),
|
||||
"workgroup prefix reread arguments, visited an inactive key or retained stale candidates");
|
||||
Check(std::ranges::find(snapshot.specialization_reads, std::pair{memory.base, uint64_t{12u}}) !=
|
||||
snapshot.specialization_reads.end(),
|
||||
"indirect dispatch arguments escaped writable-alias tracking");
|
||||
if (!empty)
|
||||
Check(std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair{memory.base + selector, uint64_t{(dimensions[0] - 1u) * 12u + 4u}}) !=
|
||||
snapshot.specialization_reads.end(),
|
||||
"batched selector prefix escaped writable-alias tracking");
|
||||
}
|
||||
const auto valid = specialization;
|
||||
memory.words[selector / 4u] = 0x40000001u; // key*192 wraps to the valid second row.
|
||||
Check(materialize() && specialization == valid,
|
||||
"GPU descriptor key lost its native U32 multiplication wrap");
|
||||
memory.words[selector / 4u] = 22369622u;
|
||||
Check(!materialize(), "an actually selected numeric descriptor was silently filtered");
|
||||
memory.words[selector / 4u] = 1u;
|
||||
user_data[5] = 0u;
|
||||
user_data[6] = 14u; // Second key has fewer than four in-bounds bytes: SMEM returns zero.
|
||||
Check(materialize() && snapshot.buffers.size() == plan.info.buffers.size() + 2u,
|
||||
"partial scalar selector did not zero-fill its OOB DWORD");
|
||||
user_data[6] = 0u;
|
||||
Check(materialize() && snapshot.buffers.size() == plan.info.buffers.size() &&
|
||||
memory.reads == 3u,
|
||||
"empty selector V# did not select key zero without reading payload");
|
||||
// Existing vector selectors use the V# stride; each scalar snapshot read
|
||||
// independently rounds its resulting byte offset down to a DWORD.
|
||||
for (const uint32_t stride : {5u, 6u}) {
|
||||
for (const auto root : roots) {
|
||||
auto& indirect = *plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor;
|
||||
indirect.selector->stride = 0u;
|
||||
indirect.selector_first = Value(1u);
|
||||
}
|
||||
user_data[5] = stride << 16u;
|
||||
user_data[6] = 5u;
|
||||
memory.words[0] = 4u;
|
||||
std::fill_n(memory.words.begin() + selector / 4u, 8u, 0xdeadbeefu);
|
||||
for (uint32_t index = 0; index < 4u; ++index)
|
||||
memory.words[selector / 4u + ((index + 1u) * stride) / 4u] = index & 1u;
|
||||
Check(materialize() && memory.reads == 7u &&
|
||||
snapshot.buffers.size() == plan.info.buffers.size() + 2u,
|
||||
"batched vector selector changed independent scalar DWORD alignment");
|
||||
}
|
||||
for (const auto root : roots) {
|
||||
auto& indirect = *plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor;
|
||||
indirect.selector->stride = 12u;
|
||||
indirect.selector_first = Value(0u);
|
||||
}
|
||||
user_data[5] = 12u << 16u;
|
||||
user_data[6] = MaxIndirectDescriptorProbes;
|
||||
std::fill(memory.words.begin() + selector / 4u, memory.words.end(), 0u);
|
||||
memory.words[0] = MaxIndirectDescriptorProbes;
|
||||
Check(materialize() && memory.reads == 9u,
|
||||
"strided selector prefix was not read in bounded contiguous batches");
|
||||
memory.words[0] = MaxIndirectDescriptorProbes + 1u;
|
||||
Check(!materialize() && memory.reads == 1u,
|
||||
"selector cap was checked after reading its GPU payload");
|
||||
memory.words[0] = 2u;
|
||||
for (const auto root : roots)
|
||||
plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor->selector->offset = UINT32_MAX - 3u;
|
||||
Check(!materialize() && memory.reads == 1u,
|
||||
"selector offset arithmetic accepted a wrapped prefix span");
|
||||
for (const auto root : roots)
|
||||
plan.descriptor_sources[plan.info.buffers[root].source].indirect_descriptor->selector->offset = 0u;
|
||||
runtime.indirect_args_address = 0u;
|
||||
const std::array direct_counts{1u, 1u, 1u};
|
||||
runtime.workgroup_counts = std::span(direct_counts).first(1u);
|
||||
Check(!materialize() && memory.reads == 0u,
|
||||
"workgroup prefix accepted an incomplete dispatch extent");
|
||||
runtime.workgroup_counts = direct_counts;
|
||||
memory.fail_address = memory.base;
|
||||
Check(materialize() && memory.watched_reads == 0u,
|
||||
"direct workgroup counts unnecessarily read indirect arguments");
|
||||
const auto ordinary = MakeBufferRecordFixture(1u, false, true);
|
||||
const auto ordinary_plan = ExtractResourcePlan(ordinary->program);
|
||||
runtime.indirect_args_address = memory.base;
|
||||
user_data[2] = 0u;
|
||||
Check(MaterializeResources(ordinary_plan, runtime, snapshot, specialization),
|
||||
"a plan without workgroup counts read the indirect argument pointer");
|
||||
}
|
||||
|
||||
void TestGpuSelectedFormattedRecordStore() {
|
||||
namespace Prospero = Libs::Graphics::Prospero;
|
||||
const auto make = [](uint32_t stride = 592u, uint32_t shift = 16u, bool split = false,
|
||||
@@ -7024,6 +7168,7 @@ int main() {
|
||||
Run("bounded formatted buffer store", TestBoundedFormattedBufferStore);
|
||||
Run("bounded buffer records", TestBoundedBufferRecords);
|
||||
Run("GPU-selected buffer records", TestGpuSelectedBufferRecords);
|
||||
Run("workgroup-selected buffer records", TestWorkgroupSelectedBufferRecords);
|
||||
Run("GPU-selected formatted record store", TestGpuSelectedFormattedRecordStore);
|
||||
Run("finite buffer selection", TestFiniteBufferSelection);
|
||||
Run("packed scalar buffer selection", TestPackedScalarBufferSelection);
|
||||
|
||||
@@ -1684,12 +1684,14 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
|
||||
ShaderRecompiler::IR::ExtractResourcePlan(translated.program);
|
||||
ShaderRecompiler::IR::ResourceSnapshot resources;
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization;
|
||||
const std::array workgroup_counts{test.dispatch_x, test.dispatch_y, test.dispatch_z};
|
||||
const ShaderRecompiler::IR::SrtRuntime runtime{
|
||||
.user_data = options.user_data,
|
||||
.shader_base = reinterpret_cast<uint64_t>(test.code.data()),
|
||||
.read_memory = ReadTestMemory,
|
||||
.userdata = const_cast<std::vector<u32> *>(&test.initial),
|
||||
.read_specialization_memory = ReadTestMemory,
|
||||
.workgroup_counts = workgroup_counts,
|
||||
};
|
||||
Require(test.name, "resource materialization",
|
||||
ShaderRecompiler::IR::MaterializeResources(
|
||||
@@ -10960,6 +10962,128 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
// GPU-written dispatch counts bound the stride12 selector prefix; only
|
||||
// those keys select the two real descriptor members in each stride192 row.
|
||||
constexpr auto record_args = base + 0xd0000;
|
||||
constexpr auto record_keys = base + 0xd1000;
|
||||
constexpr auto record_table = base + 0xd2000;
|
||||
constexpr auto record_output = base + 0xd7000;
|
||||
std::vector<u32> record_shader{
|
||||
EncodeSop2(0x26, 16, 13, InlineU32(12)),
|
||||
EncodeSmem0(0x08, 16, 0), EncodeSmem1(0, 16), EncodeSopp(0x0c, 0),
|
||||
EncodeSop2(0x26, 17, 16, 255), 192u,
|
||||
EncodeSmem0(0x0a, 20, 2), EncodeSmem1(160, 17),
|
||||
EncodeSmem0(0x0a, 24, 2), EncodeSmem1(176, 17), EncodeSopp(0x0c, 0),
|
||||
EncodeMubuf0(0x83, 0, true, false), EncodeMubuf1(4, 5, 0),
|
||||
EncodeSopp(0x0c, 0)};
|
||||
AppendVop3(&record_shader, 0x346, 8, 13, 12, Vgpr(0));
|
||||
record_shader.insert(record_shader.end(), {
|
||||
EncodeMubuf0(0x1d, 0, true, false), EncodeMubuf1(4, 2, 8)});
|
||||
AppendVMovLiteral(&record_shader, 10, 0x5au);
|
||||
AppendSMovLiteral(&record_shader, 126, 0x55555555u);
|
||||
AppendSMovLiteral(&record_shader, 127, 0x55555555u);
|
||||
record_shader.insert(record_shader.end(), {
|
||||
EncodeMubuf0(0x18, 0, true, false, true), EncodeMubuf1(10, 6, 0)});
|
||||
AppendEnd(&record_shader);
|
||||
ShaderMapUserData(reinterpret_cast<uint64_t>(record_shader.data()),
|
||||
{.type = Prospero::ShaderBinaryType::kCs,
|
||||
.code_size_bytes = static_cast<u32>(record_shader.size() * sizeof(u32))});
|
||||
std::array<u32, 96> records;
|
||||
records.fill(0xdeadbeefu);
|
||||
const std::array<u32, 4> numeric_descriptor{0x640u, 3u, 0x3ba3d70au, 0x3a23d70au};
|
||||
std::copy(numeric_descriptor.begin(), numeric_descriptor.end(), records.begin() + 76u);
|
||||
for (u32 row = 0; row < 2u; ++row) {
|
||||
const auto payload = base + 0xd3000 + row * 0x1000;
|
||||
const auto target = base + 0xd5000 + row * 0x1000;
|
||||
const std::array<u32, 8> descriptors{
|
||||
static_cast<u32>(payload), static_cast<u32>(payload >> 32u) | (8u << 16u), 4u,
|
||||
(u32(Prospero::BufferFormat::k16_16_16_16Float) << 12u) | DstSel(4, 5, 6, 7),
|
||||
static_cast<u32>(target), static_cast<u32>(target >> 32u) | (1u << 16u), 4u, 0x5204u};
|
||||
std::copy(descriptors.begin(), descriptors.end(), records.begin() + row * 48u + 40u);
|
||||
const std::array<u32, 8> data{0x40003c00u + row, 0xc0003800u + row,
|
||||
0x44004200u + row, 0xc400bc00u + row, 0x48004600u + row, 0xc800c600u + row,
|
||||
0x4c004a00u + row, 0xcc00ca00u + row};
|
||||
LibKernel::Memory::WriteBacking(payload, data.data(), sizeof(data));
|
||||
}
|
||||
LibKernel::Memory::WriteBacking(record_table, records.data(), sizeof(records));
|
||||
const auto produce = [&](uint64_t address, u32 stride, std::array<u32, 3> values) {
|
||||
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(writer.data()),
|
||||
.num_thread_x = 1, .num_thread_y = 1, .num_thread_z = 1,
|
||||
.wave_size = 64, .user_sgpr = 15});
|
||||
for (u32 word = 0; word < 3u; ++word) {
|
||||
set_buffer(word * 4u, address + word * stride, 4u);
|
||||
shaders.SetCsUserSgpr(12u + word, values[word], HW::UserSgprType::Unknown);
|
||||
}
|
||||
processor.DispatchDirect(1, 1, 1, 0x41u);
|
||||
};
|
||||
for (const u32 wave : {32u, 64u}) {
|
||||
for (const auto dimensions : {std::array{2u, 1u, 1u}, std::array{0u, 1u, 1u},
|
||||
std::array{2u, 0u, 1u}, std::array{2u, 1u, 0u},
|
||||
std::array{1u, 1u, 1u}, std::array{2u, 1u, 1u}}) {
|
||||
const bool empty = std::ranges::find(dimensions, 0u) != dimensions.end();
|
||||
const std::array<u32, 3> stale_args{};
|
||||
std::array<u32, 7> stale_keys;
|
||||
stale_keys.fill(sentinel);
|
||||
Require(name, "record fixture reset", context.InvalidateMemory(record_args, 0x8000),
|
||||
"record fixture escaped its mapped allocation");
|
||||
LibKernel::Memory::WriteBacking(record_args, stale_args.data(), sizeof(stale_args));
|
||||
LibKernel::Memory::WriteBacking(record_keys, stale_keys.data(), sizeof(stale_keys));
|
||||
std::vector<u32> expected_output(wave * 4u, sentinel);
|
||||
LibKernel::Memory::WriteBacking(record_output, expected_output.data(), expected_output.size() * 4u);
|
||||
const std::array<u32, 4> untouched{sentinel, sentinel, sentinel, sentinel};
|
||||
for (u32 row = 0; row < 2u; ++row)
|
||||
LibKernel::Memory::WriteBacking(base + 0xd5000 + row * 0x1000, untouched.data(), sizeof(untouched));
|
||||
produce(record_args, 4u, dimensions);
|
||||
produce(record_keys, 12u, {1u, 0u, 22369622u});
|
||||
std::array<u32, 3> cpu_args{};
|
||||
std::array<u32, 7> cpu_keys{};
|
||||
Require(name, "GPU record dependencies remain pending",
|
||||
cache.HasGpuDirtyBytes(record_args, 12u) && cache.HasGpuDirtyBytes(record_keys, 16u) &&
|
||||
LibKernel::Memory::TryReadBacking(record_args, cpu_args.data(), sizeof(cpu_args)) &&
|
||||
LibKernel::Memory::TryReadBacking(record_keys, cpu_keys.data(), sizeof(cpu_keys)) &&
|
||||
cpu_args == stale_args && cpu_keys == stale_keys,
|
||||
"the producer did not leave dispatch counts and selectors GPU-owned");
|
||||
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(record_shader.data()),
|
||||
.num_thread_x = wave, .num_thread_y = 1, .num_thread_z = 1,
|
||||
.user_sgpr = 13, .tgid_x_en = true});
|
||||
set_buffer(0, record_keys, 3u, 12u);
|
||||
set_buffer(4, record_table, 2u, 192u);
|
||||
set_buffer(8, record_output, wave * 2u, 8u);
|
||||
shaders.SetCsUserSgpr(12, std::countr_zero(wave), HW::UserSgprType::Unknown);
|
||||
processor.DispatchIndirect(record_args, 0x41u |
|
||||
(wave == 32u ? 1u << Pm4::COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT : 0u));
|
||||
cache.ReadMemory(record_output, expected_output.size() * 4u);
|
||||
std::vector<u32> actual_output(expected_output.size());
|
||||
Require(name, "record output readback", LibKernel::Memory::TryReadBacking(
|
||||
record_output, actual_output.data(), actual_output.size() * 4u),
|
||||
"indirect record output could not be read back");
|
||||
if (!empty)
|
||||
for (u32 group = 0; group < dimensions[0]; ++group) {
|
||||
const auto row = 1u - group;
|
||||
for (u32 lane = 0; lane < wave; ++lane) {
|
||||
const auto out = (group * wave + lane) * 2u;
|
||||
expected_output[out] = expected_output[out + 1u] = 0u;
|
||||
if (lane < 4u)
|
||||
Require(name, "unchanged formatted input", LibKernel::Memory::TryReadBacking(
|
||||
base + 0xd3000 + row * 0x1000 + lane * 8u, expected_output.data() + out, 8u),
|
||||
"the formatted source escaped its mapped allocation");
|
||||
}
|
||||
}
|
||||
Require(name, "coherent exact workgroup selector prefix", actual_output == expected_output,
|
||||
"GPU counts or selectors were stale, over-enumerated, or selected the wrong record");
|
||||
for (u32 row = 0; row < 2u; ++row) {
|
||||
const auto target = base + 0xd5000 + row * 0x1000;
|
||||
cache.ReadMemory(target, sizeof(untouched));
|
||||
auto expected = untouched;
|
||||
if (!empty && (dimensions[0] == 2u || row == 1u)) expected[0] = 0xa55aa55au;
|
||||
std::array<u32, 4> actual{};
|
||||
Require(name, "selected byte store and canaries", LibKernel::Memory::TryReadBacking(
|
||||
target, actual.data(), sizeof(actual)) && actual == expected,
|
||||
"masked/OOB record stores changed an inactive row or adjacent bytes");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exchange values across two waves in the last 512 bytes of each
|
||||
// workgroup's 64-KiB LDS, on a host restricted to 48-KiB native shared memory.
|
||||
constexpr auto lds_output = base + 0xb0000;
|
||||
@@ -37009,8 +37133,10 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) {
|
||||
test.dispatch_x = keys.size();
|
||||
test.bda_mappings = {{0u, 0u}};
|
||||
auto &code = test.code;
|
||||
code.insert(code.end(), {EncodeSop2(0x26, 32, 24, InlineU32(12)),
|
||||
EncodeSmem0(0x08, 32, 0), EncodeSmem1(0u, 32), EncodeSopp(0x0c, 0),
|
||||
// A vector load keeps this regression on the unbounded affine-table path.
|
||||
code.insert(code.end(), {EncodeVop1(0x01, 2, 24),
|
||||
EncodeMubuf0(0x0c, 0, true, false), EncodeMubuf1(3, 0, 2),
|
||||
EncodeSopp(0x0c, 0), EncodeVop1(0x02, 32, Vgpr(3)),
|
||||
EncodeSop2(0x26, 33, 32, 255), 192u,
|
||||
EncodeSop2(0x02, 33, 33, 255), 96u,
|
||||
EncodeSmem0(0x0a, 36, 2), EncodeSmem1(144u, 33),
|
||||
@@ -37113,6 +37239,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) {
|
||||
for (const bool wide : {false, true})
|
||||
for (const u32 wave : {32u, 64u}) {
|
||||
const std::array keys{wide ? 31u : 5u, 0u, 3u, 1u, 4u, 2u, 6u};
|
||||
const auto selector_index = [wide](u32 group) { return wide ? group ^ 1u : group; };
|
||||
TestCase test;
|
||||
test.name = wide ? "GpuSelectedFormattedRgba16Store" : "GpuSelectedFormattedRecordStore";
|
||||
test.initial.assign((slot * 7u + keys.size() * wave * 4u) / 4u, 0xdeadbeefu);
|
||||
@@ -37136,7 +37263,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) {
|
||||
records[0] = table;
|
||||
std::copy_n(records.begin(), 4, test.user_data.begin() + 32u);
|
||||
if (wide) { test.user_data[32] = table - 48u; test.user_data[33] = 0u; }
|
||||
auto selector_buffer = MakeStructuredStorageBufferData(0u, keys.size() * 4u);
|
||||
auto selector_buffer = MakeStructuredStorageBufferData(0u, (keys.size() + u32(wide)) * 4u);
|
||||
selector_buffer[0] = selectors;
|
||||
std::copy_n(selector_buffer.begin(), 4, test.user_data.begin() + 4u);
|
||||
auto output = MakeStructuredStorageBufferData(0u, keys.size() * wave * 4u);
|
||||
@@ -37152,7 +37279,10 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) {
|
||||
test.compute_info.workgroup_register = 44u;
|
||||
test.dispatch_x = keys.size();
|
||||
auto &code = test.code;
|
||||
code.push_back(EncodeSop2(0x1e, 23, 44, InlineU32(2)));
|
||||
// Keep the finite32-key proof independent of workgroup-prefix materialization:
|
||||
// this fixture changes GPU keys after compiling its complete candidate domain.
|
||||
if (wide) code.push_back(EncodeSop2(0x12, 23, 44, InlineU32(1)));
|
||||
code.push_back(EncodeSop2(0x1e, 23, wide ? 23u : 44u, InlineU32(2)));
|
||||
if (wide) {
|
||||
code.insert(code.end(), {EncodeSmem0(0x08, 22, 2), EncodeSmem1(0, 23),
|
||||
EncodeSop2(0x27, 22, 22, 255), (5u << 16u) | 16u,
|
||||
@@ -37219,7 +37349,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) {
|
||||
producer.compute_info.wave_size = 32u;
|
||||
for (u32 group = 0; group < keys.size(); ++group) {
|
||||
AppendVMovLiteral(&producer.code, 1, (keys[group] << 16u) | 0xbeefu);
|
||||
AppendVMovLiteral(&producer.code, 2, selectors + group * 4u);
|
||||
AppendVMovLiteral(&producer.code, 2, selectors + selector_index(group) * 4u);
|
||||
AppendBufferStoreDword(&producer.code, 1, 2);
|
||||
}
|
||||
AppendEnd(&producer.code);
|
||||
@@ -37228,7 +37358,7 @@ void CheckGpuSelectedFormattedRecordStore(VulkanHarness &vulkan) {
|
||||
auto expected_bytes = std::as_writable_bytes(std::span(test.expected));
|
||||
for (u32 group = 0; group < keys.size(); ++group) {
|
||||
const auto row = keys[group];
|
||||
test.expected[selectors / 4u + group] = (row << 16u) | 0xbeefu;
|
||||
test.expected[selectors / 4u + selector_index(group)] = (row << 16u) | 0xbeefu;
|
||||
for (u32 lane = 0; lane < wave; ++lane) {
|
||||
const bool valid = wide ? row == 0u || row == 3u : row < 6u;
|
||||
test.expected[slot * 7u / 4u + group * wave + lane] =
|
||||
|
||||
Reference in New Issue
Block a user