From 3ba0e971e9b61eeac6e19ae6016bec4f358d2127 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 4 Oct 2026 11:42:09 +0400 Subject: [PATCH] shader: materialize exact buffer capacity through shared clean reads Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com> --- .../renderer/pipeline/descriptors.cpp | 22 +-- src/graphics/host_gpu/renderer/render.h | 4 +- .../host_gpu/renderer/renderCompute.cpp | 5 +- .../shader/recompiler/ShaderRecompiler.cpp | 4 - .../backend/spirv/spirvEmitterMemory.cpp | 10 +- src/graphics/shader/recompiler/ir/Program.cpp | 12 ++ src/graphics/shader/recompiler/ir/ShaderIR.h | 4 +- .../ir/passes/ConstantPropagation.cpp | 5 + .../ir/passes/ResourceMaterialization.cpp | 47 +++--- .../ir/passes/ResourceMaterialization.h | 4 - .../recompiler/ir/passes/ResourceTracking.cpp | 35 +--- .../shader/recompiler/ir/passes/SrtWalker.cpp | 93 ++++++++++- tests/ResourceMaterializationTests.cpp | 151 +++++++++++++----- tests/ResourceTrackingTests.cpp | 28 ++++ tests/ShaderRecompilerComputeTests.cpp | 85 +++++++--- 15 files changed, 333 insertions(+), 176 deletions(-) diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index a7cc9238d..e592cc6c3 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -805,8 +805,7 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, } } -void RenderExecutor::FindBuffers(PreparedBindings& prepared, uint64_t dispatch_threads_x, - uint32_t group_size_x) { +void RenderExecutor::FindBuffers(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); const auto& program = *prepared.runtime->program; @@ -823,23 +822,8 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared, uint64_t dispatch_t for (const auto resource: resources) { auto descriptor = DecodeNativeDescriptor(snapshot.buffers[resource]); const auto address = descriptor.Base48(); - if (address == 0u) { - prepared.buffer_sources.push_back({}); - continue; - } - auto requested_size = descriptor.GetSize(); - const auto& buffer = program.info.buffers[resource]; - if (buffer.dispatch_stride != 0u) { - if (dispatch_threads_x == 0u || dispatch_threads_x > uint64_t{UINT32_MAX} + 1u || - buffer.dispatch_stride != group_size_x) { - EXIT("GPU-sized buffer requires a known matching X dispatch extent\n"); - } - requested_size = (dispatch_threads_x - 1u) * descriptor.Stride() + buffer.max_byte_extent; - if (requested_size > uint64_t{UINT32_MAX} + 1u) { - EXIT("GPU-sized buffer dispatch wraps its 32-bit byte address\n"); - } - } - if (requested_size == 0) { + const auto requested_size = descriptor.GetSize(); + if (address == 0 || requested_size == 0) { prepared.buffer_sources.push_back({}); continue; } diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index 71e83be67..b8db5b081 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -161,9 +161,7 @@ public: uint32_t mode); void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared); - void FindBuffers(PreparedBindings& bindings, - uint64_t dispatch_threads_x = 0, - uint32_t group_size_x = 0); + void FindBuffers(PreparedBindings& bindings); void RebindBuffers(PreparedBindings& bindings); void RebindImages(PreparedBindings& bindings); void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 457184b62..4f9c0673e 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -306,10 +306,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, std::copy(std::begin(input_info.dispatch_threads_num), std::end(input_info.dispatch_threads_num), bindings.shader_data.begin() + program.bindings.dispatch_thread_dword); } - const auto dispatch_threads_x = use_thread_dimensions - ? uint64_t{input_info.dispatch_threads_num[0]} - : uint64_t{thread_group_x} * input_info.threads_num[0]; - FindBuffers(bindings, dispatch_threads_x, input_info.threads_num[0]); + FindBuffers(bindings); if (program.info.uses_dma) { m_context.PrepareBda(); } diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index a24caf876..9b74c9a98 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -641,7 +641,6 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o for (auto& inst: *block) { const auto op = inst.GetOpcode(); uint32_t first = 0; - bool native_buffer_count = false; if (op == IR::ValueOpcode::GetBufferResource) { if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) { return ir.memory_info[use.user->Flags().index].kind == @@ -650,8 +649,6 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o continue; } const auto resource = inst.Flags(); - native_buffer_count = resource < ir.info.buffers.size() && - ir.info.buffers[resource].dispatch_stride != 0u; first = resource < ir.info.buffers.size() && ir.info.buffers[resource].indirect_root == resource ? 1u @@ -664,7 +661,6 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o continue; } for (size_t index = first; index < inst.NumArgs(); index++) { - if (native_buffer_count && index == 2u) continue; inst.SetArg(index, IR::Value(0u)); } } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index f2e6d0068..66f58e480 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -506,15 +506,7 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) { void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t data) { const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem)); - auto in_bounds = EmitMemoryElementInBounds(ctx.state, resource, index); - if (mem.kind == IR::ResourceKind::Buffer && - ctx.state.program.info.buffers[mem.resource].dispatch_stride != 0u) { - const auto& handle = *inst.Arg(0).ResolveInstruction(); - in_bounds = AndCondition( - ctx.state, in_bounds, - Binary(ctx.state, spv::OpULessThan, TypeBool(ctx.state), ctx.Arg(inst, 1), ctx.Arg(handle, 2))); - } - EmitIfCondition(ctx.state, in_bounds, [&]() { + EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { StoreWordInBounds(ctx, resource, index, data); }); } diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index 3123f62ae..647a4d9d8 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -190,6 +190,18 @@ Value ResolveInvariantPhi(const ResourcePlan& program, Value value) { return invariant; } +Value ResolveActiveU32(Value value, Value active) { + for (uint32_t depth = 0; depth <= 32; ++depth) { + value = value.Resolve(); + const auto* inst = value.TryInstruction(); + if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 || + inst->Arg(0).Resolve() != active) + return value; + value = inst->Arg(1); + } + return {}; +} + bool HasShaderMemoryWrites(const Program& program) { for (const auto* block: program.blocks) { for (const auto& inst: *block) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index d9a584494..56a6233f4 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -106,9 +106,6 @@ struct BufferResource { uint32_t source = 0; uint32_t first_use_pc = 0; uint32_t max_byte_extent = 0; - // Nonzero proves index = local_x + workgroup_x * dispatch_stride. NUM_RECORDS - // stays on the GPU; host binding capacity comes from the known dispatch extent. - uint32_t dispatch_stride = 0; uint32_t packed_stride = 0; Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid; uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7); @@ -625,6 +622,7 @@ void ValidateProgram(const Program& program, bool require_ssa); void ResolveControlFlowIdentities(Program& program); bool EquivalentValue(const ResourcePlan& program, Value left, Value right); Value ResolveInvariantPhi(const ResourcePlan& program, Value value); +Value ResolveActiveU32(Value value, Value active); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 5878a5873..30840daad 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -199,6 +199,11 @@ void FoldInstruction(Block& block, Block::iterator instruction, std::unordered_set& lowered_ancillary) { auto& inst = *instruction; switch (inst.GetOpcode()) { + case ValueOpcode::StoreBufferU32: { + const auto data = ResolveActiveU32(Arg(inst, 4), Arg(inst, 5)); + if (!data.IsEmpty() && data != inst.Arg(4)) inst.SetArg(4, data); + return; + } case ValueOpcode::Phi: FoldPhi(inst); return; case ValueOpcode::SelectU1: if (!FoldSelect(inst) && IsImmediate(Arg(inst, 2), Type::U1) && diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index e70b09f2a..e55afcf8f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -762,23 +762,11 @@ static bool IsFullDispatchPredicate(Value value, uint32_t depth = 0) { return axis.IsImmediate() && axis.U32() < 3 && extent->Arg(0).Resolve() == axis; } -static Value FillValue(Value value, Value guard) { - for (uint32_t depth = 0; depth <= 32; ++depth) { - value = value.Resolve(); - const auto* inst = value.TryInstruction(); - if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 || - inst->Arg(0).Resolve() != guard) - return value; - value = inst->Arg(1); - } - return {}; -} - // Nonnegative affine coefficients for constant, local and workgroup coordinates. Reject modular // arithmetic that could wrap; runtime coverage also bounds the largest invocation index. -std::optional> AffineIndex(Value value, uint32_t axis, Value guard, - uint32_t depth) { - value = FillValue(value, guard); +static std::optional> FillIndex(Value value, uint32_t axis, Value guard, + uint32_t depth = 0) { + value = ResolveActiveU32(value, guard); if (depth > 32 || value.GetType() != Type::U32) { return {}; } @@ -802,8 +790,8 @@ std::optional> AffineIndex(Value value, uint32_t axis, V op != ValueOpcode::ShiftLeftLogical32) { return {}; } - auto left = AffineIndex(inst->Arg(0), axis, guard, depth + 1); - auto right = AffineIndex(inst->Arg(1), axis, guard, depth + 1); + auto left = FillIndex(inst->Arg(0), axis, guard, depth + 1); + auto right = FillIndex(inst->Arg(1), axis, guard, depth + 1); if (!left || !right) { return {}; } @@ -870,7 +858,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) { const auto* address = store->Arg(1).ResolveInstruction(); if (address == nullptr || address->GetOpcode() != ValueOpcode::MakeImageAddress) return {}; for (uint32_t axis = 0; axis < 3; ++axis) { - const auto index = AffineIndex(address->Arg(axis), axis, guard); + const auto index = FillIndex(address->Arg(axis), axis, guard); if (!index || (*index)[0] != 0) return {}; if (axis < 2) { if ((*index)[1] != 1 || (*index)[2] == 0) return {}; @@ -900,14 +888,14 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) { memory.data_bits != 32 || memory.data_dwords != static_cast(store_op - stores.begin() + 1)) return {}; - const auto address = AffineIndex(store->Arg(1), 0, guard); + const auto address = FillIndex(store->Arg(1), 0, guard); if (!address || (*address)[0] != 0 || (*address)[1] != 1 || (*address)[2] == 0) return {}; result.fill.kind = UniformFillKind::Buffer; result.fill.group_stride[0] = static_cast((*address)[2]); result.fill.words = memory.data_dwords; data = store->Arg(4); } - data = FillValue(data, guard); + data = ResolveActiveU32(data, guard); const auto* vector = data.TryInstruction(); constexpr std::array composites {ValueOpcode::CompositeConstructU32x2, ValueOpcode::CompositeConstructU32x3, @@ -916,7 +904,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) { (vector == nullptr || vector->GetOpcode() != composites[result.fill.words - 2])) return {}; for (uint32_t i = 0; i < result.fill.words; ++i) { - const auto word = FillValue(result.fill.words == 1 ? data : vector->Arg(i), guard); + const auto word = ResolveActiveU32(result.fill.words == 1 ? data : vector->Arg(i), guard); if (word.GetType() != Type::U32 || !ValidateRuntimeValue(program, word, RuntimeValueType::Integer)) return {}; @@ -943,6 +931,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) { if (source == nullptr) { return value; } + if (source->GetOpcode() == ValueOpcode::ReadFirstLane && + ValidateRuntimeValue(program, source->Arg(0))) { + // Uniform values need no new EXEC context; preserve their shared evaluation memo. + return Clone(source->Arg(0)); + } if (source->GetOpcode() == ValueOpcode::Phi) { const auto invariant = ResolveInvariantPhi(program, value); if (!invariant.IsEmpty() && invariant != value) { @@ -952,6 +945,10 @@ ResourcePlan ExtractResourcePlan(const Program& program) { if (const auto found = cloned.find(source); found != cloned.end()) { return Value(found->second); } + if (source->GetOpcode() == ValueOpcode::LoadBufferU32) { + plan.requires_specialization_memory = true; + plan.capture_specialization_reads = true; + } auto& target = plan.value_storage.emplace_back(source->GetOpcode(), source->Flags()); cloned.emplace(source, &target); @@ -1042,7 +1039,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) { } } } - plan.capture_specialization_reads = capture_indirect_reads || !plan.control_flow.empty(); + plan.capture_specialization_reads |= capture_indirect_reads || !plan.control_flow.empty(); if (plan.capture_specialization_reads) { // Clean writable addresses prove that shader writes cannot overlap captured reads. for (const auto& buffer: plan.info.buffers) { @@ -1138,12 +1135,6 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime auto packed_stride = descriptor.PackedStride(); const auto stride = packed_stride & 0x3fffu; const bool swizzle = stride != 0u && ((packed_stride >> 14u) & 1u) != 0u; - if (base.dispatch_stride != 0u && descriptor.Base48() != 0u && - (descriptor.OutOfBounds() != 0u || stride == 0u || swizzle || - (packed_stride & (1u << 20u)) != 0u || base.max_byte_extent > stride)) { - return SpecializationFail(fmt::format( - "dispatch-sized buffer {} requires an unswizzled structured DWORD layout", i)); - } if (stride == 0u) { packed_stride &= ~((1u << 14u) | (3u << 16u)); } else if (!swizzle) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index 2e2013dfb..6ad72cb36 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -43,10 +43,6 @@ struct ResourceSpecialization { // its values and is independent of the translated shader CFG. ResourcePlan ExtractResourcePlan(const Program& program); -// Nonnegative constant/local/workgroup coefficients under the instruction's active guard. -std::optional> AffineIndex(Value value, uint32_t axis, Value guard, - uint32_t depth = 0); - // Refreshes cached resources and specialization in place. A failed refresh must not be used. bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index e381f76b8..7fde52b2c 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -3,7 +3,6 @@ #include "common/assert.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" -#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include @@ -2043,32 +2042,9 @@ private: } } - bool TryMakeDispatchBuffer(const Inst& inst, const Inst& handle, uint32_t& source, - uint32_t& dispatch_stride) { - const auto flags = inst.Flags(); - const auto& memory = m_program.memory_info[flags.index]; - if (m_program.stage != ShaderType::Compute || inst.GetOpcode() != ValueOpcode::StoreBufferU32 || - memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted || - !memory.idxen || memory.offen || memory.data_bits != 32u || memory.data_dwords != 1u || - inst.Arg(2).Resolve() != Value(0u) || inst.Arg(3).Resolve() != Value(0u)) return false; - const auto index = AffineIndex(inst.Arg(1), 0, inst.Arg(5).Resolve()); - if (!index || (*index)[0] != 0u || (*index)[1] != 1u || (*index)[2] == 0u) return false; - DescriptorSource descriptor; - MakeSource(handle, 4u, false, false, memory.resource * 4u, descriptor, flags.pc); - // Binding capacity is derived from the dispatch. The native handle keeps its count. - descriptor.dwords[2] = Value(0u); - uint32_t bad_dword = 0; - if (!ValidateSource(descriptor, bad_dword)) return false; - source = InternSource(descriptor); - dispatch_stride = static_cast((*index)[2]); - return true; - } - - uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc, - uint32_t dispatch_stride) { + uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) { for (uint32_t i = 0; i < m_info.buffers.size(); i++) { - if (m_info.buffers[i].source == source && - m_info.buffers[i].dispatch_stride == dispatch_stride) { + if (m_info.buffers[i].source == source) { Merge(m_info.buffers[i], memory, op, pc); return i; } @@ -2079,7 +2055,6 @@ private: BufferResource resource; resource.source = source; resource.first_use_pc = pc; - resource.dispatch_stride = dispatch_stride; Merge(resource, memory, op, pc); m_info.buffers.push_back(resource); return static_cast(m_info.buffers.size() - 1); @@ -2235,14 +2210,12 @@ private: uint32_t resource = 0; if (buffer != BufferAccess::None) { - uint32_t dispatch_stride = 0; handle = inst.Arg(0).Resolve().TryInstruction(); const auto* indirect = handle == nullptr ? nullptr : FindIndirectDescriptor(*handle); if (indirect != nullptr) { source = indirect->source; } else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, - memory.resource * 4u, handle, source) && - !TryMakeDispatchBuffer(inst, *handle, source, dispatch_stride)) { + memory.resource * 4u, handle, source)) { if (memory.kind != (op == ValueOpcode::ReadConstBuffer ? ResourceKind::ScalarBuffer : ResourceKind::Buffer) || !memory.SupportsIndirectBufferLoad(op)) { @@ -2254,7 +2227,7 @@ private: m_info.uses_dma = true; return; } - resource = AddBuffer(source, memory, op, flags.pc, dispatch_stride); + resource = AddBuffer(source, memory, op, flags.pc); if (resource == UINT32_MAX) { Fail(flags.pc, "buffer resource limit exceeded"); } diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index f3dff66e0..4ec6beca7 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -41,16 +41,29 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { bool IsRawRead(const ResourcePlan& values, const Inst& inst) { const auto op = inst.GetOpcode(); - if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) { + if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer && + op != ValueOpcode::LoadBufferU32) { return false; } const auto index = inst.Flags().index; if (index >= values.memory_info.size()) { return false; } - const auto kind = values.memory_info[index].kind; - return (op == ValueOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) || - (op == ValueOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer); + const auto& memory = values.memory_info[index]; + if (op != ValueOpcode::LoadBufferU32) { + return (op == ValueOpcode::LoadAddressU32 && memory.kind == ResourceKind::ScalarAddress) || + (op == ValueOpcode::ReadConstBuffer && memory.kind == ResourceKind::ScalarBuffer); + } + if (inst.NumArgs() != 5u || memory.kind != ResourceKind::Buffer || memory.typed || + memory.formatted || memory.coherent || memory.data_bits != 32u || + memory.data_dwords != 1u || memory.idxen || memory.offen || memory.offset != 0u || + inst.Arg(4).GetType() != Type::U1) { + return false; + } + for (uint32_t arg = 1; arg <= 3; ++arg) { + if (inst.Arg(arg).Resolve() != Value(0u)) return false; + } + return true; } bool IsDescriptorHandle(ValueOpcode opcode) { @@ -89,6 +102,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) { case ValueOpcode::ISub64: case ValueOpcode::IMul32: case ValueOpcode::IMul64: + case ValueOpcode::UMulHi: case ValueOpcode::UMin32: case ValueOpcode::ShiftLeftLogical32: case ValueOpcode::ShiftLeftLogical64: @@ -105,6 +119,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) { case ValueOpcode::SelectU32: case ValueOpcode::SelectF32: case ValueOpcode::ULessThan32: + case ValueOpcode::ULessThanEqual32: case ValueOpcode::IEqual32: case ValueOpcode::UGreaterThan32: case ValueOpcode::SGreaterThanEqual32: @@ -117,6 +132,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) { case ValueOpcode::FPOrdGreaterThanEqual32: case ValueOpcode::FPIsNan32: case ValueOpcode::FPMul32: + case ValueOpcode::FPRecipIFlag32: case ValueOpcode::FPTrunc32: return true; default: return false; } @@ -157,6 +173,7 @@ private: default: return false; } } + if (require_uniform && !m_active_mask.IsEmpty() && value == m_active_mask) return true; // Integer-only dependency checks do not depend on the active EXEC mask. if (!require_uniform && m_validated_dependencies.contains(inst)) return true; if (!m_visiting.insert(inst).second) { @@ -246,7 +263,8 @@ private: } return finish(true); } - if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) { + if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer || + op == ValueOpcode::LoadBufferU32) { const auto expected = op == ValueOpcode::LoadAddressU32 ? ValueOpcode::GetAddressResource : ValueOpcode::GetBufferResource; @@ -255,6 +273,11 @@ private: handle->GetOpcode() != expected) { return finish(false); } + if (op == ValueOpcode::LoadBufferU32) { + const auto guard = inst->Arg(4).Resolve(); + return finish(Validate(inst->Arg(0)) && + ((!m_active_mask.IsEmpty() && guard == m_active_mask) || Validate(guard))); + } } else if (op == ValueOpcode::CompositeExtractU64) { const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {}; if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) { @@ -341,6 +364,10 @@ bool SrtWalker::EvaluateWide(Value value, uint64_t& result) { default: return false; } } + if (!m_active_mask.IsEmpty() && value == m_active_mask) { + result = 1u; + return true; + } auto* inst = value.TryInstruction(); if (inst == nullptr) { return false; @@ -430,6 +457,16 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { return false; } const auto& mem = m_program.memory_info[flags.index]; + const bool vector = inst.GetOpcode() == ValueOpcode::LoadBufferU32; + const auto guard = vector ? inst.Arg(4).Resolve() : Value {}; + if (vector && (guard.IsImmediate() || guard != m_active_mask)) { + uint64_t enabled = 0; + if (!Arg(inst, 4, enabled)) return false; + if (enabled == 0u) { + result = 0u; + return true; + } + } const auto* handle = inst.Arg(0).ResolveInstruction(); if (handle == nullptr) { return false; @@ -443,7 +480,7 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { const auto base = (high << 32u) | static_cast(low); const auto immediate = static_cast(static_cast(mem.offset)); uint64_t address = 0; - if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { + if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer || vector) { uint64_t records = 0; uint64_t word3 = 0; if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) { @@ -455,6 +492,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { const auto byte_offset = (static_cast(immediate) & ~uint64_t {3}) + (static_cast(offset) & ~3u); const auto stride = (static_cast(high) >> 16u) & 0x3fffu; + if (vector) { + // Only the uniform, unswizzled structured DWORD address is evaluated on the host. + if ((high & (1u << 31u)) != 0u || (word3 & ((1u << 23u) | 0xf0000000u)) != 0u) + return false; + if (stride == 0u || records == 0u || ((word3 >> 12u) & 0x7fu) == 0u) { + result = 0u; + return true; + } + } const auto size = stride == 0u ? static_cast(static_cast(records)) : static_cast(stride) * static_cast(records); @@ -470,11 +516,13 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) { } } uint32_t word = 0; - if (m_runtime.read_memory != nullptr) { - if (!m_runtime.read_memory(m_runtime.userdata, address, {&word, 1})) { + const auto reader = vector ? m_runtime.read_specialization_memory : m_runtime.read_memory; + if (reader != nullptr) { + if (!reader(m_runtime.userdata, address, {&word, 1})) { return false; } } else { + if (vector) return false; std::memcpy(&word, reinterpret_cast(address), sizeof(word)); } result = word; @@ -502,6 +550,9 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; case ValueOpcode::Phi: return EvaluatePhi(inst, result); case ValueOpcode::ReadFirstLane: { + if (!m_active_mask.IsEmpty() && inst.Arg(1).Resolve() == m_active_mask) { + return EvaluateWide(inst.Arg(0), result); + } const auto clean_runtime = CleanRuntime(m_runtime); SrtWalker clean_active(m_program, clean_runtime, {}, nullptr, inst.Arg(1)); SrtWalker active(m_program, m_runtime, m_clean_flat_slots, &clean_active, @@ -534,7 +585,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { } case ValueOpcode::LoadAddressU32: case ValueOpcode::ReadConstBuffer: + case ValueOpcode::LoadBufferU32: if (IsRawRead(m_program, inst)) { + if (inst.GetOpcode() == ValueOpcode::LoadBufferU32 && m_clean_evaluator != nullptr && + m_clean_evaluator->m_active_mask == m_active_mask) { + return m_clean_evaluator->EvaluateWide(Value(const_cast(&inst)), result); + } return EvaluateRawRead(inst, result); } break; @@ -574,6 +630,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { return true; } return false; + case ValueOpcode::UMulHi: + if (binary()) { + result = (static_cast(static_cast(a)) * static_cast(b)) >> 32u; + return true; + } + return false; case ValueOpcode::UMin32: if (binary()) { result = std::min(static_cast(a), static_cast(b)); @@ -609,6 +671,15 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { return true; } return false; + case ValueOpcode::FPRecipIFlag32: + if (Arg(inst, 0, a)) { + const auto exponent = (a >> 23u) & 0xffu; + // Normal positive powers of two have exact normal reciprocals in every FP mode. + if ((a & 0x807fffffu) != 0u || exponent == 0u || exponent >= 254u) return false; + result = (254u - exponent) << 23u; + return true; + } + return false; case ValueOpcode::FPIsNan32: if (Arg(inst, 0, a)) { result = std::isnan(Float32(a)); @@ -779,6 +850,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) { return true; } return false; + case ValueOpcode::ULessThanEqual32: + if (binary()) { + result = static_cast(a) <= static_cast(b); + return true; + } + return false; case ValueOpcode::UGreaterThan32: if (binary()) { result = static_cast(a) > static_cast(b); diff --git a/tests/ResourceMaterializationTests.cpp b/tests/ResourceMaterializationTests.cpp index bd2cae4ee..a9ecfe690 100644 --- a/tests/ResourceMaterializationTests.cpp +++ b/tests/ResourceMaterializationTests.cpp @@ -1,8 +1,10 @@ #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include #include #include +#include #include namespace { @@ -196,45 +198,6 @@ void TestRawScalarAddressPreservesHighBits() { } } -void TestDispatchBufferKeepsOnlyBindingLayoutOnHost() { - using namespace Libs::Graphics::ShaderRecompiler::IR; - auto plan = UserDataBufferPlan(); - plan.info.buffers[0].dispatch_stride = 64; - plan.info.buffers[0].max_byte_extent = 20; - auto &source = plan.descriptor_sources[0]; - source.dwords[1] = Value(20u << 16u); - source.dwords[3] = Value(0x16204u); - std::array user_data{0x1000u}; - uint32_t reads = 0; - const SrtRuntime runtime{.user_data = user_data, - .read_memory = RejectSpecializationRead, - .userdata = &reads, - .read_specialization_memory = RejectSpecializationRead}; - ResourceSnapshot snapshot; - ResourceSpecialization specialization; - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.buffers[0].dwords[2] == 0 && - specialization.buffers[0].packed_stride == 20 && reads == 0, - "native record count was required to materialize its stable binding layout"); - for (const auto words : {std::array{0u, 0x16204u}, - {16u << 16u, 0x16204u}, - {(20u << 16u) | (1u << 31u), 0x16204u}, - {20u << 16u, 0x16204u | (1u << 23u)}, - {20u << 16u, 0x16204u | (1u << 28u)}}) { - source.dwords[1] = Value(words[0]); - source.dwords[3] = Value(words[1]); - Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "dispatch buffer accepted a layout outside its native bounds proof"); - } - user_data[0] = 0; - source.dwords[1] = Value(0u); - source.dwords[3] = Value(0u); - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - specialization.buffers[0].packed_stride == 0, - "inactive dispatch buffer did not preserve null binding semantics"); - Check(reads == 0, "dispatch buffer layout validation read GPU data on the host"); -} - void TestIntegerRuntimeValueFollowsSrtReads() { using namespace Libs::Graphics::ShaderRecompiler::IR; auto plan = SrtPlan(0x10000); @@ -284,6 +247,113 @@ void TestIntegerRuntimeValueFollowsSrtReads() { "cyclic SRT read-first-lane dependency was accepted"); } +void TestUniformVectorDescriptorRead() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = program.resource_tracking_complete = true; + auto &block = AddValueBlock(program); + auto &handle = block.AppendNewInst(ValueOpcode::GetBufferResource, + {Value(0x1000u), Value(4u << 16u), Value(1u), Value(0x16204u)}); + program.memory_info.push_back({.kind = ResourceKind::Buffer}); + auto &count = block.AppendNewInst(ValueOpcode::LoadBufferU32, + {Value(&handle), Value(0u), Value(0u), Value(0u), Value(true)}); + count.SetFlags(MemoryFlags{.index = 0}); + // The captured indirect kernel shares one read across sibling scalar lane reads, + // enclosed by a different EXEC mask. Its resource plan must share that read too. + auto &lane = block.AppendNewInst(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}); + auto &active = block.AppendNewInst(ValueOpcode::ULessThan32, {Value(&lane), Value(32u)}); + auto &first = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(true)}); + auto &next = block.AppendNewInst(ValueOpcode::IAdd32, {Value(&count), Value(1u)}); + auto &second = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&next), Value(true)}); + auto &sum = block.AppendNewInst(ValueOpcode::IAdd32, {Value(&first), Value(&second)}); + auto &small = block.AppendNewInst(ValueOpcode::ULessThanEqual32, {Value(&count), Value(72u)}); + auto &selected = block.AppendNewInst(ValueOpcode::SelectU32, {Value(&small), Value(&sum), Value(&count)}); + auto &masked = block.AppendNewInst(ValueOpcode::SelectU32, {Value(&active), Value(&selected), Value(0u)}); + auto &records = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&masked), Value(&active)}); + DescriptorSource source; + source.dword_count = 4; + source.dwords = {Value(0x2000u), Value(4u << 16u), Value(&records), Value(0x16204u)}; + program.descriptor_sources.push_back(source); + program.info.buffers.push_back({.source = 0, .written = true}); + Check(ValidateRuntimeValue(program, Value(&count)), "uniform DWORD count was rejected"); + struct Reads { uint32_t value = 72; uint32_t strict = 0; uint32_t ordinary = 0; bool clean = true; } reads; + const SrtRuntime runtime{ + .read_memory = [](void *data, uint64_t, std::span words) { + ++static_cast(data)->ordinary; + words[0] = 999; + return true; + }, + .userdata = &reads, + .read_specialization_memory = [](void *data, uint64_t address, std::span words) { + auto &reads = *static_cast(data); + ++reads.strict; + if (!reads.clean || address != 0x1000u || words.size() != 1) return false; + words[0] = reads.value; + return true; + }}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + for (const bool written : {false, true}) { + program.info.buffers[0].written = written; + auto plan = ExtractResourcePlan(program); + Check(plan.control_flow.empty() && plan.requires_specialization_memory && + plan.capture_specialization_reads, + "vector descriptor read depended on incidental control-flow capture"); + reads = {}; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.buffers[0].dwords[2] == 145 && reads.strict == 1 && reads.ordinary == 0 && + snapshot.specialization_reads == + std::vector>{{0x1000u, 4u}}, + "vector descriptor input was not read and captured exactly once"); + reads.clean = false; + reads.strict = 0; + Check(!MaterializeResources(plan, runtime, snapshot, specialization) && + reads.strict == 1 && reads.ordinary == 0, + "dirty vector descriptor input fell back to an ordinary memory read"); + } + count.SetArg(4, Value(false)); + auto &inactive = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(false)}); + reads.strict = 0; + uint32_t result = 99; + Check(SrtWalker(program, runtime).Evaluate(Value(&inactive), result) && + result == 0 && reads.strict == 0 && reads.ordinary == 0, + "literal false EXEC read vector memory"); + count.SetArg(4, Value(true)); + handle.SetArg(3, Value(0x204u)); + Check(SrtWalker(program, runtime).Evaluate(Value(&count), result) && + result == 0 && reads.strict == 0 && reads.ordinary == 0, + "invalid vector buffer format read memory"); + handle.SetArg(3, Value(0x16204u)); + count.SetArg(1, Value(&lane)); + Check(!ValidateRuntimeValue(program, Value(&count)), + "varying vector address was treated as a uniform descriptor read"); +} + +void TestExactReciprocalDescriptorArithmetic() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + auto &block = AddValueBlock(program); + for (const float divisor : {64.f, 128.f, 256.f, 512.f, + std::numeric_limits::min(), + std::bit_cast(253u << 23u)}) { + auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)}); + uint32_t result = 0; + Check(SrtWalker(program, {}).Evaluate(Value(&reciprocal), result) && + result == std::bit_cast(1.f / divisor), + "power-of-two reciprocal was not exact"); + } + for (const float divisor : {0.f, 3.f, -64.f, std::numeric_limits::infinity(), + std::numeric_limits::denorm_min(), + std::bit_cast(254u << 23u)}) { + auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)}); + uint32_t result = 0; + Check(!SrtWalker(program, {}).Evaluate(Value(&reciprocal), result), + "unsupported reciprocal rounding or exceptional input was accepted"); + } +} + void TestUnbasedFlatCacheHitMaterializes() { using namespace Libs::Graphics::ShaderRecompiler::IR; auto plan = UnbasedFlatPlan(); @@ -527,8 +597,9 @@ void DbgExit(int) { std::abort(); } int main() { TestMappedSrtUsesDirectReaderByDefault(); TestRawScalarAddressPreservesHighBits(); - TestDispatchBufferKeepsOnlyBindingLayoutOnHost(); TestIntegerRuntimeValueFollowsSrtReads(); + TestUniformVectorDescriptorRead(); + TestExactReciprocalDescriptorArithmetic(); TestUnbasedFlatCacheHitMaterializes(); TestWrittenDescriptorUsesStrictReaderOnce(); TestFailedMaterializationRejectsStage(); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index ba80e58cd..bf8a820dc 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -2728,6 +2728,33 @@ void TestInvariantLoopPhi() { "loop-invariant descriptor phi was not evaluated through typed SSA"); } +void TestBufferStoreUsesItsOwnActiveValue() { + Fixture fixture; + const auto lane = fixture.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}); + const auto guard = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(16u)}); + const auto other_guard = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(32u)}); + const auto inner = fixture.Emit(ValueOpcode::SelectU32, {guard, Value(9u), lane}); + const auto data = fixture.Emit(ValueOpcode::SelectU32, {guard, inner, lane}); + const auto handle = fixture.Buffer({Value(0x1000u), Value(4u << 16u), + Value(64u), Value(0x16204u)}); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + const auto store = [&](Value active) { + return fixture.Emit(ValueOpcode::StoreBufferU32, + {handle, Value(0u), Value(0u), Value(0u), data, active}, + fixture.AddMemory(memory, 0)); + }; + const auto matching = store(guard); + const auto different = store(other_guard); + ConstantPropagationPass(fixture.program.blocks); + Check(matching.Instruction()->Arg(4).Resolve() == Value(9u), + "buffer store retained data from its inactive lanes"); + Check(different.Instruction()->Arg(4).Resolve() == data.Resolve() && + data.Resolve().Instruction()->GetOpcode() == ValueOpcode::SelectU32, + "buffer store changed a shared value outside its own EXEC mask"); +} + void TestBoundedRelativeRegisterWrites() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; enum class Variant { Bounded, BoundClobber, DescriptorClobber, EntryBypass }; @@ -3639,6 +3666,7 @@ int main() { Run("finite image bit scan sentinel", TestFiniteImageBitScanSentinel); Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue); Run("invariant loop phi", TestInvariantLoopPhi); + Run("buffer store active value", TestBufferStoreUsesItsOwnActiveValue); Run("bounded relative register writes", TestBoundedRelativeRegisterWrites); Run("DMA address materialization", TestDmaAddressMaterialization); Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 6328e62b8..ca3c78b5a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -31282,11 +31282,11 @@ void CheckComputeThreadDimensions(VulkanHarness &vulkan) { } } -void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) { +void CheckRuntimeBufferRecords(VulkanHarness &vulkan) { using namespace ShaderRecompiler::IR; TestCase test; - test.name = "GpuComputedBufferRecords"; - // Captured b71ef7c579835a0d: read a GPU counter, divide it into chunks, + test.name = "RuntimeBufferRecords"; + // Captured b71ef7c579835a0d: read an input count, divide it into chunks, // construct a structured descriptor, then clear each live record's last DWORD. test.code = { 0xbfa00003u, 0xd7460000u, 0x04010c0eu, 0x8805ff0du, 0x00040000u, 0xbe84030cu, @@ -31303,17 +31303,17 @@ void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) { 0xe0700000u, 0x80000500u, 0xbefe046au, 0x7e0a0280u, 0x7e000502u, 0x7e020501u, 0x7e040504u, 0x7e060503u, 0xe0702010u, 0x80000500u, 0xbf810000u, }; - constexpr u32 guest_base = 0x1000; - test.user_data[0] = guest_base + 16; + constexpr u32 count_address = 4; + test.user_data[0] = 16; test.user_data[1] = 4u << 16; test.user_data[2] = 1; test.user_data[3] = 0x5204; - test.user_data[8] = guest_base + 20; - test.user_data[12] = guest_base; + test.user_data[8] = 20; + test.user_data[12] = count_address; test.has_user_data = true; - test.storage_buffer_offsets = {0, 16, 20}; + test.storage_buffer_offsets = {count_address, 16, 20}; test.initial.assign(5 + 5 * 130, 0xdeadbeefu); - test.initial[0] = 72; + test.initial[1] = 72; test.has_compute_info = true; test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; @@ -31326,23 +31326,62 @@ void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) { test.compute_info.dispatch_thread_dimensions = true; test.compute_info.dispatch_threads_num[0] = 2; test.compute_info.dispatch_threads_num[1] = test.compute_info.dispatch_threads_num[2] = 1; - test.required_spirv = {"OpFDiv", "OpUMulExtended"}; - test.forbidden_spirv = {"get_bda_pointer", "flattened_srt"}; + test.forbidden_spirv = {"get_bda_pointer", "flattened_srt", "OpFDiv", "OpUMulExtended"}; const auto compiled = CompileCase(test, vulkan.SubgroupSize()); - Require(test.name, "native record count", + Require(test.name, "runtime record count", compiled.program.info.buffers.size() == 3 && - compiled.program.info.buffers[2].dispatch_stride == 64 && - compiled.resources.buffers[2].dwords[2] == 0 && - compiled.resources.specialization_reads.empty(), - "GPU count entered the host descriptor snapshot or lost its dispatch bound"); - // Reuse the same compiled shader as both GPU data and dispatch coverage change. + compiled.resources.buffers[2].dwords[2] == 2 && + compiled.resources.specialization_reads == + std::vector>{{count_address, 4}}, + "descriptor capacity lost its exact clean input read"); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.wave_size = 64; + options.user_data = test.user_data; + options.input_info.compute = &test.compute_info; + auto translated = ShaderRecompiler::TranslateProgram(test.code, options); + auto plan = ExtractResourcePlan(translated.program); + struct CountMemory { u32 value = 72; u32 reads = 0; bool clean = true; } memory; + SrtRuntime runtime{.user_data = test.user_data, .userdata = &memory}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Require(test.name, "strict reader required", + !MaterializeResources(plan, runtime, snapshot, specialization), + "vector descriptor input bypassed the clean-memory reader"); + runtime.read_specialization_memory = [](void *data, uint64_t address, std::span words) { + auto &memory = *static_cast(data); + ++memory.reads; + if (!memory.clean || address != count_address || words.size() != 1) return false; + words[0] = memory.value; + return true; + }; + memory.clean = false; + Require(test.name, "dirty input rejected", + !MaterializeResources(plan, runtime, snapshot, specialization), + "GPU-dirty count was accepted for host descriptor evaluation"); + memory.clean = true; + ResourceSpecialization initial_specialization; + // Reuse the same resource plan and compiled shader as the count and coverage change. for (const u32 count : {0u, 72u, 512u, 8192u, 262144u}) { + memory.value = count; + memory.reads = 0; + const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512; + const u32 records = (count + chunk - 1) / chunk; + Require(test.name, "cached descriptor refresh", + MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1 && + snapshot.buffers[2].dwords[2] == records && + snapshot.specialization_reads == + std::vector>{{count_address, 4}}, + "count refresh repeated a memory read or retained an old descriptor capacity"); + if (count == 0) initial_specialization = specialization; + Require(test.name, "capacity is not specialization", + specialization == initial_specialization, + "changing record capacity would recompile the cached shader"); + test.initial.assign(5 + 5 * records, 0xdeadbeefu); for (const u32 threads : {2u, 3u, 65u}) { - test.initial[0] = count; + test.initial[1] = count; test.compute_info.dispatch_threads_num[0] = threads; test.dispatch_x = (threads + 63) / 64; - const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512; - const u32 records = (count + chunk - 1) / chunk; test.expected = test.initial; test.expected[4] = 0; for (u32 index = 0; index < std::min(threads, records); ++index) { @@ -31352,7 +31391,7 @@ void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) { vulkan.Dispatch(test, compiled, output); const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size()); vulkan.DestroyBuffer(&output); - CompareWords(test, "GPU records and padded invocation bounds", test.expected, actual); + CompareWords(test, "record capacity and padded invocation bounds", test.expected, actual); } } std::printf("[compute] %-32s ok\n", test.name); @@ -38454,7 +38493,7 @@ int main(int argc, char **argv) { RunCase(&vulkan, ScalarBufferFromLoopReadlane(32)); RunCase(&vulkan, ScalarBufferFromLoopReadlane(64)); CheckIndirectBufferStore(vulkan); - CheckGpuComputedBufferRecords(vulkan); + CheckRuntimeBufferRecords(vulkan); RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors()); @@ -39030,7 +39069,7 @@ int main(int argc, char **argv) { CheckPixelParameterAliases(); CheckRectListShaders(); CheckIndirectBufferStore(vulkan); - CheckGpuComputedBufferRecords(vulkan); + CheckRuntimeBufferRecords(vulkan); CheckComputeThreadDimensions(vulkan); CheckIndirectImageKeySwitch(); CheckWave64WholeWaveResults();