mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 02:47:02 +02:00
shader: materialize exact buffer capacity through shared clean reads
Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>
This commit is contained in:
@@ -805,8 +805,7 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
|
||||
}
|
||||
}
|
||||
|
||||
void RenderExecutor::FindBuffers(PreparedBindings& prepared, uint64_t dispatch_threads_x,
|
||||
uint32_t group_size_x) {
|
||||
void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime);
|
||||
const auto& program = *prepared.runtime->program;
|
||||
@@ -823,23 +822,8 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared, uint64_t dispatch_t
|
||||
for (const auto resource: resources) {
|
||||
auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[resource]);
|
||||
const auto address = descriptor.Base48();
|
||||
if (address == 0u) {
|
||||
prepared.buffer_sources.push_back({});
|
||||
continue;
|
||||
}
|
||||
auto requested_size = descriptor.GetSize();
|
||||
const auto& buffer = program.info.buffers[resource];
|
||||
if (buffer.dispatch_stride != 0u) {
|
||||
if (dispatch_threads_x == 0u || dispatch_threads_x > uint64_t{UINT32_MAX} + 1u ||
|
||||
buffer.dispatch_stride != group_size_x) {
|
||||
EXIT("GPU-sized buffer requires a known matching X dispatch extent\n");
|
||||
}
|
||||
requested_size = (dispatch_threads_x - 1u) * descriptor.Stride() + buffer.max_byte_extent;
|
||||
if (requested_size > uint64_t{UINT32_MAX} + 1u) {
|
||||
EXIT("GPU-sized buffer dispatch wraps its 32-bit byte address\n");
|
||||
}
|
||||
}
|
||||
if (requested_size == 0) {
|
||||
const auto requested_size = descriptor.GetSize();
|
||||
if (address == 0 || requested_size == 0) {
|
||||
prepared.buffer_sources.push_back({});
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -161,9 +161,7 @@ public:
|
||||
uint32_t mode);
|
||||
|
||||
void PrepareBindings(const ShaderStageRuntime& runtime, PreparedBindings& prepared);
|
||||
void FindBuffers(PreparedBindings& bindings,
|
||||
uint64_t dispatch_threads_x = 0,
|
||||
uint32_t group_size_x = 0);
|
||||
void FindBuffers(PreparedBindings& bindings);
|
||||
void RebindBuffers(PreparedBindings& bindings);
|
||||
void RebindImages(PreparedBindings& bindings);
|
||||
void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
|
||||
|
||||
@@ -306,10 +306,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
std::copy(std::begin(input_info.dispatch_threads_num), std::end(input_info.dispatch_threads_num),
|
||||
bindings.shader_data.begin() + program.bindings.dispatch_thread_dword);
|
||||
}
|
||||
const auto dispatch_threads_x = use_thread_dimensions
|
||||
? uint64_t{input_info.dispatch_threads_num[0]}
|
||||
: uint64_t{thread_group_x} * input_info.threads_num[0];
|
||||
FindBuffers(bindings, dispatch_threads_x, input_info.threads_num[0]);
|
||||
FindBuffers(bindings);
|
||||
if (program.info.uses_dma) {
|
||||
m_context.PrepareBda();
|
||||
}
|
||||
|
||||
@@ -641,7 +641,6 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
|
||||
for (auto& inst: *block) {
|
||||
const auto op = inst.GetOpcode();
|
||||
uint32_t first = 0;
|
||||
bool native_buffer_count = false;
|
||||
if (op == IR::ValueOpcode::GetBufferResource) {
|
||||
if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) {
|
||||
return ir.memory_info[use.user->Flags<IR::MemoryFlags>().index].kind ==
|
||||
@@ -650,8 +649,6 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
|
||||
continue;
|
||||
}
|
||||
const auto resource = inst.Flags<uint32_t>();
|
||||
native_buffer_count = resource < ir.info.buffers.size() &&
|
||||
ir.info.buffers[resource].dispatch_stride != 0u;
|
||||
first = resource < ir.info.buffers.size() &&
|
||||
ir.info.buffers[resource].indirect_root == resource
|
||||
? 1u
|
||||
@@ -664,7 +661,6 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
|
||||
continue;
|
||||
}
|
||||
for (size_t index = first; index < inst.NumArgs(); index++) {
|
||||
if (native_buffer_count && index == 2u) continue;
|
||||
inst.SetArg(index, IR::Value(0u));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,15 +506,7 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t data) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
auto in_bounds = EmitMemoryElementInBounds(ctx.state, resource, index);
|
||||
if (mem.kind == IR::ResourceKind::Buffer &&
|
||||
ctx.state.program.info.buffers[mem.resource].dispatch_stride != 0u) {
|
||||
const auto& handle = *inst.Arg(0).ResolveInstruction();
|
||||
in_bounds = AndCondition(
|
||||
ctx.state, in_bounds,
|
||||
Binary(ctx.state, spv::OpULessThan, TypeBool(ctx.state), ctx.Arg(inst, 1), ctx.Arg(handle, 2)));
|
||||
}
|
||||
EmitIfCondition(ctx.state, in_bounds, [&]() {
|
||||
EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
StoreWordInBounds(ctx, resource, index, data);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -190,6 +190,18 @@ Value ResolveInvariantPhi(const ResourcePlan& program, Value value) {
|
||||
return invariant;
|
||||
}
|
||||
|
||||
Value ResolveActiveU32(Value value, Value active) {
|
||||
for (uint32_t depth = 0; depth <= 32; ++depth) {
|
||||
value = value.Resolve();
|
||||
const auto* inst = value.TryInstruction();
|
||||
if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 ||
|
||||
inst->Arg(0).Resolve() != active)
|
||||
return value;
|
||||
value = inst->Arg(1);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
bool HasShaderMemoryWrites(const Program& program) {
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
|
||||
@@ -106,9 +106,6 @@ struct BufferResource {
|
||||
uint32_t source = 0;
|
||||
uint32_t first_use_pc = 0;
|
||||
uint32_t max_byte_extent = 0;
|
||||
// Nonzero proves index = local_x + workgroup_x * dispatch_stride. NUM_RECORDS
|
||||
// stays on the GPU; host binding capacity comes from the known dispatch extent.
|
||||
uint32_t dispatch_stride = 0;
|
||||
uint32_t packed_stride = 0;
|
||||
Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid;
|
||||
uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7);
|
||||
@@ -625,6 +622,7 @@ void ValidateProgram(const Program& program, bool require_ssa);
|
||||
void ResolveControlFlowIdentities(Program& program);
|
||||
bool EquivalentValue(const ResourcePlan& program, Value left, Value right);
|
||||
Value ResolveInvariantPhi(const ResourcePlan& program, Value value);
|
||||
Value ResolveActiveU32(Value value, Value active);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::IR
|
||||
|
||||
|
||||
@@ -199,6 +199,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
|
||||
std::unordered_set<Inst*>& lowered_ancillary) {
|
||||
auto& inst = *instruction;
|
||||
switch (inst.GetOpcode()) {
|
||||
case ValueOpcode::StoreBufferU32: {
|
||||
const auto data = ResolveActiveU32(Arg(inst, 4), Arg(inst, 5));
|
||||
if (!data.IsEmpty() && data != inst.Arg(4)) inst.SetArg(4, data);
|
||||
return;
|
||||
}
|
||||
case ValueOpcode::Phi: FoldPhi(inst); return;
|
||||
case ValueOpcode::SelectU1:
|
||||
if (!FoldSelect(inst) && IsImmediate(Arg(inst, 2), Type::U1) &&
|
||||
|
||||
@@ -762,23 +762,11 @@ static bool IsFullDispatchPredicate(Value value, uint32_t depth = 0) {
|
||||
return axis.IsImmediate() && axis.U32() < 3 && extent->Arg(0).Resolve() == axis;
|
||||
}
|
||||
|
||||
static Value FillValue(Value value, Value guard) {
|
||||
for (uint32_t depth = 0; depth <= 32; ++depth) {
|
||||
value = value.Resolve();
|
||||
const auto* inst = value.TryInstruction();
|
||||
if (inst == nullptr || inst->GetOpcode() != ValueOpcode::SelectU32 ||
|
||||
inst->Arg(0).Resolve() != guard)
|
||||
return value;
|
||||
value = inst->Arg(1);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// Nonnegative affine coefficients for constant, local and workgroup coordinates. Reject modular
|
||||
// arithmetic that could wrap; runtime coverage also bounds the largest invocation index.
|
||||
std::optional<std::array<uint64_t, 3>> AffineIndex(Value value, uint32_t axis, Value guard,
|
||||
uint32_t depth) {
|
||||
value = FillValue(value, guard);
|
||||
static std::optional<std::array<uint64_t, 3>> FillIndex(Value value, uint32_t axis, Value guard,
|
||||
uint32_t depth = 0) {
|
||||
value = ResolveActiveU32(value, guard);
|
||||
if (depth > 32 || value.GetType() != Type::U32) {
|
||||
return {};
|
||||
}
|
||||
@@ -802,8 +790,8 @@ std::optional<std::array<uint64_t, 3>> AffineIndex(Value value, uint32_t axis, V
|
||||
op != ValueOpcode::ShiftLeftLogical32) {
|
||||
return {};
|
||||
}
|
||||
auto left = AffineIndex(inst->Arg(0), axis, guard, depth + 1);
|
||||
auto right = AffineIndex(inst->Arg(1), axis, guard, depth + 1);
|
||||
auto left = FillIndex(inst->Arg(0), axis, guard, depth + 1);
|
||||
auto right = FillIndex(inst->Arg(1), axis, guard, depth + 1);
|
||||
if (!left || !right) {
|
||||
return {};
|
||||
}
|
||||
@@ -870,7 +858,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
|
||||
const auto* address = store->Arg(1).ResolveInstruction();
|
||||
if (address == nullptr || address->GetOpcode() != ValueOpcode::MakeImageAddress) return {};
|
||||
for (uint32_t axis = 0; axis < 3; ++axis) {
|
||||
const auto index = AffineIndex(address->Arg(axis), axis, guard);
|
||||
const auto index = FillIndex(address->Arg(axis), axis, guard);
|
||||
if (!index || (*index)[0] != 0) return {};
|
||||
if (axis < 2) {
|
||||
if ((*index)[1] != 1 || (*index)[2] == 0) return {};
|
||||
@@ -900,14 +888,14 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
|
||||
memory.data_bits != 32 ||
|
||||
memory.data_dwords != static_cast<uint32_t>(store_op - stores.begin() + 1))
|
||||
return {};
|
||||
const auto address = AffineIndex(store->Arg(1), 0, guard);
|
||||
const auto address = FillIndex(store->Arg(1), 0, guard);
|
||||
if (!address || (*address)[0] != 0 || (*address)[1] != 1 || (*address)[2] == 0) return {};
|
||||
result.fill.kind = UniformFillKind::Buffer;
|
||||
result.fill.group_stride[0] = static_cast<uint32_t>((*address)[2]);
|
||||
result.fill.words = memory.data_dwords;
|
||||
data = store->Arg(4);
|
||||
}
|
||||
data = FillValue(data, guard);
|
||||
data = ResolveActiveU32(data, guard);
|
||||
const auto* vector = data.TryInstruction();
|
||||
constexpr std::array composites {ValueOpcode::CompositeConstructU32x2,
|
||||
ValueOpcode::CompositeConstructU32x3,
|
||||
@@ -916,7 +904,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
|
||||
(vector == nullptr || vector->GetOpcode() != composites[result.fill.words - 2]))
|
||||
return {};
|
||||
for (uint32_t i = 0; i < result.fill.words; ++i) {
|
||||
const auto word = FillValue(result.fill.words == 1 ? data : vector->Arg(i), guard);
|
||||
const auto word = ResolveActiveU32(result.fill.words == 1 ? data : vector->Arg(i), guard);
|
||||
if (word.GetType() != Type::U32 ||
|
||||
!ValidateRuntimeValue(program, word, RuntimeValueType::Integer))
|
||||
return {};
|
||||
@@ -943,6 +931,11 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
||||
if (source == nullptr) {
|
||||
return value;
|
||||
}
|
||||
if (source->GetOpcode() == ValueOpcode::ReadFirstLane &&
|
||||
ValidateRuntimeValue(program, source->Arg(0))) {
|
||||
// Uniform values need no new EXEC context; preserve their shared evaluation memo.
|
||||
return Clone(source->Arg(0));
|
||||
}
|
||||
if (source->GetOpcode() == ValueOpcode::Phi) {
|
||||
const auto invariant = ResolveInvariantPhi(program, value);
|
||||
if (!invariant.IsEmpty() && invariant != value) {
|
||||
@@ -952,6 +945,10 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
||||
if (const auto found = cloned.find(source); found != cloned.end()) {
|
||||
return Value(found->second);
|
||||
}
|
||||
if (source->GetOpcode() == ValueOpcode::LoadBufferU32) {
|
||||
plan.requires_specialization_memory = true;
|
||||
plan.capture_specialization_reads = true;
|
||||
}
|
||||
auto& target =
|
||||
plan.value_storage.emplace_back(source->GetOpcode(), source->Flags<uint64_t>());
|
||||
cloned.emplace(source, &target);
|
||||
@@ -1042,7 +1039,7 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
||||
}
|
||||
}
|
||||
}
|
||||
plan.capture_specialization_reads = capture_indirect_reads || !plan.control_flow.empty();
|
||||
plan.capture_specialization_reads |= capture_indirect_reads || !plan.control_flow.empty();
|
||||
if (plan.capture_specialization_reads) {
|
||||
// Clean writable addresses prove that shader writes cannot overlap captured reads.
|
||||
for (const auto& buffer: plan.info.buffers) {
|
||||
@@ -1138,12 +1135,6 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
auto packed_stride = descriptor.PackedStride();
|
||||
const auto stride = packed_stride & 0x3fffu;
|
||||
const bool swizzle = stride != 0u && ((packed_stride >> 14u) & 1u) != 0u;
|
||||
if (base.dispatch_stride != 0u && descriptor.Base48() != 0u &&
|
||||
(descriptor.OutOfBounds() != 0u || stride == 0u || swizzle ||
|
||||
(packed_stride & (1u << 20u)) != 0u || base.max_byte_extent > stride)) {
|
||||
return SpecializationFail(fmt::format(
|
||||
"dispatch-sized buffer {} requires an unswizzled structured DWORD layout", i));
|
||||
}
|
||||
if (stride == 0u) {
|
||||
packed_stride &= ~((1u << 14u) | (3u << 16u));
|
||||
} else if (!swizzle) {
|
||||
|
||||
@@ -43,10 +43,6 @@ struct ResourceSpecialization {
|
||||
// its values and is independent of the translated shader CFG.
|
||||
ResourcePlan ExtractResourcePlan(const Program& program);
|
||||
|
||||
// Nonnegative constant/local/workgroup coefficients under the instruction's active guard.
|
||||
std::optional<std::array<uint64_t, 3>> AffineIndex(Value value, uint32_t axis, Value guard,
|
||||
uint32_t depth = 0);
|
||||
|
||||
// Refreshes cached resources and specialization in place. A failed refresh must not be used.
|
||||
bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime,
|
||||
ResourceSnapshot& snapshot, ResourceSpecialization& specialization);
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -2043,32 +2042,9 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
bool TryMakeDispatchBuffer(const Inst& inst, const Inst& handle, uint32_t& source,
|
||||
uint32_t& dispatch_stride) {
|
||||
const auto flags = inst.Flags<MemoryFlags>();
|
||||
const auto& memory = m_program.memory_info[flags.index];
|
||||
if (m_program.stage != ShaderType::Compute || inst.GetOpcode() != ValueOpcode::StoreBufferU32 ||
|
||||
memory.kind != ResourceKind::Buffer || memory.typed || memory.formatted ||
|
||||
!memory.idxen || memory.offen || memory.data_bits != 32u || memory.data_dwords != 1u ||
|
||||
inst.Arg(2).Resolve() != Value(0u) || inst.Arg(3).Resolve() != Value(0u)) return false;
|
||||
const auto index = AffineIndex(inst.Arg(1), 0, inst.Arg(5).Resolve());
|
||||
if (!index || (*index)[0] != 0u || (*index)[1] != 1u || (*index)[2] == 0u) return false;
|
||||
DescriptorSource descriptor;
|
||||
MakeSource(handle, 4u, false, false, memory.resource * 4u, descriptor, flags.pc);
|
||||
// Binding capacity is derived from the dispatch. The native handle keeps its count.
|
||||
descriptor.dwords[2] = Value(0u);
|
||||
uint32_t bad_dword = 0;
|
||||
if (!ValidateSource(descriptor, bad_dword)) return false;
|
||||
source = InternSource(descriptor);
|
||||
dispatch_stride = static_cast<uint32_t>((*index)[2]);
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc,
|
||||
uint32_t dispatch_stride) {
|
||||
uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
|
||||
for (uint32_t i = 0; i < m_info.buffers.size(); i++) {
|
||||
if (m_info.buffers[i].source == source &&
|
||||
m_info.buffers[i].dispatch_stride == dispatch_stride) {
|
||||
if (m_info.buffers[i].source == source) {
|
||||
Merge(m_info.buffers[i], memory, op, pc);
|
||||
return i;
|
||||
}
|
||||
@@ -2079,7 +2055,6 @@ private:
|
||||
BufferResource resource;
|
||||
resource.source = source;
|
||||
resource.first_use_pc = pc;
|
||||
resource.dispatch_stride = dispatch_stride;
|
||||
Merge(resource, memory, op, pc);
|
||||
m_info.buffers.push_back(resource);
|
||||
return static_cast<uint32_t>(m_info.buffers.size() - 1);
|
||||
@@ -2235,14 +2210,12 @@ private:
|
||||
uint32_t resource = 0;
|
||||
|
||||
if (buffer != BufferAccess::None) {
|
||||
uint32_t dispatch_stride = 0;
|
||||
handle = inst.Arg(0).Resolve().TryInstruction();
|
||||
const auto* indirect = handle == nullptr ? nullptr : FindIndirectDescriptor(*handle);
|
||||
if (indirect != nullptr) {
|
||||
source = indirect->source;
|
||||
} else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc,
|
||||
memory.resource * 4u, handle, source) &&
|
||||
!TryMakeDispatchBuffer(inst, *handle, source, dispatch_stride)) {
|
||||
memory.resource * 4u, handle, source)) {
|
||||
if (memory.kind != (op == ValueOpcode::ReadConstBuffer ? ResourceKind::ScalarBuffer
|
||||
: ResourceKind::Buffer) ||
|
||||
!memory.SupportsIndirectBufferLoad(op)) {
|
||||
@@ -2254,7 +2227,7 @@ private:
|
||||
m_info.uses_dma = true;
|
||||
return;
|
||||
}
|
||||
resource = AddBuffer(source, memory, op, flags.pc, dispatch_stride);
|
||||
resource = AddBuffer(source, memory, op, flags.pc);
|
||||
if (resource == UINT32_MAX) {
|
||||
Fail(flags.pc, "buffer resource limit exceeded");
|
||||
}
|
||||
|
||||
@@ -41,16 +41,29 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
|
||||
|
||||
bool IsRawRead(const ResourcePlan& values, const Inst& inst) {
|
||||
const auto op = inst.GetOpcode();
|
||||
if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) {
|
||||
if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer &&
|
||||
op != ValueOpcode::LoadBufferU32) {
|
||||
return false;
|
||||
}
|
||||
const auto index = inst.Flags<MemoryFlags>().index;
|
||||
if (index >= values.memory_info.size()) {
|
||||
return false;
|
||||
}
|
||||
const auto kind = values.memory_info[index].kind;
|
||||
return (op == ValueOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) ||
|
||||
(op == ValueOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer);
|
||||
const auto& memory = values.memory_info[index];
|
||||
if (op != ValueOpcode::LoadBufferU32) {
|
||||
return (op == ValueOpcode::LoadAddressU32 && memory.kind == ResourceKind::ScalarAddress) ||
|
||||
(op == ValueOpcode::ReadConstBuffer && memory.kind == ResourceKind::ScalarBuffer);
|
||||
}
|
||||
if (inst.NumArgs() != 5u || memory.kind != ResourceKind::Buffer || memory.typed ||
|
||||
memory.formatted || memory.coherent || memory.data_bits != 32u ||
|
||||
memory.data_dwords != 1u || memory.idxen || memory.offen || memory.offset != 0u ||
|
||||
inst.Arg(4).GetType() != Type::U1) {
|
||||
return false;
|
||||
}
|
||||
for (uint32_t arg = 1; arg <= 3; ++arg) {
|
||||
if (inst.Arg(arg).Resolve() != Value(0u)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool IsDescriptorHandle(ValueOpcode opcode) {
|
||||
@@ -89,6 +102,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
|
||||
case ValueOpcode::ISub64:
|
||||
case ValueOpcode::IMul32:
|
||||
case ValueOpcode::IMul64:
|
||||
case ValueOpcode::UMulHi:
|
||||
case ValueOpcode::UMin32:
|
||||
case ValueOpcode::ShiftLeftLogical32:
|
||||
case ValueOpcode::ShiftLeftLogical64:
|
||||
@@ -105,6 +119,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
|
||||
case ValueOpcode::SelectU32:
|
||||
case ValueOpcode::SelectF32:
|
||||
case ValueOpcode::ULessThan32:
|
||||
case ValueOpcode::ULessThanEqual32:
|
||||
case ValueOpcode::IEqual32:
|
||||
case ValueOpcode::UGreaterThan32:
|
||||
case ValueOpcode::SGreaterThanEqual32:
|
||||
@@ -117,6 +132,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) {
|
||||
case ValueOpcode::FPOrdGreaterThanEqual32:
|
||||
case ValueOpcode::FPIsNan32:
|
||||
case ValueOpcode::FPMul32:
|
||||
case ValueOpcode::FPRecipIFlag32:
|
||||
case ValueOpcode::FPTrunc32: return true;
|
||||
default: return false;
|
||||
}
|
||||
@@ -157,6 +173,7 @@ private:
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
if (require_uniform && !m_active_mask.IsEmpty() && value == m_active_mask) return true;
|
||||
// Integer-only dependency checks do not depend on the active EXEC mask.
|
||||
if (!require_uniform && m_validated_dependencies.contains(inst)) return true;
|
||||
if (!m_visiting.insert(inst).second) {
|
||||
@@ -246,7 +263,8 @@ private:
|
||||
}
|
||||
return finish(true);
|
||||
}
|
||||
if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
|
||||
if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer ||
|
||||
op == ValueOpcode::LoadBufferU32) {
|
||||
const auto expected = op == ValueOpcode::LoadAddressU32
|
||||
? ValueOpcode::GetAddressResource
|
||||
: ValueOpcode::GetBufferResource;
|
||||
@@ -255,6 +273,11 @@ private:
|
||||
handle->GetOpcode() != expected) {
|
||||
return finish(false);
|
||||
}
|
||||
if (op == ValueOpcode::LoadBufferU32) {
|
||||
const auto guard = inst->Arg(4).Resolve();
|
||||
return finish(Validate(inst->Arg(0)) &&
|
||||
((!m_active_mask.IsEmpty() && guard == m_active_mask) || Validate(guard)));
|
||||
}
|
||||
} else if (op == ValueOpcode::CompositeExtractU64) {
|
||||
const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
|
||||
if (!index.IsImmediate() || index.GetType() != Type::U32 || index.U32() >= 2u) {
|
||||
@@ -341,6 +364,10 @@ bool SrtWalker::EvaluateWide(Value value, uint64_t& result) {
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
if (!m_active_mask.IsEmpty() && value == m_active_mask) {
|
||||
result = 1u;
|
||||
return true;
|
||||
}
|
||||
auto* inst = value.TryInstruction();
|
||||
if (inst == nullptr) {
|
||||
return false;
|
||||
@@ -430,6 +457,16 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
|
||||
return false;
|
||||
}
|
||||
const auto& mem = m_program.memory_info[flags.index];
|
||||
const bool vector = inst.GetOpcode() == ValueOpcode::LoadBufferU32;
|
||||
const auto guard = vector ? inst.Arg(4).Resolve() : Value {};
|
||||
if (vector && (guard.IsImmediate() || guard != m_active_mask)) {
|
||||
uint64_t enabled = 0;
|
||||
if (!Arg(inst, 4, enabled)) return false;
|
||||
if (enabled == 0u) {
|
||||
result = 0u;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const auto* handle = inst.Arg(0).ResolveInstruction();
|
||||
if (handle == nullptr) {
|
||||
return false;
|
||||
@@ -443,7 +480,7 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
|
||||
const auto base = (high << 32u) | static_cast<uint32_t>(low);
|
||||
const auto immediate = static_cast<int64_t>(static_cast<int32_t>(mem.offset));
|
||||
uint64_t address = 0;
|
||||
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
|
||||
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer || vector) {
|
||||
uint64_t records = 0;
|
||||
uint64_t word3 = 0;
|
||||
if (handle->NumArgs() != 4u || !Arg(*handle, 2, records) || !Arg(*handle, 3, word3)) {
|
||||
@@ -455,6 +492,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
|
||||
const auto byte_offset =
|
||||
(static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
|
||||
const auto stride = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
|
||||
if (vector) {
|
||||
// Only the uniform, unswizzled structured DWORD address is evaluated on the host.
|
||||
if ((high & (1u << 31u)) != 0u || (word3 & ((1u << 23u) | 0xf0000000u)) != 0u)
|
||||
return false;
|
||||
if (stride == 0u || records == 0u || ((word3 >> 12u) & 0x7fu) == 0u) {
|
||||
result = 0u;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const auto size = stride == 0u
|
||||
? static_cast<uint64_t>(static_cast<uint32_t>(records))
|
||||
: static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
|
||||
@@ -470,11 +516,13 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
|
||||
}
|
||||
}
|
||||
uint32_t word = 0;
|
||||
if (m_runtime.read_memory != nullptr) {
|
||||
if (!m_runtime.read_memory(m_runtime.userdata, address, {&word, 1})) {
|
||||
const auto reader = vector ? m_runtime.read_specialization_memory : m_runtime.read_memory;
|
||||
if (reader != nullptr) {
|
||||
if (!reader(m_runtime.userdata, address, {&word, 1})) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if (vector) return false;
|
||||
std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
|
||||
}
|
||||
result = word;
|
||||
@@ -502,6 +550,9 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
|
||||
case ValueOpcode::Phi: return EvaluatePhi(inst, result);
|
||||
case ValueOpcode::ReadFirstLane: {
|
||||
if (!m_active_mask.IsEmpty() && inst.Arg(1).Resolve() == m_active_mask) {
|
||||
return EvaluateWide(inst.Arg(0), result);
|
||||
}
|
||||
const auto clean_runtime = CleanRuntime(m_runtime);
|
||||
SrtWalker clean_active(m_program, clean_runtime, {}, nullptr, inst.Arg(1));
|
||||
SrtWalker active(m_program, m_runtime, m_clean_flat_slots, &clean_active,
|
||||
@@ -534,7 +585,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
}
|
||||
case ValueOpcode::LoadAddressU32:
|
||||
case ValueOpcode::ReadConstBuffer:
|
||||
case ValueOpcode::LoadBufferU32:
|
||||
if (IsRawRead(m_program, inst)) {
|
||||
if (inst.GetOpcode() == ValueOpcode::LoadBufferU32 && m_clean_evaluator != nullptr &&
|
||||
m_clean_evaluator->m_active_mask == m_active_mask) {
|
||||
return m_clean_evaluator->EvaluateWide(Value(const_cast<Inst*>(&inst)), result);
|
||||
}
|
||||
return EvaluateRawRead(inst, result);
|
||||
}
|
||||
break;
|
||||
@@ -574,6 +630,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::UMulHi:
|
||||
if (binary()) {
|
||||
result = (static_cast<uint64_t>(static_cast<uint32_t>(a)) * static_cast<uint32_t>(b)) >> 32u;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::UMin32:
|
||||
if (binary()) {
|
||||
result = std::min(static_cast<uint32_t>(a), static_cast<uint32_t>(b));
|
||||
@@ -609,6 +671,15 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::FPRecipIFlag32:
|
||||
if (Arg(inst, 0, a)) {
|
||||
const auto exponent = (a >> 23u) & 0xffu;
|
||||
// Normal positive powers of two have exact normal reciprocals in every FP mode.
|
||||
if ((a & 0x807fffffu) != 0u || exponent == 0u || exponent >= 254u) return false;
|
||||
result = (254u - exponent) << 23u;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::FPIsNan32:
|
||||
if (Arg(inst, 0, a)) {
|
||||
result = std::isnan(Float32(a));
|
||||
@@ -779,6 +850,12 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::ULessThanEqual32:
|
||||
if (binary()) {
|
||||
result = static_cast<uint32_t>(a) <= static_cast<uint32_t>(b);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::UGreaterThan32:
|
||||
if (binary()) {
|
||||
result = static_cast<uint32_t>(a) > static_cast<uint32_t>(b);
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
|
||||
|
||||
#include <bit>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
|
||||
namespace {
|
||||
@@ -196,45 +198,6 @@ void TestRawScalarAddressPreservesHighBits() {
|
||||
}
|
||||
}
|
||||
|
||||
void TestDispatchBufferKeepsOnlyBindingLayoutOnHost() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
auto plan = UserDataBufferPlan();
|
||||
plan.info.buffers[0].dispatch_stride = 64;
|
||||
plan.info.buffers[0].max_byte_extent = 20;
|
||||
auto &source = plan.descriptor_sources[0];
|
||||
source.dwords[1] = Value(20u << 16u);
|
||||
source.dwords[3] = Value(0x16204u);
|
||||
std::array<uint32_t, 1> user_data{0x1000u};
|
||||
uint32_t reads = 0;
|
||||
const SrtRuntime runtime{.user_data = user_data,
|
||||
.read_memory = RejectSpecializationRead,
|
||||
.userdata = &reads,
|
||||
.read_specialization_memory = RejectSpecializationRead};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.buffers[0].dwords[2] == 0 &&
|
||||
specialization.buffers[0].packed_stride == 20 && reads == 0,
|
||||
"native record count was required to materialize its stable binding layout");
|
||||
for (const auto words : {std::array<uint32_t, 2>{0u, 0x16204u},
|
||||
{16u << 16u, 0x16204u},
|
||||
{(20u << 16u) | (1u << 31u), 0x16204u},
|
||||
{20u << 16u, 0x16204u | (1u << 23u)},
|
||||
{20u << 16u, 0x16204u | (1u << 28u)}}) {
|
||||
source.dwords[1] = Value(words[0]);
|
||||
source.dwords[3] = Value(words[1]);
|
||||
Check(!MaterializeResources(plan, runtime, snapshot, specialization),
|
||||
"dispatch buffer accepted a layout outside its native bounds proof");
|
||||
}
|
||||
user_data[0] = 0;
|
||||
source.dwords[1] = Value(0u);
|
||||
source.dwords[3] = Value(0u);
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
specialization.buffers[0].packed_stride == 0,
|
||||
"inactive dispatch buffer did not preserve null binding semantics");
|
||||
Check(reads == 0, "dispatch buffer layout validation read GPU data on the host");
|
||||
}
|
||||
|
||||
void TestIntegerRuntimeValueFollowsSrtReads() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
auto plan = SrtPlan(0x10000);
|
||||
@@ -284,6 +247,113 @@ void TestIntegerRuntimeValueFollowsSrtReads() {
|
||||
"cyclic SRT read-first-lane dependency was accepted");
|
||||
}
|
||||
|
||||
void TestUniformVectorDescriptorRead() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
Program program;
|
||||
program.stage = Libs::Graphics::ShaderType::Compute;
|
||||
program.srt_plan_complete = program.resource_tracking_complete = true;
|
||||
auto &block = AddValueBlock(program);
|
||||
auto &handle = block.AppendNewInst(ValueOpcode::GetBufferResource,
|
||||
{Value(0x1000u), Value(4u << 16u), Value(1u), Value(0x16204u)});
|
||||
program.memory_info.push_back({.kind = ResourceKind::Buffer});
|
||||
auto &count = block.AppendNewInst(ValueOpcode::LoadBufferU32,
|
||||
{Value(&handle), Value(0u), Value(0u), Value(0u), Value(true)});
|
||||
count.SetFlags(MemoryFlags{.index = 0});
|
||||
// The captured indirect kernel shares one read across sibling scalar lane reads,
|
||||
// enclosed by a different EXEC mask. Its resource plan must share that read too.
|
||||
auto &lane = block.AppendNewInst(ValueOpcode::GetBuiltin,
|
||||
{Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
|
||||
auto &active = block.AppendNewInst(ValueOpcode::ULessThan32, {Value(&lane), Value(32u)});
|
||||
auto &first = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(true)});
|
||||
auto &next = block.AppendNewInst(ValueOpcode::IAdd32, {Value(&count), Value(1u)});
|
||||
auto &second = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&next), Value(true)});
|
||||
auto &sum = block.AppendNewInst(ValueOpcode::IAdd32, {Value(&first), Value(&second)});
|
||||
auto &small = block.AppendNewInst(ValueOpcode::ULessThanEqual32, {Value(&count), Value(72u)});
|
||||
auto &selected = block.AppendNewInst(ValueOpcode::SelectU32, {Value(&small), Value(&sum), Value(&count)});
|
||||
auto &masked = block.AppendNewInst(ValueOpcode::SelectU32, {Value(&active), Value(&selected), Value(0u)});
|
||||
auto &records = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&masked), Value(&active)});
|
||||
DescriptorSource source;
|
||||
source.dword_count = 4;
|
||||
source.dwords = {Value(0x2000u), Value(4u << 16u), Value(&records), Value(0x16204u)};
|
||||
program.descriptor_sources.push_back(source);
|
||||
program.info.buffers.push_back({.source = 0, .written = true});
|
||||
Check(ValidateRuntimeValue(program, Value(&count)), "uniform DWORD count was rejected");
|
||||
struct Reads { uint32_t value = 72; uint32_t strict = 0; uint32_t ordinary = 0; bool clean = true; } reads;
|
||||
const SrtRuntime runtime{
|
||||
.read_memory = [](void *data, uint64_t, std::span<uint32_t> words) {
|
||||
++static_cast<Reads *>(data)->ordinary;
|
||||
words[0] = 999;
|
||||
return true;
|
||||
},
|
||||
.userdata = &reads,
|
||||
.read_specialization_memory = [](void *data, uint64_t address, std::span<uint32_t> words) {
|
||||
auto &reads = *static_cast<Reads *>(data);
|
||||
++reads.strict;
|
||||
if (!reads.clean || address != 0x1000u || words.size() != 1) return false;
|
||||
words[0] = reads.value;
|
||||
return true;
|
||||
}};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
for (const bool written : {false, true}) {
|
||||
program.info.buffers[0].written = written;
|
||||
auto plan = ExtractResourcePlan(program);
|
||||
Check(plan.control_flow.empty() && plan.requires_specialization_memory &&
|
||||
plan.capture_specialization_reads,
|
||||
"vector descriptor read depended on incidental control-flow capture");
|
||||
reads = {};
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.buffers[0].dwords[2] == 145 && reads.strict == 1 && reads.ordinary == 0 &&
|
||||
snapshot.specialization_reads ==
|
||||
std::vector<std::pair<uint64_t, uint64_t>>{{0x1000u, 4u}},
|
||||
"vector descriptor input was not read and captured exactly once");
|
||||
reads.clean = false;
|
||||
reads.strict = 0;
|
||||
Check(!MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
reads.strict == 1 && reads.ordinary == 0,
|
||||
"dirty vector descriptor input fell back to an ordinary memory read");
|
||||
}
|
||||
count.SetArg(4, Value(false));
|
||||
auto &inactive = block.AppendNewInst(ValueOpcode::ReadFirstLane, {Value(&count), Value(false)});
|
||||
reads.strict = 0;
|
||||
uint32_t result = 99;
|
||||
Check(SrtWalker(program, runtime).Evaluate(Value(&inactive), result) &&
|
||||
result == 0 && reads.strict == 0 && reads.ordinary == 0,
|
||||
"literal false EXEC read vector memory");
|
||||
count.SetArg(4, Value(true));
|
||||
handle.SetArg(3, Value(0x204u));
|
||||
Check(SrtWalker(program, runtime).Evaluate(Value(&count), result) &&
|
||||
result == 0 && reads.strict == 0 && reads.ordinary == 0,
|
||||
"invalid vector buffer format read memory");
|
||||
handle.SetArg(3, Value(0x16204u));
|
||||
count.SetArg(1, Value(&lane));
|
||||
Check(!ValidateRuntimeValue(program, Value(&count)),
|
||||
"varying vector address was treated as a uniform descriptor read");
|
||||
}
|
||||
|
||||
void TestExactReciprocalDescriptorArithmetic() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
Program program;
|
||||
auto &block = AddValueBlock(program);
|
||||
for (const float divisor : {64.f, 128.f, 256.f, 512.f,
|
||||
std::numeric_limits<float>::min(),
|
||||
std::bit_cast<float>(253u << 23u)}) {
|
||||
auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)});
|
||||
uint32_t result = 0;
|
||||
Check(SrtWalker(program, {}).Evaluate(Value(&reciprocal), result) &&
|
||||
result == std::bit_cast<uint32_t>(1.f / divisor),
|
||||
"power-of-two reciprocal was not exact");
|
||||
}
|
||||
for (const float divisor : {0.f, 3.f, -64.f, std::numeric_limits<float>::infinity(),
|
||||
std::numeric_limits<float>::denorm_min(),
|
||||
std::bit_cast<float>(254u << 23u)}) {
|
||||
auto &reciprocal = block.AppendNewInst(ValueOpcode::FPRecipIFlag32, {Value::F32(divisor)});
|
||||
uint32_t result = 0;
|
||||
Check(!SrtWalker(program, {}).Evaluate(Value(&reciprocal), result),
|
||||
"unsupported reciprocal rounding or exceptional input was accepted");
|
||||
}
|
||||
}
|
||||
|
||||
void TestUnbasedFlatCacheHitMaterializes() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
auto plan = UnbasedFlatPlan();
|
||||
@@ -527,8 +597,9 @@ void DbgExit(int) { std::abort(); }
|
||||
int main() {
|
||||
TestMappedSrtUsesDirectReaderByDefault();
|
||||
TestRawScalarAddressPreservesHighBits();
|
||||
TestDispatchBufferKeepsOnlyBindingLayoutOnHost();
|
||||
TestIntegerRuntimeValueFollowsSrtReads();
|
||||
TestUniformVectorDescriptorRead();
|
||||
TestExactReciprocalDescriptorArithmetic();
|
||||
TestUnbasedFlatCacheHitMaterializes();
|
||||
TestWrittenDescriptorUsesStrictReaderOnce();
|
||||
TestFailedMaterializationRejectsStage();
|
||||
|
||||
@@ -2728,6 +2728,33 @@ void TestInvariantLoopPhi() {
|
||||
"loop-invariant descriptor phi was not evaluated through typed SSA");
|
||||
}
|
||||
|
||||
void TestBufferStoreUsesItsOwnActiveValue() {
|
||||
Fixture fixture;
|
||||
const auto lane = fixture.Emit(ValueOpcode::GetBuiltin,
|
||||
{Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)});
|
||||
const auto guard = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(16u)});
|
||||
const auto other_guard = fixture.Emit(ValueOpcode::ULessThan32, {lane, Value(32u)});
|
||||
const auto inner = fixture.Emit(ValueOpcode::SelectU32, {guard, Value(9u), lane});
|
||||
const auto data = fixture.Emit(ValueOpcode::SelectU32, {guard, inner, lane});
|
||||
const auto handle = fixture.Buffer({Value(0x1000u), Value(4u << 16u),
|
||||
Value(64u), Value(0x16204u)});
|
||||
MemoryInfo memory;
|
||||
memory.kind = ResourceKind::Buffer;
|
||||
const auto store = [&](Value active) {
|
||||
return fixture.Emit(ValueOpcode::StoreBufferU32,
|
||||
{handle, Value(0u), Value(0u), Value(0u), data, active},
|
||||
fixture.AddMemory(memory, 0));
|
||||
};
|
||||
const auto matching = store(guard);
|
||||
const auto different = store(other_guard);
|
||||
ConstantPropagationPass(fixture.program.blocks);
|
||||
Check(matching.Instruction()->Arg(4).Resolve() == Value(9u),
|
||||
"buffer store retained data from its inactive lanes");
|
||||
Check(different.Instruction()->Arg(4).Resolve() == data.Resolve() &&
|
||||
data.Resolve().Instruction()->GetOpcode() == ValueOpcode::SelectU32,
|
||||
"buffer store changed a shared value outside its own EXEC mask");
|
||||
}
|
||||
|
||||
void TestBoundedRelativeRegisterWrites() {
|
||||
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
|
||||
enum class Variant { Bounded, BoundClobber, DescriptorClobber, EntryBypass };
|
||||
@@ -3639,6 +3666,7 @@ int main() {
|
||||
Run("finite image bit scan sentinel", TestFiniteImageBitScanSentinel);
|
||||
Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
|
||||
Run("invariant loop phi", TestInvariantLoopPhi);
|
||||
Run("buffer store active value", TestBufferStoreUsesItsOwnActiveValue);
|
||||
Run("bounded relative register writes", TestBoundedRelativeRegisterWrites);
|
||||
Run("DMA address materialization", TestDmaAddressMaterialization);
|
||||
Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma);
|
||||
|
||||
@@ -31282,11 +31282,11 @@ void CheckComputeThreadDimensions(VulkanHarness &vulkan) {
|
||||
}
|
||||
}
|
||||
|
||||
void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) {
|
||||
void CheckRuntimeBufferRecords(VulkanHarness &vulkan) {
|
||||
using namespace ShaderRecompiler::IR;
|
||||
TestCase test;
|
||||
test.name = "GpuComputedBufferRecords";
|
||||
// Captured b71ef7c579835a0d: read a GPU counter, divide it into chunks,
|
||||
test.name = "RuntimeBufferRecords";
|
||||
// Captured b71ef7c579835a0d: read an input count, divide it into chunks,
|
||||
// construct a structured descriptor, then clear each live record's last DWORD.
|
||||
test.code = {
|
||||
0xbfa00003u, 0xd7460000u, 0x04010c0eu, 0x8805ff0du, 0x00040000u, 0xbe84030cu,
|
||||
@@ -31303,17 +31303,17 @@ void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) {
|
||||
0xe0700000u, 0x80000500u, 0xbefe046au, 0x7e0a0280u, 0x7e000502u, 0x7e020501u,
|
||||
0x7e040504u, 0x7e060503u, 0xe0702010u, 0x80000500u, 0xbf810000u,
|
||||
};
|
||||
constexpr u32 guest_base = 0x1000;
|
||||
test.user_data[0] = guest_base + 16;
|
||||
constexpr u32 count_address = 4;
|
||||
test.user_data[0] = 16;
|
||||
test.user_data[1] = 4u << 16;
|
||||
test.user_data[2] = 1;
|
||||
test.user_data[3] = 0x5204;
|
||||
test.user_data[8] = guest_base + 20;
|
||||
test.user_data[12] = guest_base;
|
||||
test.user_data[8] = 20;
|
||||
test.user_data[12] = count_address;
|
||||
test.has_user_data = true;
|
||||
test.storage_buffer_offsets = {0, 16, 20};
|
||||
test.storage_buffer_offsets = {count_address, 16, 20};
|
||||
test.initial.assign(5 + 5 * 130, 0xdeadbeefu);
|
||||
test.initial[0] = 72;
|
||||
test.initial[1] = 72;
|
||||
test.has_compute_info = true;
|
||||
test.compute_info.threads_num[0] = 64;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
@@ -31326,23 +31326,62 @@ void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) {
|
||||
test.compute_info.dispatch_thread_dimensions = true;
|
||||
test.compute_info.dispatch_threads_num[0] = 2;
|
||||
test.compute_info.dispatch_threads_num[1] = test.compute_info.dispatch_threads_num[2] = 1;
|
||||
test.required_spirv = {"OpFDiv", "OpUMulExtended"};
|
||||
test.forbidden_spirv = {"get_bda_pointer", "flattened_srt"};
|
||||
test.forbidden_spirv = {"get_bda_pointer", "flattened_srt", "OpFDiv", "OpUMulExtended"};
|
||||
const auto compiled = CompileCase(test, vulkan.SubgroupSize());
|
||||
Require(test.name, "native record count",
|
||||
Require(test.name, "runtime record count",
|
||||
compiled.program.info.buffers.size() == 3 &&
|
||||
compiled.program.info.buffers[2].dispatch_stride == 64 &&
|
||||
compiled.resources.buffers[2].dwords[2] == 0 &&
|
||||
compiled.resources.specialization_reads.empty(),
|
||||
"GPU count entered the host descriptor snapshot or lost its dispatch bound");
|
||||
// Reuse the same compiled shader as both GPU data and dispatch coverage change.
|
||||
compiled.resources.buffers[2].dwords[2] == 2 &&
|
||||
compiled.resources.specialization_reads ==
|
||||
std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
|
||||
"descriptor capacity lost its exact clean input read");
|
||||
ShaderRecompiler::CompileOptions options;
|
||||
options.stage = ShaderType::Compute;
|
||||
options.wave_size = 64;
|
||||
options.user_data = test.user_data;
|
||||
options.input_info.compute = &test.compute_info;
|
||||
auto translated = ShaderRecompiler::TranslateProgram(test.code, options);
|
||||
auto plan = ExtractResourcePlan(translated.program);
|
||||
struct CountMemory { u32 value = 72; u32 reads = 0; bool clean = true; } memory;
|
||||
SrtRuntime runtime{.user_data = test.user_data, .userdata = &memory};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
Require(test.name, "strict reader required",
|
||||
!MaterializeResources(plan, runtime, snapshot, specialization),
|
||||
"vector descriptor input bypassed the clean-memory reader");
|
||||
runtime.read_specialization_memory = [](void *data, uint64_t address, std::span<u32> words) {
|
||||
auto &memory = *static_cast<CountMemory *>(data);
|
||||
++memory.reads;
|
||||
if (!memory.clean || address != count_address || words.size() != 1) return false;
|
||||
words[0] = memory.value;
|
||||
return true;
|
||||
};
|
||||
memory.clean = false;
|
||||
Require(test.name, "dirty input rejected",
|
||||
!MaterializeResources(plan, runtime, snapshot, specialization),
|
||||
"GPU-dirty count was accepted for host descriptor evaluation");
|
||||
memory.clean = true;
|
||||
ResourceSpecialization initial_specialization;
|
||||
// Reuse the same resource plan and compiled shader as the count and coverage change.
|
||||
for (const u32 count : {0u, 72u, 512u, 8192u, 262144u}) {
|
||||
memory.value = count;
|
||||
memory.reads = 0;
|
||||
const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512;
|
||||
const u32 records = (count + chunk - 1) / chunk;
|
||||
Require(test.name, "cached descriptor refresh",
|
||||
MaterializeResources(plan, runtime, snapshot, specialization) && memory.reads == 1 &&
|
||||
snapshot.buffers[2].dwords[2] == records &&
|
||||
snapshot.specialization_reads ==
|
||||
std::vector<std::pair<uint64_t, uint64_t>>{{count_address, 4}},
|
||||
"count refresh repeated a memory read or retained an old descriptor capacity");
|
||||
if (count == 0) initial_specialization = specialization;
|
||||
Require(test.name, "capacity is not specialization",
|
||||
specialization == initial_specialization,
|
||||
"changing record capacity would recompile the cached shader");
|
||||
test.initial.assign(5 + 5 * records, 0xdeadbeefu);
|
||||
for (const u32 threads : {2u, 3u, 65u}) {
|
||||
test.initial[0] = count;
|
||||
test.initial[1] = count;
|
||||
test.compute_info.dispatch_threads_num[0] = threads;
|
||||
test.dispatch_x = (threads + 63) / 64;
|
||||
const u32 chunk = count < 512 ? 64 : count < 8192 ? 128 : count < 262144 ? 256 : 512;
|
||||
const u32 records = (count + chunk - 1) / chunk;
|
||||
test.expected = test.initial;
|
||||
test.expected[4] = 0;
|
||||
for (u32 index = 0; index < std::min(threads, records); ++index) {
|
||||
@@ -31352,7 +31391,7 @@ void CheckGpuComputedBufferRecords(VulkanHarness &vulkan) {
|
||||
vulkan.Dispatch(test, compiled, output);
|
||||
const auto actual = vulkan.ReadBuffer(test.name, output, test.expected.size());
|
||||
vulkan.DestroyBuffer(&output);
|
||||
CompareWords(test, "GPU records and padded invocation bounds", test.expected, actual);
|
||||
CompareWords(test, "record capacity and padded invocation bounds", test.expected, actual);
|
||||
}
|
||||
}
|
||||
std::printf("[compute] %-32s ok\n", test.name);
|
||||
@@ -38454,7 +38493,7 @@ int main(int argc, char **argv) {
|
||||
RunCase(&vulkan, ScalarBufferFromLoopReadlane(32));
|
||||
RunCase(&vulkan, ScalarBufferFromLoopReadlane(64));
|
||||
CheckIndirectBufferStore(vulkan);
|
||||
CheckGpuComputedBufferRecords(vulkan);
|
||||
CheckRuntimeBufferRecords(vulkan);
|
||||
RunCase(&vulkan, BufferLoadsGpuSelectedDescriptors());
|
||||
RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors());
|
||||
RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors());
|
||||
@@ -39030,7 +39069,7 @@ int main(int argc, char **argv) {
|
||||
CheckPixelParameterAliases();
|
||||
CheckRectListShaders();
|
||||
CheckIndirectBufferStore(vulkan);
|
||||
CheckGpuComputedBufferRecords(vulkan);
|
||||
CheckRuntimeBufferRecords(vulkan);
|
||||
CheckComputeThreadDimensions(vulkan);
|
||||
CheckIndirectImageKeySwitch();
|
||||
CheckWave64WholeWaveResults();
|
||||
|
||||
Reference in New Issue
Block a user