mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-19 18:36:53 +02:00
Fix scalar buffer reads across sparse guest mappings
This commit is contained in:
nmzik
@@ -81,8 +81,7 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
|
||||
}
|
||||
uses_gds |= memory.kind == IR::ResourceKind::Gds;
|
||||
uses_lds |= memory.kind == IR::ResourceKind::Lds;
|
||||
} else if (memory.kind == IR::ResourceKind::Buffer ||
|
||||
memory.kind == IR::ResourceKind::ScalarBuffer) {
|
||||
} else if (memory.kind == IR::ResourceKind::Buffer) {
|
||||
if (memory.resource >= program.info.buffers.size()) {
|
||||
Fail(program, "typed shader contains an invalid buffer resource");
|
||||
}
|
||||
|
||||
@@ -867,7 +867,7 @@ void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
|
||||
});
|
||||
}
|
||||
|
||||
uint32_t LoadIndirectScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
uint32_t LoadScalarBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& state = ctx.state;
|
||||
const auto& handle = *inst.Arg(0).ResolveInstruction();
|
||||
const auto word1 = ctx.Arg(handle, 1);
|
||||
@@ -1545,29 +1545,8 @@ uint32_t EmitReadConst(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
}
|
||||
|
||||
void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto mem = ctx.Memory(inst);
|
||||
if (mem.planning_only) return;
|
||||
if (mem.kind == IR::ResourceKind::IndirectBuffer) {
|
||||
ctx.Define(inst, LoadIndirectScalarBuffer(ctx, inst));
|
||||
return;
|
||||
}
|
||||
auto& state = ctx.state;
|
||||
mem.kind = IR::ResourceKind::ScalarBuffer;
|
||||
auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
|
||||
ConstantU32(state, 2));
|
||||
if (mem.offset >= 4u) {
|
||||
index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
|
||||
}
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
// Scalar buffer addressing aligns the base and each offset independently.
|
||||
const auto element = Binary(
|
||||
state, spv::OpIAdd, TypeU32(state), index,
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byte_offset, ConstantU32(state, 2)));
|
||||
const auto condition = EmitMemoryElementInBounds(state, access, element);
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
|
||||
return EmitNative<spv::OpLoad, IR::Type::U32>(
|
||||
state, EmitMemoryElementPointer(state, access, element));
|
||||
}));
|
||||
if (ctx.Memory(inst).planning_only) return;
|
||||
ctx.Define(inst, LoadScalarBuffer(ctx, inst));
|
||||
}
|
||||
|
||||
uint32_t EmitGetBufferResourceIndex(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
|
||||
@@ -269,10 +269,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
|
||||
case IR::ResourceKind::Flat:
|
||||
case IR::ResourceKind::Global:
|
||||
EXIT("physical address memory must use the BDA emitter\n");
|
||||
case IR::ResourceKind::ScalarBuffer:
|
||||
case IR::ResourceKind::Buffer: {
|
||||
const auto bits = mem.kind == IR::ResourceKind::Buffer
|
||||
? IR::StorageBufferElementBits(state.program, mem) : 32u;
|
||||
const auto bits = IR::StorageBufferElementBits(state.program, mem);
|
||||
auto* buffer = &state.storage_buffers[2];
|
||||
if (bits == 8u) {
|
||||
buffer = &state.storage_buffers[0];
|
||||
|
||||
@@ -128,7 +128,6 @@ struct BufferResource {
|
||||
bool written = false;
|
||||
bool atomic = false;
|
||||
bool formatted = false;
|
||||
bool scalar = false;
|
||||
bool live = false;
|
||||
uint32_t indirect_root = NoIndirectBuffer;
|
||||
uint32_t indirect_mapping_offset = 0;
|
||||
|
||||
@@ -1178,7 +1178,7 @@ static UniformFillPlan AnalyzeUniformFill(const Program& program) {
|
||||
}
|
||||
if (store == nullptr || visited.size() != program.blocks.size()) return {};
|
||||
for (const auto& buffer: program.info.buffers) {
|
||||
if (buffer.read && (!buffer.scalar || buffer.written)) return {};
|
||||
if (buffer.read) return {};
|
||||
}
|
||||
const auto& memory = program.memory_info.at(store->Flags<MemoryFlags>().index);
|
||||
UniformFillPlan result;
|
||||
@@ -1589,7 +1589,6 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
base.formatted ? descriptor.Format() : Prospero::BufferFormat::kInvalid;
|
||||
buffer.descriptor_swizzle = base.formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7);
|
||||
buffer.zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u;
|
||||
buffer.scalar_empty = base.scalar && descriptor.GetSize() == 0u;
|
||||
}
|
||||
snapshot.images.resize(program.info.images.size());
|
||||
specialization.images.resize(program.info.images.size());
|
||||
@@ -1787,17 +1786,6 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
|
||||
}
|
||||
|
||||
auto& memory_info = program.memory_info;
|
||||
// Resolve empty scalar-table words before folding the indirect descriptors that use them.
|
||||
for (auto* block: program.blocks) {
|
||||
for (auto& inst: *block) {
|
||||
if (inst.GetOpcode() != ValueOpcode::ReadConstBuffer) continue;
|
||||
const auto& memory = memory_info[inst.Flags<MemoryFlags>().index];
|
||||
if (memory.kind == ResourceKind::ScalarBuffer &&
|
||||
specialization.buffers[memory.resource].scalar_empty &&
|
||||
buffers[memory.resource].indirect_root == BufferResource::NoIndirectBuffer)
|
||||
inst.ReplaceUsesWith(Value(0u));
|
||||
}
|
||||
}
|
||||
const ImageRemap image_remap(specialization);
|
||||
for (auto* block: program.blocks) {
|
||||
for (auto it = block->begin(); it != block->end(); ++it) {
|
||||
|
||||
@@ -13,7 +13,6 @@ struct ResourceSpecialization {
|
||||
Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid;
|
||||
uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7);
|
||||
bool zero_stride_oob = false;
|
||||
bool scalar_empty = false;
|
||||
uint32_t indirect_root = BufferResource::NoIndirectBuffer;
|
||||
uint32_t indirect_mapping_offset = 0;
|
||||
uint32_t indirect_search_iterations = 0;
|
||||
|
||||
@@ -3735,8 +3735,6 @@ private:
|
||||
resource.written = resource.written || write;
|
||||
resource.atomic = resource.atomic || atomic;
|
||||
resource.formatted = resource.formatted || memory.formatted;
|
||||
resource.scalar = resource.scalar || op == ValueOpcode::ReadConstBuffer ||
|
||||
memory.kind == ResourceKind::ScalarBuffer;
|
||||
}
|
||||
|
||||
uint32_t AddImage(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
|
||||
@@ -3883,6 +3881,24 @@ private:
|
||||
if (memory.planning_only || IsIndirectPlanningMemory(flags.index)) {
|
||||
return;
|
||||
}
|
||||
if (op == ValueOpcode::ReadConstBuffer) {
|
||||
const auto* handle = inst.Arg(0).Resolve().TryInstruction();
|
||||
if (memory.kind != ResourceKind::ScalarBuffer || handle == nullptr ||
|
||||
handle->GetOpcode() != ValueOpcode::GetBufferResource || handle->NumArgs() != 4u) {
|
||||
Fail(flags.pc, "scalar buffer read requires a four-DWORD buffer descriptor");
|
||||
}
|
||||
// Scalar descriptors can span disjoint guest mappings. Keep their raw GPU operands
|
||||
// separate from handles that vector-resource planning may rewrite.
|
||||
auto* block = inst.Parent();
|
||||
const auto where = std::ranges::find_if(block->Instructions(),
|
||||
[&](const Inst& candidate) { return &candidate == &inst; });
|
||||
auto& scalar = *block->PrependNewInst(where, ValueOpcode::GetBufferResource,
|
||||
{handle->Arg(0), handle->Arg(1), handle->Arg(2), handle->Arg(3)});
|
||||
inst.SetArg(0, Value(&scalar));
|
||||
m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer;
|
||||
m_info.uses_dma = true;
|
||||
return;
|
||||
}
|
||||
Inst* handle = nullptr;
|
||||
uint32_t source = 0;
|
||||
uint32_t resource = 0;
|
||||
@@ -3894,12 +3910,11 @@ private:
|
||||
source = indirect->source;
|
||||
} else if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc,
|
||||
memory.resource * 4u, handle, source)) {
|
||||
if (memory.kind != (op == ValueOpcode::ReadConstBuffer ? ResourceKind::ScalarBuffer
|
||||
: ResourceKind::Buffer) ||
|
||||
if (memory.kind != ResourceKind::Buffer ||
|
||||
!memory.SupportsIndirectBufferLoad(op)) {
|
||||
Fail(flags.pc,
|
||||
"buffer descriptor is not a valid runtime value; GPU-selected access "
|
||||
"requires a scalar, raw 8/16-bit, raw DWORD x1/x2/x3/x4, or formatted X load");
|
||||
"requires a raw 8/16-bit, raw DWORD x1/x2/x3/x4, or formatted X load");
|
||||
}
|
||||
m_program.memory_info[flags.index].kind = ResourceKind::IndirectBuffer;
|
||||
m_info.uses_dma = true;
|
||||
|
||||
@@ -307,7 +307,7 @@ void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs
|
||||
if (shared_access != SharedAccess::None) {
|
||||
info.uses_lds |= kind == ResourceKind::Lds;
|
||||
info.uses_gds |= kind == ResourceKind::Gds;
|
||||
} else if (kind == ResourceKind::Buffer || kind == ResourceKind::ScalarBuffer) {
|
||||
} else if (kind == ResourceKind::Buffer) {
|
||||
if (memory.resource >= info.buffers.size()) {
|
||||
return Fail("buffer operation has invalid resource metadata");
|
||||
}
|
||||
|
||||
@@ -89,7 +89,8 @@ bool IsRawRead(const ResourcePlan& values, const Inst& inst) {
|
||||
const auto& memory = values.memory_info[index];
|
||||
if (op != ValueOpcode::LoadBufferU32) {
|
||||
return (op == ValueOpcode::LoadAddressU32 && memory.kind == ResourceKind::ScalarAddress) ||
|
||||
(op == ValueOpcode::ReadConstBuffer && memory.kind == ResourceKind::ScalarBuffer);
|
||||
(op == ValueOpcode::ReadConstBuffer &&
|
||||
(memory.kind == ResourceKind::ScalarBuffer || memory.kind == ResourceKind::IndirectBuffer));
|
||||
}
|
||||
if (inst.NumArgs() != 5u || memory.kind != ResourceKind::Buffer || memory.typed ||
|
||||
memory.formatted || memory.coherent || memory.data_bits != 32u ||
|
||||
|
||||
@@ -1024,115 +1024,6 @@ void TestCyclicControlPredicatesRemainUnknown() {
|
||||
"cyclic control lost a possible branch during reachability");
|
||||
}
|
||||
|
||||
void TestEmptyScalarBufferSpecialization() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
struct Fixture {
|
||||
Program program;
|
||||
std::vector<std::pair<Inst *, Type>> uses;
|
||||
Inst *nonnull = nullptr;
|
||||
};
|
||||
const auto make_fixture = [] {
|
||||
Fixture fixture;
|
||||
auto &program = fixture.program;
|
||||
program.stage = Libs::Graphics::ShaderType::Compute;
|
||||
program.srt_plan_complete = program.resource_tracking_complete = true;
|
||||
program.user_data_count = 4u;
|
||||
auto &block = AddValueBlock(program);
|
||||
DescriptorSource source;
|
||||
source.dword_count = 4u;
|
||||
for (uint32_t word = 0; word < 4u; ++word)
|
||||
source.dwords[word] = Value(&block.AppendNewInst(ValueOpcode::GetUserData,
|
||||
{Value(static_cast<ScalarReg>(word))}));
|
||||
program.descriptor_sources.push_back(source);
|
||||
program.info.buffers.push_back({.source = 0u, .read = true, .scalar = true});
|
||||
const auto table = Value(&block.AppendNewInst(ValueOpcode::GetBufferResource,
|
||||
{source.dwords[0], source.dwords[1], source.dwords[2], source.dwords[3]}));
|
||||
std::array<Value, 4> words;
|
||||
for (uint32_t word = 0; word < 4u; ++word) {
|
||||
program.memory_info.push_back({.kind = ResourceKind::ScalarBuffer, .resource = 0u,
|
||||
.offset = word * 4u});
|
||||
auto &read = block.AppendNewInst(ValueOpcode::ReadConstBuffer, {table, Value(0u)});
|
||||
read.SetFlags(MemoryFlags{.index = word});
|
||||
words[word] = Value(&read);
|
||||
}
|
||||
const auto handle = Value(&block.AppendNewInst(ValueOpcode::GetBufferResource,
|
||||
{words[0], words[1], words[2], words[3]}));
|
||||
for (const auto opcode : {ValueOpcode::ReadConstBuffer, ValueOpcode::LoadBufferU8,
|
||||
ValueOpcode::LoadBufferU16, ValueOpcode::LoadBufferU32,
|
||||
ValueOpcode::LoadBufferU32x2, ValueOpcode::LoadBufferU32x3,
|
||||
ValueOpcode::LoadBufferU32x4}) {
|
||||
const auto index = static_cast<uint32_t>(program.memory_info.size());
|
||||
program.memory_info.push_back({.kind = ResourceKind::IndirectBuffer, .resource = UINT32_MAX,
|
||||
.data_dwords = BufferComponentCount(opcode),
|
||||
.data_bits = opcode == ValueOpcode::LoadBufferU8 ? 8u : opcode == ValueOpcode::LoadBufferU16 ? 16u : 32u,
|
||||
.formatted = opcode == ValueOpcode::LoadBufferU32});
|
||||
auto &read = opcode == ValueOpcode::ReadConstBuffer
|
||||
? block.AppendNewInst(opcode, {handle, Value(0u)})
|
||||
: block.AppendNewInst(opcode, {handle, Value(0u), Value(0u), Value(0u), Value(true)});
|
||||
read.SetFlags(MemoryFlags{.index = index});
|
||||
fixture.uses.emplace_back(&block.AppendNewInst(ValueOpcode::Identity, {Value(&read)}), read.GetType());
|
||||
}
|
||||
const auto nonnull = Value(&block.AppendNewInst(ValueOpcode::GetBufferResource,
|
||||
{words[0], words[1], words[2], Value(1u)}));
|
||||
auto &read = block.AppendNewInst(ValueOpcode::ReadConstBuffer, {nonnull, Value(0u)});
|
||||
read.SetFlags(MemoryFlags{.index = 4u});
|
||||
fixture.nonnull = &block.AppendNewInst(ValueOpcode::Identity, {Value(&read)});
|
||||
return fixture;
|
||||
};
|
||||
auto fixture = make_fixture();
|
||||
auto plan = ExtractResourcePlan(fixture.program);
|
||||
std::array<uint32_t, 4> descriptor{0x1000u, 8u << 16u, 0u, 3u << 28u};
|
||||
uint32_t reads = 0u;
|
||||
const SrtRuntime runtime{.user_data = descriptor, .read_memory = RejectSpecializationRead,
|
||||
.userdata = &reads, .read_specialization_memory = RejectSpecializationRead};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization, empty, populated;
|
||||
for (const uint32_t records : {0u, 1u, 0x20000000u, 0u}) {
|
||||
descriptor[2] = records;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) && reads == 0u &&
|
||||
specialization.buffers[0].scalar_empty == (records == 0u),
|
||||
"scalar emptiness read table contents or truncated its 64-bit byte size");
|
||||
if (records == 0u) {
|
||||
if (empty.buffers.empty()) empty = specialization;
|
||||
else Check(specialization == empty, "returning to an empty scalar table changed its specialization");
|
||||
}
|
||||
else if (records == 1u) populated = specialization;
|
||||
else Check(specialization == populated, "nonempty scalar size created a shader permutation");
|
||||
auto current = make_fixture();
|
||||
ApplyResourceSpecialization(current.program, specialization);
|
||||
for (const auto &[use, type] : current.uses) {
|
||||
const auto value = use->Arg(0).Resolve();
|
||||
Check(value.GetType() == type, "empty scalar-table folding changed the load result type");
|
||||
if (records != 0u) {
|
||||
Check(value.TryInstruction() != nullptr && BufferAccessOf(value.Instruction()->GetOpcode()) == BufferAccess::Read,
|
||||
"nonempty scalar table lost a runtime buffer load");
|
||||
} else if (value.IsImmediate()) {
|
||||
Check(value == Value(uint8_t{0}) || value == Value(uint16_t{0}) || value == Value(0u),
|
||||
"empty scalar table did not produce a typed zero");
|
||||
} else {
|
||||
for (size_t component = 0; component < value.Instruction()->NumArgs(); ++component)
|
||||
Check(value.Instruction()->Arg(component) == Value(0u),
|
||||
"empty scalar table retained a nonzero vector component");
|
||||
}
|
||||
}
|
||||
Check(current.nonnull->Arg(0).Resolve().TryInstruction() != nullptr,
|
||||
"a partly nonzero indirect descriptor was treated as unbound");
|
||||
}
|
||||
Check(empty != populated, "empty-to-populated scalar transition reused the folded specialization");
|
||||
plan.info.buffers[0].scalar = false;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
!specialization.buffers[0].scalar_empty,
|
||||
"vector-only resource acquired scalar-empty specialization");
|
||||
empty.buffers[0].indirect_root = 0u;
|
||||
empty.buffers.push_back(empty.buffers[0]);
|
||||
empty.buffers[1].scalar_empty = false;
|
||||
ApplyResourceSpecialization(fixture.program, empty);
|
||||
for (const auto &[use, type] : fixture.uses)
|
||||
Check(use->Arg(0).GetType() == type && use->Arg(0).Resolve().TryInstruction() != nullptr &&
|
||||
BufferAccessOf(use->Arg(0).ResolveInstruction()->GetOpcode()) == BufferAccess::Read,
|
||||
"empty root folded a scalar read with nonempty candidates");
|
||||
}
|
||||
|
||||
void TestMixedSamplerVariantsShareRuntimeDescriptor() {
|
||||
using namespace Libs::Graphics::ShaderRecompiler::IR;
|
||||
auto program = MixedSamplerProgram();
|
||||
@@ -1332,7 +1223,6 @@ int main() {
|
||||
for (const bool independent_mask : {false, true})
|
||||
TestEmptyTraversalSkipsUnmappedDescriptor(independent_mask);
|
||||
TestCyclicControlPredicatesRemainUnknown();
|
||||
TestEmptyScalarBufferSpecialization();
|
||||
TestMixedSamplerVariantsShareRuntimeDescriptor();
|
||||
TestSignedStorageSpecialization();
|
||||
for (const bool mixed : {false, true}) TestIndirectImageSamplerClasses(mixed);
|
||||
|
||||
@@ -312,7 +312,8 @@ void TestInvariantIndirectImageMaterialization(bool nested) {
|
||||
EliminateDeadCode(fixture->program.blocks);
|
||||
ValidateProgram(fixture->program, true);
|
||||
const auto &source = *plan.descriptor_sources[plan.info.images[0].source].indirect_descriptor;
|
||||
Check(!source.selector && fixture->program.info.buffers.size() == (nested ? 3u : 2u) &&
|
||||
Check(!source.selector && fixture->program.info.buffers.empty() &&
|
||||
fixture->program.info.uses_dma &&
|
||||
std::ranges::any_of(*fixture->block, [](const Inst &inst) {
|
||||
return inst.GetOpcode() == ValueOpcode::ReadConstBuffer;
|
||||
}),
|
||||
@@ -3313,9 +3314,12 @@ void TestUniformScalarBufferImage() {
|
||||
fixture.PlanAndTrack();
|
||||
const auto plan = ExtractResourcePlan(fixture.program);
|
||||
|
||||
Check(std::ranges::none_of(plan.descriptor_sources, [](const DescriptorSource& source) {
|
||||
return source.indirect_descriptor.has_value();
|
||||
}), "draw-uniform scalar image was expanded to GPU-selected candidates");
|
||||
Check(plan.info.uses_dma && plan.info.buffers.empty() &&
|
||||
fixture.program.memory_info[key.Instruction()->Flags<MemoryFlags>().index].kind ==
|
||||
ResourceKind::IndirectBuffer &&
|
||||
std::ranges::none_of(plan.descriptor_sources, [](const DescriptorSource& source) {
|
||||
return source.indirect_descriptor.has_value();
|
||||
}), "draw-uniform scalar image lost its BDA read or expanded GPU-selected candidates");
|
||||
std::array<uint32_t, 9> user_data{0x1000u, 0u, 0x100u, 0u,
|
||||
0x2000u, 0u, 0x100u, 0u, 0u};
|
||||
LinearTestMemory memory;
|
||||
@@ -4642,7 +4646,6 @@ void TestComputeBufferFill() {
|
||||
bool conditional = false;
|
||||
bool shifted = false;
|
||||
bool extra_store = false;
|
||||
bool clean = false;
|
||||
bool branch = false;
|
||||
};
|
||||
const auto Run = [](Options options) {
|
||||
@@ -4670,12 +4673,9 @@ void TestComputeBufferFill() {
|
||||
if (options.shifted)
|
||||
index = fixture.Emit(ValueOpcode::IAdd32, {index, Value(1u)});
|
||||
Value value(0u);
|
||||
TestMemory memory;
|
||||
memory.words[0] = 0x40404040u;
|
||||
if (options.scalar) {
|
||||
const auto input =
|
||||
fixture.Buffer({Value(static_cast<uint32_t>(memory.base)),
|
||||
Value(4u << 16), Value(1u), Value(0x14204u)});
|
||||
fixture.Buffer({Value(0x1000u), Value(4u << 16), Value(1u), Value(0x14204u)});
|
||||
MemoryInfo load;
|
||||
load.kind = ResourceKind::ScalarBuffer;
|
||||
value = fixture.Emit(ValueOpcode::ReadConstBuffer, {input, Value(0u)},
|
||||
@@ -4703,48 +4703,26 @@ void TestComputeBufferFill() {
|
||||
std::array<uint32_t, 4> userdata{0x200000u, 4u << 16, 0x4000u, 0x14204u};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
const auto Read = +[](void *data, uint64_t address, std::span<uint32_t> words) {
|
||||
auto &memory = *static_cast<TestMemory *>(data);
|
||||
if (address != memory.base || words.size() != 1u)
|
||||
return false;
|
||||
++memory.reads;
|
||||
words[0] = memory.words[0];
|
||||
return true;
|
||||
};
|
||||
Check(MaterializeResources(
|
||||
plan,
|
||||
{.user_data = userdata,
|
||||
.read_memory = Read,
|
||||
.userdata = &memory,
|
||||
.read_specialization_memory = options.clean ? Read : nullptr},
|
||||
snapshot, specialization),
|
||||
Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization),
|
||||
"fill fixture did not materialize");
|
||||
const bool expected = !options.conditional && !options.shifted &&
|
||||
!options.extra_store && !options.branch &&
|
||||
(!options.scalar || options.clean);
|
||||
!options.scalar;
|
||||
Check((snapshot.uniform_fill.words != 0) == expected,
|
||||
"fill proof accepted an unsafe store or missed the real GTA3 clear");
|
||||
if (expected) {
|
||||
Check(snapshot.uniform_fill.words == 1 &&
|
||||
snapshot.uniform_fill.group_stride[0] == 64 &&
|
||||
snapshot.uniform_fill.value ==
|
||||
(options.scalar ? 0x40404040u : 0u),
|
||||
snapshot.uniform_fill.value == 0u,
|
||||
"fill proof lost address coverage or the actual stored scalar");
|
||||
if (options.scalar && options.clean) {
|
||||
Check(MaterializeResources(plan,
|
||||
{.user_data = userdata, .read_memory = Read, .userdata = &memory},
|
||||
snapshot, specialization) && snapshot.uniform_fill.words == 0,
|
||||
"an unavailable clean value retained a previous uniform fill");
|
||||
}
|
||||
}
|
||||
};
|
||||
Run({});
|
||||
Run({.scalar = true, .clean = true});
|
||||
Run({.scalar = true});
|
||||
Run({.conditional = true, .clean = true});
|
||||
Run({.shifted = true, .clean = true});
|
||||
Run({.extra_store = true, .clean = true});
|
||||
Run({.clean = true, .branch = true});
|
||||
Run({.conditional = true});
|
||||
Run({.shifted = true});
|
||||
Run({.extra_store = true});
|
||||
Run({.branch = true});
|
||||
}
|
||||
|
||||
void TestDenseBufferTracking() {
|
||||
@@ -4819,8 +4797,8 @@ void TestScalarAndVectorBufferAlias() {
|
||||
MemoryInfo scalar;
|
||||
scalar.kind = ResourceKind::ScalarBuffer;
|
||||
const auto scalar_flags = fixture.AddMemory(scalar, 4);
|
||||
fixture.Emit(ValueOpcode::ReadConstBuffer, {descriptor, fixture.UserData(4)},
|
||||
scalar_flags);
|
||||
const auto scalar_read = fixture.Emit(ValueOpcode::ReadConstBuffer,
|
||||
{descriptor, fixture.UserData(4)}, scalar_flags);
|
||||
MemoryInfo vector;
|
||||
vector.kind = ResourceKind::Buffer;
|
||||
const auto vector_flags = fixture.AddMemory(vector, 8);
|
||||
@@ -4829,12 +4807,18 @@ void TestScalarAndVectorBufferAlias() {
|
||||
vector_flags);
|
||||
fixture.PlanAndTrack();
|
||||
|
||||
Check(fixture.program.info.buffers.size() == 1 &&
|
||||
fixture.program.info.buffers[0].scalar,
|
||||
"typed scalar and vector uses of one descriptor were split");
|
||||
Check(fixture.program.memory_info[scalar_flags.index].resource == 0 &&
|
||||
Check(fixture.program.info.buffers.size() == 1 && fixture.program.info.uses_dma &&
|
||||
fixture.program.memory_info[scalar_flags.index].kind == ResourceKind::IndirectBuffer &&
|
||||
fixture.program.memory_info[vector_flags.index].resource == 0,
|
||||
"scalar/vector alias did not share a dense index");
|
||||
"scalar BDA access changed the shared vector descriptor binding");
|
||||
TestMemory memory;
|
||||
memory.words[1] = 0x12345678u;
|
||||
const std::array<uint32_t, 5> user_data{0x1000u, 0u, 32u, 0u, 4u};
|
||||
SrtWalker walker(fixture.program, {.user_data = user_data, .read_memory = ReadTestMemory,
|
||||
.userdata = &memory});
|
||||
uint32_t loaded = 0;
|
||||
Check(walker.Evaluate(scalar_read, loaded) && loaded == memory.words[1] && memory.reads == 1u,
|
||||
"scalar BDA read lost its raw descriptor or dynamic uniform offset");
|
||||
}
|
||||
|
||||
void TestRuntimeUnsignedMinDescriptor() {
|
||||
@@ -6333,7 +6317,7 @@ void TestGpuSelectedFormattedRecordStore() {
|
||||
}
|
||||
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
|
||||
const auto& memory = fixture->program.memory_info[inst.Flags<MemoryFlags>().index];
|
||||
Check(memory.kind == ResourceKind::ScalarBuffer && !memory.planning_only,
|
||||
Check(memory.kind == ResourceKind::IndirectBuffer && !memory.planning_only,
|
||||
"store planning consumed descriptor words still used by the BDA load");
|
||||
++scalar_reads;
|
||||
}
|
||||
@@ -7765,9 +7749,9 @@ void TestConditionalBufferMaterialization() {
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.buffers.size() == 2 &&
|
||||
snapshot.buffers[1].dword_count == 4 &&
|
||||
snapshot.buffers[1].dwords == std::array<uint32_t, 8>{},
|
||||
snapshot.buffers.size() == 1 &&
|
||||
snapshot.buffers[0].dword_count == 4 &&
|
||||
snapshot.buffers[0].dwords == std::array<uint32_t, 8>{},
|
||||
"untaken scalar branch materialized stale buffer words");
|
||||
Check(snapshot.user_data == std::vector<uint32_t>(user_data.begin(), user_data.end()),
|
||||
"resource reachability changed native shader user data");
|
||||
@@ -7782,9 +7766,9 @@ void TestConditionalBufferMaterialization() {
|
||||
runtime.user_data = user_data;
|
||||
const auto CheckActive = [&] {
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.buffers.size() == 2 &&
|
||||
snapshot.buffers.size() == 1 &&
|
||||
std::equal(user_data.begin() + 4, user_data.end(),
|
||||
snapshot.buffers[1].dwords.begin()),
|
||||
snapshot.buffers[0].dwords.begin()),
|
||||
"potentially executed buffer descriptor was discarded");
|
||||
};
|
||||
CheckActive();
|
||||
@@ -7871,6 +7855,15 @@ void TestGuardedScalarDescriptorReads() {
|
||||
memory.data.words[0xb0 / 4] = 4u;
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
if (!exec) {
|
||||
for (const uint32_t enabled : {0u, 1u, 0u}) {
|
||||
memory.data.words[7] = enabled;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
memory.null_reads == 0 && snapshot.flattened_srt.empty(),
|
||||
"disabled GPU address loads introduced CPU descriptor reads");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!shared) {
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) && memory.null_reads == 0 &&
|
||||
std::ranges::all_of(snapshot.flattened_srt, [](auto word) { return word == 0; }),
|
||||
@@ -7883,11 +7876,8 @@ void TestGuardedScalarDescriptorReads() {
|
||||
memory.data.watched_address = 0x10a8u;
|
||||
memory.data.watched_reads = 0;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
std::ranges::any_of(snapshot.buffers, [](const auto& descriptor) {
|
||||
return descriptor.dwords[0] == 0x2000u;
|
||||
}) &&
|
||||
(!exec || (memory.data.watched_reads == 1 &&
|
||||
std::ranges::find(snapshot.flattened_srt, 0x2000u) != snapshot.flattened_srt.end())),
|
||||
memory.data.watched_reads == 1 &&
|
||||
std::ranges::find(snapshot.flattened_srt, 0x2000u) != snapshot.flattened_srt.end(),
|
||||
"active or shared descriptor read was omitted after refreshing the cached plan");
|
||||
if (!shared) {
|
||||
memory.data.words[7] = 0;
|
||||
@@ -7916,9 +7906,9 @@ void TestConservativeBufferReachability() {
|
||||
if (use != ConditionalBufferUse::Writable)
|
||||
std::copy_n(user_data.begin() + 4, expected.size(), expected.begin());
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.buffers.size() == 2 &&
|
||||
snapshot.buffers.size() == (use == ConditionalBufferUse::Writable ? 2u : 1u) &&
|
||||
std::equal(expected.begin(), expected.end(),
|
||||
snapshot.buffers[1].dwords.begin()),
|
||||
snapshot.buffers.back().dwords.begin()),
|
||||
"shared or loop-dependent resource was pruned, or inactive scalar branch retained");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30517,7 +30517,7 @@ TestCase ScalarMemoryLoadVariants() {
|
||||
std::vector<u32> expected = initial;
|
||||
expected.insert(expected.end(), initial.begin(), initial.end());
|
||||
|
||||
return {"ScalarMemoryLoadVariants",
|
||||
TestCase test{"ScalarMemoryLoadVariants",
|
||||
code,
|
||||
initial,
|
||||
expected,
|
||||
@@ -30526,6 +30526,8 @@ TestCase ScalarMemoryLoadVariants() {
|
||||
O::S_BUFFER_LOAD_DWORDX2, O::S_BUFFER_LOAD_DWORDX4,
|
||||
O::S_BUFFER_LOAD_DWORDX8, O::S_BUFFER_LOAD_DWORDX16, O::V_MOV_B32,
|
||||
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
|
||||
test.bda_mappings = {{0u, 0u}};
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase ScalarBufferOffsetAlignmentAndCarry() {
|
||||
@@ -30548,6 +30550,7 @@ TestCase ScalarBufferOffsetAlignmentAndCarry() {
|
||||
{O::S_MOV_B32, O::S_ADD_U32, O::S_BUFFER_LOAD_DWORD,
|
||||
O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
|
||||
test.forbidden_spirv = {"flattened_srt"};
|
||||
test.bda_mappings = {{0u, 0u}};
|
||||
test.compute_info.threads_num[0] = 1;
|
||||
test.compute_info.threads_num[1] = 1;
|
||||
test.compute_info.threads_num[2] = 1;
|
||||
@@ -31181,7 +31184,7 @@ TestCase BufferLoadsGpuSelectedDescriptors(u32 width, u32 wave = 32u) {
|
||||
}
|
||||
}
|
||||
AppendEnd(&test.code);
|
||||
test.bda_mappings = {{GuestBase, 0}};
|
||||
test.bda_mappings = {{0u, 0u}, {GuestBase, 0}};
|
||||
test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
|
||||
O::V_READFIRSTLANE_B32, O::S_MUL_I32, O::S_BUFFER_LOAD_DWORDX4,
|
||||
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
|
||||
@@ -31365,7 +31368,7 @@ TestCase BufferLoadGpuSelectedDescriptors(u32 wave, ShaderOpcode load) {
|
||||
code.push_back(EncodeSopc(0x0a, 28, 29));
|
||||
code.push_back(EncodeSopp(0x05, loop - static_cast<u32>(code.size()) - 1u));
|
||||
AppendEnd(&code);
|
||||
test.bda_mappings = {{GuestBase, 0}};
|
||||
test.bda_mappings = {{0u, 0u}, {GuestBase, 0}};
|
||||
test.fault_free_address = GuestBase + 0x100000u;
|
||||
test.compute_info.threads_num[0] = 1;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
@@ -31417,11 +31420,17 @@ void CheckSharedIndirectBufferLoads(VulkanHarness &vulkan) {
|
||||
auto compiled = CompileCase(test, vulkan.SubgroupSize());
|
||||
struct Profile { u32 components = 0, uses = 0, definitions = 0, calls = 0, reads = 0; };
|
||||
std::map<std::string, Profile> profiles;
|
||||
const u32 halves = wave == 64u && vulkan.SubgroupSize() == 32u ? 2u : 1u;
|
||||
u32 scalar_reads = 0;
|
||||
for (const auto *block : compiled.program.blocks)
|
||||
for (const auto &inst : *block) {
|
||||
const auto components = BufferComponentCount(inst.GetOpcode());
|
||||
if (components == 0u || inst.GetOpcode() == ValueOpcode::ReadConstBuffer) continue;
|
||||
if (components == 0u) continue;
|
||||
const auto &memory = compiled.program.memory_info[inst.Flags<MemoryFlags>().index];
|
||||
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
|
||||
scalar_reads += !memory.planning_only;
|
||||
continue;
|
||||
}
|
||||
if (memory.kind != ResourceKind::IndirectBuffer || memory.formatted) continue;
|
||||
const auto name = "load_indirect_buffer_" + std::to_string(memory.data_bits) + "x" +
|
||||
std::to_string(components) + (memory.coherent ? "_coherent" : "");
|
||||
@@ -31452,11 +31461,11 @@ void CheckSharedIndirectBufferLoads(VulkanHarness &vulkan) {
|
||||
}
|
||||
offset += words.size();
|
||||
}
|
||||
Require(test.name, "raw load function scope", functions.size() == profiles.size() && inline_reads == 0u,
|
||||
Require(test.name, "raw load function scope", functions.size() == profiles.size() && inline_reads == scalar_reads * halves,
|
||||
"a raw load profile was unused, missing or still emitted inline");
|
||||
for (const auto &[name, profile] : profiles)
|
||||
Require(test.name, name.c_str(), profile.uses != 0u && profile.definitions == 1u &&
|
||||
profile.calls == profile.uses * (wave == 64u && vulkan.SubgroupSize() == 32u ? 2u : 1u) &&
|
||||
profile.calls == profile.uses * halves &&
|
||||
profile.reads == profile.components,
|
||||
"raw load helpers duplicated definitions, calls or component reads");
|
||||
auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true);
|
||||
@@ -33760,6 +33769,48 @@ TestCase ScalarBufferFromLoopReadlane(u32 wave_size) {
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase ScalarBufferLoadsDisjointPages() {
|
||||
using O = ShaderOpcode;
|
||||
constexpr u32 GuestBase = 0x10000, PageSize = 0x10000, PageStride = 0x400000;
|
||||
TestCase test;
|
||||
test.name = "ScalarBufferLoadsDisjointPages";
|
||||
auto &code = test.code;
|
||||
code.push_back(EncodeSop2(0x1e, 12, 4, InlineU32(22)));
|
||||
code.push_back(EncodeSmem0(0x0a, 20, 4));
|
||||
code.push_back(EncodeSmem1(256, 12));
|
||||
code.push_back(EncodeVop1(0x01, 0, 4));
|
||||
code.push_back(EncodeVop2(0x1a, 0, InlineU32(2), 0));
|
||||
for (u32 component = 0; component < 4; ++component)
|
||||
AppendStoreSgprAtLaneDwordOffset(&code, 20 + component, 0, component);
|
||||
AppendEnd(&code);
|
||||
|
||||
test.initial.resize(2 * PageSize / sizeof(u32));
|
||||
for (u32 page = 0; page < 2; ++page) {
|
||||
for (u32 component = 0; component < 4; ++component) {
|
||||
const u32 value = 0x12340000u + page * 16 + component;
|
||||
test.initial[(page * PageSize + 256) / sizeof(u32) + component] = value;
|
||||
test.expected.push_back(value);
|
||||
}
|
||||
}
|
||||
test.user_data = MakeNativeUserData(nullptr);
|
||||
test.user_data[8] = GuestBase;
|
||||
test.user_data[10] = 32 * 1024 * 1024;
|
||||
test.has_user_data = true;
|
||||
// A large guest descriptor covers separate allocations, while its SSBO view ends at the first.
|
||||
test.storage_buffer_range_bytes = PageSize;
|
||||
test.bda_mappings = {{GuestBase, 0}, {GuestBase + PageStride, PageSize}};
|
||||
test.opcodes = {O::S_LSHL_B32, O::S_BUFFER_LOAD_DWORDX4, O::V_MOV_B32,
|
||||
O::V_LSHLREV_B32, O::V_ADD_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
|
||||
test.compute_info.wave_size = 32;
|
||||
test.compute_info.threads_num[0] = 1;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
test.compute_info.group_id[0] = true;
|
||||
test.compute_info.workgroup_register = 4;
|
||||
test.has_compute_info = true;
|
||||
test.dispatch_x = 2;
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase FlatVirtualAddressRebasesGuestAllocation() {
|
||||
using O = ShaderOpcode;
|
||||
|
||||
@@ -38031,7 +38082,7 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) {
|
||||
}
|
||||
// Compile against poison selectors. A separate GPU dispatch writes the actual keys.
|
||||
auto compiled = CompileCase(test, vulkan.SubgroupSize());
|
||||
if (table_bytes == 0u || components != 0u)
|
||||
if (components != 0u)
|
||||
test.forbidden_spirv = {"convert_indirect_buffer_format", "load_indirect_buffer_",
|
||||
"indirect_buffer_format_geometry"};
|
||||
else
|
||||
@@ -38047,8 +38098,8 @@ void CheckGpuSelectedRecordBufferAccess(VulkanHarness &vulkan) {
|
||||
if (opcode == spv::OpFunctionCall && words[3] == bda_function) ++bda_calls;
|
||||
offset += words.size();
|
||||
}
|
||||
Require(test.name, "empty descriptor table specialization", (bda_calls == 0u) == (table_bytes == 0u || components != 0u),
|
||||
"empty table retained BDA reads or nonempty table lost GPU descriptor loads");
|
||||
Require(test.name, "GPU descriptor reads", (bda_calls == 0u) == (components != 0u),
|
||||
"CPU-planned descriptors retained BDA reads or runtime descriptors lost them");
|
||||
for (const auto &descriptor : compiled.resources.buffers) {
|
||||
ShaderBufferResource resource{};
|
||||
std::copy_n(descriptor.dwords.begin(), 4u, resource.fields);
|
||||
@@ -42023,6 +42074,7 @@ std::vector<TestCase> MakeCases() {
|
||||
AddCase(ScalarLoadAlignsDynamicBase);
|
||||
cases.push_back(ScalarBufferFromLoopReadlane(32));
|
||||
cases.push_back(ScalarBufferFromLoopReadlane(64));
|
||||
AddCase(ScalarBufferLoadsDisjointPages);
|
||||
AddCase(BufferLoadStore);
|
||||
AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset);
|
||||
AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset);
|
||||
@@ -48020,6 +48072,9 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--indirect-buffer-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
RunCase(&vulkan, ScalarBufferLoadsDisjointPages());
|
||||
RunCase(&vulkan, ScalarMemoryLoadVariants());
|
||||
RunCase(&vulkan, ScalarBufferOffsetAlignmentAndCarry());
|
||||
RunCase(&vulkan, ScalarBufferFromLoopReadlane(32));
|
||||
RunCase(&vulkan, ScalarBufferFromLoopReadlane(64));
|
||||
CheckBoundedFormattedBufferCopy(vulkan);
|
||||
|
||||
+109
-62
@@ -5417,8 +5417,7 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() {
|
||||
Check(index < program.memory_info.size(),
|
||||
"typed scalar memory operation has invalid metadata");
|
||||
const auto &memory = program.memory_info[index];
|
||||
Check(memory.kind == kind && memory.resource == 0u &&
|
||||
!memory.planning_only,
|
||||
Check(memory.kind == kind && !memory.planning_only,
|
||||
"typed scalar memory operation has the wrong live domain");
|
||||
count++;
|
||||
}
|
||||
@@ -5478,17 +5477,15 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() {
|
||||
|
||||
auto buffer = RecompileForTest(buffer_shader, buffer_options);
|
||||
const auto& buffer_counts = buffer.program.bindings.descriptor_counts;
|
||||
Check(buffer.program.info.buffers.size() == 1u &&
|
||||
buffer.program.info.buffers[0].scalar &&
|
||||
buffer.program.info.buffers[0].descriptor_index == 0 &&
|
||||
!buffer.program.info.uses_dma &&
|
||||
buffer_counts[static_cast<size_t>(BindingKind::Buffers)] == 1 &&
|
||||
buffer_counts[static_cast<size_t>(BindingKind::BdaPagetable)] == 0 &&
|
||||
Check(buffer.program.info.buffers.empty() && buffer.program.info.uses_dma &&
|
||||
buffer_counts[static_cast<size_t>(BindingKind::Buffers)] == 0 &&
|
||||
buffer_counts[static_cast<size_t>(BindingKind::BdaPagetable)] == 1 &&
|
||||
buffer_counts[static_cast<size_t>(BindingKind::FaultBuffer)] == 1 &&
|
||||
buffer_counts[static_cast<size_t>(BindingKind::FlattenedSrt)] == 0,
|
||||
"descriptor scalar load did not use only the buffer domain");
|
||||
"descriptor scalar load did not use only the DMA domain");
|
||||
Check(count_live_memory_ops(
|
||||
buffer.program, ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer,
|
||||
ShaderRecompiler::IR::ResourceKind::ScalarBuffer) == 1u,
|
||||
ShaderRecompiler::IR::ResourceKind::IndirectBuffer) == 1u,
|
||||
"descriptor scalar load did not remain a live typed buffer operation");
|
||||
CheckSpirvBinaryValidates(buffer.spirv);
|
||||
}
|
||||
@@ -9027,6 +9024,55 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() {
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
}
|
||||
|
||||
std::vector<std::pair<uint32_t, std::array<uint32_t, 4>>> ScalarDescriptorsOnUniformPath(
|
||||
TestCompileResult& result, std::span<const uint32_t> user_data) {
|
||||
using namespace ShaderRecompiler::IR;
|
||||
using ShaderRecompiler::CFG::TerminatorKind;
|
||||
// These acyclic fixtures have uniform branches and descriptor loads already flattened.
|
||||
for (auto* block : result.program.blocks) {
|
||||
for (auto& inst : *block) {
|
||||
if (inst.GetOpcode() != ValueOpcode::ReadConst) continue;
|
||||
const auto slot = inst.Arg(1).Resolve().U32();
|
||||
Check(slot < result.resources.flattened_srt.size(), "scalar descriptor has an invalid SRT slot");
|
||||
inst.ReplaceUsesWith(Value(result.resources.flattened_srt[slot]));
|
||||
}
|
||||
}
|
||||
SrtWalker walker(result.program, {.user_data = user_data});
|
||||
std::vector<std::pair<uint32_t, std::array<uint32_t, 4>>> descriptors;
|
||||
Block* previous = nullptr;
|
||||
auto* block = result.program.blocks.front();
|
||||
for (size_t step = 0; step < result.program.blocks.size(); ++step) {
|
||||
for (auto& inst : *block) {
|
||||
if (inst.GetOpcode() == ValueOpcode::Phi) {
|
||||
size_t incoming = 0;
|
||||
while (incoming < inst.NumArgs() && inst.PhiBlock(incoming) != previous) ++incoming;
|
||||
Check(incoming < inst.NumArgs(), "descriptor Phi lost the executed predecessor");
|
||||
inst.ReplaceUsesWith(inst.Arg(incoming));
|
||||
} else if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
|
||||
const auto* handle = inst.Arg(0).ResolveInstruction();
|
||||
Check(handle && handle->GetOpcode() == ValueOpcode::GetBufferResource && handle->NumArgs() == 4u,
|
||||
"scalar read lost its raw buffer descriptor");
|
||||
auto& [pc, words] = descriptors.emplace_back();
|
||||
pc = inst.Flags<MemoryFlags>().pc;
|
||||
for (uint32_t word = 0; word < words.size(); ++word)
|
||||
Check(walker.Evaluate(handle->Arg(word), words[word]), "scalar descriptor word is not evaluable");
|
||||
}
|
||||
}
|
||||
const auto& term = block->terminator;
|
||||
if (term.kind == TerminatorKind::Return) return descriptors;
|
||||
Check(term.kind == TerminatorKind::Branch || term.kind == TerminatorKind::ConditionalBranch,
|
||||
"descriptor fixture has unsupported control flow");
|
||||
uint32_t condition = 1u;
|
||||
if (term.kind == TerminatorKind::ConditionalBranch)
|
||||
Check(walker.Evaluate(block->condition, condition), "descriptor branch is not uniform");
|
||||
previous = block;
|
||||
block = condition != 0u ? term.true_block : term.false_block;
|
||||
Check(block != nullptr, "descriptor fixture lost its branch target");
|
||||
}
|
||||
Check(false, "descriptor fixture unexpectedly contains a loop");
|
||||
return descriptors;
|
||||
}
|
||||
|
||||
void TestSharedReturnPreservesDescriptorDominance() {
|
||||
const uint32_t shader[] = {
|
||||
EncodeSmem0(0x02, 32, 14), (125u << 25u) | 0x60u,
|
||||
@@ -9072,9 +9118,12 @@ void TestSharedReturnPreservesDescriptorDominance() {
|
||||
auto options = MakeCompileOptions(ShaderType::Pixel);
|
||||
options.user_data = user_data;
|
||||
auto result = RecompileForTest(shader, options, ReadHostTestMemory);
|
||||
Check(!result.program.dispatcher_fallback &&
|
||||
result.resources.buffers.size() == 1u &&
|
||||
result.resources.buffers[0].dwords[0] == 0x2000u,
|
||||
Check(!result.program.dispatcher_fallback && result.resources.buffers.empty() &&
|
||||
result.program.info.uses_dma,
|
||||
"surviving scalar operation did not retain structured BDA access");
|
||||
const auto descriptors = ScalarDescriptorsOnUniformPath(result, user_data);
|
||||
Check(descriptors.size() == 1u && descriptors[0].first == 0x20u &&
|
||||
std::equal(descriptors[0].second.begin(), descriptors[0].second.end(), table.begin() + 0x50u / 4u),
|
||||
"return-only descriptor reached the surviving buffer operation");
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
}
|
||||
@@ -9131,24 +9180,18 @@ void TestNestedSelectionPreservesDescriptorSources() {
|
||||
auto options = MakeCompileOptions(ShaderType::Compute);
|
||||
options.user_data_base = 8;
|
||||
options.user_data = user_data;
|
||||
const auto result = RecompileForTest(shader, options, ReadHostTestMemory);
|
||||
Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 4u,
|
||||
"nested descriptor selection lost its structured resource bindings");
|
||||
for (size_t i = 0; i < result.program.info.buffers.size(); ++i) {
|
||||
const auto pc = result.program.info.buffers[i].first_use_pc;
|
||||
Check(pc == 8u * 4u || pc == 14u * 4u || pc == direct_arm * 4u || pc == join * 4u,
|
||||
"nested descriptor selection lost a native source location");
|
||||
const auto selected_pc = (mode == 0u ? 8u : mode == 1u ? 14u : direct_arm) * 4u;
|
||||
const auto *expected = pc == 8u * 4u ? table.data()
|
||||
: pc == 14u * 4u ? table.data() + 4
|
||||
: pc == direct_arm * 4u ? user_data.data() + 8 : user_data.data() + 4;
|
||||
const auto &descriptor = result.resources.buffers[i];
|
||||
const bool unused = pc != selected_pc && pc != join * 4u;
|
||||
Check(descriptor.dword_count == 4 &&
|
||||
(std::equal(expected, expected + 4, descriptor.dwords.begin()) ||
|
||||
(unused && std::ranges::all_of(descriptor.dwords, [](auto word) { return word == 0; }))),
|
||||
"nested descriptor selection chose another arm's descriptor");
|
||||
}
|
||||
auto result = RecompileForTest(shader, options, ReadHostTestMemory);
|
||||
Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 1u &&
|
||||
result.program.info.buffers[0].first_use_pc == join * 4u &&
|
||||
std::equal(user_data.begin() + 4, user_data.begin() + 8,
|
||||
result.resources.buffers[0].dwords.begin()),
|
||||
"nested descriptor selection lost its structured output binding");
|
||||
const auto descriptors = ScalarDescriptorsOnUniformPath(result, user_data);
|
||||
const auto selected_pc = (mode == 0u ? 8u : mode == 1u ? 14u : direct_arm) * 4u;
|
||||
const auto* expected = mode == 0u ? table.data() : mode == 1u ? table.data() + 4 : user_data.data() + 8;
|
||||
Check(descriptors.size() == 1u && descriptors[0].first == selected_pc &&
|
||||
std::equal(expected, expected + 4, descriptors[0].second.begin()),
|
||||
"nested descriptor selection chose another arm's descriptor");
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
}
|
||||
}
|
||||
@@ -9207,16 +9250,16 @@ void TestSharedExitPreservesNativeDescriptorSources() {
|
||||
user_data[2] = arm;
|
||||
auto options = MakeCompileOptions(ShaderType::Pixel);
|
||||
options.user_data = user_data;
|
||||
const auto result = RecompileForTest(shader, options, ReadHostTestMemory);
|
||||
Check(!result.program.dispatcher_fallback && result.resources.buffers.size() == 2u,
|
||||
"shared-exit descriptor selection lost its structured resource bindings");
|
||||
for (size_t i = 0; i < result.program.info.buffers.size(); ++i) {
|
||||
const auto pc = result.program.info.buffers[i].first_use_pc;
|
||||
const uint32_t expected = pc == 14u * 4u ? table[8]
|
||||
: pc == 22u * 4u ? table[12] : 0u;
|
||||
Check(expected != 0u && result.resources.buffers[i].dwords[0] == expected,
|
||||
"shared-exit descriptor selection retained an overwritten native source");
|
||||
}
|
||||
auto result = RecompileForTest(shader, options, ReadHostTestMemory);
|
||||
Check(!result.program.dispatcher_fallback && result.resources.buffers.empty() &&
|
||||
result.program.info.uses_dma,
|
||||
"shared-exit descriptor selection lost its structured scalar reads");
|
||||
const auto descriptors = ScalarDescriptorsOnUniformPath(result, user_data);
|
||||
const auto selected_pc = (arm == 0u ? 22u : 14u) * 4u;
|
||||
const auto* expected = table.data() + (arm == 0u ? 12u : 8u);
|
||||
Check(descriptors.size() == 1u && descriptors[0].first == selected_pc &&
|
||||
std::equal(expected, expected + 4, descriptors[0].second.begin()),
|
||||
"shared-exit descriptor selection retained an overwritten native source");
|
||||
CheckSpirvBinaryValidates(result.spirv);
|
||||
}
|
||||
}
|
||||
@@ -14121,10 +14164,12 @@ void TestNewShaderRecompilerNativeBindingPlan() {
|
||||
storage = kind;
|
||||
}
|
||||
}
|
||||
Check(counts[static_cast<size_t>(BindingKind::Buffers)] == 2 &&
|
||||
Check(counts[static_cast<size_t>(BindingKind::Buffers)] == 1 &&
|
||||
result.program.info.buffers.size() == 1u &&
|
||||
result.program.info.buffers[0].descriptor_index == 0 &&
|
||||
result.program.info.buffers[1].descriptor_index == 1,
|
||||
"native emission did not allocate scalar/vector buffer slots");
|
||||
counts[static_cast<size_t>(BindingKind::BdaPagetable)] == 1 &&
|
||||
counts[static_cast<size_t>(BindingKind::FaultBuffer)] == 1,
|
||||
"native emission did not allocate the vector buffer and scalar BDA slots");
|
||||
Check(sampled && counts[static_cast<size_t>(*sampled)] == 1,
|
||||
"native emission did not allocate the sampled image");
|
||||
Check(storage && counts[static_cast<size_t>(*storage)] == 1,
|
||||
@@ -14135,7 +14180,8 @@ void TestNewShaderRecompilerNativeBindingPlan() {
|
||||
"native emission did not allocate the sampler");
|
||||
Check(SpirvContainsOpcode(result.spirv, 86),
|
||||
"SPIR-V binary does not combine separate image/sampler descriptors");
|
||||
for (const auto kind : {BindingKind::Buffers, *sampled, BindingKind::Samplers}) {
|
||||
for (const auto kind : {BindingKind::Buffers, BindingKind::BdaPagetable,
|
||||
BindingKind::FaultBuffer, *sampled, BindingKind::Samplers}) {
|
||||
Check(SpirvHasDecorationValue(result.spirv, 33u,
|
||||
ShaderRecompiler::IR::NativeBinding(result.program.stage, kind)),
|
||||
"SPIR-V lacks a resource Binding decoration");
|
||||
@@ -14158,10 +14204,10 @@ void TestNewShaderRecompilerNativeBindingPlan() {
|
||||
for (auto *block : result.program.blocks) {
|
||||
const auto found = std::ranges::find_if(*block, [](const auto &inst) {
|
||||
return inst.GetOpcode() ==
|
||||
ShaderRecompiler::IR::ValueOpcode::GetBufferResource;
|
||||
ShaderRecompiler::IR::ValueOpcode::StoreBufferU32;
|
||||
});
|
||||
if (found != block->end()) {
|
||||
buffer_handle = &*found;
|
||||
buffer_handle = found->Arg(0).ResolveInstruction();
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -14335,13 +14381,11 @@ void TestComputeImageFill() {
|
||||
0xbf8cc07fu, 0x7e06026au, 0xd7460001u, 0x0405060du,
|
||||
0x7e04020eu, 0xf0200128u, 0x00000300u, 0xbf810000u,
|
||||
};
|
||||
uint32_t value = 0x7bu;
|
||||
const auto address = reinterpret_cast<uint64_t>(&value);
|
||||
constexpr uint32_t value = 0x7bu;
|
||||
const std::array<uint32_t, 12> userdata = {
|
||||
0x20021000u, 0xc0500000u, 0x01d8c347u, 0xd1800204u,
|
||||
0u, 0x00700000u, 0u, 0u,
|
||||
static_cast<uint32_t>(address),
|
||||
static_cast<uint32_t>(address >> 32) | (4u << 16), 1u, 0x14204u,
|
||||
0x1000u, 4u << 16u, 1u, 0x14204u,
|
||||
};
|
||||
ShaderComputeInputInfo compute{};
|
||||
compute.threads_num[0] = compute.threads_num[1] = 8;
|
||||
@@ -14352,9 +14396,14 @@ void TestComputeImageFill() {
|
||||
auto options = MakeCompileOptions(ShaderType::Compute);
|
||||
options.input_info.compute = &compute;
|
||||
options.user_data = userdata;
|
||||
enum class Mutation { None, Offset, WrongAxis, VaryingValue, Predicate, ExtraStore, A16 };
|
||||
const auto Run = [&](Mutation mutation, bool clean = true) {
|
||||
auto translated = ShaderRecompiler::TranslateProgram(shader, options);
|
||||
enum class Mutation { None, RuntimeScalar, Offset, WrongAxis, VaryingValue, Predicate, ExtraStore, A16 };
|
||||
const auto Run = [&](Mutation mutation) {
|
||||
auto code = std::to_array(shader);
|
||||
if (mutation != Mutation::RuntimeScalar) {
|
||||
code[2] = EncodeSMovB32(106, 255);
|
||||
code[3] = value;
|
||||
}
|
||||
auto translated = ShaderRecompiler::TranslateProgram(code, options);
|
||||
auto &program = translated.program;
|
||||
for (auto *block : program.blocks) {
|
||||
const auto found = std::ranges::find(*block, ValueOpcode::ImageWrite, &Inst::GetOpcode);
|
||||
@@ -14365,7 +14414,8 @@ void TestComputeImageFill() {
|
||||
Check(coordinates && values, "stencil fill fixture lost its image operands");
|
||||
const auto x = coordinates->Arg(0);
|
||||
switch (mutation) {
|
||||
case Mutation::None: break;
|
||||
case Mutation::None:
|
||||
case Mutation::RuntimeScalar: break;
|
||||
case Mutation::Offset:
|
||||
coordinates->SetArg(0, Value(&*block->PrependNewInst(found, ValueOpcode::IAdd32,
|
||||
{x, Value(1u)})));
|
||||
@@ -14390,15 +14440,12 @@ void TestComputeImageFill() {
|
||||
auto plan = ExtractResourcePlan(program);
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
Check(MaterializeResources(plan,
|
||||
{.user_data = userdata, .read_memory = ReadHostTestMemory,
|
||||
.read_specialization_memory = clean ? ReadHostTestMemory : nullptr},
|
||||
snapshot, specialization),
|
||||
Check(MaterializeResources(plan, {.user_data = userdata}, snapshot, specialization),
|
||||
"captured stencil clear did not materialize");
|
||||
const auto &fill = snapshot.uniform_fill;
|
||||
const bool expected = mutation == Mutation::None && clean;
|
||||
const bool expected = mutation == Mutation::None;
|
||||
Check((fill.kind == UniformFillKind::Image) == expected,
|
||||
"image fill proof accepted an unsafe store or missed the captured clear");
|
||||
"image fill proof accepted a runtime read or unsafe store, or missed a constant clear");
|
||||
if (expected) {
|
||||
Check(fill.resource == 0 && fill.words == 1 && fill.value == value &&
|
||||
fill.group_stride == std::array<uint32_t, 3>{8, 8, 1},
|
||||
@@ -14406,7 +14453,7 @@ void TestComputeImageFill() {
|
||||
}
|
||||
};
|
||||
Run(Mutation::None);
|
||||
Run(Mutation::None, false);
|
||||
Run(Mutation::RuntimeScalar);
|
||||
for (const auto mutation : {Mutation::Offset, Mutation::WrongAxis, Mutation::VaryingValue,
|
||||
Mutation::Predicate, Mutation::ExtraStore, Mutation::A16})
|
||||
Run(mutation);
|
||||
|
||||
Reference in New Issue
Block a user