diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index ce3b40799..a11fa40fd 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -860,7 +860,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o const auto resource = inst.Flags(); first = resource < ir.info.images.size() && ir.info.images[resource].indirect_root == resource ? 1u : 0u; - } else if (op != IR::ValueOpcode::GetSamplerResource) { + } else if (op == IR::ValueOpcode::GetSamplerResource) { + const auto resource = inst.Flags(); + first = resource < ir.info.samplers.size() && + !ir.info.samplers[resource].indirect_resources.empty() ? 1u : 0u; + } else { continue; } for (size_t index = first; index < inst.NumArgs(); index++) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 776b41498..f77142891 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -137,6 +137,15 @@ uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) { } } +namespace { +void DecorateNonUniformDescriptor(EmitterState& state, uint32_t id) { + state.builder.RequireExtension("SPV_EXT_descriptor_indexing"); + state.builder.RequireCapability(spv::CapabilityShaderNonUniform); + state.builder.RequireCapability(spv::CapabilitySampledImageArrayNonUniformIndexing); + state.builder.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); +} +} // namespace + uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip, uint32_t array_index) { const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index); @@ -145,29 +154,28 @@ uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mi state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type, image, pointer); if (array_index != 0u) { - state.builder.RequireExtension("SPV_EXT_descriptor_indexing"); - state.builder.RequireCapability(spv::CapabilityShaderNonUniform); - state.builder.RequireCapability(spv::CapabilitySampledImageArrayNonUniformIndexing); - state.builder.AddAnnotation(spv::OpDecorate, image, spv::DecorationNonUniform); + DecorateNonUniformDescriptor(state, image); } return image; } -uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) { +uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t array_index) { if (sampler >= state.program.info.samplers.size()) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Samplers, sampler, "sampler resource index is out of range"); } const auto pointer = DescriptorElementPointer( - state, state.sampler_pointer_type, state.sampler_variable, ConstantU32(state, sampler), + state, state.sampler_pointer_type, state.sampler_variable, + array_index == 0u ? ConstantU32(state, sampler) : array_index, IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); const auto sampler_id = state.builder.AllocateId(); state.builder.AddFunction(spv::OpLoad, state.sampler_type, sampler_id, pointer); + if (array_index != 0u) DecorateNonUniformDescriptor(state, sampler_id); return sampler_id; } uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, uint32_t mip, - uint32_t array_index) { + uint32_t array_index, bool sampler_dynamic) { const auto& image_resource = state.program.info.images.at(resource); EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled); const auto image = LoadImageDescriptor(state, resource, mip, array_index); @@ -179,8 +187,8 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type); } state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id); - if (array_index != 0u) { - state.builder.AddAnnotation(spv::OpDecorate, sampled_image, spv::DecorationNonUniform); + if (array_index != 0u || sampler_dynamic) { + DecorateNonUniformDescriptor(state, sampled_image); } return sampled_image; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 278995ef4..ceeb45697 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -898,14 +898,25 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { operands[1] = AddressF32(ctx, mem, *address, layout.bias); operand_count = 2; } - const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler); + const auto& samplers = state.program.info.samplers.at(mem.sampler).indirect_resources; + uint32_t sampler_index = 0; + if (!samplers.empty()) { + const auto* handle = inst.Arg(1).ResolveInstruction(); + if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetSamplerResource) + ctx.Fail(inst, "has invalid finite sampler key provenance"); + sampler_index = EmitIndexSwitch( + state, ctx.Def(handle->Arg(0)), static_cast(samplers.size()), + TypeU32(state), [&](uint32_t ordinal) { return ConstantU32(state, samplers[ordinal]); }); + } + const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler, sampler_index); const auto EmitSample = [&](uint32_t resource, uint32_t array_index) { const auto& candidate = state.program.info.images[resource]; const auto coord = CoordF32(ctx, mem, *address, layout.coord, ImageDimensionInfoFor(candidate.dimension).coordinate_components, candidate.cube); - const auto sampled = MakeSampledImage(state, resource, sampler_id, 0u, array_index); + const auto sampled = MakeSampledImage(state, resource, sampler_id, 0u, array_index, + sampler_index != 0u); const auto sample = state.builder.AllocateId(); state.builder.AddFunction(opcode, result_type, sample, sampled, coord, std::span(&dref_value, dref ? 1u : 0u), diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 411b1ee25..57529c881 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -322,10 +322,11 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip = 0, uint32_t array_index = 0); -uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler); +uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t array_index = 0); uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampler_id, - uint32_t mip = 0, uint32_t array_index = 0); + uint32_t mip = 0, uint32_t array_index = 0, + bool sampler_dynamic = false); void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord, uint32_t texel); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index ea4cf7bf1..82fd77666 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -161,6 +161,7 @@ struct SamplerResource { uint32_t first_use_pc = 0; // Native filtering/border variants share the original sampler's runtime descriptor. uint32_t snapshot_index = 0; + std::vector indirect_resources; bool force_point_filtering = false; bool depth_compare = false; bool integer_border = false; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 49a6993d5..ebd5dc1aa 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -635,6 +635,10 @@ bool BuildSamplerPlan(const ShaderInfo& base, SamplerPlan& plan) { return false; } usage[pair.sampler] |= 1u << static_cast(ClassifySampler(base.images[pair.image])); + for (const auto candidate: base.samplers[pair.sampler].indirect_resources) { + if (candidate >= base.samplers.size()) return false; + usage[candidate] |= usage[pair.sampler]; + } } for (uint32_t index = 0; index < base.samplers.size(); index++) { auto& mapping = plan.mapping[index]; @@ -1224,7 +1228,14 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime } snapshot.samplers.resize(program.info.samplers.size()); for (uint32_t i = 0; i < program.info.samplers.size(); ++i) { - if (!evaluate(program.info.samplers[i].source, snapshot.samplers[i])) { + auto source = program.info.samplers[i].source; + if (source >= program.descriptor_sources.size()) return false; + const auto& descriptor = program.descriptor_sources[source]; + if (descriptor.indirect_descriptor) { + if (descriptor.indirect_descriptor->sources.empty()) return false; + source = descriptor.indirect_descriptor->sources[0]; + } + if (!evaluate(source, snapshot.samplers[i])) { return false; } if (program.info.samplers[i].gather_lod) { @@ -1316,11 +1327,20 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger; samplers[index].integer_border = binding.type != SamplerClass::Float; } + for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) { + const auto& binding = sampler_plan.bindings[index]; + for (auto& candidate: samplers[index].indirect_resources) { + candidate = sampler_plan.mapping[candidate][static_cast(binding.type)]; + EXIT_IF(candidate == UINT32_MAX); + } + } for (auto& pair: sampled_pairs) { const auto type = static_cast(ClassifySampler(images[pair.image])); pair.sampler = sampler_plan.mapping[pair.sampler][type]; EXIT_IF(pair.sampler == UINT32_MAX); samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare; + for (const auto candidate: samplers[pair.sampler].indirect_resources) + samplers[candidate].depth_compare |= images[pair.image].depth_compare; } auto& memory_info = program.memory_info; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 3c476834a..c50e6acc1 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -430,7 +430,7 @@ public: plan.handle->SetArg(word, Value(0u)); continue; } - if (plan.handle->GetOpcode() == ValueOpcode::GetBufferResource) { + if (plan.handle->NumArgs() == 4u) { plan.handle->SetArg(0, plan.key); for (uint32_t word = 1; word < 4u; ++word) plan.handle->SetArg(word, Value(0u)); } else { @@ -1944,17 +1944,21 @@ private: return found; } - bool TryMakeFiniteImage(Inst& handle, IndirectDescriptorPlan& plan) { - if (handle.GetOpcode() != ValueOpcode::GetImageResource || handle.NumArgs() != 8u) + bool TryMakeFiniteDescriptor(Inst& handle, IndirectDescriptorPlan& plan) { + const bool sampler = handle.GetOpcode() == ValueOpcode::GetSamplerResource; + if ((!sampler && handle.GetOpcode() != ValueOpcode::GetImageResource) || + handle.NumArgs() != (sampler ? 4u : 8u)) return false; - if (std::ranges::any_of(handle.Uses(), [](const Use& use) { + if (std::ranges::any_of(handle.Uses(), [&](const Use& use) { const auto op = use.user->GetOpcode(); - return op != ValueOpcode::ImageSampleRaw && op != ValueOpcode::ImageGatherRaw; + if (op != ValueOpcode::ImageSampleRaw) return sampler || op != ValueOpcode::ImageGatherRaw; + return sampler && (m_program.memory_info[use.user->Flags().index] + .image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0u; })) return false; bool has_phi = false; for (size_t word = 0; word < handle.NumArgs(); ++word) has_phi |= handle.Arg(word).Resolve().IsPhi(); if (!has_phi) return false; - // Normalize the eight synchronized descriptor words to one GPU ordinal. + // Normalize synchronized descriptor words to one GPU ordinal. // Build the complete graph before changing IR so an unsupported leaf is transactional. struct Choice { DescriptorSource descriptor; @@ -1970,7 +1974,15 @@ private: const auto index = static_cast(choices.size()); choices.push_back({descriptor}); uint32_t bad = 0; - if (ValidateSource(descriptor, bad)) return index; + if (ValidateSource(descriptor, bad)) { + // Finite inline samplers have literal leaves; live descriptor tables use the + // existing direct sampler planning path. + if (sampler && !std::ranges::all_of( + std::span(descriptor.dwords).first(descriptor.dword_count), + [](Value word) { return word.IsImmediate() && word.GetType() == Type::U32; })) + return UINT32_MAX; + return index; + } const auto* branch = descriptor.dwords[bad].TryInstruction(); if (branch == nullptr || branch->Parent() == nullptr || branch->GetOpcode() != ValueOpcode::Phi || @@ -2007,14 +2019,14 @@ private: return index; }; DescriptorSource root; - root.dword_count = 8u; + root.dword_count = handle.NumArgs(); for (uint32_t word = 0; word < root.dword_count; ++word) root.dwords[word] = handle.Arg(word); if (visit(visit, root) == UINT32_MAX || choices.front().branch == nullptr) return false; - DescriptorSource image_source; - image_source.dword_count = 8u; - image_source.dwords.fill(Value(0u)); - image_source.indirect_descriptor.emplace(DescriptorSource::IndirectDescriptor{}); - auto& sources = image_source.indirect_descriptor->sources; + DescriptorSource finite_source; + finite_source.dword_count = root.dword_count; + finite_source.dwords.fill(Value(0u)); + finite_source.indirect_descriptor.emplace(DescriptorSource::IndirectDescriptor{}); + auto& sources = finite_source.indirect_descriptor->sources; for (auto& choice: choices) { if (choice.branch != nullptr) continue; const auto source = InternSource(choice.descriptor); @@ -2043,9 +2055,9 @@ private: } } plan.handle = &handle; - plan.source = InternSource(image_source); + plan.source = InternSource(finite_source); plan.key = choices.front().key; - plan.roots = image_source.dwords; + plan.roots = finite_source.dwords; plan.reads.fill(nullptr); return true; } @@ -2076,27 +2088,33 @@ private: inst.NumArgs() == 0u) { continue; } - auto* handle = inst.Arg(0).Resolve().TryInstruction(); - if (handle == nullptr || FindIndirectDescriptor(*handle) != nullptr) { - continue; - } const auto flags = inst.Flags(); const auto& memory = m_program.memory_info[flags.index]; - DescriptorSource descriptor; - MakeSource(*handle, inst.GetOpcode() == ValueOpcode::StoreBufferU32 ? 4u : 8u, - false, false, memory.resource * 4u, descriptor, flags.pc); - uint32_t bad_dword = 0; - if (ValidateSource(descriptor, bad_dword)) continue; - IndirectDescriptorPlan plan; - if (TryMakeIndirectImage(*handle, descriptor, plan) || TryMakeFiniteImage(*handle, plan) || - TryMakeIndirectBuffer(*handle, descriptor, plan)) { - for (const auto& previous: m_indirect_descriptors) { - if (plan.reads[0] != nullptr && previous.reads[0] == plan.reads[0] && - !EquivalentValue(m_program, previous.key, plan.key)) - Fail(flags.pc, "shared descriptor read has incompatible keys"); + const auto plan_handle = [&](Value value, uint32_t width, uint32_t base, bool sampler) { + auto* handle = value.Resolve().TryInstruction(); + if (handle == nullptr || FindIndirectDescriptor(*handle) != nullptr) return; + DescriptorSource descriptor; + MakeSource(*handle, width, sampler, false, base, descriptor, flags.pc); + uint32_t bad_dword = 0; + if (ValidateSource(descriptor, bad_dword)) return; + IndirectDescriptorPlan plan; + if (sampler ? TryMakeFiniteDescriptor(*handle, plan) + : (TryMakeIndirectImage(*handle, descriptor, plan) || + TryMakeFiniteDescriptor(*handle, plan) || + TryMakeIndirectBuffer(*handle, descriptor, plan))) { + for (const auto& previous: m_indirect_descriptors) { + if (plan.reads[0] != nullptr && previous.reads[0] == plan.reads[0] && + !EquivalentValue(m_program, previous.key, plan.key)) + Fail(flags.pc, "shared descriptor read has incompatible keys"); + } + m_indirect_descriptors.push_back(std::move(plan)); } - m_indirect_descriptors.push_back(std::move(plan)); - } + }; + plan_handle(inst.Arg(0), inst.GetOpcode() == ValueOpcode::StoreBufferU32 ? 4u : 8u, + memory.resource * 4u, false); + if (inst.GetOpcode() == ValueOpcode::ImageSampleRaw && + (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) == 0u) + plan_handle(inst.Arg(1), 4u, memory.sampler * 4u, true); } } } @@ -2225,7 +2243,20 @@ private: return UINT32_MAX; } m_info.samplers.push_back({source, pc}); - return static_cast(m_info.samplers.size() - 1); + const auto root = static_cast(m_info.samplers.size() - 1); + const auto* descriptor = Source(source); + if (descriptor != nullptr && descriptor->indirect_descriptor) { + const auto& sources = descriptor->indirect_descriptor->sources; + if (sources.empty()) Fail(pc, "finite sampler has no candidates"); + std::vector resources {root}; + for (size_t i = 1; i < sources.size(); ++i) { + const auto candidate = AddSampler(sources[i], pc); + if (candidate == UINT32_MAX) return UINT32_MAX; + resources.push_back(candidate); + } + m_info.samplers[root].indirect_resources = std::move(resources); + } + return root; } void AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc) { @@ -2385,8 +2416,15 @@ private: uint32_t sampler_source = 0; const bool sample_adjust = (memory.image_sample_flags & Decoder::ImageSampleFlagAdjust) != 0; - GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, - memory.sampler * 4u, sampler_handle, sampler_source, true, sample_adjust); + sampler_handle = inst.Arg(1).Resolve().TryInstruction(); + const auto* selected_sampler = sampler_handle != nullptr + ? FindIndirectDescriptor(*sampler_handle) : nullptr; + if (selected_sampler != nullptr) { + sampler_source = selected_sampler->source; + } else { + GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, + memory.sampler * 4u, sampler_handle, sampler_source, true, sample_adjust); + } sampler = AddSampler(sampler_source, flags.pc); if (sampler == UINT32_MAX) { Fail(flags.pc, "sampler resource limit exceeded"); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 06b4315a3..cd13266a3 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -2648,6 +2648,129 @@ void TestFiniteImagePhiCycle() { } } +void TestFiniteInlineSampler() { + namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; + Fixture fixture; + auto* entry = fixture.block; + auto* defined = fixture.AddBlock(); + auto* merge = fixture.AddBlock(); + entry->AddBranch(defined); + entry->AddBranch(merge); + defined->AddBranch(merge); + const auto lane = fixture.Emit(ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::LocalInvocationId)), Value(0u)}); + entry->condition = fixture.Emit(ValueOpcode::IEqual32, {lane, Value(0u)}); + entry->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = defined, .false_block = merge}; + defined->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = merge}; + merge->terminator.kind = CFG::TerminatorKind::Return; + fixture.block = merge; + auto& lod = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + auto& filter = merge->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + lod.AddPhiOperand(entry, Value(0u)); + lod.AddPhiOperand(defined, Value(0x00fff000u)); + filter.AddPhiOperand(defined, Value(0x0a500000u)); + filter.AddPhiOperand(entry, Value(0u)); + const auto sampler = fixture.Sampler({Value(0u), Value(&lod), Value(&filter), Value(0u)}); + const auto direct = fixture.Sampler({Value(0u), Value(0u), Value(0u), Value(0u)}); + const auto previous = fixture.Sampler( + {Value(0u), Value(0x00fff000u), Value(0x0a500000u), Value(0u)}); + using F = Libs::Graphics::Prospero::BufferFormat; + for (const auto format : {F::k32Float, F::k32SInt}) { + const auto image = fixture.Image({Value(0x1000u), Value(uint32_t(format) << 20u), + Value(0u), Value(Libs::Graphics::DstSel(4, 5, 6, 7) | + (uint32_t(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u)), + Value(0u), Value(0u), Value(0u), Value(0u)}); + MemoryInfo memory; + memory.kind = ResourceKind::Image; + memory.image_dimension = Decoder::ImageDimension::Dim2D; + if (format == F::k32Float) + fixture.Emit(ValueOpcode::ImageSampleRaw, {image, previous, fixture.ImageAddress()}, + fixture.AddMemory(memory, 0x440u)); + fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, fixture.ImageAddress()}, + fixture.AddMemory(memory, 0x1dd0u)); + if (format == F::k32Float) + fixture.Emit(ValueOpcode::ImageSampleRaw, {image, direct, fixture.ImageAddress()}, + fixture.AddMemory(memory, 0x1dd8u)); + } + fixture.PlanAndTrack(); + const auto root = fixture.program.memory_info[1].sampler; + const auto source = fixture.program.info.samplers[root].source; + const auto& finite = fixture.program.descriptor_sources[source].indirect_descriptor; + Check(finite && finite->sources.size() == 2 && fixture.program.info.samplers.size() == 3, + "finite inline sampler lost either state or aliased the direct sampler"); + const auto* key = sampler.Instruction()->Arg(0).Instruction(); + Check(key->GetOpcode() == ValueOpcode::Phi && key->Parent() == merge && + key->PhiBlock(0) == entry && key->PhiBlock(1) == defined && + key->Arg(0).U32() == 0 && key->Arg(1).U32() == 1, + "inline sampler DWORD correlation did not preserve its GPU selector"); + const auto plan = ExtractResourcePlan(fixture.program); + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + const SrtRuntime runtime{.read_specialization_memory = ReadTestMemory}; + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "finite inline sampler could not materialize literal choices"); + const auto candidates = fixture.program.info.samplers[root].indirect_resources; + Check(candidates.size() == 2 && snapshot.samplers[candidates[0]].dwords[1] == 0 && + snapshot.samplers[candidates[0]].dwords[2] == 0 && + snapshot.samplers[candidates[1]].dwords[1] == 0x00fff000u && + snapshot.samplers[candidates[1]].dwords[2] == 0x0a500000u, + "finite sampler materialization mixed the two correlated DWORD states"); + Check(candidates[1] == fixture.program.memory_info[0].sampler && candidates[1] < root, + "finite sampler did not reuse the earlier constant candidate"); + ApplyResourceSpecialization(fixture.program, specialization); + for (const uint32_t operation : {1u, 3u}) { + const auto selected = fixture.program.memory_info[operation].sampler; + const auto& choices = fixture.program.info.samplers[selected].indirect_resources; + Check(choices.size() == 2, "sampler class specialization lost a finite candidate"); + for (uint32_t ordinal = 0; ordinal < choices.size(); ++ordinal) { + const auto& choice = fixture.program.info.samplers[choices[ordinal]]; + Check(choice.snapshot_index == candidates[ordinal] && + choice.force_point_filtering == (operation == 3u) && + choice.integer_border == (operation == 3u), + "finite sampler class remapping changed candidate identity or filtering"); + } + } + Check(fixture.program.info.samplers[fixture.program.memory_info[2].sampler] + .indirect_resources.empty(), + "direct sampler was changed into a dynamic selection"); + auto invalid = ExtractResourcePlan(fixture.program); + invalid.info.samplers[0].source = static_cast(invalid.descriptor_sources.size()); + Check(!MaterializeResources(invalid, runtime, snapshot, specialization), + "invalid sampler source bypassed materialization validation"); + + Fixture adjusted; + auto* initial = adjusted.block; + auto* alternate = adjusted.AddBlock(); + auto* join = adjusted.AddBlock(); + initial->AddBranch(alternate); + initial->AddBranch(join); + alternate->AddBranch(join); + initial->condition = adjusted.Emit(ValueOpcode::IEqual32, + {adjusted.Emit(ValueOpcode::LaneId), Value(0u)}); + initial->terminator = {.kind = CFG::TerminatorKind::ConditionalBranch, + .true_block = alternate, .false_block = join}; + alternate->terminator = {.kind = CFG::TerminatorKind::Branch, .true_block = join}; + join->terminator.kind = CFG::TerminatorKind::Return; + auto& selected = join->AppendNewInst(ValueOpcode::Phi, {}, uint64_t(Type::U32)); + selected.AddPhiOperand(initial, Value(0u)); + selected.AddPhiOperand(alternate, Value(0x00fff000u)); + adjusted.block = join; + const auto shared = adjusted.Sampler({Value(0u), Value(&selected), Value(0u), Value(0u)}); + const auto image = adjusted.Image({Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u), Value(0u)}); + for (const uint32_t flags : {0u, uint32_t(Decoder::ImageSampleFlagAdjust)}) { + MemoryInfo memory; + memory.kind = ResourceKind::Image; + memory.image_dimension = Decoder::ImageDimension::Dim2D; + memory.image_sample_flags = flags; + adjusted.Emit(ValueOpcode::ImageSampleRaw, {image, shared, adjusted.ImageAddress()}, + adjusted.AddMemory(memory, 0x1dd0u)); + } + CheckFatal([&] { adjusted.PlanAndTrack(); }, "not a valid runtime value", + "finite sampler planning accepted a shared SampleAdjust handle"); +} + void TestFiniteImageBitScanSentinel() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; for (const bool nonzero : {false, true}) { @@ -3608,6 +3731,7 @@ int main() { Run("writable descriptor phi", TestWritableDescriptorPhi); Run("conditional sampler phi", TestConditionalSamplerPhi); Run("finite image phi cycle", TestFiniteImagePhiCycle); + Run("finite inline sampler", TestFiniteInlineSampler); Run("finite image bit scan sentinel", TestFiniteImageBitScanSentinel); Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue); Run("invariant loop phi", TestInvariantLoopPhi); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index b2f7f435b..2b7709893 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -15356,7 +15356,8 @@ public: const Image *storage_image = nullptr, const Image *storage_image_uint = nullptr, vk::Sampler sampler = nullptr, - std::span sampled_resources = {}) { + std::span sampled_resources = {}, + std::span sampler_resources = {}) { using Kind = ShaderRecompiler::IR::DescriptorBindingKind; const auto &layout = compiled.program.bindings; auto shader_data = compiled.packed_user_data; @@ -15671,11 +15672,12 @@ public: writes.push_back(write); } if (Count(Kind::Samplers) != 0) { - Require(test.name, "dispatch", sampler != nullptr, - "sampler descriptor requested but no sampler was provided"); + Require(test.name, "dispatch", sampler_resources.empty() + ? sampler != nullptr : sampler_resources.size() == Count(Kind::Samplers), + "sampler resources must match the native sampler array"); sampler_infos.resize(Count(Kind::Samplers)); - for (auto &info : sampler_infos) { - info.sampler = sampler; + for (u32 index = 0; index < sampler_infos.size(); ++index) { + sampler_infos[index].sampler = sampler_resources.empty() ? sampler : sampler_resources[index]; } vk::WriteDescriptorSet write{}; write.dstSet = descriptor_set; @@ -34947,6 +34949,122 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { std::printf("[compute] %-32s ok\n", test.name); } +void CheckFiniteInlineSamplerPhi(VulkanHarness &vulkan) { + TestCase test; + test.name = "FiniteInlineSamplerPhi"; + auto &code = test.code; + code.insert(code.end(), {EncodeSop1(0x04, 36, InlineU32(0)), + EncodeSop1(0x04, 38, InlineU32(0)), EncodeSop1(0x04, 24, 126), + EncodeVop1(0x01, 10, 16), EncodeVopc(0xd5, InlineU32(0), 10)}); + const auto skip_sampler = code.size(); + code.push_back(EncodeSopp(0x08)); + // Actual correlated inline sampler words from 905b2e2c: zero or {0,fff000,a500000,0}. + code.insert(code.end(), {EncodeSop2(0x24, 37, InlineU32(12), InlineU32(12)), + EncodeSop1(0x04, 38, 255), 0x0a500000u}); + code[skip_sampler] = EncodeSopp(0x08, code.size() - skip_sampler - 1u); + code.push_back(EncodeSop1(0x04, 126, 24)); + AppendVMovLiteral(&code, 20, std::bit_cast(0.5f)); + AppendVMovLiteral(&code, 21, std::bit_cast(0.5f)); + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 2, 0x87654321u); + code.insert(code.end(), {EncodeVopc(0xd4, InlineU32(2), 10), + EncodeMimg0(0x27, 1), EncodeMimg1(1, 20, 0, 9), + EncodeMimg0(0x27, 1), EncodeMimg1(2, 20, 2, 9), + EncodeSop1(0x04, 126, 24), EncodeVop1(0x01, 30, 16), + EncodeVop2(0x1a, 31, InlineU32(3), 30), + EncodeMubuf0(0x1c), EncodeMubuf1(1, 12, 31), + EncodeMubuf0(0x1c, 4), EncodeMubuf1(2, 12, 31)}); + AppendEnd(&code); + test.initial.assign(7, 0xa5a5a5a5u); + test.expected = {std::bit_cast(4.0f), 0xfffffff9u, + std::bit_cast(2.0f), 0xfffffff9u, 0x12345678u, 0x87654321u, 0xa5a5a5a5u}; + test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32_32_32_32Float); + test.user_data[3] |= static_cast(Prospero::ImageType::kColor2D) << 28u; + const auto signed_image = MakeSampledTextureData(Prospero::BufferFormat::k32SInt); + std::copy_n(signed_image.begin(), 8, test.user_data.begin() + 8); + test.user_data[11] |= static_cast(Prospero::ImageType::kColor2D) << 28u; + test.user_data[50] = test.initial.size() * sizeof(u32); + test.user_data[51] = 3u << 28u; + test.has_user_data = true; + test.compute_info.threads_num[0] = test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = 32; + test.compute_info.group_id[0] = true; + test.compute_info.workgroup_register = 16; + test.has_compute_info = true; + test.dispatch_x = 3; + test.opcodes = {ShaderOpcode::S_MOV_B64, ShaderOpcode::S_BFM_B32, + ShaderOpcode::S_CBRANCH_EXECZ, ShaderOpcode::V_MOV_B32, ShaderOpcode::V_CMPX_NE_U32, + ShaderOpcode::V_CMPX_GT_U32, ShaderOpcode::V_LSHLREV_B32, + ShaderOpcode::IMAGE_SAMPLE, ShaderOpcode::BUFFER_STORE_DWORD, ShaderOpcode::S_ENDPGM}; + test.required_spirv = {"OpPhi", "NonUniform", "OpImageSampleExplicitLod"}; + auto compiled = CompileCase(test, vulkan.SubgroupSize()); + std::set nonuniform_ids; + std::vector> sampled_ids; + for (size_t offset = 5; offset < compiled.spirv.size();) { + const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u); + if (static_cast(words[0] & 0xffffu) == spv::OpDecorate && + words[2] == spv::DecorationNonUniform) nonuniform_ids.insert(words[1]); + if (static_cast(words[0] & 0xffffu) == spv::OpSampledImage) + sampled_ids.emplace_back(words[2], words[4]); + offset += words.size(); + } + Require(test.name, "dynamic sampler decorations", sampled_ids.size() >= 2 && + std::ranges::all_of(sampled_ids, [&](const auto &ids) { + return nonuniform_ids.contains(ids.first) && nonuniform_ids.contains(ids.second); + }), + "fetched sampler or sampled image lost its NonUniform decoration"); + const auto &samplers = compiled.program.info.samplers; + u32 roots = 0; + for (const auto &root : samplers) { + if (root.indirect_resources.empty()) continue; + ++roots; + Require(test.name, "correlated sampler candidates", root.indirect_resources.size() == 2, + "finite sampler was folded or lost a correlated branch state"); + for (const auto child : root.indirect_resources) + Require(test.name, "sampler class remapping", child < samplers.size() && + samplers[child].force_point_filtering == root.force_point_filtering && + samplers[child].integer_border == root.integer_border, + "a finite candidate retained another image class's sampler"); + } + Require(test.name, "finite sampler class variants", roots == 2 && samplers.size() == 4 && + compiled.resources.samplers.size() == 2, + "the two-state sampler did not retain both float and signed image classes"); + constexpr std::array zero_sampler{}; + constexpr std::array linear_sampler{0, 0x00fff000u, 0x0a500000u, 0}; + Require(test.name, "inline sampler states", + std::ranges::all_of(compiled.resources.samplers, [&](const auto &value) { + return std::equal(zero_sampler.begin(), zero_sampler.end(), value.dwords.begin()) || + std::equal(linear_sampler.begin(), linear_sampler.end(), value.dwords.begin()); + }) && compiled.resources.samplers[0] != compiled.resources.samplers[1], + "descriptor words were mixed between the two native sampler states"); + auto pixels = MakeRgbaImage(4, 4); + for (u32 y = 0; y < 4; ++y) + for (u32 x = 2; x < 4; ++x) SetRgbaPixel(&pixels, 4, x, y, std::bit_cast(4.0f), 0, 0, 0); + std::array images{ + vulkan.CreateImageMips(test.name, 4, 4, vk::Format::eR32G32B32A32Sfloat, + vk::ImageUsageFlagBits::eSampled, {pixels}, 4, vk::ImageLayout::eShaderReadOnlyOptimal, + vk::ImageType::e2D, vk::ImageViewType::e2D, 1), + vulkan.CreateImageMips(test.name, 4, 4, vk::Format::eR32Sint, + vk::ImageUsageFlagBits::eSampled, {std::vector(16, 0xfffffff9u)}, 1, + vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2D, 1)}; + std::vector native_samplers; + for (const auto &resource : samplers) { + ShaderSamplerResource descriptor{}; + std::copy_n(compiled.resources.samplers[resource.snapshot_index].dwords.begin(), 4, descriptor.fields); + const auto filter = !resource.force_point_filtering && descriptor.XyMagFilter() == 1 + ? vk::Filter::eLinear : vk::Filter::eNearest; + native_samplers.push_back(vulkan.CreateSampler(test.name, 1, filter)); + } + auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size()); + vulkan.Dispatch(test, compiled, output, nullptr, nullptr, nullptr, nullptr, nullptr, images, native_samplers); + CompareWords(test, "point, linear, remapped signed sampler and inactive output", test.expected, + vulkan.ReadBuffer(test.name, output, test.expected.size())); + vulkan.DestroyBuffer(&output); + for (const auto sampler : native_samplers) vulkan.Device().destroySampler(sampler); + for (auto &image : images) vulkan.DestroyImage(&image); + std::printf("[compute] %-32s ok\n", test.name); +} + void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { constexpr const char *name = "IndirectImageKeySwitch"; constexpr uint32_t mapping_capacity = 1793u; @@ -43191,6 +43309,7 @@ int main(int argc, char **argv) { CheckImageSamplerSpecialization(); VulkanHarness vulkan; CheckIndirectImageKeySwitch(vulkan); + CheckFiniteInlineSamplerPhi(vulkan); RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives()); return 0; } @@ -43354,6 +43473,7 @@ int main(int argc, char **argv) { CheckRuntimeBufferRecords(vulkan); CheckComputeThreadDimensions(vulkan); CheckIndirectImageKeySwitch(vulkan); + CheckFiniteInlineSamplerPhi(vulkan); CheckWave64WholeWaveResults(); CheckPs5GameExampleImageClearRuntimeShape(); vulkan.CheckSchedulerTimeline();