From e1ac893a0ced026d1439642717f19f702ace4d4a Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 9 Oct 2026 02:17:38 +0400 Subject: [PATCH] shader: collect usage once and emit descriptor slots directly --- CMakeLists.txt | 1 - .../renderer/pipeline/descriptors.cpp | 135 ++-- .../host_gpu/renderer/pipeline/descriptors.h | 2 - .../host_gpu/renderer/pipeline/shaders.cpp | 9 +- src/graphics/host_gpu/renderer/render.h | 1 - .../host_gpu/renderer/renderCompute.cpp | 5 +- .../shader/recompiler/ShaderRecompiler.cpp | 8 +- .../recompiler/backend/spirv/SpirvEmitter.cpp | 343 +++------ .../recompiler/backend/spirv/SpirvEmitter.h | 6 +- .../backend/spirv/spirvEmitterAnalysis.cpp | 25 +- .../backend/spirv/spirvEmitterBvh.cpp | 2 +- .../backend/spirv/spirvEmitterFlow.cpp | 4 +- .../backend/spirv/spirvEmitterImage.cpp | 11 +- .../backend/spirv/spirvEmitterInternal.h | 37 +- .../backend/spirv/spirvEmitterMemory.cpp | 3 +- .../spirv/spirvEmitterMemoryHelpers.cpp | 20 +- .../backend/spirv/spirvEmitterModule.cpp | 110 ++- .../backend/spirv/spirvEmitterProgram.cpp | 19 +- src/graphics/shader/recompiler/ir/Program.cpp | 11 + src/graphics/shader/recompiler/ir/ShaderIR.h | 61 +- .../recompiler/ir/passes/BindingLayout.cpp | 197 ----- .../recompiler/ir/passes/BindingLayout.h | 23 - .../ir/passes/ResourceMaterialization.cpp | 3 +- .../ir/passes/ShaderInfoCollection.cpp | 720 +++++++++++------- .../ir/passes/ShaderInfoCollection.h | 3 +- tests/ResourceTrackingTests.cpp | 147 +--- tests/ShaderRecompilerComputeTests.cpp | 325 ++++---- tests/shaderCfgTests.cpp | 245 +++--- 28 files changed, 1068 insertions(+), 1408 deletions(-) delete mode 100644 src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp delete mode 100644 src/graphics/shader/recompiler/ir/passes/BindingLayout.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 03ee62b0c..ac75a8dc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -477,7 +477,6 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/BindingLayout.cpp" ) target_link_libraries(resource_tracking_tests fmt::fmt) target_include_directories(resource_tracking_tests PRIVATE ${inc_headers}) diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index c65f738f8..401edb8be 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -26,7 +26,6 @@ #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include "kernel/memory.h" @@ -76,10 +75,6 @@ vk::DescriptorType NativeDescriptorType(BindingKind kind) { EXIT("invalid native descriptor binding kind"); } -uint32_t NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) { - return binding.resources.empty() ? 1u : static_cast(binding.resources.size()); -} - vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element) { vk::ImageView view = nullptr; if (texture.mip_views.empty()) { @@ -799,12 +794,11 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, m_context, program, i, snapshot.samplers[program.info.samplers[i].snapshot_index])); } prepared.shader_data.reserve(program.bindings.ShaderDataDwords()); - for (const auto reg: program.bindings.user_data_registers) { + for (const auto reg: program.info.user_data_registers) { prepared.shader_data.push_back(snapshot.user_data[reg - program.user_data_base]); } prepared.shader_data.resize(program.bindings.ShaderDataDwords()); - if (ShaderRecompiler::IR::FindBinding( - program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) { + if (program.bindings.descriptor_counts[static_cast(BindingKind::Gds)] != 0) { prepared.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle(); } } @@ -817,19 +811,20 @@ void RenderExecutor::FindBuffers(std::span stages) { EXIT_IF(prepared.runtime == nullptr || !*prepared.runtime); const auto& program = *prepared.runtime->program; const auto& snapshot = *prepared.runtime->resources; - prepared.buffer_sources.clear(); const auto& layout = program.bindings; - if (layout.memory_offset_count == 0) { + prepared.buffer_sources.resize(layout.descriptor_counts[static_cast(BindingKind::Buffers)]); + if (layout.descriptor_counts[static_cast(BindingKind::Buffers)] == 0) { continue; } - const auto& resources = layout.descriptors.front().resources; - prepared.buffer_sources.reserve(resources.size()); - for (const auto resource: resources) { + for (uint32_t resource = 0; resource < program.info.buffers.size(); ++resource) { + const auto& metadata = program.info.buffers[resource]; + if (metadata.descriptor_index == UINT32_MAX) continue; + auto& source = prepared.buffer_sources[metadata.descriptor_index]; const auto descriptor = DecodeNativeDescriptor(snapshot.buffers[resource]); const auto address = descriptor.Base48(); auto size = descriptor.GetSize(); if (address == 0 || size == 0) { - prepared.buffer_sources.push_back({}); + source = {}; continue; } if (descriptor.NumRecords() == UINT32_MAX) { @@ -853,7 +848,7 @@ void RenderExecutor::FindBuffers(std::span stages) { } } size = Libs::LibKernel::Memory::ClampRangeSize(address, size); - prepared.buffer_sources.push_back({address, size, cache.FindBuffer(address, size)}); + source = {address, size, cache.FindBuffer(address, size)}; } } } @@ -864,10 +859,9 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { const auto& program = *prepared.runtime->program; const auto& snapshot = *prepared.runtime->resources; const auto& layout = program.bindings; - EXIT_IF(prepared.buffer_sources.size() != layout.memory_offset_count); + EXIT_IF(prepared.buffer_sources.size() != layout.descriptor_counts[static_cast(BindingKind::Buffers)]); - prepared.buffers.clear(); - prepared.buffers.reserve(layout.memory_offset_count); + prepared.buffers.resize(layout.descriptor_counts[static_cast(BindingKind::Buffers)]); EXIT_IF(prepared.shader_data.size() != layout.ShaderDataDwords()); std::fill(prepared.shader_data.begin() + layout.memory_offset_dword, prepared.shader_data.end(), 0); @@ -876,20 +870,18 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { const auto shift = (index % 4u) * 8u; prepared.shader_data[dword] |= offset << shift; }; - for (uint32_t i = 0; i < layout.memory_offset_count; i++) { - const auto resource = layout.descriptors.front().resources[i]; + for (const auto& resource: program.info.buffers) { + const auto index = resource.descriptor_index; + if (index == UINT32_MAX) continue; uint32_t buffer_offset = 0; - prepared.buffers.push_back(NativeStorageBuffer(m_context, prepared.buffer_sources[i], - program.info.buffers[resource], - buffer_offset)); - pack_memory_offset(i, buffer_offset); + prepared.buffers[index] = NativeStorageBuffer(m_context, prepared.buffer_sources[index], + resource, buffer_offset); + pack_memory_offset(index, buffer_offset); } - if (ShaderRecompiler::IR::FindBinding( - layout, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) != nullptr) { + if (layout.descriptor_counts[static_cast(BindingKind::FlattenedSrt)] != 0) { prepared.flattened_srt = NativeUpload(m_context, snapshot.flattened_srt); } - if (ShaderRecompiler::IR::FindBinding( - program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::ShaderData) != nullptr) { + if (layout.descriptor_counts[static_cast(BindingKind::ShaderData)] != 0) { prepared.shader_data_buffer = NativeUpload(m_context, prepared.shader_data); } } @@ -982,6 +974,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, KYTY_PROFILER_FUNCTION(); auto vk_buffer = buffer.Handle(); size_t descriptor_count = 0; + size_t image_descriptor_count = 0; size_t write_count = 0; ShaderRecompiler::IR::PushData push_data; bool has_push_data = false; @@ -995,9 +988,15 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, for (const auto* prepared: prepared_bindings) { EXIT_IF(prepared == nullptr || prepared->runtime == nullptr || !*prepared->runtime); const auto& program = *prepared->runtime->program; - write_count += program.bindings.descriptors.size(); - for (const auto& binding: program.bindings.descriptors) { - descriptor_count += NativeDescriptorCount(binding); + write_count += std::popcount(program.bindings.descriptor_mask); + for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) { + const auto index = std::countr_zero(mask); + const auto count = program.bindings.descriptor_counts[index]; + descriptor_count += count; + if (index >= ShaderRecompiler::IR::FirstImageBinding && + index <= static_cast(BindingKind::Samplers)) { + image_descriptor_count += count; + } } const auto shader_stage = NativeShaderStage(program.stage); push_stages |= shader_stage; @@ -1029,10 +1028,9 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, } for (const auto* writer: prepared_bindings) { const auto& program = *writer->runtime->program; - for (uint32_t i = 0; i < writer->buffer_sources.size(); ++i) { - const auto resource = program.bindings.descriptors.front().resources[i]; - if (!program.info.buffers[resource].written) continue; - const auto& written = writer->buffer_sources[i]; + for (const auto& resource: program.info.buffers) { + if (!resource.written || resource.descriptor_index == UINT32_MAX) continue; + const auto& written = writer->buffer_sources[resource.descriptor_index]; if (written.size != 0 && ImageRangeOverlaps(address, size, written.address, written.size)) { EXIT("scalar resource reads overlap a shader buffer write\n"); @@ -1042,11 +1040,11 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, } } m_descriptor_buffers.clear(); - m_descriptor_images.clear(); + m_descriptor_images.resize(image_descriptor_count); m_descriptor_writes.clear(); m_descriptor_buffers.reserve(descriptor_count); - m_descriptor_images.reserve(descriptor_count); m_descriptor_writes.reserve(write_count); + size_t image_cursor = 0; for (auto* prepared: prepared_bindings) { const auto& program = *prepared->runtime->program; @@ -1111,24 +1109,24 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, binding.layout = image.backing.state.layout; } - m_image_occurrences.assign(descriptors.images.size(), 0); - for (const auto& binding: program.bindings.descriptors) { + std::array image_starts {}; + for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) { + const auto index = std::countr_zero(mask); + const auto kind = static_cast(index); vk::WriteDescriptorSet write {}; - write.dstBinding = ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind); - write.descriptorType = NativeDescriptorType(binding.kind); - write.descriptorCount = NativeDescriptorCount(binding); + write.dstBinding = ShaderRecompiler::IR::NativeBinding(program.stage, kind); + write.descriptorType = NativeDescriptorType(kind); + write.descriptorCount = program.bindings.descriptor_counts[index]; const auto buffer_start = m_descriptor_buffers.size(); - const auto image_start = m_descriptor_images.size(); - if (ShaderRecompiler::IR::ImageBindingResourceClass(binding.kind) != + const auto image_start = image_cursor; + if (ShaderRecompiler::IR::ImageBindingResourceClass(kind) != ShaderRecompiler::IR::ImageResourceClass::None) { - for (const auto resource: binding.resources) { - m_descriptor_images.push_back(MakeImageInfo( - descriptors.images.at(resource), m_image_occurrences.at(resource)++)); - } + image_starts[ShaderRecompiler::IR::ImageBindingIndex(kind)] = image_start; + image_cursor += write.descriptorCount; } else { - switch (binding.kind) { + switch (kind) { case BindingKind::Buffers: - EXIT_IF(descriptors.buffers.size() != binding.resources.size()); + EXIT_IF(descriptors.buffers.size() != write.descriptorCount); for (const auto& view: descriptors.buffers) { EXIT_IF(view.buffer == nullptr); m_descriptor_buffers.push_back(view); @@ -1136,12 +1134,11 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, break; case BindingKind::BdaPagetable: case BindingKind::FaultBuffer: { - auto& cache = m_context.GetBufferCache(); - const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable + auto& cache = m_context.GetBufferCache(); + const auto* bda_buffer = kind == BindingKind::BdaPagetable ? cache.GetBdaPageTableBuffer() : cache.GetFaultBuffer(); - m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, - bda_buffer->Size()); + m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size()); break; } case BindingKind::FlattenedSrt: @@ -1149,11 +1146,11 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, case BindingKind::SharedMemory: case BindingKind::Gds: { const vk::DescriptorBufferInfo* view = &descriptors.gds; - if (binding.kind == BindingKind::FlattenedSrt) { + if (kind == BindingKind::FlattenedSrt) { view = &descriptors.flattened_srt; - } else if (binding.kind == BindingKind::ShaderData) { + } else if (kind == BindingKind::ShaderData) { view = &descriptors.shader_data_buffer; - } else if (binding.kind == BindingKind::SharedMemory) { + } else if (kind == BindingKind::SharedMemory) { view = &descriptors.shared_memory; } EXIT_IF(view->buffer == nullptr); @@ -1161,11 +1158,9 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, break; } case BindingKind::Samplers: - for (const auto resource: binding.resources) { - const auto sampler = descriptors.samplers.at(resource); + for (const auto sampler: descriptors.samplers) { EXIT_IF(sampler == nullptr); - m_descriptor_images.emplace_back(sampler, nullptr, - vk::ImageLayout::eUndefined); + m_descriptor_images[image_cursor++] = {sampler, nullptr, vk::ImageLayout::eUndefined}; } break; case BindingKind::Count: EXIT("invalid descriptor binding kind"); @@ -1174,17 +1169,21 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, if (m_descriptor_buffers.size() != buffer_start) { write.pBufferInfo = m_descriptor_buffers.data() + buffer_start; } - if (m_descriptor_images.size() != image_start) { + if (image_cursor != image_start) { write.pImageInfo = m_descriptor_images.data() + image_start; } m_descriptor_writes.push_back(write); } - for (uint32_t i = 0; i < descriptors.images.size(); i++) { - const auto expected = - descriptors.images[i].mip_views.empty() - ? 1u - : static_cast(descriptors.images[i].mip_views.size()); - EXIT_IF(m_image_occurrences[i] != expected); + for (uint32_t i = 0; i < program.info.images.size(); ++i) { + const auto& resource = program.info.images[i]; + const auto& texture = descriptors.images[i]; + EXIT_IF(resource.mip_count != (texture.mip_views.empty() ? 1u : texture.mip_views.size())); + const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(resource); + const auto start = image_starts[ShaderRecompiler::IR::ImageBindingIndex(kind)] + + resource.descriptor_index; + for (uint32_t mip = 0; mip < resource.mip_count; ++mip) { + m_descriptor_images[start + mip] = MakeImageInfo(texture, mip); + } } const auto shader_data_dwords = program.bindings.ShaderDataDwords(); diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.h b/src/graphics/host_gpu/renderer/pipeline/descriptors.h index 766b24dd9..3869bf8a0 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.h +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.h @@ -50,8 +50,6 @@ struct PreparedBindings { [[nodiscard]] vk::DescriptorType NativeDescriptorType(ShaderRecompiler::IR::DescriptorBindingKind kind); -[[nodiscard]] uint32_t -NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding); [[nodiscard]] vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element = 0); diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index f564ee8ac..c7ee54095 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -19,6 +19,7 @@ #include "graphics/shader/shader.h" #include +#include #include #include #include @@ -180,10 +181,12 @@ static vk::BlendOp GetBlendOp(uint32_t op) { static void AddLayoutBindings(std::vector& descriptor_bindings, const ShaderRecompiler::IR::CompiledShaderInfo& program, vk::ShaderStageFlagBits stage) { - for (const auto& binding: program.bindings.descriptors) { + for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) { + const auto index = std::countr_zero(mask); + const auto kind = static_cast(index); descriptor_bindings.push_back( - {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind), - NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), stage, nullptr}); + {ShaderRecompiler::IR::NativeBinding(program.stage, kind), NativeDescriptorType(kind), + program.bindings.descriptor_counts[index], stage, nullptr}); } } diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index b8ef32c30..af56471c3 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -216,7 +216,6 @@ private: std::vector m_descriptor_buffers; std::vector m_descriptor_images; std::vector m_descriptor_writes; - std::vector m_image_occurrences; friend class CommandProcessor; friend struct RenderExecutorTestAccess; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 38edecfab..bddb6d5c2 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -20,7 +20,6 @@ #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/shader.h" #include "kernel/eventQueue.h" #include "kernel/pthread.h" @@ -198,8 +197,8 @@ bool RenderExecutor::TryConsumeComputeImageClear(const ShaderComputeInputInfo& i static void BindSharedMemory(RenderContext& context, ShaderComputeInputInfo& input, PreparedBindings& bindings, uint64_t indirect_args = 0) { - if (ShaderRecompiler::IR::FindBinding(input.stage.program->bindings, - ShaderRecompiler::IR::DescriptorBindingKind::SharedMemory) == nullptr) { + if (input.stage.program->bindings.descriptor_counts[ + static_cast(ShaderRecompiler::IR::DescriptorBindingKind::SharedMemory)] == 0) { return; } auto& cache = context.GetBufferCache(); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index cb07edfda..b600adbd4 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -8,7 +8,6 @@ #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/frontend/translate/Translate.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" @@ -670,20 +669,17 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o IR::EliminateDeadCode(ir.blocks); IR::CollectShaderInfo(ir, options.input_info); - IR::AllocateBindings(ir, push_data_start_dword, - ir.stage == ShaderType::Compute && options.input_info.compute != nullptr && - options.input_info.compute->lds_storage); std::string ir_dump; if (options.dump_ir) { ir_dump = MakeIrDump(translated.cfg_dump, ir); if (options.early_dump) { - LOGF("%s native IR and bindings (early):\n%s", GetDumpLabel(options), ir_dump.c_str()); + LOGF("%s native IR (early):\n%s", GetDumpLabel(options), ir_dump.c_str()); } } LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", GetDumpLabel(options), StageName(ir.stage), ir.shader_hash); - auto spirv = Spirv::EmitProgram(ir, options.input_info); + auto spirv = Spirv::EmitProgram(ir, options.input_info, push_data_start_dword); LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(ir.stage), ir.shader_hash, diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index e5600fe86..9198f7765 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -3,7 +3,6 @@ #include "common/assert.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include #include @@ -19,92 +18,14 @@ namespace { } void ValidateNativeProgram(const IR::Program& program, bool lds_storage) { - using Kind = IR::DescriptorBindingKind; - constexpr auto KindCount = static_cast(Kind::Count); - std::array, KindCount> expected; - std::array present {}; - const auto Dense = [](size_t size) { - std::vector values(size); - for (uint32_t i = 0; i < values.size(); i++) { - values[i] = i; - } - return values; + uint64_t live_buffers = 0; + bool uses_lds = false; + bool uses_gds = false; + const auto uses_mapping = [](const auto& resource) { + return resource.indirect_root != UINT32_MAX; }; - auto Expect = [&](Kind kind, std::vector resources = {}) { - const auto index = static_cast(kind); - present[index] = true; - expected[index] = std::move(resources); - }; - for (uint32_t i = 0; i < program.info.images.size(); i++) { - const auto kind = IR::DescriptorBindingForImage(program.info.images[i]); - if (!kind.has_value()) { - Fail(program, "native shader plan has an invalid image class"); - } - present[static_cast(*kind)] = true; - const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::Dynamic; - const auto count = dynamic ? program.info.images[i].mip_count : 1u; - if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) { - Fail(program, "native shader plan has an invalid image mip descriptor count"); - } - expected[static_cast(*kind)].insert(expected[static_cast(*kind)].end(), - count, i); - } - if (!program.info.samplers.empty()) { - Expect(Kind::Samplers, Dense(program.info.samplers.size())); - } - auto& buffers = expected[static_cast(Kind::Buffers)]; - const auto shared = IR::CollectMemoryResources(program, buffers); - present[static_cast(Kind::Buffers)] = !buffers.empty(); - if (shared.gds) { - Expect(Kind::Gds); - } - if (shared.lds && lds_storage) { - Expect(Kind::SharedMemory); - } - if (program.info.uses_dma) { - Expect(Kind::BdaPagetable); - Expect(Kind::FaultBuffer); - } - if (IR::UsesFlattenedSrt(program)) { - Expect(Kind::FlattenedSrt); - } - if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) { - Expect(Kind::ShaderData); - } - - std::array seen {}; - for (const auto& binding: program.bindings.descriptors) { - const auto kind = static_cast(binding.kind); - if (kind >= KindCount || seen[kind] || !present[kind] || - binding.resources != expected[kind]) { - Fail(program, "native descriptor groups do not match shader topology"); - } - seen[kind] = true; - } - for (size_t i = 0; i < KindCount; i++) { - if (present[i] != seen[i]) { - Fail(program, "native shader plan is missing a required descriptor group"); - } - } - const auto has_shader_data_storage = present[static_cast(Kind::ShaderData)]; - const auto shader_data_dwords = program.bindings.ShaderDataDwords(); - const auto user_data_dwords = program.bindings.user_data_registers.size(); - const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart; - if ((program.bindings.UsesPushData() && - !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) || - (has_dispatch_threads && (program.stage != ShaderType::Compute || - program.bindings.dispatch_thread_dword != user_data_dwords)) || - program.bindings.memory_offset_dword != user_data_dwords + (has_dispatch_threads ? 3u : 0u) || - program.bindings.memory_offset_count != buffers.size() || - has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) || - !std::is_sorted(program.bindings.user_data_registers.begin(), - program.bindings.user_data_registers.end()) || - std::adjacent_find(program.bindings.user_data_registers.begin(), - program.bindings.user_data_registers.end()) != - program.bindings.user_data_registers.end()) { - Fail(program, "native shader-data layout is inconsistent"); - } - + bool uses_flattened_srt = std::ranges::any_of(program.info.buffers, uses_mapping) || + std::ranges::any_of(program.info.images, uses_mapping); const auto planning_only_handle = [&](const IR::Inst& handle) { return !handle.Uses().empty() && std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) { @@ -141,6 +62,39 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) { }; for (const auto* block: program.blocks) { for (const auto& inst: *block) { + const auto op = inst.GetOpcode(); + uses_flattened_srt |= op == IR::ValueOpcode::ReadConst; + if (IR::BufferAccessOf(op) != IR::BufferAccess::None || + IR::SharedAccessOf(op) != IR::SharedAccess::None || + IR::AddressOpcodeInfoOf(op).access != IR::AddressAccess::None) { + const auto index = inst.Flags().index; + if (index >= program.memory_info.size()) { + Fail(program, "typed shader contains invalid memory metadata"); + } + const auto& memory = program.memory_info[index]; + if (!memory.planning_only) { + uses_lds |= memory.kind == IR::ResourceKind::FlatLocal; + if (IR::SharedAccessOf(op) != IR::SharedAccess::None) { + if (memory.kind != IR::ResourceKind::Lds && memory.kind != IR::ResourceKind::Gds) { + Fail(program, "typed shader contains invalid shared-memory metadata"); + } + uses_gds |= memory.kind == IR::ResourceKind::Gds; + uses_lds |= memory.kind == IR::ResourceKind::Lds; + } else if (memory.kind == IR::ResourceKind::Buffer || + memory.kind == IR::ResourceKind::ScalarBuffer) { + if (memory.resource >= program.info.buffers.size()) { + Fail(program, "typed shader contains an invalid buffer resource"); + } + live_buffers |= uint64_t{1} << memory.resource; + for (const auto child: program.info.buffers[memory.resource].indirect_resources) { + if (child >= program.info.buffers.size()) { + Fail(program, "typed shader contains an invalid indirect buffer resource"); + } + live_buffers |= uint64_t{1} << child; + } + } + } + } const auto dense = inst.Flags(); switch (inst.GetOpcode()) { case IR::ValueOpcode::GetBufferResource: @@ -186,163 +140,64 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) { } } } + using Kind = IR::DescriptorBindingKind; + std::array(Kind::Count)> expected {}; + auto& buffer_count = expected[static_cast(Kind::Buffers)]; + for (uint32_t index = 0; index < program.info.buffers.size(); ++index) { + const auto slot = (live_buffers & (uint64_t{1} << index)) != 0 ? buffer_count++ : UINT32_MAX; + if (program.info.buffers[index].descriptor_index != slot) { + Fail(program, "native buffer descriptor index does not match shader topology"); + } + } + for (const auto& image: program.info.images) { + const auto kind = IR::DescriptorBindingForImage(image); + if (!kind.has_value()) { + Fail(program, "native shader plan has an invalid image class"); + } + if (image.mip_count == 0u || + (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) { + Fail(program, "native shader plan has an invalid image mip descriptor count"); + } + auto& count = expected[static_cast(*kind)]; + if (image.descriptor_index != count) { + Fail(program, "native image descriptor index does not match shader topology"); + } + count += image.mip_count; + } + expected[static_cast(Kind::Samplers)] = static_cast(program.info.samplers.size()); + expected[static_cast(Kind::Gds)] = uses_gds; + expected[static_cast(Kind::SharedMemory)] = uses_lds && lds_storage; + expected[static_cast(Kind::BdaPagetable)] = program.info.uses_dma; + expected[static_cast(Kind::FaultBuffer)] = program.info.uses_dma; + expected[static_cast(Kind::FlattenedSrt)] = uses_flattened_srt; + expected[static_cast(Kind::ShaderData)] = + program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData(); + uint64_t descriptor_mask = 0; + for (uint32_t index = 0; index < expected.size(); ++index) { + if (expected[index] != 0) descriptor_mask |= uint64_t{1} << index; + } + if (program.bindings.descriptor_counts != expected || + program.bindings.descriptor_mask != descriptor_mask) { + Fail(program, "native descriptor counts do not match shader topology"); + } + const auto shader_data_dwords = program.bindings.ShaderDataDwords(); + const auto& registers = program.info.user_data_registers; + const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart; + if ((program.bindings.UsesPushData() && + !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) || + (has_dispatch_threads && (program.stage != ShaderType::Compute || + program.bindings.dispatch_thread_dword != registers.size())) || + program.bindings.memory_offset_dword != registers.size() + (has_dispatch_threads ? 3u : 0u) || + !std::is_sorted(registers.begin(), registers.end()) || + std::adjacent_find(registers.begin(), registers.end()) != registers.end()) { + Fail(program, "native shader-data layout is inconsistent"); + } } } // namespace -Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) { - SpirvRequirements requirements {}; - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - requirements.float64 |= inst.GetType() == IR::Type::F64; - if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic && - inst.GetType() == IR::Type::U64) { - requirements.buffer_int64_atomics = true; - } - const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access; - if (address_access != IR::AddressAccess::None) { - const auto memory_index = inst.Flags().index; - if (memory_index >= program.memory_info.size()) { - Fail(program, "address operation has invalid memory metadata"); - } - const auto& memory = program.memory_info[memory_index]; - const auto kind = memory.kind; - if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) { - if (program.scratch_dwords == 0) { - Fail(program, "scratch operation has no per-thread storage"); - } - requirements.function_scratch = true; - if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute && - program.stage != ShaderType::Mesh) { - requirements.function_lds = true; - } - } else if (address_access == IR::AddressAccess::Write) { - Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking"); - } else { - // Descriptor stores can alias coherent physical-address loads. - requirements.coherent_buffers |= memory.coherent; - } - } - if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { - const auto memory_index = inst.Flags().index; - if (memory_index >= program.memory_info.size()) { - Fail(program, "buffer operation has invalid memory metadata"); - } - const auto& memory = program.memory_info[memory_index]; - if (memory.kind == IR::ResourceKind::IndirectBuffer && - inst.GetOpcode() != IR::ValueOpcode::ReadConstBuffer) { - requirements.subgroup_local_invocation_id = true; - } - if (memory.kind == IR::ResourceKind::Buffer) { - requirements.coherent_buffers |= memory.coherent; - if (memory.resource >= program.info.buffers.size()) { - Fail(program, "buffer operation has invalid resource metadata"); - } - const auto bits = StorageBufferElementBits(program, memory); - requirements.buffer_u8 |= bits == 8u; - requirements.buffer_u16 |= bits == 16u; - if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { - if (program.stage != ShaderType::Compute) { - Fail(program, "buffer ADD_TID is only valid for compute shaders"); - } - requirements.subgroup_local_invocation_id = true; - } - } - } - const auto shared_access = IR::SharedAccessOf(inst.GetOpcode()); - if (shared_access != IR::SharedAccess::None) { - const auto index = inst.Flags().index; - if (index >= program.memory_info.size()) { - Fail(program, "shared operation has invalid memory metadata"); - } - const auto kind = program.memory_info[index].kind; - if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { - Fail(program, "shared operation has invalid resource kind"); - } - if (shared_access == IR::SharedAccess::Atomic && - IR::SharedComponentCount(inst.GetOpcode()) == 2u) { - if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) { - Fail(program, "64-bit shared atomics require compute LDS"); - } - requirements.shared_int64_atomics = true; - } - if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh && - kind == IR::ResourceKind::Lds) { - requirements.function_lds = true; - } - if (shared_access == IR::SharedAccess::Append || - shared_access == IR::SharedAccess::Consume) { - requirements.subgroup_ballot = true; - requirements.subgroup_shuffle = true; - requirements.subgroup_local_invocation_id = true; - } - } - switch (inst.GetOpcode()) { - case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break; - case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break; - case IR::ValueOpcode::ConditionRef: - requirements.subgroup_ballot |= - inst.Flags() != CFG::BranchCondition::ScalarInstruction; - break; - case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break; - case IR::ValueOpcode::DppMoveU32: - case IR::ValueOpcode::ReadFirstLane: - case IR::ValueOpcode::ReadLane: { - requirements.subgroup_ballot = true; - requirements.subgroup_shuffle = true; - if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) { - requirements.subgroup_local_invocation_id = true; - } - break; - } - case IR::ValueOpcode::DppUpdateU32: - case IR::ValueOpcode::WriteLane: { - requirements.subgroup_ballot = true; - requirements.subgroup_local_invocation_id = true; - break; - } - case IR::ValueOpcode::Permlane16U32: { - requirements.subgroup_ballot = true; - requirements.subgroup_shuffle = true; - requirements.subgroup_local_invocation_id = true; - break; - } - case IR::ValueOpcode::SwizzleU32: - case IR::ValueOpcode::PermuteU32: - case IR::ValueOpcode::BpermuteU32: { - requirements.subgroup_ballot = true; - requirements.subgroup_shuffle = true; - requirements.subgroup_local_invocation_id = true; - break; - } - case IR::ValueOpcode::LaneId: - requirements.subgroup_local_invocation_id |= - program.stage != ShaderType::TessellationControl; - break; - case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break; - case IR::ValueOpcode::ImageGatherRaw: - requirements.image_gather_extended = true; - break; - case IR::ValueOpcode::SetAttribute: { - const auto index = inst.Flags().index; - if (index >= program.export_info.size()) { - Fail(program, "attribute export has invalid metadata"); - } - if (program.stage == ShaderType::Pixel && - program.export_info[index].vm) { - requirements.pixel_valid_mask = true; - } - break; - } - default: break; - } - } - } - return requirements; -} - -std::vector EmitProgram(const IR::Program& program, - ShaderStageInputInfo input_info) { +std::vector EmitProgram(IR::Program& program, ShaderStageInputInfo input_info, + uint32_t push_data_start_dword) { using namespace Emitter; if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && @@ -352,12 +207,14 @@ std::vector EmitProgram(const IR::Program& program, Fail(program, "binary SPIR-V emitter received an unsupported shader stage"); } if (!program.srt_plan_complete || !program.resource_tracking_complete || - !program.shader_info_complete || !program.binding_layout_complete) { + !program.shader_info_complete) { Fail(program, "SPIR-V emitter requires a fully planned native shader program"); } - ValidateNativeProgram(program, program.stage == ShaderType::Compute && - input_info.compute != nullptr && input_info.compute->lds_storage); + if (program.info.buffers.size() > IR::ShaderInfo::MaxBuffers) { + Fail(program, "native shader exceeds the buffer resource limit"); + } IR::ValidateProgram(program, true); + program.bindings = {.push_data_start_dword = push_data_start_dword}; EmitterState state(program, input_info); const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info); state.lane_count = @@ -365,6 +222,8 @@ std::vector EmitProgram(const IR::Program& program, ? 2u : 1u; DefineModule(state); + ValidateNativeProgram(program, program.stage == ShaderType::Compute && + input_info.compute != nullptr && input_info.compute->lds_storage); EmitProgram(state); state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func, "main", state.interface_variables); diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h index 1f6adb3db..0626148e7 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h @@ -9,9 +9,9 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv { -// Resets and writes IR backend IDs; do not emit the same program concurrently. -std::vector EmitProgram(const IR::Program& program, - ShaderStageInputInfo input_info); +// Resets and writes backend IDs and descriptor indices; do not emit the same program concurrently. +std::vector EmitProgram(IR::Program& program, ShaderStageInputInfo input_info, + uint32_t push_data_start_dword = 0); } // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 15af1ee60..0a39f9b3c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -59,21 +59,6 @@ uint32_t ConstantU32(EmitterState& state, uint32_t value); std::abort(); } -uint32_t ResourceForDescriptor(const EmitterState& state, IR::DescriptorBindingKind kind, - uint32_t resource) { - const auto* descriptor = IR::FindBinding(state.program.bindings, kind); - if (descriptor == nullptr) { - ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated"); - } - const auto found = - std::find(descriptor->resources.begin(), descriptor->resources.end(), resource); - if (found == descriptor->resources.end()) { - ExitDescriptorBindingFailure(state, kind, resource, - "resource is absent from descriptor group"); - } - return static_cast(found - descriptor->resources.begin()); -} - uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type, uint32_t variable_id, uint32_t array_index, IR::DescriptorBindingKind kind, uint32_t resource, @@ -162,13 +147,15 @@ uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mi } uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) { - const auto array_index = - ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler); + if (sampler >= state.program.info.samplers.size()) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Samplers, sampler, + "sampler resource index is out of range"); + } const auto sampler_type = state.builder.Type(spv::OpTypeSampler); const auto pointer_type = state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type); const auto pointer = DescriptorElementPointer( - state, pointer_type, state.sampler_variable, ConstantU32(state, array_index), + state, pointer_type, state.sampler_variable, ConstantU32(state, sampler), IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); const auto sampler_id = state.builder.AllocateId(); state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer); @@ -200,7 +187,7 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t const auto kind = IR::DescriptorBindingForImage(image); EXIT_IF(!kind.has_value()); if (array_index == 0u) { - array_index = ConstantU32(state, ResourceForDescriptor(state, *kind, resource) + mip); + array_index = ConstantU32(state, image.descriptor_index + mip); } const auto pointer_type = state.builder.Type( spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image)); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp index aab43a23a..bdc49c800 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp @@ -223,7 +223,7 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half, } // namespace void DefineBvhIntersect(EmitterState& s) { - if (!s.requirements.bvh) return; + if (!s.program.info.bvh) return; const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s); const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3); const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp index 26fb09d81..f53790ea4 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp @@ -10,7 +10,7 @@ namespace { bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) { const auto register_index = IR::RegIndex(reg); - const auto& registers = state.program.bindings.user_data_registers; + const auto& registers = state.program.info.user_data_registers; const auto found = std::lower_bound(registers.begin(), registers.end(), register_index); if (found == registers.end() || *found != register_index) { return false; @@ -472,7 +472,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto& exp = ctx.Export(inst); const auto exec = ctx.Arg(inst, 1); - if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask && + if (state.program.stage == ShaderType::Pixel && exp.vm && state.program.info.pixel_valid_mask && state.pixel_valid_mask_variable != 0) { const auto value = state.builder.AllocateId(); state.builder.AddFunction(spv::OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index d140a1331..c04c8cae1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -867,7 +867,6 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { const auto resource = image.indirect_resources[ordinal]; const auto& candidate = state.program.info.images[resource]; const auto kind = *IR::DescriptorBindingForImage(candidate); - const auto& binding = *IR::FindBinding(state.program.bindings, kind); if (!runs.empty()) { auto& run = runs.back(); const auto& first = state.program.info.images[run.resource]; @@ -875,17 +874,16 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { if (candidate.dimension == first.dimension && candidate.cube == first.cube && IR::DescriptorBindingForImage(first) == kind && candidate.mip_count == 1u && first.mip_count == 1u && - (ordinal == 1u || (next_slot < binding.resources.size() && - binding.resources[next_slot] == resource))) { + (ordinal == 1u || candidate.descriptor_index == next_slot)) { // Native roots precede appended children; only the root may have a slot gap. if (ordinal == 1u) - run.slot_bias = ResourceForDescriptor(state, kind, resource) - ordinal; + run.slot_bias = candidate.descriptor_index - ordinal; ++run.count; continue; } } runs.push_back({ordinal, 1u, resource, - ResourceForDescriptor(state, kind, resource) - ordinal}); + candidate.descriptor_index - ordinal}); } const auto EmitRun = [&](uint32_t index) { const auto& run = runs[index]; @@ -893,8 +891,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { auto slot = Binary(state, spv::OpIAdd, TypeU32(state), selected, ConstantU32(state, run.slot_bias)); if (run.first == 0u) { - const auto kind = *IR::DescriptorBindingForImage(image); - const auto root_slot = ResourceForDescriptor(state, kind, mem.resource); + const auto root_slot = image.descriptor_index; if (root_slot != run.slot_bias) { const auto is_root = Binary(state, spv::OpIEqual, TypeBool(state), selected, ConstantU32(state, 0u)); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 6c1f80352..2f5df6ebd 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -6,7 +6,6 @@ #include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include @@ -58,42 +57,21 @@ constexpr std::array ImageDimensions {{ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); -struct SpirvRequirements { - bool bvh = false; - bool subgroup_ballot = false; - bool subgroup_barrier = false; - bool subgroup_shuffle = false; - bool subgroup_local_invocation_id = false; - bool compute_derivatives = false; - bool image_gather_extended = false; - bool function_lds = false; - bool function_scratch = false; - bool pixel_valid_mask = false; - bool buffer_int64_atomics = false; - bool buffer_u8 = false; - bool buffer_u16 = false; - bool shared_int64_atomics = false; - bool coherent_buffers = false; - bool float64 = false; -}; - -SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program); - struct EmitterState { - EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_) + EmitterState(IR::Program& program_, ShaderStageInputInfo input_info_) : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u), - program(program_), input_info(input_info_), - requirements(AnalyzeProgramRequirements(program_)) { + program(program_), input_info(input_info_) { if (ShaderWorkgroupInput(program.stage, input_info) != nullptr) { lds_storage_class = spv::StorageClassWorkgroup; } - if (IR::FindBinding(program.bindings, IR::DescriptorBindingKind::SharedMemory) != nullptr) { + if (program.info.uses_lds && program.stage == ShaderType::Compute && + input_info.compute != nullptr && input_info.compute->lds_storage) { lds_storage_class = spv::StorageClassStorageBuffer; } } Builder builder; - const IR::Program& program; + IR::Program& program; ShaderStageInputInfo input_info; uint32_t void_type = 0; uint32_t bool_type = 0; @@ -113,7 +91,6 @@ struct EmitterState { uint32_t tess_inner_variable = 0; uint32_t tess_patch_base = 0; - const SpirvRequirements requirements; uint32_t lane_count = 1; uint32_t lane_half = 0; uint32_t storage_buffer_variable = 0; @@ -313,9 +290,6 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state); IR::DescriptorBindingKind kind, uint32_t resource, const char* reason); -uint32_t ResourceForDescriptor(const EmitterState& state, IR::DescriptorBindingKind kind, - uint32_t resource); - uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type, uint32_t variable_id, uint32_t array_index, IR::DescriptorBindingKind kind, uint32_t resource, @@ -410,7 +384,6 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index); uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem); Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem); -uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem); void EmitMemoryOffsets(EmitterState& state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 5f0b4bb51..ed742a57e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -251,8 +251,7 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3 uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { if (mem.kind == IR::ResourceKind::Buffer) { - const auto resource = ResourceForDescriptor(ctx.state, IR::DescriptorBindingKind::Buffers, - mem.resource); + const auto resource = ctx.state.program.info.buffers.at(mem.resource).descriptor_index; const auto prefix = ctx.state.memory_byte_offsets[resource]; const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), BufferByteAddress(ctx, inst, mem), prefix); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index e6d9e1222..60d1292e2 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -102,17 +102,9 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR:: return state.program.info.buffers[mem.resource].descriptor_format; } -uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) { - if (!mem.formatted) return mem.data_bits; - const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format) - : program.info.buffers[mem.resource].descriptor_format; - const auto info = Format::GetFormatInfo(format); - return info.type == Format::ComponentType::Unknown || info.packed_bitfield - ? 32u : info.component_bits[0]; -} - void EmitMemoryOffsets(EmitterState& state) { - for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) { + const auto count = state.program.bindings.descriptor_counts[static_cast(IR::DescriptorBindingKind::Buffers)]; + for (uint32_t i = 0; i < count; i++) { const auto word = EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u); const auto shift = ConstantU32(state, (i % 4u) * 8u); @@ -146,7 +138,7 @@ void EnsureLdsStorage(EmitterState& state) { state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); return variable; }; - if (state.requirements.shared_int64_atomics) { + if (state.program.info.shared_int64_atomics) { state.lds_variable = define(TypeU32(state), 4u); state.lds_u64_variable = define(TypeU64(state), 8u); state.builder.AddName(state.lds_u64_variable, "lds_qwords"); @@ -167,7 +159,7 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, "storage buffer descriptor array was not emitted"); } const auto array_index = - ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); + state.program.info.buffers.at(mem.resource).descriptor_index; MemoryResourceAccess access { .kind = mem.kind, .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone}; @@ -214,7 +206,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: case IR::ResourceKind::ScalarBuffer: case IR::ResourceKind::Buffer: { const auto bits = mem.kind == IR::ResourceKind::Buffer - ? StorageBufferElementBits(state.program, mem) : 32u; + ? IR::StorageBufferElementBits(state.program, mem) : 32u; const auto variable = bits == 8u ? state.storage_buffer_u8_variable : bits == 16u ? state.storage_buffer_u16_variable : state.storage_buffer_variable; @@ -245,7 +237,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces TypeStorageBufferElementPointer(state, 32)); } const auto pointer = state.builder.AllocateId(); - if (access.kind == IR::ResourceKind::Lds && state.requirements.shared_int64_atomics) { + if (access.kind == IR::ResourceKind::Lds && state.program.info.shared_int64_atomics) { state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class), pointer, access.object_pointer, ConstantU32(state, 0), index); return pointer; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 05fb01277..e30044c97 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -200,6 +200,43 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) { } void DefineDescriptors(EmitterState& state) { + using Kind = IR::DescriptorBindingKind; + auto& info = state.program.info; + auto& bindings = state.program.bindings; + auto& counts = bindings.descriptor_counts; + auto& buffer_count = counts[static_cast(Kind::Buffers)]; + for (uint32_t index = 0; index < info.buffers.size(); ++index) { + info.buffers[index].descriptor_index = (info.live_buffers & (uint64_t{1} << index)) != 0 + ? buffer_count++ : UINT32_MAX; + } + std::array image_types {}; + for (auto& image: info.images) { + const auto kind = IR::DescriptorBindingForImage(image); + if (!kind.has_value() || image.mip_count == 0u || + (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) { + EXIT("shader image has an invalid descriptor type or mip count"); + } + auto& count = counts[static_cast(*kind)]; + image.descriptor_index = count; + if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = ℑ + count += image.mip_count; + } + counts[static_cast(Kind::Samplers)] = static_cast(info.samplers.size()); + counts[static_cast(Kind::Gds)] = info.uses_gds; + counts[static_cast(Kind::SharedMemory)] = + info.uses_lds && state.lds_storage_class == spv::StorageClassStorageBuffer; + counts[static_cast(Kind::BdaPagetable)] = info.uses_dma; + counts[static_cast(Kind::FaultBuffer)] = info.uses_dma; + counts[static_cast(Kind::FlattenedSrt)] = info.uses_flattened_srt; + bindings.memory_offset_dword = static_cast(info.user_data_registers.size()); + if (info.uses_dispatch_threads) { + bindings.dispatch_thread_dword = bindings.memory_offset_dword; + bindings.memory_offset_dword += 3u; + } + bindings.push_data_start_dword = + IR::PushData::StartFor(bindings.push_data_start_dword, bindings.ShaderDataDwords()); + counts[static_cast(Kind::ShaderData)] = + bindings.ShaderDataDwords() != 0 && !bindings.UsesPushData(); if (state.program.bindings.UsesPushData() || state.program.stage == ShaderType::Mesh) { const auto type = PushConstantBlockType(state); state.push_constant_variable = state.builder.DefineGlobalVariable( @@ -207,7 +244,10 @@ void DefineDescriptors(EmitterState& state) { state.builder.AddName(type, "BufferResource"); state.builder.AddName(state.push_constant_variable, "vsharp"); } - for (const auto& binding: state.program.bindings.descriptors) { + for (uint32_t index = 0; index < counts.size(); ++index) { + if (counts[index] == 0) continue; + bindings.descriptor_mask |= uint64_t{1} << index; + const auto kind = static_cast(index); const auto Define = [&](uint32_t type, const char* name, spv::StorageClass storage = spv::StorageClassStorageBuffer) { const auto variable = @@ -215,27 +255,27 @@ void DefineDescriptors(EmitterState& state) { state.builder.AddName(variable, name); state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0); state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, - IR::NativeBinding(state.program.stage, binding.kind)); + IR::NativeBinding(state.program.stage, kind)); return variable; }; const auto ArrayType = [&](uint32_t type) { return state.builder.Type( spv::OpTypeArray, type, - ConstantU32(state, static_cast(binding.resources.size()))); + ConstantU32(state, counts[index])); }; - switch (binding.kind) { + switch (kind) { case IR::DescriptorBindingKind::Buffers: state.storage_buffer_variable = Define(ArrayType(StorageBufferType(state)), "buffers"); - if (state.requirements.buffer_u8) { + if (state.program.info.buffer_u8) { state.storage_buffer_u8_variable = Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8"); } - if (state.requirements.buffer_u16) { + if (state.program.info.buffer_u16) { state.storage_buffer_u16_variable = Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16"); } - if (state.requirements.buffer_int64_atomics) { + if (state.program.info.buffer_int64_atomics) { state.storage_buffer_u64_variable = Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64"); } @@ -248,7 +288,7 @@ void DefineDescriptors(EmitterState& state) { } // RDNA2 stores publish to L2 even without GLC; every alias of the buffer // must participate in visibility for cache-bypassing polling loads. - if (state.requirements.coherent_buffers) { + if (state.program.info.coherent_buffers) { state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent); } } @@ -276,7 +316,7 @@ void DefineDescriptors(EmitterState& state) { case IR::DescriptorBindingKind::SharedMemory: state.lds_variable = Define(StorageBufferType(state), "lds_dwords"); state.builder.AddAnnotation(spv::OpDecorate, state.lds_variable, spv::DecorationCoherent); - if (state.requirements.shared_int64_atomics) { + if (state.program.info.shared_int64_atomics) { state.lds_u64_variable = Define(StorageBufferType(state, 64), "lds_qwords"); state.builder.AddAnnotation(spv::OpDecorate, state.lds_u64_variable, spv::DecorationCoherent); for (const auto variable: {state.lds_variable, state.lds_u64_variable}) { @@ -285,11 +325,11 @@ void DefineDescriptors(EmitterState& state) { } break; default: { - EXIT_IF(IR::ImageBindingResourceClass(binding.kind) == + EXIT_IF(IR::ImageBindingResourceClass(kind) == IR::ImageResourceClass::None); - const auto& image = state.program.info.images.at(binding.resources.front()); - const auto name = "image_" + std::to_string(static_cast(binding.kind)); - state.image_variables[IR::ImageBindingIndex(binding.kind)] = + const auto& image = *image_types[IR::ImageBindingIndex(kind)]; + const auto name = "image_" + std::to_string(static_cast(kind)); + state.image_variables[IR::ImageBindingIndex(kind)] = Define(ArrayType(ImageType(state, image)), name.c_str(), spv::StorageClassUniformConstant); if (image.dimension == ImageDimension::Dim1D || @@ -437,7 +477,7 @@ void DefineInputs(EmitterState& state) { state.inputs.push_back({{kind, 0, components, name}}); } }; - if (IR::FindBinding(state.program.bindings, IR::DescriptorBindingKind::SharedMemory) != nullptr) { + if (state.program.info.uses_lds && state.lds_storage_class == spv::StorageClassStorageBuffer) { add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID"); add_builtin(IR::StageInputKind::NumWorkgroups, 3, "gl_NumWorkGroups"); } @@ -524,7 +564,7 @@ void DefineInputs(EmitterState& state) { builtin); } } - if (state.requirements.subgroup_local_invocation_id) { + if (state.program.info.subgroup_local_invocation_id) { const auto variable = DefineInterfaceVariable(state, TypeU32(state), spv::StorageClassInput, "gl_SubgroupInvocationID"); state.subgroup_local_invocation_id_variable = variable; @@ -647,10 +687,10 @@ void DefineModule(EmitterState& state) { DefineOutputs(state); DefineTessellationInterfaces(state); DefineDescriptors(state); - if (state.requirements.function_lds) { + if (state.program.info.function_lds) { state.lds_variable = state.builder.AllocateId(); } - if (state.requirements.function_scratch) { + if (state.program.info.function_scratch) { for (uint32_t half = 0; half < state.lane_count; half++) { state.scratch_variable[half] = state.builder.AllocateId(); } @@ -675,11 +715,11 @@ void DefineModule(EmitterState& state) { state.builder.RequireCapability(spv::CapabilityShader); state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve); - if (state.requirements.buffer_u8) { + if (state.program.info.buffer_u8) { state.builder.RequireExtension("SPV_KHR_8bit_storage"); state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess); } - if (state.requirements.buffer_u16) { + if (state.program.info.buffer_u16) { state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess); } if (state.program.info.uses_dma) { @@ -687,11 +727,11 @@ void DefineModule(EmitterState& state) { state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses); state.builder.RequireExtension("SPV_KHR_physical_storage_buffer"); } - if (state.requirements.buffer_int64_atomics || state.requirements.shared_int64_atomics) { + if (state.program.info.buffer_int64_atomics || state.program.info.shared_int64_atomics) { state.builder.RequireCapability(spv::CapabilityInt64); state.builder.RequireCapability(spv::CapabilityInt64Atomics); } - if (state.requirements.shared_int64_atomics && + if (state.program.info.shared_int64_atomics && state.lds_storage_class == spv::StorageClassWorkgroup) { state.builder.RequireVersion(0x00010400u); state.builder.RequireExtension("SPV_KHR_workgroup_memory_explicit_layout"); @@ -714,21 +754,21 @@ void DefineModule(EmitterState& state) { if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) { state.builder.RequireCapability(spv::CapabilitySampleRateShading); } - if (state.requirements.image_gather_extended) { + if (state.program.info.image_gather_extended) { state.builder.RequireCapability(spv::CapabilityImageGatherExtended); } - if (state.lane_count == 2 || state.requirements.subgroup_barrier || - state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle || - state.requirements.subgroup_local_invocation_id) { + if (state.lane_count == 2 || state.program.info.subgroup_barrier || + state.program.info.subgroup_ballot || state.program.info.subgroup_shuffle || + state.program.info.subgroup_local_invocation_id) { state.builder.RequireCapability(spv::CapabilityGroupNonUniform); } - if (state.lane_count == 2 || state.requirements.subgroup_ballot) { + if (state.lane_count == 2 || state.program.info.subgroup_ballot) { state.builder.RequireCapability(spv::CapabilityGroupNonUniformBallot); } - if (state.requirements.subgroup_shuffle) { + if (state.program.info.subgroup_shuffle) { state.builder.RequireCapability(spv::CapabilityGroupNonUniformShuffle); } - if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) { + if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) { state.builder.RequireCapability(spv::CapabilityComputeDerivativeGroupQuadsKHR); state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives"); } @@ -751,7 +791,7 @@ void DefineModule(EmitterState& state) { // contract prevents host compilers from treating synthesized IEEE values as finite. state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve, 32u); - if (state.requirements.float64) { + if (state.program.info.float64) { EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute && state.input_info.compute->float_mode != 0xc0); // Use native rounding for MODE=0xc0, consistent with ordinary FP32 arithmetic. @@ -763,8 +803,8 @@ void DefineModule(EmitterState& state) { // state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u); } if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) { - uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u; - uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u; + uint32_t local_x = state.program.info.compute_derivatives ? 2u : 1u; + uint32_t local_y = state.program.info.compute_derivatives ? 2u : 1u; uint32_t local_z = 1u; local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x; local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y; @@ -772,7 +812,7 @@ void DefineModule(EmitterState& state) { if (state.lane_count == 2) { local_x = ((local_x * local_y * local_z + 63u) / 64u) * 32u; local_y = local_z = 1u; - if (state.requirements.compute_derivatives) { + if (state.program.info.compute_derivatives) { local_y = local_x / 2u; local_x = 2u; } @@ -791,14 +831,14 @@ void DefineModule(EmitterState& state) { state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeEarlyFragmentTests); } } - if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) { + if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) { state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDerivativeGroupQuadsKHR); } state.builder.AddName(state.main_func, "main"); - if (state.requirements.function_lds) { + if (state.program.info.function_lds) { state.builder.AddName(state.lds_variable, "lds_dwords"); } - if (state.requirements.function_scratch) { + if (state.program.info.function_scratch) { for (uint32_t half = 0; half < state.lane_count; half++) { state.builder.AddName(state.scratch_variable[half], "scratch_dwords"); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index c4c99a21c..ca10e541e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -618,7 +618,7 @@ void EmitProgram(EmitterState& state) { high.half = 1; } std::optional dispatcher; - if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) { + if (state.program.stage == ShaderType::Pixel && state.program.info.pixel_valid_mask) { state.pixel_valid_mask_variable = state.builder.AllocateId(); state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active"); } @@ -686,28 +686,23 @@ void EmitProgram(EmitterState& state) { } DefineGetBdaPointer(state); DefineBvhIntersect(state); - for (const auto* block: program.blocks) { - if (std::ranges::any_of(*block, [](const IR::Inst& inst) { - return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32; - })) { - ctx.scratch_u32_variable = state.builder.AllocateId(); - if (state.lane_count == 2) { - high.scratch_u32_variable = state.builder.AllocateId(); - } - break; + if (program.info.uses_swizzle) { + ctx.scratch_u32_variable = state.builder.AllocateId(); + if (state.lane_count == 2) { + high.scratch_u32_variable = state.builder.AllocateId(); } } state.builder.AddFunction(spv::OpFunction, TypeVoid(state), state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func, spv::FunctionControlMaskNone, TypeFunction(state)); EmitLabel(state, state.entry_label); - if (state.requirements.function_lds) { + if (state.program.info.function_lds) { state.builder.AddFunction( spv::OpVariable, TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)), state.lds_variable, spv::StorageClassFunction); } - if (state.requirements.function_scratch) { + if (state.program.info.function_scratch) { for (uint32_t half = 0; half < state.lane_count; half++) { state.builder.AddFunction( spv::OpVariable, diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index 647a4d9d8..0cdb4bba6 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -1,5 +1,6 @@ #include "common/assert.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/BufferFormat.h" #include #include @@ -202,6 +203,16 @@ Value ResolveActiveU32(Value value, Value active) { return {}; } +uint32_t StorageBufferElementBits(const Program& program, const MemoryInfo& memory) { + if (!memory.formatted) return memory.data_bits; + const auto format = memory.typed + ? Format::DecodeTBufferFormat(memory.data_format, memory.number_format) + : program.info.buffers[memory.resource].descriptor_format; + const auto info = Format::GetFormatInfo(format); + return info.type == Format::ComponentType::Unknown || info.packed_bitfield + ? 32u : info.component_bits[0]; +} + bool HasShaderMemoryWrites(const Program& program) { for (const auto* block: program.blocks) { for (const auto& inst: *block) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 1c6c2828e..cdde351c5 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -104,6 +104,7 @@ struct BufferResource { static constexpr uint32_t NoIndirectBuffer = UINT32_MAX; uint32_t source = 0; + uint32_t descriptor_index = UINT32_MAX; uint32_t first_use_pc = 0; uint32_t max_byte_extent = 0; uint32_t packed_stride = 0; @@ -131,6 +132,7 @@ struct ImageResource { static constexpr uint32_t NoIndirectImage = UINT32_MAX; uint32_t source = 0; + uint32_t descriptor_index = UINT32_MAX; uint32_t first_use_pc = 0; ImageResourceClass resource_class = ImageResourceClass::None; Prospero::TextureNumericClass numeric_class = Prospero::TextureNumericClass::Unsupported; @@ -424,23 +426,16 @@ DescriptorBindingForImage(const ImageResource& image) { return static_cast(base + dimension); } -struct DescriptorBinding { - DescriptorBindingKind kind = DescriptorBindingKind::Buffers; - std::vector resources; - - bool operator==(const DescriptorBinding& other) const = default; -}; - -struct BindingLayout { - uint32_t push_data_start_dword = PushData::NoStart; - uint32_t dispatch_thread_dword = PushData::NoStart; - uint32_t memory_offset_dword = 0; - uint32_t memory_offset_count = 0; - std::vector user_data_registers; - std::vector descriptors; +struct Bindings { + std::array(DescriptorBindingKind::Count)> descriptor_counts {}; + uint64_t descriptor_mask = 0; + uint32_t push_data_start_dword = PushData::NoStart; + uint32_t dispatch_thread_dword = PushData::NoStart; + uint32_t memory_offset_dword = 0; [[nodiscard]] uint32_t ShaderDataDwords() const { - return memory_offset_dword + (memory_offset_count + 3u) / 4u; + return memory_offset_dword + + (descriptor_counts[static_cast(DescriptorBindingKind::Buffers)] + 3u) / 4u; } [[nodiscard]] bool UsesPushData() const { return push_data_start_dword != PushData::NoStart; @@ -451,7 +446,7 @@ struct BindingLayout { } } - bool operator==(const BindingLayout& other) const = default; + bool operator==(const Bindings& other) const = default; }; struct ShaderInfo { @@ -466,11 +461,34 @@ struct ShaderInfo { std::vector sampled_pairs; std::vector inputs; std::vector outputs; + std::vector user_data_registers; + uint64_t live_buffers = 0; std::array vertex_fetch_components {}; int32_t vertex_offset_sgpr = -1; int32_t instance_offset_sgpr = -1; - bool has_bitwise_xor = false; - bool uses_dma = false; + bool has_bitwise_xor = false; + bool uses_dma = false; + bool uses_swizzle = false; + bool uses_lds = false; + bool uses_gds = false; + bool uses_flattened_srt = false; + bool uses_dispatch_threads = false; + bool bvh = false; + bool subgroup_ballot = false; + bool subgroup_barrier = false; + bool subgroup_shuffle = false; + bool subgroup_local_invocation_id = false; + bool compute_derivatives = false; + bool image_gather_extended = false; + bool function_lds = false; + bool function_scratch = false; + bool pixel_valid_mask = false; + bool buffer_int64_atomics = false; + bool buffer_u8 = false; + bool buffer_u16 = false; + bool shared_int64_atomics = false; + bool coherent_buffers = false; + bool float64 = false; bool operator==(const ShaderInfo& other) const = default; }; @@ -542,7 +560,7 @@ struct CompiledShaderInfo { uint32_t param_export_mask = 0; bool has_address_writes = false; ShaderInfo info; - BindingLayout bindings; + Bindings bindings; }; struct UniformFillPlan { @@ -622,11 +640,12 @@ struct Program: ResourcePlan { std::vector export_info; bool has_address_writes = false; bool shader_info_complete = false; - BindingLayout bindings; - bool binding_layout_complete = false; + Bindings bindings; }; +uint32_t StorageBufferElementBits(const Program& program, const MemoryInfo& memory); + std::string ProgramToString(const Program& program); bool HasShaderMemoryWrites(const Program& program); diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp deleted file mode 100644 index 226d5260a..000000000 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ /dev/null @@ -1,197 +0,0 @@ -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" - -#include "common/assert.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -[[noreturn]] void BindingFail(const char* message) { - EXIT("shader binding layout failed: %s", message); - std::abort(); -} - -void CollectShaderData(const Program& program, BindingLayout& layout) { - std::array registers {}; - bool uses_dispatch_threads = false; - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - if (!inst.HasUses()) { - continue; - } - uses_dispatch_threads |= inst.GetOpcode() == ValueOpcode::GetDispatchThreadExtent; - if (inst.GetOpcode() != ValueOpcode::GetUserData) { - continue; - } - if (inst.Arg(0).GetType() != Type::ScalarReg) { - BindingFail("typed shader contains an invalid user-data register"); - } - const auto index = RegIndex(inst.Arg(0).ScalarRegister()); - if (index >= NumScalarRegs) { - BindingFail("typed shader contains an invalid user-data register"); - } - registers[index] = true; - } - } - for (uint32_t index = 0; index < registers.size(); index++) { - if (registers[index]) { - layout.user_data_registers.push_back(index); - } - } - layout.memory_offset_dword = static_cast(layout.user_data_registers.size()); - if (uses_dispatch_threads) { - layout.dispatch_thread_dword = layout.memory_offset_dword; - layout.memory_offset_dword += 3u; - } -} - -void AddBinding(BindingLayout& layout, DescriptorBindingKind kind, - std::vector resources = {}) { - layout.descriptors.push_back({kind, std::move(resources)}); -} - -} // namespace - -SharedMemoryResources CollectMemoryResources(const Program& program, std::vector& buffers) { - std::array live_buffers {}; - SharedMemoryResources shared; - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - const auto op = inst.GetOpcode(); - if (BufferAccessOf(op) == BufferAccess::None && - SharedAccessOf(op) == SharedAccess::None && - AddressOpcodeInfoOf(op).access == AddressAccess::None) { - continue; - } - const auto index = inst.Flags().index; - if (index >= program.memory_info.size()) { - BindingFail("typed shader contains invalid memory metadata"); - } - const auto& memory = program.memory_info[index]; - if (memory.planning_only) { - continue; - } - shared.lds |= memory.kind == ResourceKind::FlatLocal; - if (SharedAccessOf(op) != SharedAccess::None) { - if (memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) { - BindingFail("typed shader contains invalid shared-memory metadata"); - } - shared.gds |= memory.kind == ResourceKind::Gds; - shared.lds |= memory.kind == ResourceKind::Lds; - } else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) { - EXIT_IF(memory.resource >= program.info.buffers.size()); - live_buffers.at(memory.resource) = true; - for (const auto child: program.info.buffers[memory.resource].indirect_resources) { - live_buffers.at(child) = true; - } - } - } - } - for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - if (live_buffers.at(i)) { - buffers.push_back(i); - } - } - return shared; -} - -bool UsesFlattenedSrt(const Program& program) { - const auto uses_mapping = [](const auto& resource) { - return resource.indirect_root != UINT32_MAX; - }; - return std::ranges::any_of(program.blocks, [](const Block* block) { - return std::ranges::any_of(*block, [](const Inst& inst) { - return inst.GetOpcode() == ValueOpcode::ReadConst; - }); - }) || std::ranges::any_of(program.info.buffers, uses_mapping) || - std::ranges::any_of(program.info.images, uses_mapping); -} - -void AllocateBindings(Program& program, uint32_t push_data_start_dword, bool lds_storage) { - if (!program.shader_info_complete || program.binding_layout_complete) { - EXIT("shader binding layout failed: %s", !program.shader_info_complete - ? "shader info is not ready" - : "binding layout already allocated"); - } - BindingLayout next; - std::vector buffers; - const auto shared = CollectMemoryResources(program, buffers); - CollectShaderData(program, next); - next.memory_offset_count = static_cast(buffers.size()); - next.push_data_start_dword = - PushData::StartFor(push_data_start_dword, next.ShaderDataDwords()); - - if (!buffers.empty()) { - // Draw binding accesses this first group directly when memory_offset_count is nonzero. - AddBinding(next, DescriptorBindingKind::Buffers, std::move(buffers)); - } - - std::array, ImageBindingCount> image_groups; - for (uint32_t i = 0; i < program.info.images.size(); i++) { - const auto kind = DescriptorBindingForImage(program.info.images[i]); - if (!kind.has_value()) { - EXIT("shader binding layout failed: image %u has an invalid binding class", i); - } - const auto group = ImageBindingIndex(*kind); - if (group >= image_groups.size()) { - EXIT("shader binding layout failed: image %u has an unmapped binding class", i); - } - auto& resources = image_groups[group]; - const auto dynamic = program.info.images[i].mip_mode == ImageMipMode::Dynamic; - const auto count = dynamic ? program.info.images[i].mip_count : 1u; - if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) { - EXIT("shader binding layout failed: image %u has invalid specialized mip count %u", i, - program.info.images[i].mip_count); - } - resources.insert(resources.end(), count, i); - } - for (uint32_t i = 0; i < image_groups.size(); i++) { - if (!image_groups[i].empty()) { - AddBinding(next, static_cast(FirstImageBinding + i), - std::move(image_groups[i])); - } - } - - if (!program.info.samplers.empty()) { - std::vector resources(program.info.samplers.size()); - for (uint32_t i = 0; i < resources.size(); i++) { - resources[i] = i; - } - AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources)); - } - if (shared.gds) { - AddBinding(next, DescriptorBindingKind::Gds); - } - if (shared.lds && lds_storage) { - AddBinding(next, DescriptorBindingKind::SharedMemory); - } - if (program.info.uses_dma) { - AddBinding(next, DescriptorBindingKind::BdaPagetable); - AddBinding(next, DescriptorBindingKind::FaultBuffer); - } - if (UsesFlattenedSrt(program)) { - AddBinding(next, DescriptorBindingKind::FlattenedSrt); - } - - if (next.ShaderDataDwords() != 0 && !next.UsesPushData()) { - AddBinding(next, DescriptorBindingKind::ShaderData); - } - - program.bindings = std::move(next); - program.binding_layout_complete = true; -} - -const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind) { - for (const auto& binding: layout.descriptors) { - if (binding.kind == kind) { - return &binding; - } - } - return nullptr; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h deleted file mode 100644 index 566e2f043..000000000 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -namespace Libs::Graphics::ShaderRecompiler::IR { - -void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0, - bool lds_storage = false); - -struct SharedMemoryResources { - bool lds = false; - bool gds = false; -}; - -SharedMemoryResources CollectMemoryResources(const Program& program, std::vector& buffers); -bool UsesFlattenedSrt(const Program& program); - -const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind); - -} // namespace Libs::Graphics::ShaderRecompiler::IR - -#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ */ diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 467450992..d50f32de7 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -1229,8 +1229,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime } void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) { - EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete || - program.binding_layout_complete); + EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete); EXIT_IF(program.info.buffers.size() > specialization.buffers.size() || program.info.images.size() > specialization.images.size()); diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 9528c7753..29aecffe1 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -75,128 +75,11 @@ void ValidateOptions(const Program& program, ShaderStageInputInfo input_info) { } } -void ValidateValueReferences(const Program& program, ShaderStageInputInfo input_info) { - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - switch (inst.GetOpcode()) { - case ValueOpcode::GetAttribute: { - if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 || - !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) { - return Fail("typed attribute reference is not constant"); - } - if ((program.stage == ShaderType::Vertex || - program.stage == ShaderType::Local) && - (inst.Arg(1).U32() >= 4u || - inst.Arg(0).U32() >= - static_cast(input_info.vertex->resources_num))) { - return Fail("vertex input reference is out of range"); - } - if (program.stage == ShaderType::Pixel && - (inst.Arg(1).U32() >= 4u || - inst.Arg(0).U32() >= input_info.pixel->input_num)) { - return Fail("pixel input reference is out of range"); - } - break; - } - case ValueOpcode::GetInterpolationParameter: { - if (program.stage != ShaderType::Pixel || !inst.Arg(0).IsImmediate() || - inst.Arg(0).GetType() != Type::U32 || !inst.Arg(1).IsImmediate() || - inst.Arg(1).GetType() != Type::U32 || !inst.Arg(2).IsImmediate() || - inst.Arg(2).GetType() != Type::U32) { - return Fail("interpolation parameter reference is invalid"); - } - if (inst.Arg(0).U32() >= input_info.pixel->input_num || - inst.Arg(1).U32() >= 4u || inst.Arg(2).U32() >= 3u) { - return Fail("interpolation parameter reference is out of range"); - } - break; - } - case ValueOpcode::GetBuiltin: { - if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 || - !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) { - return Fail("typed builtin reference is not constant"); - } - const auto kind = static_cast(inst.Arg(0).U32()); - const auto component = inst.Arg(1).U32(); - switch (kind) { - case StageInputKind::PackedAncillary: - return Fail("packed pixel ancillary input has an unsupported live use"); - case StageInputKind::Layer: - case StageInputKind::SampleId: - if (program.stage != ShaderType::Pixel || component != 0u) { - return Fail("typed pixel scalar input is invalid"); - } - break; - case StageInputKind::VertexIndex: - case StageInputKind::InvocationId: - case StageInputKind::PrimitiveId: - case StageInputKind::InstanceIndex: - case StageInputKind::FrontFacing: - case StageInputKind::LocalInvocationIndex: - if (component != 0u) { - return Fail("typed scalar builtin component is out of range"); - } - break; - case StageInputKind::FragCoord: - if (component >= 4u) { - return Fail("typed fragment-coordinate component is out of range"); - } - break; - case StageInputKind::BaryCoordSmooth: - case StageInputKind::BaryCoordSmoothSample: - case StageInputKind::BaryCoordSmoothCentroid: - case StageInputKind::BaryCoordNoPerspective: - if (component >= 2u) { - return Fail("typed barycentric component is out of range"); - } - break; - case StageInputKind::TessCoord: - case StageInputKind::WorkgroupId: - case StageInputKind::LocalInvocationId: - case StageInputKind::GlobalInvocationId: - if (component >= 3u) { - return Fail("typed invocation builtin component is out of range"); - } - break; - case StageInputKind::Parameter: - default: return Fail("typed builtin kind is invalid"); - } - break; - } - case ValueOpcode::SetAttribute: { - const auto index = inst.Flags().index; - if (index >= program.export_info.size()) { - return Fail("typed export metadata index is out of range"); - } - const auto& exp = program.export_info[index]; - if (exp.kind == ExportTargetKind::Position && exp.index != 0 && exp.en != 0 && - program.stage != ShaderType::Vertex && program.stage != ShaderType::Mesh && - program.stage != ShaderType::TessellationEvaluation) { - return Fail("auxiliary position export requires a vertex, mesh, or " - "tessellation evaluation shader"); - } - break; - } - default: break; - } - } - } -} -void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex, - ShaderInfo& info) { +void CollectVertexInputs(const ShaderVertexInputInfo* vertex, ShaderInfo& info, + const std::array& used_components) { AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex"); AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex"); - uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {}; - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - if (inst.GetOpcode() == ValueOpcode::GetAttribute) { - const auto attr = inst.Arg(0).U32(); - const auto chan = inst.Arg(1).U32(); - used_components[attr] = std::max(used_components[attr], chan + 1u); - } - } - } for (uint32_t attr = 0; attr < static_cast(vertex->resources_num); attr++) { if (used_components[attr] != 0) { AddInput(info, StageInputKind::Parameter, attr, used_components[attr], @@ -205,28 +88,15 @@ void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* ve } } -void CollectPixelInputs(const Program& program, const ShaderPixelInputInfo* pixel, - ShaderInfo& info) { +void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info, + std::array& per_vertex, + const std::array& used_components) { if (pixel->HasPositionInput()) { AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord"); } if (pixel->ps_front_face) { AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing"); } - std::array per_vertex {}; - std::array interpolated {}; - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - if (inst.GetOpcode() == ValueOpcode::GetAttribute) { - interpolated[inst.Arg(0).U32()] = true; - } else if (inst.GetOpcode() == ValueOpcode::GetInterpolationParameter) { - const auto input = inst.Arg(0).U32(); - const auto mode = inst.Arg(2).U32(); - per_vertex[input] = - per_vertex[input] || mode < 2u || !ShaderPixelParameterIsFlat(*pixel, input); - } - } - } // Mixed interpolation modes share raw vertices at their guest export slot. // Rectangle expansion alone supplies separate flat and smooth outputs. for (uint32_t input = 0; input < pixel->input_num; input++) { @@ -245,7 +115,7 @@ void CollectPixelInputs(const Program& program, const ShaderPixelInputInfo* pixe per_vertex[input]); } for (uint32_t input = 0; input < pixel->input_num; input++) { - if (interpolated[input] && per_vertex[input] && + if (used_components[input] != 0 && per_vertex[input] && !ShaderPixelParameterIsFlat(*pixel, input)) { const auto kind = pixel->ps_no_perspective ? StageInputKind::BaryCoordNoPerspective : StageInputKind::BaryCoordSmooth; @@ -271,147 +141,411 @@ void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& inf } } -void CollectBuiltinInputs(const Program& program, ShaderInfo& info) { - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - if (program.stage == ShaderType::TessellationControl && - inst.GetOpcode() == ValueOpcode::LaneId) { - AddInput(info, StageInputKind::InvocationId, 0, 1, "gl_InvocationID"); - } - if (inst.GetOpcode() != ValueOpcode::GetBuiltin) { - continue; - } - const auto kind = static_cast(inst.Arg(0).U32()); - switch (kind) { - case StageInputKind::VertexIndex: - AddInput(info, kind, 0, 1, "gl_VertexIndex"); - break; - case StageInputKind::InstanceIndex: - AddInput(info, kind, 0, 1, "gl_InstanceIndex"); - break; - case StageInputKind::InvocationId: - AddInput(info, kind, 0, 1, "gl_InvocationID"); - break; - case StageInputKind::PrimitiveId: - AddInput(info, kind, 0, 1, "gl_PrimitiveID"); - break; - case StageInputKind::TessCoord: AddInput(info, kind, 0, 3, "gl_TessCoord"); break; - case StageInputKind::FragCoord: AddInput(info, kind, 0, 4, "gl_FragCoord"); break; - case StageInputKind::FrontFacing: - AddInput(info, kind, 0, 1, "gl_FrontFacing"); - break; - case StageInputKind::Layer: AddInput(info, kind, 0, 1, "gl_Layer"); break; - case StageInputKind::SampleId: AddInput(info, kind, 0, 1, "gl_SampleID"); break; - case StageInputKind::BaryCoordSmoothSample: - AddInput(info, StageInputKind::SampleId, 0, 1, "gl_SampleID"); - [[fallthrough]]; - case StageInputKind::BaryCoordSmooth: - case StageInputKind::BaryCoordSmoothCentroid: - AddInput(info, StageInputKind::BaryCoordSmooth, 0, 3, "gl_BaryCoordKHR"); - break; - case StageInputKind::BaryCoordNoPerspective: - AddInput(info, kind, 0, 3, "gl_BaryCoordNoPerspKHR"); - break; - case StageInputKind::WorkgroupId: - AddInput(info, kind, 0, 3, "gl_WorkGroupID"); - break; - case StageInputKind::LocalInvocationId: - AddInput(info, kind, 0, 3, "gl_LocalInvocationID"); - break; - case StageInputKind::LocalInvocationIndex: - AddInput(info, kind, 0, 1, "gl_LocalInvocationIndex"); - break; - case StageInputKind::GlobalInvocationId: - AddInput(info, kind, 0, 3, "gl_GlobalInvocationID"); - break; - case StageInputKind::PackedAncillary: - case StageInputKind::Parameter: break; - } - } +void AddBuiltinInput(ShaderInfo& info, StageInputKind kind) { + switch (kind) { + case StageInputKind::VertexIndex: + AddInput(info, kind, 0, 1, "gl_VertexIndex"); + break; + case StageInputKind::InstanceIndex: + AddInput(info, kind, 0, 1, "gl_InstanceIndex"); + break; + case StageInputKind::InvocationId: + AddInput(info, kind, 0, 1, "gl_InvocationID"); + break; + case StageInputKind::PrimitiveId: + AddInput(info, kind, 0, 1, "gl_PrimitiveID"); + break; + case StageInputKind::TessCoord: AddInput(info, kind, 0, 3, "gl_TessCoord"); break; + case StageInputKind::FragCoord: AddInput(info, kind, 0, 4, "gl_FragCoord"); break; + case StageInputKind::FrontFacing: + AddInput(info, kind, 0, 1, "gl_FrontFacing"); + break; + case StageInputKind::Layer: AddInput(info, kind, 0, 1, "gl_Layer"); break; + case StageInputKind::SampleId: AddInput(info, kind, 0, 1, "gl_SampleID"); break; + case StageInputKind::BaryCoordSmoothSample: + AddInput(info, StageInputKind::SampleId, 0, 1, "gl_SampleID"); + [[fallthrough]]; + case StageInputKind::BaryCoordSmooth: + case StageInputKind::BaryCoordSmoothCentroid: + AddInput(info, StageInputKind::BaryCoordSmooth, 0, 3, "gl_BaryCoordKHR"); + break; + case StageInputKind::BaryCoordNoPerspective: + AddInput(info, kind, 0, 3, "gl_BaryCoordNoPerspKHR"); + break; + case StageInputKind::WorkgroupId: + AddInput(info, kind, 0, 3, "gl_WorkGroupID"); + break; + case StageInputKind::LocalInvocationId: + AddInput(info, kind, 0, 3, "gl_LocalInvocationID"); + break; + case StageInputKind::LocalInvocationIndex: + AddInput(info, kind, 0, 1, "gl_LocalInvocationIndex"); + break; + case StageInputKind::GlobalInvocationId: + AddInput(info, kind, 0, 3, "gl_GlobalInvocationID"); + break; + case StageInputKind::PackedAncillary: + case StageInputKind::Parameter: break; } } -void CollectOutputs(const Program& program, ShaderStageInputInfo input_info, ShaderInfo& info) { +void CollectOutput(const Program& program, ShaderStageInputInfo input_info, + ShaderInfo& info, const Inst& inst) { const bool alpha_remap = program.stage == ShaderType::Pixel && input_info.pixel != nullptr && input_info.pixel->alpha_blend_source != ShaderAlphaBlendSource::None; - for (const auto* block: program.blocks) { - for (const auto& inst: *block) { - if (inst.GetOpcode() != ValueOpcode::SetAttribute) { - continue; + const auto& export_info = program.export_info[inst.Flags().index]; + if (export_info.kind == ExportTargetKind::MrtZ) { + if (program.stage == ShaderType::Pixel && (export_info.en & 0x1u) != 0 && + input_info.pixel->ps_depth_export_enable) { + AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth"); + } + if (program.stage == ShaderType::Pixel && (export_info.en & 0x4u) != 0 && + input_info.pixel->ps_sample_mask_export_enable) { + AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask"); + } + return; + } + if (export_info.en == 0) { + return; + } + switch (export_info.kind) { + case ExportTargetKind::Position: + if (export_info.index == 0) { + AddOutput(info, StageOutputKind::Position, 0, 0, "out_position"); + break; } - const auto& export_info = program.export_info[inst.Flags().index]; - if (export_info.kind == ExportTargetKind::MrtZ) { - if (program.stage == ShaderType::Pixel && (export_info.en & 0x1u) != 0 && - input_info.pixel->ps_depth_export_enable) { - AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth"); - } - if (program.stage == ShaderType::Pixel && (export_info.en & 0x4u) != 0 && - input_info.pixel->ps_sample_mask_export_enable) { - AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask"); - } - continue; + if (export_info.compr) { + return Fail("compressed auxiliary position export is unsupported"); } - if (export_info.en == 0) { - continue; - } - switch (export_info.kind) { - case ExportTargetKind::Position: - if (export_info.index == 0) { - AddOutput(info, StageOutputKind::Position, 0, 0, "out_position"); - break; - } - if (export_info.compr) { - return Fail("compressed auxiliary position export is unsupported"); - } - for (uint32_t component = 0; component < 4; component++) { - if ((export_info.en & (1u << component)) == 0) { - continue; - } - const auto output = DecodePositionExportComponent( - input_info.vertex->pa_cl_vs_out_cntl, export_info.index, component); - if (output.viewport) { - AddOutput(info, StageOutputKind::ViewportIndex, 0, 0, "gl_ViewportIndex"); - } - if (output.point_size) { - AddOutput(info, StageOutputKind::PointSize, 0, 0, "gl_PointSize"); - } - if (output.layer) { - AddOutput(info, StageOutputKind::Layer, 0, 0, "gl_Layer"); - } - if (output.clip_distance != UINT32_MAX) { - AddOutput(info, StageOutputKind::ClipDistance, output.clip_distance, 0, - "gl_ClipDistance"); - } - if (output.cull_distance != UINT32_MAX) { - AddOutput(info, StageOutputKind::CullDistance, output.cull_distance, 0, - "gl_CullDistance"); - } - } - break; - case ExportTargetKind::Parameter: - AddOutput(info, StageOutputKind::Parameter, export_info.index, - export_info.index, fmt::format("out_param_{}", export_info.index)); - break; - case ExportTargetKind::Mrt: { - if (alpha_remap && export_info.index != 0) { - break; - } - const auto slot = input_info.pixel->dual_source_blending - ? 0u : ShaderPixelExportTarget(input_info.pixel->target_shader_mask, - export_info.index); - if (slot >= 8) { - break; - } - AddOutput(info, StageOutputKind::Mrt, export_info.index, slot, - fmt::format("out_mrt_{}", export_info.index)); - if (alpha_remap) { - AddOutput(info, StageOutputKind::Mrt, 1, 0, "out_mrt_1"); - } - break; + for (uint32_t component = 0; component < 4; component++) { + if ((export_info.en & (1u << component)) == 0) { + continue; + } + const auto output = DecodePositionExportComponent( + input_info.vertex->pa_cl_vs_out_cntl, export_info.index, component); + if (output.viewport) { + AddOutput(info, StageOutputKind::ViewportIndex, 0, 0, "gl_ViewportIndex"); + } + if (output.point_size) { + AddOutput(info, StageOutputKind::PointSize, 0, 0, "gl_PointSize"); + } + if (output.layer) { + AddOutput(info, StageOutputKind::Layer, 0, 0, "gl_Layer"); + } + if (output.clip_distance != UINT32_MAX) { + AddOutput(info, StageOutputKind::ClipDistance, output.clip_distance, 0, + "gl_ClipDistance"); + } + if (output.cull_distance != UINT32_MAX) { + AddOutput(info, StageOutputKind::CullDistance, output.cull_distance, 0, + "gl_CullDistance"); + } + } + break; + case ExportTargetKind::Parameter: + AddOutput(info, StageOutputKind::Parameter, export_info.index, + export_info.index, fmt::format("out_param_{}", export_info.index)); + break; + case ExportTargetKind::Mrt: { + if (alpha_remap && export_info.index != 0) { + break; + } + const auto slot = input_info.pixel->dual_source_blending + ? 0u : ShaderPixelExportTarget(input_info.pixel->target_shader_mask, + export_info.index); + if (slot >= 8) { + break; + } + AddOutput(info, StageOutputKind::Mrt, export_info.index, slot, + fmt::format("out_mrt_{}", export_info.index)); + if (alpha_remap) { + AddOutput(info, StageOutputKind::Mrt, 1, 0, "out_mrt_1"); + } + break; + } + default: break; + } +} + +struct InputUsage { + std::array components {}; + std::array per_vertex {}; + std::array user_data {}; + std::array(StageInputKind::Parameter)> builtins {}; + uint32_t builtin_mask = 0; + uint32_t builtin_count = 0; + + void AddBuiltin(StageInputKind kind) { + const auto bit = 1u << static_cast(kind); + if ((builtin_mask & bit) == 0) { + builtin_mask |= bit; + builtins[builtin_count++] = kind; + } + } +}; + +void Visit(Program& program, ShaderStageInputInfo input_info, InputUsage& inputs, + const Inst& inst) { + auto& info = program.info; + const auto op = inst.GetOpcode(); + const auto buffer_access = BufferAccessOf(op); + const auto address_access = AddressOpcodeInfoOf(op).access; + const auto shared_access = SharedAccessOf(op); + info.float64 |= inst.GetType() == Type::F64; + info.buffer_int64_atomics |= buffer_access == BufferAccess::Atomic && + inst.GetType() == Type::U64; + if (buffer_access != BufferAccess::None || address_access != AddressAccess::None || + shared_access != SharedAccess::None) { + const auto index = inst.Flags().index; + if (index >= program.memory_info.size()) { + return Fail("typed shader contains invalid memory metadata"); + } + const auto& memory = program.memory_info[index]; + const auto kind = memory.kind; + if (!memory.planning_only) { + info.uses_lds |= kind == ResourceKind::FlatLocal; + if (shared_access != SharedAccess::None) { + info.uses_lds |= kind == ResourceKind::Lds; + info.uses_gds |= kind == ResourceKind::Gds; + } else if (kind == ResourceKind::Buffer || kind == ResourceKind::ScalarBuffer) { + if (memory.resource >= info.buffers.size() || memory.resource >= ShaderInfo::MaxBuffers) { + return Fail("buffer operation has invalid resource metadata"); + } + info.live_buffers |= uint64_t{1} << memory.resource; + for (const auto child: info.buffers[memory.resource].indirect_resources) { + if (child >= info.buffers.size()) { + return Fail("indirect buffer operation has invalid resource metadata"); + } + info.live_buffers |= uint64_t{1} << child; } - default: break; } } + if (address_access != AddressAccess::None) { + if (kind == ResourceKind::Scratch || kind == ResourceKind::FlatLocal) { + if (program.scratch_dwords == 0) { + return Fail("scratch operation has no per-thread storage"); + } + info.function_scratch = true; + info.function_lds |= kind == ResourceKind::FlatLocal && + program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh; + } else if (address_access == AddressAccess::Write) { + return Fail("writable FLAT/GLOBAL addresses require GPU ownership tracking"); + } else { + info.coherent_buffers |= memory.coherent; + } + } + if (buffer_access != BufferAccess::None) { + if (kind == ResourceKind::IndirectBuffer && op != ValueOpcode::ReadConstBuffer) { + info.subgroup_local_invocation_id = true; + } + if (kind == ResourceKind::Buffer) { + info.coherent_buffers |= memory.coherent; + if (memory.resource >= info.buffers.size()) { + return Fail("buffer operation has invalid resource metadata"); + } + const auto bits = StorageBufferElementBits(program, memory); + info.buffer_u8 |= bits == 8u; + info.buffer_u16 |= bits == 16u; + if ((info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { + if (program.stage != ShaderType::Compute) { + return Fail("buffer ADD_TID is only valid for compute shaders"); + } + info.subgroup_local_invocation_id = true; + } + } + } + if (shared_access != SharedAccess::None) { + if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) { + return Fail("shared operation has invalid resource kind"); + } + if (shared_access == SharedAccess::Atomic && SharedComponentCount(op) == 2u) { + if (kind != ResourceKind::Lds || program.stage != ShaderType::Compute) { + return Fail("64-bit shared atomics require compute LDS"); + } + info.shared_int64_atomics = true; + } + info.function_lds |= kind == ResourceKind::Lds && + program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh; + if (shared_access == SharedAccess::Append || shared_access == SharedAccess::Consume) { + info.subgroup_ballot = true; + info.subgroup_shuffle = true; + info.subgroup_local_invocation_id = true; + } + } + } + switch (inst.GetOpcode()) { + case ValueOpcode::GetAttribute: { + if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 || + !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) { + return Fail("typed attribute reference is not constant"); + } + if ((program.stage == ShaderType::Vertex || + program.stage == ShaderType::Local) && + (inst.Arg(1).U32() >= 4u || + inst.Arg(0).U32() >= + static_cast(input_info.vertex->resources_num))) { + return Fail("vertex input reference is out of range"); + } + if (program.stage == ShaderType::Pixel && + (inst.Arg(1).U32() >= 4u || + inst.Arg(0).U32() >= input_info.pixel->input_num)) { + return Fail("pixel input reference is out of range"); + } + if (program.stage == ShaderType::Vertex || program.stage == ShaderType::Local || + program.stage == ShaderType::Pixel) { + auto& components = inputs.components[inst.Arg(0).U32()]; + components = std::max(components, static_cast(inst.Arg(1).U32() + 1u)); + } + break; + } + case ValueOpcode::GetInterpolationParameter: { + if (program.stage != ShaderType::Pixel || !inst.Arg(0).IsImmediate() || + inst.Arg(0).GetType() != Type::U32 || !inst.Arg(1).IsImmediate() || + inst.Arg(1).GetType() != Type::U32 || !inst.Arg(2).IsImmediate() || + inst.Arg(2).GetType() != Type::U32) { + return Fail("interpolation parameter reference is invalid"); + } + if (inst.Arg(0).U32() >= input_info.pixel->input_num || + inst.Arg(1).U32() >= 4u || inst.Arg(2).U32() >= 3u) { + return Fail("interpolation parameter reference is out of range"); + } + const auto input = inst.Arg(0).U32(); + inputs.per_vertex[input] |= inst.Arg(2).U32() < 2u || + !ShaderPixelParameterIsFlat(*input_info.pixel, input); + break; + } + case ValueOpcode::GetBuiltin: { + if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 || + !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) { + return Fail("typed builtin reference is not constant"); + } + const auto kind = static_cast(inst.Arg(0).U32()); + const auto component = inst.Arg(1).U32(); + switch (kind) { + case StageInputKind::PackedAncillary: + return Fail("packed pixel ancillary input has an unsupported live use"); + case StageInputKind::Layer: + case StageInputKind::SampleId: + if (program.stage != ShaderType::Pixel || component != 0u) { + return Fail("typed pixel scalar input is invalid"); + } + break; + case StageInputKind::VertexIndex: + case StageInputKind::InvocationId: + case StageInputKind::PrimitiveId: + case StageInputKind::InstanceIndex: + case StageInputKind::FrontFacing: + case StageInputKind::LocalInvocationIndex: + if (component != 0u) { + return Fail("typed scalar builtin component is out of range"); + } + break; + case StageInputKind::FragCoord: + if (component >= 4u) { + return Fail("typed fragment-coordinate component is out of range"); + } + break; + case StageInputKind::BaryCoordSmooth: + case StageInputKind::BaryCoordSmoothSample: + case StageInputKind::BaryCoordSmoothCentroid: + case StageInputKind::BaryCoordNoPerspective: + if (component >= 2u) { + return Fail("typed barycentric component is out of range"); + } + break; + case StageInputKind::TessCoord: + case StageInputKind::WorkgroupId: + case StageInputKind::LocalInvocationId: + case StageInputKind::GlobalInvocationId: + if (component >= 3u) { + return Fail("typed invocation builtin component is out of range"); + } + break; + case StageInputKind::Parameter: + default: return Fail("typed builtin kind is invalid"); + } + inputs.AddBuiltin(kind); + break; + } + case ValueOpcode::SetAttribute: { + const auto index = inst.Flags().index; + if (index >= program.export_info.size()) { + return Fail("typed export metadata index is out of range"); + } + const auto& exp = program.export_info[index]; + if (exp.kind == ExportTargetKind::Position && exp.index != 0 && exp.en != 0 && + program.stage != ShaderType::Vertex && program.stage != ShaderType::Mesh && + program.stage != ShaderType::TessellationEvaluation) { + return Fail("auxiliary position export requires a vertex, mesh, or " + "tessellation evaluation shader"); + } + info.pixel_valid_mask |= program.stage == ShaderType::Pixel && exp.vm; + CollectOutput(program, input_info, info, inst); + break; + } + case ValueOpcode::GetUserData: { + if (!inst.HasUses()) break; + if (inst.Arg(0).GetType() != Type::ScalarReg) { + return Fail("typed shader contains an invalid user-data register"); + } + const auto index = RegIndex(inst.Arg(0).ScalarRegister()); + if (index >= NumScalarRegs) { + return Fail("typed shader contains an invalid user-data register"); + } + inputs.user_data[index] = true; + break; + } + case ValueOpcode::GetDispatchThreadExtent: + info.uses_dispatch_threads |= inst.HasUses(); + break; + case ValueOpcode::ReadConst: info.uses_flattened_srt = true; break; + case ValueOpcode::BitwiseXor32: info.has_bitwise_xor = true; break; + case ValueOpcode::StoreCompletion: info.subgroup_barrier = true; break; + case ValueOpcode::BvhIntersect: info.bvh = true; break; + case ValueOpcode::ConditionRef: + info.subgroup_ballot |= + inst.Flags() != CFG::BranchCondition::ScalarInstruction; + break; + case ValueOpcode::Ballot: info.subgroup_ballot = true; break; + case ValueOpcode::DppMoveU32: + case ValueOpcode::ReadFirstLane: + case ValueOpcode::ReadLane: { + info.subgroup_ballot = true; + info.subgroup_shuffle = true; + if (inst.GetOpcode() == ValueOpcode::DppMoveU32) { + info.subgroup_local_invocation_id = true; + } + break; + } + case ValueOpcode::DppUpdateU32: + case ValueOpcode::WriteLane: { + info.subgroup_ballot = true; + info.subgroup_local_invocation_id = true; + break; + } + case ValueOpcode::SwizzleU32: + info.uses_swizzle = true; + [[fallthrough]]; + case ValueOpcode::Permlane16U32: + case ValueOpcode::PermuteU32: + case ValueOpcode::BpermuteU32: { + info.subgroup_ballot = true; + info.subgroup_shuffle = true; + info.subgroup_local_invocation_id = true; + break; + } + case ValueOpcode::LaneId: + if (program.stage == ShaderType::TessellationControl) { + inputs.AddBuiltin(StageInputKind::InvocationId); + } else { + info.subgroup_local_invocation_id = true; + } + break; + case ValueOpcode::ImageQueryLod: info.compute_derivatives = true; break; + case ValueOpcode::ImageGatherRaw: + info.image_gather_extended = true; + break; + default: break; } } @@ -423,29 +557,49 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) { : "shader info already collected"); } ValidateOptions(program, input_info); - ValidateValueReferences(program, input_info); - - auto& next = program.info; - next.inputs.clear(); - next.outputs.clear(); - next.has_bitwise_xor = - std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto* block) { - return std::any_of(block->begin(), block->end(), [](const auto& inst) { - return inst.GetOpcode() == ValueOpcode::BitwiseXor32; - }); - }); + auto& info = program.info; + if (info.buffers.size() > ShaderInfo::MaxBuffers) { + return Fail("buffer resource limit exceeded"); + } + info.inputs.clear(); + info.outputs.clear(); + info.user_data_registers.clear(); + info.live_buffers = 0; + info.has_bitwise_xor = info.uses_lds = info.uses_gds = info.uses_flattened_srt = + info.uses_dispatch_threads = info.uses_swizzle = false; + info.bvh = info.subgroup_ballot = info.subgroup_barrier = info.subgroup_shuffle = + info.subgroup_local_invocation_id = info.compute_derivatives = info.image_gather_extended = + info.function_lds = info.function_scratch = info.pixel_valid_mask = info.buffer_int64_atomics = + info.buffer_u8 = info.buffer_u16 = info.shared_int64_atomics = info.coherent_buffers = info.float64 = false; + InputUsage inputs; + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + Visit(program, input_info, inputs, inst); + } + } + for (uint32_t index = 0; index < inputs.user_data.size(); index++) { + if (inputs.user_data[index]) info.user_data_registers.push_back(index); + } + const auto uses_mapping = [](const auto& resource) { + return resource.indirect_root != UINT32_MAX; + }; + info.uses_flattened_srt |= std::ranges::any_of(info.buffers, uses_mapping) || + std::ranges::any_of(info.images, uses_mapping); switch (program.stage) { case ShaderType::Vertex: - case ShaderType::Local: CollectVertexInputs(program, input_info.vertex, next); break; + case ShaderType::Local: CollectVertexInputs(input_info.vertex, info, inputs.components); break; case ShaderType::TessellationControl: case ShaderType::TessellationEvaluation: case ShaderType::Mesh: break; - case ShaderType::Pixel: CollectPixelInputs(program, input_info.pixel, next); break; - case ShaderType::Compute: CollectComputeInputs(input_info.compute, next); break; + case ShaderType::Pixel: + CollectPixelInputs(input_info.pixel, info, inputs.per_vertex, inputs.components); + break; + case ShaderType::Compute: CollectComputeInputs(input_info.compute, info); break; default: return Fail("unsupported shader stage for info collection"); } - CollectBuiltinInputs(program, next); - CollectOutputs(program, input_info, next); + for (uint32_t i = 0; i < inputs.builtin_count; i++) { + AddBuiltinInput(info, inputs.builtins[i]); + } program.shader_info_complete = true; } diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h index c4c052de5..a567ecabb 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h @@ -5,8 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -// Completes the immutable shader interface after resource tracking. On failure Program::info and -// all completion state remain unchanged. +// Collects the shader interface, resource usage and instruction features after resource tracking. void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index d694be4c8..d5d16a2b3 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1,6 +1,5 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" #include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" #include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" @@ -2009,12 +2008,8 @@ void TestFmaskLoadSpecialization() { "inactive FMASK load did not preserve the execution mask"); ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program); - const auto kind = DescriptorBindingForImage(fixture.program.info.images[0]); - Check(kind.has_value() && - FindBinding(fixture.program.bindings, *kind)->resources == - std::vector{0}, - "FMASK allocated an ordinary image descriptor"); + Check(DescriptorBindingForImage(fixture.program.info.images[0]).has_value(), + "FMASK removal lost the ordinary image class"); user_data[8] = 0; user_data[1] = static_cast(Prospero::BufferFormat::k8UInt) << 20u; ResourceSpecialization rebound; @@ -2152,14 +2147,9 @@ void TestDynamicStorageMipTracking() { "base-1 through last-3 dynamic storage range was not specialized"); ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program); - const auto storage_kind = DescriptorBindingForImage(images[0]); - Check(storage_kind.has_value(), "storage image has no descriptor binding"); - const auto *storage_binding = - FindBinding(fixture.program.bindings, *storage_kind); - Check(storage_binding != nullptr && - storage_binding->resources == std::vector({0, 1, 1, 1}), - "dynamic storage mip descriptors were not expanded consecutively"); + Check(DescriptorBindingForImage(images[0]) == DescriptorBindingForImage(images[1]) && + images[0].mip_count == 1 && images[1].mip_mode == ImageMipMode::Dynamic, + "dynamic storage mip metadata did not retain one homogeneous image class"); Fixture null_fixture; const auto null_handle = null_fixture.Image( @@ -2284,10 +2274,8 @@ void TestSrtFlatteningAndRuntimeMemoization() { ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program); - Check(FindBinding(fixture.program.bindings, - DescriptorBindingKind::FlattenedSrt) != nullptr, - "flattened typed SRT reads did not receive a binding"); + Check(fixture.program.info.uses_flattened_srt, + "flattened typed SRT reads were not collected"); } void TestDynamicSrtReadRemainsExplicit() { @@ -2327,20 +2315,9 @@ void TestDynamicSrtReadRemainsExplicit() { ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program); - Check(FindBinding(fixture.program.bindings, - DescriptorBindingKind::FlattenedSrt) == nullptr && - FindBinding(fixture.program.bindings, - DescriptorBindingKind::BdaPagetable) != nullptr && - FindBinding(fixture.program.bindings, - DescriptorBindingKind::FaultBuffer) != nullptr, - "dynamic scalar read received the wrong resource bindings"); - Check(fixture.program.bindings.memory_offset_dword == - fixture.program.bindings.user_data_registers.size() && - fixture.program.bindings.memory_offset_count == 1u && - fixture.program.bindings.ShaderDataDwords() == - fixture.program.bindings.memory_offset_dword + 1u, - "unified memory-offset layout is inconsistent"); + Check(!fixture.program.info.uses_flattened_srt && fixture.program.info.uses_dma && + fixture.program.info.live_buffers == 1u, + "dynamic scalar read received the wrong resource usage"); } void TestWritableDescriptorPhi() { @@ -3240,7 +3217,7 @@ void TestConditionalIndirectImageMaterialization() { "taken indirect image branch did not require its descriptor table"); } -void TestShaderInfoAndBindingLayout() { +void TestShaderInfoCollection() { Fixture fixture; const auto handle = fixture.Buffer( {fixture.UserData(3), fixture.UserData(4), Value(64u), Value(0u)}, 4); @@ -3270,15 +3247,9 @@ void TestShaderInfoAndBindingLayout() { StageInputKind::GlobalInvocationId, "typed shader values were not reflected in shader info"); - AllocateBindings(fixture.program); - Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::Buffers) != - nullptr && - FindBinding(fixture.program.bindings, DescriptorBindingKind::Gds) != - nullptr && - FindBinding(fixture.program.bindings, - DescriptorBindingKind::ShaderData) == nullptr && - fixture.program.bindings.UsesPushData(), - "typed resources were not assigned native bindings"); + Check(fixture.program.info.live_buffers == 1u && + fixture.program.info.uses_gds && !fixture.program.info.uses_lds, + "typed resources were not collected in shader info"); Check(NativeBinding(ShaderType::Compute, DescriptorBindingKind::Buffers) == static_cast(DescriptorBindingKind::Buffers) && NativeBinding(ShaderType::Vertex, DescriptorBindingKind::Buffers) == @@ -3287,9 +3258,8 @@ void TestShaderInfoAndBindingLayout() { static_cast(DescriptorBindingKind::Count) + static_cast(DescriptorBindingKind::Buffers), "fixed stage binding ranges are inconsistent"); - Check(fixture.program.bindings.user_data_registers == - std::vector({3u, 4u}), - "binding layout did not collect live typed user-data values"); + Check(fixture.program.info.user_data_registers == std::vector({3u, 4u}), + "shader info did not collect live typed user-data values"); } void TestImageBindingAbi() { @@ -3414,60 +3384,27 @@ void TestImageBindingAbi() { Check(Invalid(image), "float atomic image received a descriptor binding"); } -void TestGraphicsPushConstantLayout() { - const auto AddUserData = [](Fixture &fixture, uint32_t count) { - for (uint32_t index = 0; index < count; index++) { - fixture.Emit(ValueOpcode::ReferenceU32, {fixture.UserData(index)}); - } - fixture.program.shader_info_complete = true; - }; - uint32_t cursor = 0; - Fixture pixel(ShaderType::Pixel); - AddUserData(pixel, 4); - AllocateBindings(pixel.program, cursor); - Check( - pixel.program.bindings.UsesPushData() && - pixel.program.bindings.push_data_start_dword == 0 && - FindBinding(pixel.program.bindings, - DescriptorBindingKind::ShaderData) == nullptr, - "pixel shader did not start the shared push-data block"); - pixel.program.bindings.AdvancePushData(cursor); +void TestLiveShaderDataCollection() { + Fixture fixture; + fixture.UserData(7); + fixture.Emit(ValueOpcode::ReferenceU32, {fixture.UserData(5)}); + fixture.Emit(ValueOpcode::ReferenceU32, {fixture.UserData(2)}); + fixture.Emit(ValueOpcode::ReferenceU32, {fixture.UserData(5)}); + fixture.Emit(ValueOpcode::GetDispatchThreadExtent, {Value(1u)}); + fixture.PlanAndTrack(); + ShaderComputeInputInfo compute{}; + CollectShaderInfo(fixture.program, {.compute = &compute}); + Check(fixture.program.info.user_data_registers == std::vector({2u, 5u}) && + !fixture.program.info.uses_dispatch_threads, + "shader data collection retained unused values or duplicated registers"); - Fixture vertex(ShaderType::Vertex); - AddUserData(vertex, 9); - AllocateBindings(vertex.program, cursor); - Check(vertex.program.bindings.UsesPushData() && - vertex.program.bindings.push_data_start_dword == 4, - "vertex shader did not follow pixel data in the shared push-data block"); - vertex.program.bindings.AdvancePushData(cursor); - Check(cursor == 13, "graphics push-data cursor advanced incorrectly"); - - Fixture edge(ShaderType::Pixel); - AddUserData(edge, NativePushConstantSize / sizeof(uint32_t)); - AllocateBindings(edge.program); - Check(edge.program.bindings.UsesPushData() && - FindBinding(edge.program.bindings, - DescriptorBindingKind::ShaderData) == nullptr, - "the full shared push-data block did not fit"); - - Fixture spill(ShaderType::Pixel); - AddUserData(spill, 20); - AllocateBindings(spill.program, cursor); - Check( - !spill.program.bindings.UsesPushData() && - spill.program.bindings.push_data_start_dword == PushData::NoStart && - FindBinding(spill.program.bindings, - DescriptorBindingKind::ShaderData) != nullptr, - "a stage that exceeded the remaining shared push data did not spill to storage"); - const auto spill_layout = spill.program.bindings; - spill.program.bindings.AdvancePushData(cursor); - Check(cursor == 13, "a spilled stage consumed shared push-data space"); - - Fixture repeated_spill(ShaderType::Pixel); - AddUserData(repeated_spill, 20); - AllocateBindings(repeated_spill.program, 20); - Check(repeated_spill.program.bindings == spill_layout, - "storage fallback retained an irrelevant attempted push-data position"); + const auto extent = fixture.Emit(ValueOpcode::GetDispatchThreadExtent, {Value(0u)}); + fixture.Emit(ValueOpcode::ReferenceU32, {extent}); + fixture.program.shader_info_complete = false; + CollectShaderInfo(fixture.program, {.compute = &compute}); + Check(fixture.program.info.uses_dispatch_threads && + fixture.program.info.user_data_registers == std::vector({2u, 5u}), + "shader data recollection lost a live dispatch extent or retained stale registers"); } void TestResourceLimitIsTransactional() { @@ -3489,12 +3426,8 @@ void TestResourceLimitIsTransactional() { "compute shader did not retain all 64 distinct buffers"); ShaderComputeInputInfo compute{}; CollectShaderInfo(accepted.program, {.compute = &compute}); - AllocateBindings(accepted.program); - const auto *binding = FindBinding(accepted.program.bindings, - DescriptorBindingKind::Buffers); - Check(binding != nullptr && binding->resources.size() == 64u && - accepted.program.bindings.memory_offset_count == 64u, - "compute shader binding layout truncated the 64 buffers"); + Check(accepted.program.info.live_buffers == UINT64_MAX, + "shader info truncated the 64 live buffers"); Fixture fixture; MemoryInfo memory; @@ -3596,9 +3529,9 @@ int main() { Run("guarded scalar descriptor reads", TestGuardedScalarDescriptorReads); Run("conservative buffer reachability", TestConservativeBufferReachability); Run("conditional indirect image", TestConditionalIndirectImageMaterialization); - Run("shader info and bindings", TestShaderInfoAndBindingLayout); + Run("shader info collection", TestShaderInfoCollection); Run("image binding ABI", TestImageBindingAbi); - Run("graphics push constants", TestGraphicsPushConstantLayout); + Run("live shader data collection", TestLiveShaderDataCollection); Run("resource limit", TestResourceLimitIsTransactional); Run("malformed memory kinds", TestMalformedMemoryKindsRejected); } catch (const std::exception &exception) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 4285822c1..80209494f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -41,7 +41,7 @@ #include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" #include "graphics/shader/rectListShader.h" #include "graphics/shader/shader.h" #include "graphics/shader/shaderCompiler.h" @@ -456,11 +456,12 @@ struct RenderExecutorTestAccess { : program.stage == ShaderType::Pixel ? vk::ShaderStageFlagBits::eFragment : vk::ShaderStageFlagBits::eCompute; - for (const auto &binding : program.bindings.descriptors) { + for (auto mask = program.bindings.descriptor_mask; mask != 0; mask &= mask - 1) { + const auto index = std::countr_zero(mask); + const auto kind = static_cast(index); layout_bindings.push_back( - {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind), - NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), - shader_stage}); + {ShaderRecompiler::IR::NativeBinding(program.stage, kind), + NativeDescriptorType(kind), program.bindings.descriptor_counts[index], shader_stage}); } } vk::DescriptorSetLayoutCreateInfo descriptor_info{}; @@ -1676,15 +1677,11 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) { image != result.program.info.images.end() && image->mip_count == test.expected_mip_descriptors, "runtime descriptor range did not specialize the mip count"); - const auto resource = - static_cast(image - result.program.info.images.begin()); const auto kind = ShaderRecompiler::IR::DescriptorBindingForImage(*image); - const auto *binding = kind ? ShaderRecompiler::IR::FindBinding( - result.program.bindings, *kind) : nullptr; Require(test.name, "dynamic mip binding", - binding != nullptr && - std::ranges::count(binding->resources, resource) == - test.expected_mip_descriptors, + kind && image->descriptor_index != UINT32_MAX && + image->descriptor_index + image->mip_count <= + result.program.bindings.descriptor_counts[static_cast(*kind)], "dynamic image did not receive one descriptor per mip"); } if (test.expand_shader_data_storage) { @@ -1697,49 +1694,45 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) { block.AppendNewInst(ShaderRecompiler::IR::ValueOpcode::ReferenceU32, {ShaderRecompiler::IR::Value(&user_data)}); } - result.program.bindings = {}; - result.program.binding_layout_complete = false; - ShaderRecompiler::IR::AllocateBindings(result.program); - const auto *shader_data = ShaderRecompiler::IR::FindBinding( - result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::ShaderData); + result.program.shader_info_complete = false; + ShaderRecompiler::IR::CollectShaderInfo(result.program, options.input_info); + result.spirv = ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info); + const auto shader_data = result.program.bindings.descriptor_counts[ + static_cast(ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)]; Require(test.name, "binding layout", - result.program.bindings.user_data_registers.size() == 33 && + result.program.info.user_data_registers.size() == 33 && !result.program.bindings.UsesPushData() && - shader_data != nullptr, + shader_data != 0, "oversized shader data did not use its storage fallback"); - result.spirv = ShaderRecompiler::Spirv::EmitProgram(result.program, - options.input_info); } Require(test.name, "SPIR-V emit", !result.spirv.empty(), "recompiler returned empty SPIR-V"); ValidateSpirv(test.name, result.spirv); CheckSpirvText(test, result.spirv); - const auto *buffer_binding = ShaderRecompiler::IR::FindBinding( - result.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Buffers); if (test.expected_buffer_resources) { + std::vector resources; + for (u32 i = 0; i < result.program.info.buffers.size(); ++i) { + if (result.program.info.buffers[i].descriptor_index != UINT32_MAX) resources.push_back(i); + } Require(test.name, "live buffer bindings", - buffer_binding != nullptr && - buffer_binding->resources == *test.expected_buffer_resources, + resources == *test.expected_buffer_resources, "dead buffer changed the surviving descriptor order"); } std::vector packed_user_data; - for (const auto reg : result.program.bindings.user_data_registers) { + for (const auto reg : result.program.info.user_data_registers) { packed_user_data.push_back( resources.user_data[reg - result.program.user_data_base]); } packed_user_data.resize(result.program.bindings.ShaderDataDwords()); - const auto buffer_count = result.program.bindings.memory_offset_count; - for (u32 i = 0; i < buffer_count; i++) { - u32 offset = 0; - const auto resource = buffer_binding->resources[i]; - if (resource < test.storage_buffer_offsets.size()) { - offset = test.storage_buffer_offsets[resource]; - } + for (u32 resource = 0; resource < result.program.info.buffers.size(); ++resource) { + const auto index = result.program.info.buffers[resource].descriptor_index; + if (index == UINT32_MAX) continue; + const auto offset = resource < test.storage_buffer_offsets.size() + ? test.storage_buffer_offsets[resource] : 0u; Require(test.name, "shader data", offset < 256, "storage buffer offset is not representable"); - packed_user_data[result.program.bindings.memory_offset_dword + i / 4u] |= - offset << ((i % 4u) * 8u); + packed_user_data[result.program.bindings.memory_offset_dword + index / 4u] |= + offset << ((index % 4u) * 8u); } return {std::move(result.spirv), std::move(result.program), std::move(resources), std::move(packed_user_data)}; @@ -1835,7 +1828,7 @@ CompiledShader CompileFragmentCase(const GraphicsCase &test) { "recompiler returned empty SPIR-V"); ValidateSpirv(test.name, result.spirv); std::vector packed_user_data; - for (const auto reg : result.program.bindings.user_data_registers) { + for (const auto reg : result.program.info.user_data_registers) { packed_user_data.push_back( resources.user_data[reg - result.program.user_data_base]); } @@ -2142,12 +2135,12 @@ public: ShaderRecompiler::IR::CompiledShaderInfo vertex_program{}; vertex_program.stage = ShaderType::Vertex; vertex_program.bindings.push_data_start_dword = 2; - vertex_program.bindings.user_data_registers = {0, 1}; + vertex_program.info.user_data_registers = {0, 1}; vertex_program.bindings.memory_offset_dword = 2; ShaderRecompiler::IR::CompiledShaderInfo pixel_program{}; pixel_program.stage = ShaderType::Pixel; pixel_program.bindings.push_data_start_dword = 0; - pixel_program.bindings.user_data_registers = {0, 1}; + pixel_program.info.user_data_registers = {0, 1}; pixel_program.bindings.memory_offset_dword = 2; ShaderRecompiler::IR::ResourceSnapshot vertex_snapshot; vertex_snapshot.user_data = {0x11111111u, 0x22222222u}; @@ -11970,19 +11963,23 @@ public: block_words + 0x10000 / sizeof(uint32_t) - 4), "compressed atlas aliases reloaded stale CPU bytes over GPU-written blocks"); - const auto allocate_bindings = - [&](ShaderRecompiler::IR::Program &program) { - program.shader_info_complete = true; - ShaderRecompiler::IR::AllocateBindings(program); - }; + const auto set_image_bindings = [](ShaderRecompiler::IR::Program &program) { + for (auto &image : program.info.images) { + const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image); + const auto index = static_cast(kind); + image.descriptor_index = program.bindings.descriptor_counts[index]; + program.bindings.descriptor_counts[index] += image.mip_count; + program.bindings.descriptor_mask |= uint64_t{1} << index; + } + }; const auto make_buffer_program = [](ShaderType stage, ShaderRecompiler::IR::BufferResource resource) { ShaderRecompiler::IR::CompiledShaderInfo program{}; program.stage = stage; + resource.descriptor_index = 0; program.info.buffers.push_back(resource); - program.bindings.descriptors.push_back( - {ShaderRecompiler::IR::DescriptorBindingKind::Buffers, {0}}); - program.bindings.memory_offset_count = 1; + program.bindings.descriptor_counts[0] = 1; + program.bindings.descriptor_mask = 1; program.bindings.push_data_start_dword = 0; return program; }; @@ -12283,7 +12280,7 @@ public: } ShaderRecompiler::IR::ApplyResourceSpecialization( null_program, null_specialization); - allocate_bindings(null_program); + set_image_bindings(null_program); ShaderRecompiler::IR::CompiledShaderInfo null_info{}; null_info.stage = null_program.stage; null_info.info = std::move(null_program.info); @@ -12378,7 +12375,7 @@ public: ShaderRecompiler::Decoder::ImageDimension::Dim2D; storage_resource.written = true; storage_program.info.images.push_back(storage_resource); - allocate_bindings(storage_program); + set_image_bindings(storage_program); ShaderRecompiler::IR::DescriptorValue storage_descriptor{}; std::copy(std::begin(storage.fields), std::end(storage.fields), storage_descriptor.dwords.begin()); @@ -12914,7 +12911,7 @@ public: ShaderRecompiler::Decoder::ImageDimension::Dim2D; sampled_resource.read = true; sampled_program.info.images.push_back(sampled_resource); - allocate_bindings(sampled_program); + set_image_bindings(sampled_program); ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{}; sampled_snapshot.images.push_back(storage_descriptor); ShaderRecompiler::IR::CompiledShaderInfo sampled_info{}; @@ -12996,7 +12993,7 @@ public: vertex_sampled_program.stage = ShaderType::Vertex; vertex_sampled_program.resource_tracking_complete = true; vertex_sampled_program.info.images.push_back(sampled_resource); - allocate_bindings(vertex_sampled_program); + set_image_bindings(vertex_sampled_program); ShaderRecompiler::IR::CompiledShaderInfo vertex_sampled_info{}; vertex_sampled_info.stage = vertex_sampled_program.stage; vertex_sampled_info.info = std::move(vertex_sampled_program.info); @@ -13008,7 +13005,7 @@ public: pixel_storage_program.stage = ShaderType::Pixel; pixel_storage_program.resource_tracking_complete = true; pixel_storage_program.info.images.push_back(storage_resource); - allocate_bindings(pixel_storage_program); + set_image_bindings(pixel_storage_program); ShaderRecompiler::IR::CompiledShaderInfo pixel_storage_info{}; pixel_storage_info.stage = pixel_storage_program.stage; pixel_storage_info.info = std::move(pixel_storage_program.info); @@ -13084,7 +13081,7 @@ public: split_ir.stage = ShaderType::Vertex; split_ir.resource_tracking_complete = true; split_ir.info.images = {storage_resource, sampled_resource}; - allocate_bindings(split_ir); + set_image_bindings(split_ir); ShaderRecompiler::IR::CompiledShaderInfo split_program{}; split_program.stage = split_ir.stage; split_program.info = std::move(split_ir.info); @@ -13177,7 +13174,7 @@ public: depth_views_ir.stage = ShaderType::Pixel; depth_views_ir.resource_tracking_complete = true; depth_views_ir.info.images = {sampled_resource, sampled_resource}; - allocate_bindings(depth_views_ir); + set_image_bindings(depth_views_ir); ShaderRecompiler::IR::CompiledShaderInfo depth_views_program{}; depth_views_program.stage = depth_views_ir.stage; depth_views_program.info = std::move(depth_views_ir.info); @@ -15261,29 +15258,20 @@ public: std::copy_n(test.compute_info.dispatch_threads_num, 3, shader_data.begin() + layout.dispatch_thread_dword); } - auto Binding = [&](Kind kind) { - return ShaderRecompiler::IR::FindBinding(layout, kind); + auto Count = [&](Kind kind) { + return layout.descriptor_counts[static_cast(kind)]; }; auto Native = [](Kind kind) { return ShaderRecompiler::IR::NativeBinding(ShaderType::Compute, kind); }; - for (const auto &binding : layout.descriptors) { - const auto resource_class = - ShaderRecompiler::IR::ImageBindingResourceClass(binding.kind); - if (resource_class == ShaderRecompiler::IR::ImageResourceClass::None) { - continue; - } - const auto &image = - compiled.program.info.images.at(binding.resources.front()); + for (const auto &image : compiled.program.info.images) { bool supported = image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D || image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - if (resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) { + if (image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) { supported = supported || - image.dimension == - ShaderRecompiler::Decoder::ImageDimension::Dim1D || - image.dimension == - ShaderRecompiler::Decoder::ImageDimension::Dim1DArray; + image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D || + image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray; } Require(test.name, "dispatch", supported, "unsupported image dimension needs a matching Vulkan test view"); @@ -15311,11 +15299,10 @@ public: item.stageFlags = vk::ShaderStageFlagBits::eCompute; layout_bindings.push_back(item); }; - for (const auto &binding : layout.descriptors) { - add_layout_binding(ShaderRecompiler::IR::NativeBinding( - ShaderType::Compute, binding.kind), - NativeDescriptorType(binding.kind), - NativeDescriptorCount(binding)); + for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) { + const auto index = std::countr_zero(mask); + const auto kind = static_cast(index); + add_layout_binding(Native(kind), NativeDescriptorType(kind), layout.descriptor_counts[index]); } vk::DescriptorSetLayoutCreateInfo layout_info{}; @@ -15376,9 +15363,9 @@ public: } pool_sizes.push_back({type, count}); }; - for (const auto &binding : layout.descriptors) { - add_pool_size(NativeDescriptorType(binding.kind), - NativeDescriptorCount(binding)); + for (auto mask = layout.descriptor_mask; mask != 0; mask &= mask - 1) { + const auto index = std::countr_zero(mask); + add_pool_size(NativeDescriptorType(static_cast(index)), layout.descriptor_counts[index]); } vk::DescriptorPoolCreateInfo pool_info{}; pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; @@ -15403,11 +15390,8 @@ public: std::vector writes; std::vector buffer_infos; - std::vector sampled_infos; + std::array, ShaderRecompiler::IR::ImageBindingCount> image_infos; std::vector sampled_mip_views; - std::vector storage_infos; - std::vector storage_uint_infos; - std::vector storage_atomic_infos; std::vector sampler_infos; Buffer flattened_buffer; Buffer user_data_buffer; @@ -15419,7 +15403,7 @@ public: vk::DescriptorBufferInfo bda_pagetable_info{}; vk::DescriptorBufferInfo fault_buffer_info{}; - if (Binding(Kind::SharedMemory) != nullptr) { + if (Count(Kind::SharedMemory) != 0) { const auto bytes = uint64_t{test.compute_info.lds_size_dwords} * sizeof(u32) * test.dispatch_x * test.dispatch_y * test.dispatch_z; shared_buffer = CreateHostBuffer(test.name, bytes, vk::BufferUsageFlagBits::eStorageBuffer, {}); @@ -15433,9 +15417,9 @@ public: writes.push_back(write); } - const bool uses_bda = Binding(Kind::BdaPagetable) != nullptr; + const bool uses_bda = Count(Kind::BdaPagetable) != 0; Require(test.name, "dispatch", - uses_bda == (Binding(Kind::FaultBuffer) != nullptr), + uses_bda == (Count(Kind::FaultBuffer) != 0), "BDA page table and fault buffer must be bound together"); if (uses_bda) { EnsureBdaBuffers(test.name); @@ -15458,16 +15442,16 @@ public: } } - const auto *buffers = Binding(Kind::Buffers); - if (buffers != nullptr) { - buffer_infos.resize(buffers->resources.size()); - for (u32 i = 0; i < buffer_infos.size(); i++) { - auto &info = buffer_infos[i]; + if (Count(Kind::Buffers) != 0) { + buffer_infos.resize(Count(Kind::Buffers)); + for (u32 resource = 0; resource < compiled.program.info.buffers.size(); ++resource) { + const auto index = compiled.program.info.buffers[resource].descriptor_index; + if (index == UINT32_MAX) continue; + auto &info = buffer_infos[index]; info.buffer = buffer.buffer; info.offset = 0; info.range = buffer.size; if (test.storage_buffer_range_bytes != 0) { - const auto resource = buffers->resources[i]; const auto offset = resource < test.storage_buffer_offsets.size() ? test.storage_buffer_offsets[resource] : 0u; @@ -15486,8 +15470,7 @@ public: write.pBufferInfo = buffer_infos.data(); writes.push_back(write); } - if (const auto *flattened = Binding(Kind::FlattenedSrt); - flattened != nullptr) { + if (Count(Kind::FlattenedSrt) != 0) { flattened_buffer = CreateStorageBuffer(test.name, compiled.resources.flattened_srt, compiled.resources.flattened_srt.size()); @@ -15501,7 +15484,7 @@ public: write.pBufferInfo = &flattened_info; writes.push_back(write); } - if (const auto *user = Binding(Kind::ShaderData); user != nullptr) { + if (Count(Kind::ShaderData) != 0) { user_data_buffer = CreateStorageBuffer(test.name, shader_data, shader_data.size()); user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size}; @@ -15514,7 +15497,7 @@ public: write.pBufferInfo = &user_data_info; writes.push_back(write); } - if (const auto *gds = Binding(Kind::Gds); gds != nullptr) { + if (Count(Kind::Gds) != 0) { Require(test.name, "dispatch", gds_buffer != nullptr, "GDS descriptor requested but no GDS buffer was provided"); gds_info = {gds_buffer->buffer, 0, gds_buffer->size}; @@ -15527,51 +15510,27 @@ public: write.pBufferInfo = &gds_info; writes.push_back(write); } - std::vector sampled_bindings; - const ShaderRecompiler::IR::DescriptorBinding *storage = nullptr; - const ShaderRecompiler::IR::DescriptorBinding *storage_uint = nullptr; - const ShaderRecompiler::IR::DescriptorBinding *storage_atomic = nullptr; - for (const auto &binding : layout.descriptors) { - const auto resource_class = - ShaderRecompiler::IR::ImageBindingResourceClass(binding.kind); - if (resource_class == ShaderRecompiler::IR::ImageResourceClass::None) { - continue; - } - const auto &image = - compiled.program.info.images.at(binding.resources.front()); - if (resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) { - sampled_bindings.push_back(&binding); - } else if (image.atomic) { - storage_atomic = &binding; - } else if (image.numeric_class == Prospero::TextureNumericClass::Float) { - storage = &binding; - } else { - storage_uint = &binding; - } - } - size_t sampled_count = 0; - for (const auto *binding : sampled_bindings) sampled_count += binding->resources.size(); - sampled_infos.reserve(sampled_count); Require(test.name, "dispatch", sampled_resources.empty() || sampled_resources.size() == compiled.program.info.images.size(), "sampled resources must match the logical images"); - for (const auto *sampled : sampled_bindings) { - Require(test.name, "dispatch", sampled_image != nullptr || !sampled_resources.empty(), - "sampled image descriptor requested but no sampled image was " - "provided"); - const auto first_info = sampled_infos.size(); - sampled_infos.resize(first_info + sampled->resources.size()); - std::vector mip_indices(compiled.program.info.images.size()); - for (u32 slot = 0; slot < sampled->resources.size(); slot++) { - auto &info = sampled_infos[first_info + slot]; - const auto resource = sampled->resources[slot]; - const auto *source_image = sampled_resources.empty() ? sampled_image - : &sampled_resources[resource]; + for (u32 resource = 0; resource < compiled.program.info.images.size(); ++resource) { + const auto &image = compiled.program.info.images[resource]; + const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(image); + auto &infos = image_infos[ShaderRecompiler::IR::ImageBindingIndex(kind)]; + infos.resize(Count(kind)); + const bool sampled = image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled; + const auto *source_image = sampled + ? (sampled_resources.empty() ? sampled_image : &sampled_resources[resource]) + : (image.numeric_class == Prospero::TextureNumericClass::Float + ? storage_image : storage_image_uint); + Require(test.name, "dispatch", source_image != nullptr, + "image descriptor requested but no matching image was provided"); + for (u32 mip_index = 0; mip_index < image.mip_count; ++mip_index) { + auto &info = infos[image.descriptor_index + mip_index]; info.imageView = source_image->view; info.imageLayout = source_image->layout; - if (compiled.program.info.images[resource].mip_mode == - ShaderRecompiler::IR::ImageMipMode::Dynamic) { - const auto mip = test.sampled_image_view_base_mip + mip_indices[resource]++; + if (sampled && image.mip_mode == ShaderRecompiler::IR::ImageMipMode::Dynamic) { + const auto mip = test.sampled_image_view_base_mip + mip_index; Require(test.name, "dispatch", mip < source_image->mip_levels, "sampled mip descriptor exceeds the supplied image"); vk::ImageViewCreateInfo view{}; @@ -15590,52 +15549,27 @@ public: sampled_mip_views.push_back(info.imageView); } } + } + for (u32 group = 0; group < image_infos.size(); ++group) { + const auto &infos = image_infos[group]; + if (infos.empty()) continue; + const auto kind = static_cast(ShaderRecompiler::IR::FirstImageBinding + group); vk::WriteDescriptorSet write{}; - write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = Native(sampled->kind); - write.descriptorCount = static_cast(sampled->resources.size()); - write.descriptorType = vk::DescriptorType::eSampledImage; - write.pImageInfo = sampled_infos.data() + first_info; + write.dstBinding = Native(kind); + write.descriptorCount = static_cast(infos.size()); + write.descriptorType = NativeDescriptorType(kind); + write.pImageInfo = infos.data(); writes.push_back(write); } - const auto BindStorage = - [&](const ShaderRecompiler::IR::DescriptorBinding *binding, - const Image *image, std::vector *infos) { - if (binding == nullptr) { - return; - } - Require( - test.name, "dispatch", image != nullptr, - "storage image descriptor requested but no matching image was " - "provided"); - infos->resize(binding->resources.size()); - for (auto &info : *infos) { - info.imageView = image->view; - info.imageLayout = image->layout; - } - vk::WriteDescriptorSet write{}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = Native(binding->kind); - write.descriptorCount = static_cast(infos->size()); - write.descriptorType = vk::DescriptorType::eStorageImage; - write.pImageInfo = infos->data(); - writes.push_back(write); - }; - BindStorage(storage, storage_image, &storage_infos); - BindStorage(storage_uint, storage_image_uint, &storage_uint_infos); - BindStorage(storage_atomic, storage_image_uint, &storage_atomic_infos); - const auto *samplers = Binding(Kind::Samplers); - if (samplers != nullptr) { + if (Count(Kind::Samplers) != 0) { Require(test.name, "dispatch", sampler != nullptr, "sampler descriptor requested but no sampler was provided"); - sampler_infos.resize(samplers->resources.size()); + sampler_infos.resize(Count(Kind::Samplers)); for (auto &info : sampler_infos) { info.sampler = sampler; } vk::WriteDescriptorSet write{}; - write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; write.dstBinding = Native(Kind::Samplers); write.descriptorCount = static_cast(sampler_infos.size()); @@ -15711,7 +15645,7 @@ public: } cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z); - if (buffers != nullptr) { + if (Count(Kind::Buffers) != 0) { vk::BufferMemoryBarrier barrier{}; barrier.sType = vk::StructureType::eBufferMemoryBarrier; barrier.srcAccessMask = @@ -18886,8 +18820,7 @@ void RunCase(VulkanHarness *vulkan, const TestCase &test) { } using Kind = ShaderRecompiler::IR::DescriptorBindingKind; auto Has = [&](Kind kind) { - return ShaderRecompiler::IR::FindBinding(compiled.program.bindings, kind) != - nullptr; + return compiled.program.bindings.descriptor_counts[static_cast(kind)] != 0; }; const auto dwords = std::max( {test.initial.size(), test.expected.size(), static_cast(1)}); @@ -18900,9 +18833,8 @@ void RunCase(VulkanHarness *vulkan, const TestCase &test) { vk::Sampler sampler = nullptr; bool needs_sampled_image = false; bool needs_storage_image = false; - for (const auto &binding : compiled.program.bindings.descriptors) { - const auto resource_class = - ShaderRecompiler::IR::ImageBindingResourceClass(binding.kind); + for (const auto &image : compiled.program.info.images) { + const auto resource_class = image.resource_class; if (resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) { needs_sampled_image = true; } else if (resource_class == @@ -34642,16 +34574,16 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { program.info.buffers[0].indirect_search_iterations = 0; program.info.buffers[0].indirect_resources.clear(); } - program.binding_layout_complete = false; - AllocateBindings(program); - const auto *buffers = FindBinding(program.bindings, DescriptorBindingKind::Buffers); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); + compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); Require(test.name, "candidate bindings", - buffers != nullptr && buffers->resources == - (null_buffers == 2u ? std::vector{0u} : std::vector{0u, 1u}), + program.bindings.descriptor_counts[0] == (null_buffers == 2u ? 1u : 2u) && + program.info.buffers[0].descriptor_index == 0u && + (null_buffers == 2u || program.info.buffers[1].descriptor_index == 1u), "indirect store lost a buffer binding"); compiled.packed_user_data.resize(program.bindings.ShaderDataDwords()); compiled.packed_user_data[program.bindings.memory_offset_dword] = 12u | (108u << 8u); - compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv(test.name, compiled.spirv); if (null_buffers == 2u) { test.required_spirv.clear(); @@ -34760,8 +34692,9 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { program.info.samplers.push_back({1u, 0x10f0u}); program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u}); - AllocateBindings(program); ShaderComputeInputInfo compute{}; + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv(name, spirv); @@ -34789,8 +34722,8 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { "the sample operand lacks its nonuniform decoration"); program.info.images[0].indirect_search_iterations = 0; - program.binding_layout_complete = false; - AllocateBindings(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv("DenseIndirectImageTable", spirv); Require(name, "dense image map disassembly", tools.Disassemble(spirv, &text), @@ -34810,8 +34743,8 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { program.info.images.push_back(candidate); } program.info.images[0] = root; - program.binding_layout_complete = false; - AllocateBindings(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv(name, spirv); Require(name, "large image array disassembly", tools.Disassemble(spirv, &text), @@ -34824,8 +34757,8 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { program.info.images[root.indirect_resources[103]].dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; program.info.images[root.indirect_resources[103]].cube = true; - program.binding_layout_complete = false; - AllocateBindings(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv(name, spirv); Require(name, "mixed contiguous image runs", tools.Disassemble(spirv, &text) && @@ -34850,8 +34783,8 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { ? ShaderRecompiler::Decoder::ImageDimension::Dim2D : ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; } - program.binding_layout_complete = false; - AllocateBindings(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv(name, spirv); Require(name, "mixed SPIR-V disassembly", tools.Disassemble(spirv, &text), @@ -34914,14 +34847,13 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { {Value(&output), Value(0u), Value(&byte_offset), Value(0u), Value(&sample_x), Value(true)}); store.SetFlags(MemoryFlags{1u, 0u}); program.info.buffers.push_back({.packed_stride = 1, .written = true}); - program.binding_layout_complete = false; - AllocateBindings(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); TestCase test; test.name = "IndirectImageContiguousRuns"; CompiledShader compiled; compiled.program = std::move(program); - compiled.packed_user_data.resize(compiled.program.bindings.ShaderDataDwords()); constexpr std::array ordinals{0u, 1u, 2u, 3u, 4u, 5u, 2u, 0u}; compiled.resources.flattened_srt.push_back(ordinals.size()); compiled.resources.flattened_srt.insert(compiled.resources.flattened_srt.end(), @@ -34944,6 +34876,7 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { compute.threads_num[1] = compute.threads_num[2] = 1u; compiled.program.wave_size = wave_size; compiled.spirv = ShaderRecompiler::Spirv::EmitProgram(compiled.program, {.compute = &compute}); + compiled.packed_user_data.resize(compiled.program.bindings.ShaderDataDwords()); ValidateSpirv(test.name, compiled.spirv); const auto halves = wave_size > compute.host_subgroup_size ? 2u : 1u; Require(test.name, "native array run samples", tools.Disassemble(compiled.spirv, &text) && @@ -39491,8 +39424,8 @@ void CheckResourcePlanHandoff() { ValidateSpirv(name, compiled.spirv); const auto &bindings = compiled.program.bindings; Require(name, "GPU data requirements", - bindings.user_data_registers.empty() && bindings.memory_offset_count == 1 && - (FindBinding(bindings, DescriptorBindingKind::FlattenedSrt) != nullptr) == + compiled.program.info.user_data_registers.empty() && bindings.descriptor_counts[0] == 1 && + (bindings.descriptor_counts[static_cast(DescriptorBindingKind::FlattenedSrt)] != 0) == numeric_read, "descriptor-only shader retained SRT uploads or live scalar data was removed"); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index e94a2f5f3..9e205876f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -1294,10 +1294,14 @@ void TestNormalizedImageContracts() { "Vulkan image-view compatibility classes diverged from production"); } -void TestSpirvRequirementsAnalysis() { +void TestShaderFeatureCollection() { using namespace ShaderRecompiler::IR; + ShaderPixelInputInfo pixel_info{}; + ShaderComputeInputInfo compute_info{}; + ShaderVertexInputInfo vertex_info{}; Program program; + program.resource_tracking_complete = true; program.stage = ShaderType::Pixel; program.block_storage.push_back(std::make_unique()); auto *block = program.block_storage.back().get(); @@ -1318,8 +1322,8 @@ void TestSpirvRequirementsAnalysis() { block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)}); export_value.SetFlags(ExportFlags{.index = 0}); - const auto requirements = - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program); + CollectShaderInfo(program, {.pixel = &pixel_info}); + const auto& requirements = program.info; Check(requirements.subgroup_ballot && requirements.subgroup_shuffle && requirements.subgroup_local_invocation_id && requirements.compute_derivatives && @@ -1328,8 +1332,9 @@ void TestSpirvRequirementsAnalysis() { "consolidated SPIR-V requirements missed an IR dependency"); program.stage = ShaderType::Compute; - const auto compute_requirements = - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute_info}); + const auto& compute_requirements = program.info; Check(!compute_requirements.function_lds && !compute_requirements.pixel_valid_mask, "stage-specific SPIR-V requirements leaked into compute"); @@ -1339,19 +1344,22 @@ void TestSpirvRequirementsAnalysis() { [&] { program.stage = ShaderType::Pixel; shared.SetFlags(MemoryFlags{.index = 1}); - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.pixel = &pixel_info}); }, "invalid shared-memory requirement metadata did not terminate analysis"); ExpectFatal( [&] { program.stage = ShaderType::Pixel; export_value.SetFlags(ExportFlags{.index = 1}); - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program); + program.shader_info_complete = false; + CollectShaderInfo(program, {.pixel = &pixel_info}); }, "invalid export requirement metadata did not terminate analysis"); #endif Program add_tid; + add_tid.resource_tracking_complete = true; add_tid.stage = ShaderType::Compute; add_tid.info.buffers.resize(1); add_tid.info.buffers[0].packed_stride = 1u << 20u; @@ -1367,8 +1375,8 @@ void TestSpirvRequirementsAnalysis() { ValueOpcode::LoadBufferU32, {Value(&buffer), Value(0u), Value(0u), Value(0u), Value(true)}); load.SetFlags(MemoryFlags{.index = 0}); - const auto add_tid_requirements = - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(add_tid); + CollectShaderInfo(add_tid, {.compute = &compute_info}); + const auto& add_tid_requirements = add_tid.info; Check(add_tid_requirements.subgroup_local_invocation_id && !add_tid_requirements.subgroup_ballot && !add_tid_requirements.subgroup_shuffle, @@ -1378,20 +1386,24 @@ void TestSpirvRequirementsAnalysis() { ExpectFatal( [&] { add_tid.memory_info[0].resource = 1; - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(add_tid); + add_tid.shader_info_complete = false; + CollectShaderInfo(add_tid, {.compute = &compute_info}); }, "invalid buffer requirement metadata did not terminate analysis"); ExpectFatal( [&] { add_tid.stage = ShaderType::Vertex; - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(add_tid); + add_tid.shader_info_complete = false; + CollectShaderInfo(add_tid, {.vertex = &vertex_info}); }, "graphics buffer ADD_TID did not terminate requirements analysis"); #endif Program empty; - const auto empty_requirements = - ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(empty); + empty.stage = ShaderType::Compute; + empty.resource_tracking_complete = true; + CollectShaderInfo(empty, {.compute = &compute_info}); + const auto& empty_requirements = empty.info; Check(!empty_requirements.subgroup_ballot && !empty_requirements.subgroup_shuffle && !empty_requirements.subgroup_local_invocation_id && @@ -5209,25 +5221,13 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { raw_options.user_data = raw_user_data; auto raw = RecompileForTest(raw_shader, raw_options); - const auto *address_binding = ShaderRecompiler::IR::FindBinding( - raw.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable); - const auto *fault_binding = ShaderRecompiler::IR::FindBinding( - raw.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer); + const auto& raw_counts = raw.program.bindings.descriptor_counts; + using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; Check(raw.program.info.uses_dma && raw.program.info.buffers.empty() && - address_binding != nullptr && fault_binding != nullptr && - address_binding->resources.empty() && - fault_binding->resources.empty() && - ShaderRecompiler::IR::FindBinding( - raw.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::Buffers) == - nullptr && - ShaderRecompiler::IR::FindBinding( - raw.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) == - nullptr && - raw.program.bindings.memory_offset_count == 0u, + raw_counts[static_cast(BindingKind::BdaPagetable)] == 1 && + raw_counts[static_cast(BindingKind::FaultBuffer)] == 1 && + raw_counts[static_cast(BindingKind::Buffers)] == 0 && + raw_counts[static_cast(BindingKind::FlattenedSrt)] == 0, "raw scalar load did not use only the DMA domain"); Check(count_live_memory_ops( raw.program, ShaderRecompiler::IR::ValueOpcode::LoadAddressU32, @@ -5255,23 +5255,14 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { buffer_options.user_data = buffer_user_data; auto buffer = RecompileForTest(buffer_shader, buffer_options); - const auto *buffer_binding = ShaderRecompiler::IR::FindBinding( - buffer.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::Buffers); + const auto& buffer_counts = buffer.program.bindings.descriptor_counts; Check(buffer.program.info.buffers.size() == 1u && buffer.program.info.buffers[0].scalar && + buffer.program.info.buffers[0].descriptor_index == 0 && !buffer.program.info.uses_dma && - buffer_binding != nullptr && - buffer_binding->resources == std::vector{0u} && - ShaderRecompiler::IR::FindBinding( - buffer.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable) == - nullptr && - ShaderRecompiler::IR::FindBinding( - buffer.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) == - nullptr && - buffer.program.bindings.memory_offset_count == 1u, + buffer_counts[static_cast(BindingKind::Buffers)] == 1 && + buffer_counts[static_cast(BindingKind::BdaPagetable)] == 0 && + buffer_counts[static_cast(BindingKind::FlattenedSrt)] == 0, "descriptor scalar load did not use only the buffer domain"); Check(count_live_memory_ops( buffer.program, ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer, @@ -6066,8 +6057,7 @@ void TestNewShaderRecompilerImageLoad2DMsaa() { const auto binding_kind = ShaderRecompiler::IR::DescriptorBindingForImage( result.program.info.images[0]); Check(binding_kind.has_value() && - ShaderRecompiler::IR::FindBinding(result.program.bindings, - *binding_kind) != nullptr, + result.program.bindings.descriptor_counts[static_cast(*binding_kind)] != 0, "2D-MSAA image did not receive a multisampled descriptor binding"); Check(SpirvContainsTypeImage(result.spirv, 1, 0, 1, 1), "SPIR-V binary does not contain a multisampled 2D image type"); @@ -7232,14 +7222,8 @@ void TestNewShaderRecompilerUnbasedFlatUsesBda() { auto result = RecompileForTest(shader, options); Check(result.program.info.uses_dma && - ShaderRecompiler::IR::FindBinding( - result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable) != - nullptr && - ShaderRecompiler::IR::FindBinding( - result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer) != - nullptr, + result.program.bindings.descriptor_counts[static_cast(ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable)] != 0 && + result.program.bindings.descriptor_counts[static_cast(ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer)] != 0, "unbased FLAT did not compile through BDA"); CheckSpirvBinaryValidates(result.spirv); } @@ -7978,6 +7962,8 @@ void TestNewShaderRecompilerStructuredU64Phi() { use.Emit(IR::ValueOpcode::CompositeExtractU64, {IR::Value(&phi), IR::Value(1u)}); + program.shader_info_complete = false; + ShaderRecompiler::IR::CollectShaderInfo(program, options.input_info); auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto before = MeasureSpirv(result.spirv); @@ -9841,6 +9827,8 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { IR::IREmitter use(prologue_program.blocks[1]); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); + prologue_program.shader_info_complete = false; + IR::CollectShaderInfo(prologue_program, options.input_info); auto spirv = ShaderRecompiler::Spirv::EmitProgram(prologue_program, options.input_info); CheckSpirvBinaryValidates(spirv); @@ -9866,6 +9854,8 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { const auto sum = use.Emit(IR::ValueOpcode::IAdd32, {high, IR::Value(7u)}); use.Emit(IR::ValueOpcode::IAdd32, {sum, middle}); + program.shader_info_complete = false; + IR::CollectShaderInfo(program, options.input_info); auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); @@ -9902,6 +9892,8 @@ void TestNewShaderRecompilerPlanningOnlyLoads() { read.Instruction()->SetFlags(IR::MemoryFlags{.index = 1}); ir.Emit(IR::ValueOpcode::ReferenceU32, {load}); ir.Emit(IR::ValueOpcode::ReferenceU32, {read}); + program.shader_info_complete = false; + IR::CollectShaderInfo(program, options.input_info); const auto spirv = Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); Check(spirv == result.spirv, @@ -10035,6 +10027,8 @@ void TestNewShaderRecompilerNativeU64Translation() { ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {base, IR::Value(count)}); } + program.shader_info_complete = false; + ShaderRecompiler::IR::CollectShaderInfo(program, options.input_info); auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto source = DisassembleSpirvBinary(spirv); @@ -10072,6 +10066,8 @@ void TestNewShaderRecompilerNativeU64Translation() { use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)}); spirv.clear(); + program.shader_info_complete = false; + ShaderRecompiler::IR::CollectShaderInfo(program, options.input_info); spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto before = MeasureSpirv(result.spirv); @@ -10432,13 +10428,13 @@ void TestMeshExportStorage() { CheckSpirvBinaryValidates(result.spirv); const auto &layout = result.program.bindings; Check(layout.UsesPushData() == (push_data_start == PushData::MeshDrawDwordCount) && - layout.memory_offset_count == 1, + layout.descriptor_counts[static_cast(ShaderRecompiler::IR::DescriptorBindingKind::Buffers)] == 1, "mesh shader did not retain its user-data placement and buffer offset"); - const auto reg = std::ranges::find(layout.user_data_registers, 13u); - Check(reg != layout.user_data_registers.end(), "mesh shader lost user s13"); + const auto reg = std::ranges::find(result.program.info.user_data_registers, 13u); + Check(reg != result.program.info.user_data_registers.end(), "mesh shader lost user s13"); const auto source = DisassembleSpirvBinary(result.spirv); for (const auto dword : - {static_cast(reg - layout.user_data_registers.begin()), + {static_cast(reg - result.program.info.user_data_registers.begin()), layout.memory_offset_dword}) { const auto operand = std::string(layout.UsesPushData() ? "vsharp" : "shader_data") + @@ -13317,72 +13313,55 @@ void TestNewShaderRecompilerNativeBindingPlan() { options.user_data = user_data; auto result = RecompileForTest(shader, options); - const auto *buffers = ShaderRecompiler::IR::FindBinding( - result.program.bindings, BindingKind::Buffers); - const ShaderRecompiler::IR::DescriptorBinding *sampled = nullptr; - const ShaderRecompiler::IR::DescriptorBinding *storage = nullptr; - const ShaderRecompiler::IR::DescriptorBinding *atomic_storage = nullptr; + const auto& counts = result.program.bindings.descriptor_counts; + std::optional sampled; + std::optional storage; + std::optional atomic_storage; for (const auto &image : result.program.info.images) { const auto kind = ShaderRecompiler::IR::DescriptorBindingForImage(image); - if (!kind.has_value()) { - continue; - } - const auto *binding = - ShaderRecompiler::IR::FindBinding(result.program.bindings, *kind); - if (image.resource_class == - ShaderRecompiler::IR::ImageResourceClass::Sampled) { - sampled = binding; + Check(kind.has_value() && image.descriptor_index == 0, + "native image did not receive its descriptor slot"); + if (image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) { + sampled = kind; } else if (image.atomic) { - atomic_storage = binding; + atomic_storage = kind; } else { - storage = binding; + storage = kind; } } - const auto *samplers = ShaderRecompiler::IR::FindBinding( - result.program.bindings, BindingKind::Samplers); - Check(buffers != nullptr && buffers->resources.size() == 2, - "native binding plan did not allocate scalar/vector buffers"); - Check(sampled != nullptr && sampled->resources.size() == 1, - "native binding plan did not allocate the sampled image"); - Check(storage != nullptr && storage->resources.size() == 1, - "native binding plan did not allocate the float storage image"); - Check(atomic_storage != nullptr && atomic_storage->resources.size() == 1, - "native binding plan did not allocate the atomic storage image"); - Check(samplers != nullptr && samplers->resources.size() == 1, - "native binding plan did not allocate the sampler"); + Check(counts[static_cast(BindingKind::Buffers)] == 2 && + result.program.info.buffers[0].descriptor_index == 0 && + result.program.info.buffers[1].descriptor_index == 1, + "native emission did not allocate scalar/vector buffer slots"); + Check(sampled && counts[static_cast(*sampled)] == 1, + "native emission did not allocate the sampled image"); + Check(storage && counts[static_cast(*storage)] == 1, + "native emission did not allocate the float storage image"); + Check(atomic_storage && counts[static_cast(*atomic_storage)] == 1, + "native emission did not allocate the atomic storage image"); + Check(counts[static_cast(BindingKind::Samplers)] == 1, + "native emission did not allocate the sampler"); Check(SpirvContainsOpcode(result.spirv, 86), "SPIR-V binary does not combine separate image/sampler descriptors"); - Check(SpirvHasDecorationValue(result.spirv, 33u, - ShaderRecompiler::IR::NativeBinding( - result.program.stage, buffers->kind)), - "SPIR-V lacks storage-buffer Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 33u, - ShaderRecompiler::IR::NativeBinding( - result.program.stage, sampled->kind)), - "SPIR-V lacks sampled-image Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 33u, - ShaderRecompiler::IR::NativeBinding( - result.program.stage, samplers->kind)), - "SPIR-V lacks sampler Binding decoration"); + for (const auto kind : {BindingKind::Buffers, *sampled, BindingKind::Samplers}) { + Check(SpirvHasDecorationValue(result.spirv, 33u, + ShaderRecompiler::IR::NativeBinding(result.program.stage, kind)), + "SPIR-V lacks a resource Binding decoration"); + } Check(SpirvDecorationValueCount(result.spirv, 34u, 0u) == - result.program.bindings.descriptors.size(), + static_cast(std::popcount(result.program.bindings.descriptor_mask)), "SPIR-V resources do not all use descriptor set zero"); CheckSpirvBinaryValidates(result.spirv); -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - ExpectFatal( - [&] { - auto malformed = RecompileForTest(shader, options); - auto malformed_buffers = std::ranges::find_if( - malformed.program.bindings.descriptors, [](const auto &binding) { - return binding.kind == BindingKind::Buffers; - }); - malformed_buffers->resources.clear(); - (void)ShaderRecompiler::Spirv::EmitProgram(malformed.program, - options.input_info); - }, - "malformed native binding topology did not terminate SPIR-V emission"); + const auto bindings = result.program.bindings; + result.program.bindings = {}; + result.program.info.buffers[0].descriptor_index = UINT32_MAX; + Check(ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info) == result.spirv && + result.program.bindings == bindings && + result.program.info.buffers[0].descriptor_index == 0, + "repeated emission did not rebuild descriptor slots deterministically"); +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS ShaderRecompiler::IR::Inst *buffer_handle = nullptr; for (auto *block : result.program.blocks) { const auto found = std::ranges::find_if(*block, [](const auto &inst) { @@ -14499,12 +14478,32 @@ void TestGraphicsPushConstantPlacement() { RecompileForTest(shader, options, nullptr, nullptr, PushDataStart); Check(placed.program.bindings.UsesPushData() && placed.program.bindings.push_data_start_dword == PushDataStart && - ShaderRecompiler::IR::FindBinding(placed.program.bindings, - BindingKind::ShaderData) == nullptr, + placed.program.bindings.descriptor_counts[static_cast(BindingKind::ShaderData)] == 0, "pixel shader did not use its assigned shared push-data position"); Check(SpirvHasMemberDecorationValue(placed.spirv, OffsetDecoration, 0), "SPIR-V push-data block did not use the canonical zero base"); CheckSpirvBinaryValidates(placed.spirv); + + auto exact = RecompileForTest(shader, options, nullptr, nullptr, 28); + Check(exact.program.bindings.UsesPushData() && + exact.program.bindings.push_data_start_dword == 28 && + exact.program.bindings.ShaderDataDwords() == 4, + "exact-fit shader data did not use the remaining push space"); + uint32_t cursor = 28; + exact.program.bindings.AdvancePushData(cursor); + Check(cursor == 32, "exact-fit shader data advanced the cursor incorrectly"); + CheckSpirvBinaryValidates(exact.spirv); + + auto spill = RecompileForTest(shader, options, nullptr, nullptr, 29); + auto full = RecompileForTest(shader, options, nullptr, nullptr, 32); + Check(!spill.program.bindings.UsesPushData() && + spill.program.bindings.descriptor_counts[static_cast(BindingKind::ShaderData)] == 1 && + spill.program.bindings == full.program.bindings && spill.spirv == full.spirv, + "insufficient push space did not produce canonical storage-backed data"); + cursor = 29; + spill.program.bindings.AdvancePushData(cursor); + Check(cursor == 29, "spilled shader data consumed push space"); + CheckSpirvBinaryValidates(spill.spirv); } void TestNewShaderRecompilerUnsupportedMemoryDecode() { @@ -14689,7 +14688,7 @@ void TestCompilerStageInputOwnership() { #endif } -void TestSpirvEmissionOwnsRequirements() { +void TestSpirvRepeatedEmission() { using namespace ShaderRecompiler; const uint32_t shader[] = {EncodeSopp(0x01)}; ShaderComputeInputInfo compute{}; @@ -14705,10 +14704,12 @@ void TestSpirvEmissionOwnsRequirements() { const auto first = emitter.Emit(IR::ValueOpcode::ReadFirstLane, {value, IR::Value(true)}); emitter.Emit(IR::ValueOpcode::IAdd32, {first, lane}); + compiled.program.shader_info_complete = false; + IR::CollectShaderInfo(compiled.program, options.input_info); const auto binary = Spirv::EmitProgram(compiled.program, options.input_info); CheckSpirvBinaryValidates(binary); Check((DisassembleSpirvBinary(binary).find("BuiltIn SubgroupLocalInvocationId") != std::string::npos), - "emission reused requirements from an earlier IR version"); + "collection missed the subgroup requirement of the added IR"); Check(Spirv::EmitProgram(compiled.program, options.input_info) == binary, "repeated emission reused IDs from the previous module"); @@ -14781,9 +14782,7 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { const auto dead_gds_result = compile("dead-gds", dead_gds, {.words = 64, .instructions = 20, .labels = 4, .branches = 3}); - Check(ShaderRecompiler::IR::FindBinding( - dead_gds_result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::Gds) == nullptr, + Check(dead_gds_result.program.bindings.descriptor_counts[static_cast(ShaderRecompiler::IR::DescriptorBindingKind::Gds)] == 0, "dead GDS load retained a descriptor binding"); const uint32_t structured_phi[] = { @@ -14924,9 +14923,7 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { wide_gds_metrics.workgroup_variables == 0u && wide_gds_metrics.array_lengths == 1u, "native wide GDS fixture did not share one entry-dominating length"); - Check(ShaderRecompiler::IR::FindBinding( - wide_gds_result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr, + Check(wide_gds_result.program.bindings.descriptor_counts[static_cast(ShaderRecompiler::IR::DescriptorBindingKind::Gds)] != 0, "live native wide GDS access did not allocate its descriptor binding"); const auto count_shared = [](const auto &result, ShaderRecompiler::IR::ValueOpcode opcode) { @@ -15020,11 +15017,11 @@ int main() { TestShaderBufferResourceSize(); TestNativeShaderResourceDependencies(); TestNormalizedImageContracts(); - TestSpirvRequirementsAnalysis(); + TestShaderFeatureCollection(); TestTypedSpirvSerialization(); TestDeferredSpirvPhiPatching(); TestCompilerStageInputOwnership(); - TestSpirvEmissionOwnsRequirements(); + TestSpirvRepeatedEmission(); TestRepeatedExportsHaveOneInterface(); TestNewShaderRecompilerSpirvSizeBaselines(); TestDemandDrivenSpirvDeclarations();