From bad6726f5e12e9a707860b9456a054547730aa10 Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 21 Aug 2026 22:28:10 +0200 Subject: [PATCH] Rewrite descriptor bindings and unify graphics push constants --- .../host_gpu/renderer/colorRenderTarget.cpp | 1 - .../host_gpu/renderer/commandScheduler.h | 15 +- src/graphics/host_gpu/renderer/context.cpp | 20 +- .../host_gpu/renderer/depthRenderTarget.cpp | 5 +- .../renderer/pipeline/descriptorCache.cpp | 318 -------- .../renderer/pipeline/descriptorCache.h | 118 --- .../renderer/pipeline/descriptorHeap.cpp | 97 +++ .../renderer/pipeline/descriptorHeap.h | 45 ++ .../renderer/pipeline/descriptors.cpp | 415 +++++++--- .../host_gpu/renderer/pipeline/descriptors.h | 55 +- .../renderer/pipeline/pipelineCache.cpp | 9 +- .../renderer/pipeline/pipelineCache.h | 31 +- .../host_gpu/renderer/pipeline/shaders.cpp | 130 ++- src/graphics/host_gpu/renderer/render.h | 73 +- .../host_gpu/renderer/renderCompute.cpp | 10 +- .../host_gpu/renderer/renderContext.cpp | 5 +- .../host_gpu/renderer/renderContext.h | 6 +- src/graphics/host_gpu/renderer/renderDraw.cpp | 37 +- src/graphics/host_gpu/vulkanInstance.h | 1 + .../presentation/window/vulkanWindow.cpp | 24 +- .../shader/recompiler/ShaderRecompiler.cpp | 5 +- .../shader/recompiler/ShaderRecompiler.h | 3 +- .../recompiler/backend/spirv/SpirvEmitter.cpp | 15 +- .../backend/spirv/spirvEmitterInternal.h | 4 +- .../backend/spirv/spirvEmitterModule.cpp | 27 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 17 +- .../recompiler/ir/passes/BindingLayout.cpp | 94 +-- .../recompiler/ir/passes/BindingLayout.h | 8 +- src/graphics/shader/shader.cpp | 40 +- src/graphics/shader/shader.h | 1 - tests/ResourceTrackingTests.cpp | 143 +++- tests/ShaderRecompilerComputeTests.cpp | 743 +++++++++++------- tests/shaderCfgTests.cpp | 241 ++++-- 33 files changed, 1501 insertions(+), 1255 deletions(-) delete mode 100644 src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp delete mode 100644 src/graphics/host_gpu/renderer/pipeline/descriptorCache.h create mode 100644 src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp create mode 100644 src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h diff --git a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp index 8cfeddfc7..94ad70d9e 100644 --- a/src/graphics/host_gpu/renderer/colorRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/colorRenderTarget.cpp @@ -9,7 +9,6 @@ #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/image/textureCommon.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" diff --git a/src/graphics/host_gpu/renderer/commandScheduler.h b/src/graphics/host_gpu/renderer/commandScheduler.h index c079e9ab9..bfa7f502e 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.h +++ b/src/graphics/host_gpu/renderer/commandScheduler.h @@ -55,13 +55,14 @@ public: void DeferPriorityOperation(Common::UniqueFunction&& operation); [[nodiscard]] static bool InDeferredOperation() noexcept; - [[nodiscard]] bool Active() const noexcept { return m_current >= 0; } - void CheckActive() const; - RenderCommandBuffer& Current() const; - [[nodiscard]] uint64_t CurrentTick() const noexcept { return m_master.CurrentTick(); } - [[nodiscard]] bool IsFree(uint64_t tick); - [[nodiscard]] RenderContext& Context() const noexcept { return m_context; } - [[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; } + [[nodiscard]] bool Active() const noexcept { return m_current >= 0; } + void CheckActive() const; + RenderCommandBuffer& Current() const; + [[nodiscard]] uint64_t CurrentTick() const noexcept { return m_master.CurrentTick(); } + [[nodiscard]] bool IsFree(uint64_t tick); + [[nodiscard]] MasterSemaphore& GetMasterSemaphore() noexcept { return m_master; } + [[nodiscard]] RenderContext& Context() const noexcept { return m_context; } + [[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; } private: static constexpr size_t CommandBufferGrowStep = 4; diff --git a/src/graphics/host_gpu/renderer/context.cpp b/src/graphics/host_gpu/renderer/context.cpp index 0edceccc7..34805a904 100644 --- a/src/graphics/host_gpu/renderer/context.cpp +++ b/src/graphics/host_gpu/renderer/context.cpp @@ -9,7 +9,6 @@ #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" #include "graphics/host_gpu/renderer/image/imageView.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vma.h" @@ -30,8 +29,8 @@ void ReportVulkanFatal(const char* what, vk::Result result, uint32_t slot, uint6 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", what, VulkanToString(result).c_str(), static_cast(result), slot, submit_seq, debug_op, debug_submit, arg0, arg1, arg2, arg3, arg4); - std::printf("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64 - " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", + std::printf("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64 + " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n", what, VulkanToString(result).c_str(), static_cast(result), slot, submit_seq, debug_op, debug_submit, arg0, arg1, arg2, arg3, arg4); std::fflush(stdout); @@ -68,23 +67,11 @@ void CommandBuffer::Release() { m_slot->busy = false; m_slot->Reset(); - RecycleDescriptorsAfterFence(); m_slot = nullptr; EXIT_NOT_IMPLEMENTED(!IsInvalid()); } -void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet& set) { - m_descriptor_sets_after_fence.push_back(&set); -} - -void CommandBuffer::RecycleDescriptorsAfterFence() { - for (auto* set: m_descriptor_sets_after_fence) { - m_context.GetDescriptorCache().Recycle(*set); - } - m_descriptor_sets_after_fence.clear(); -} - void CommandBuffer::Begin() const { EXIT_IF(m_rendering); auto buffer = Handle(); @@ -218,9 +205,6 @@ void CommandBuffer::FinalizeFence(bool reset_recording) { m_slot->Reset(); } } - if (was_executed) { - RecycleDescriptorsAfterFence(); - } } void CommandBuffer::BeginRendering(const RenderState& state) const { diff --git a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp index f8d457a19..d11014250 100644 --- a/src/graphics/host_gpu/renderer/depthRenderTarget.cpp +++ b/src/graphics/host_gpu/renderer/depthRenderTarget.cpp @@ -13,7 +13,6 @@ #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/image/textureCommon.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" @@ -290,8 +289,8 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB r.stencil_clear_enable = has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable; - r.stencil_clear_value = hw.GetStencilClearValue(); - r.stencil_test_enable = has_stencil && dc.stencil_enable; + r.stencil_clear_value = hw.GetStencilClearValue(); + r.stencil_test_enable = has_stencil && dc.stencil_enable; if (r.stencil_test_enable) { const bool stencil_ops_disabled = rc.stencil_clear_enable || z.depth_view.stencil_write_disable; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp deleted file mode 100644 index b56031cf0..000000000 --- a/src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp +++ /dev/null @@ -1,318 +0,0 @@ -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" - -#include "common/assert.h" -#include "common/profiler.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -#include - -namespace Libs::Graphics { -namespace { - -using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; - -vk::ShaderStageFlags StageFlags(DescriptorCache::Stage stage) { - switch (stage) { - case DescriptorCache::Stage::Vertex: return vk::ShaderStageFlagBits::eVertex; - case DescriptorCache::Stage::Pixel: return vk::ShaderStageFlagBits::eFragment; - case DescriptorCache::Stage::Compute: return vk::ShaderStageFlagBits::eCompute; - default: EXIT("unknown descriptor stage\n"); - } -} - -bool IsSampledImage(BindingKind kind) { - switch (kind) { - case BindingKind::Sampled1D: - case BindingKind::Sampled1DArray: - case BindingKind::Sampled2D: - case BindingKind::Sampled2DArray: - case BindingKind::Sampled2DMsaa: - case BindingKind::Sampled2DMsaaArray: - case BindingKind::Sampled3D: - case BindingKind::SampledUint1D: - case BindingKind::SampledUint1DArray: - case BindingKind::SampledUint2D: - case BindingKind::SampledUint2DArray: - case BindingKind::SampledUint2DMsaa: - case BindingKind::SampledUint2DMsaaArray: - case BindingKind::SampledUint3D: return true; - default: return false; - } -} - -bool IsStorageImage(BindingKind kind) { - switch (kind) { - case BindingKind::Storage1D: - case BindingKind::Storage1DArray: - case BindingKind::Storage2D: - case BindingKind::Storage2DArray: - case BindingKind::Storage3D: - case BindingKind::StorageUint1D: - case BindingKind::StorageUint1DArray: - case BindingKind::StorageUint2D: - case BindingKind::StorageUint2DArray: - case BindingKind::StorageUint3D: - case BindingKind::StorageAtomic1D: - case BindingKind::StorageAtomic1DArray: - case BindingKind::StorageAtomic2D: - case BindingKind::StorageAtomic2DArray: - case BindingKind::StorageAtomic3D: return true; - default: return false; - } -} - -vk::DescriptorType DescriptorType(BindingKind kind) { - if (kind == BindingKind::Samplers) { - return vk::DescriptorType::eSampler; - } - if (IsSampledImage(kind)) { - return vk::DescriptorType::eSampledImage; - } - if (IsStorageImage(kind)) { - return vk::DescriptorType::eStorageImage; - } - return vk::DescriptorType::eStorageBuffer; -} - -uint32_t DescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) { - return binding.resources.empty() ? 1u : static_cast(binding.resources.size()); -} - -std::vector LayoutKey(DescriptorCache::Stage stage, - const ShaderRecompiler::IR::Program& program) { - std::vector key; - key.reserve(1u + program.bindings.descriptors.size() * 4u); - key.push_back(static_cast(stage)); - for (const auto& binding: program.bindings.descriptors) { - key.push_back(static_cast(binding.kind)); - key.push_back(binding.binding); - key.push_back(DescriptorCount(binding)); - key.push_back(static_cast(DescriptorType(binding.kind))); - } - return key; -} - -vk::DescriptorBufferInfo BufferInfo(const BufferView& view) { - EXIT_IF(view.buffer == nullptr); - return {view.buffer, view.offset, view.range}; -} - -} // namespace - -vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture, - uint32_t element) { - const auto view = - texture.mip_views.empty() - ? (element == 0u ? texture.image_view : vk::ImageView {}) - : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {}); - EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined); - return {nullptr, view, texture.layout}; -} - -DescriptorCache::~DescriptorCache() { - for (auto& [layout, sets]: m_free_sets_by_layout) { - (void)layout; - for (auto* set: sets) { - delete set; - } - } - for (const auto& [key, layout]: m_descriptor_set_layouts) { - (void)key; - m_graphics.device.destroyDescriptorSetLayout(layout, nullptr); - } - for (const auto& pool: m_pools) { - m_graphics.device.destroyDescriptorPool(pool.pool, nullptr); - } -} - -vk::DescriptorSetLayout -DescriptorCache::GetDescriptorSetLayoutInternal(Stage stage, - const ShaderRecompiler::IR::Program& program) { - const auto key = LayoutKey(stage, program); - if (const auto found = m_descriptor_set_layouts.find(key); - found != m_descriptor_set_layouts.end()) { - return found->second; - } - - std::vector bindings; - bindings.reserve(program.bindings.descriptors.size()); - for (const auto& descriptor: program.bindings.descriptors) { - bindings.push_back({descriptor.binding, DescriptorType(descriptor.kind), - DescriptorCount(descriptor), StageFlags(stage), nullptr}); - } - if (bindings.empty()) { - return nullptr; - } - - vk::DescriptorSetLayoutCreateInfo info {}; - info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; - info.bindingCount = static_cast(bindings.size()); - info.pBindings = bindings.data(); - vk::DescriptorSetLayout layout = nullptr; - const auto result = m_graphics.device.createDescriptorSetLayout(&info, nullptr, &layout); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || layout == nullptr); - m_descriptor_set_layouts.emplace(key, layout); - return layout; -} - -void DescriptorCache::CreatePool() { - KYTY_PROFILER_FUNCTION(); - constexpr uint32_t MaxSets = 512; - const vk::DescriptorPoolSize sizes[] = { - {vk::DescriptorType::eStorageBuffer, - MaxSets * (ShaderRecompiler::IR::ShaderInfo::MaxBuffers + - ShaderRecompiler::IR::ShaderInfo::MaxAddresses + 3u)}, - {vk::DescriptorType::eSampledImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages}, - {vk::DescriptorType::eStorageImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages}, - {vk::DescriptorType::eSampler, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxSamplers}, - }; - vk::DescriptorPoolCreateInfo info {}; - info.sType = vk::StructureType::eDescriptorPoolCreateInfo; - info.poolSizeCount = static_cast(std::size(sizes)); - info.pPoolSizes = sizes; - info.maxSets = MaxSets; - const auto pool_id = static_cast(m_pools.size()); - auto& pool = m_pools.emplace_back(); - const auto result = m_graphics.device.createDescriptorPool(&info, nullptr, &pool.pool); - EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || pool.pool == nullptr); - pool.next_free_pool = m_first_free_pool; - m_first_free_pool = pool_id; -} - -VulkanDescriptorSet* DescriptorCache::Allocate(Stage stage, - const ShaderRecompiler::IR::Program& program) { - KYTY_PROFILER_FUNCTION(); - Common::LockGuard lock(m_mutex); - const auto layout = GetDescriptorSetLayoutInternal(stage, program); - EXIT_IF(layout == nullptr); - auto& free_sets = m_free_sets_by_layout[layout]; - if (!free_sets.empty()) { - auto* result = free_sets.back(); - free_sets.pop_back(); - return result; - } - - constexpr uint32_t Batch = 32; - for (int attempt = 0; attempt < 2; attempt++) { - for (int pool_id = m_first_free_pool; pool_id != -1; - pool_id = m_pools[pool_id].next_free_pool) { - auto& pool = m_pools[pool_id]; - std::array layouts; - layouts.fill(layout); - std::array sets {}; - vk::DescriptorSetAllocateInfo info {}; - info.sType = vk::StructureType::eDescriptorSetAllocateInfo; - info.descriptorPool = pool.pool; - info.descriptorSetCount = Batch; - info.pSetLayouts = layouts.data(); - if (m_graphics.device.allocateDescriptorSets(&info, sets.data()) == - vk::Result::eSuccess) { - free_sets.reserve(free_sets.size() + Batch - 1u); - for (uint32_t i = 0; i + 1u < Batch; i++) { - free_sets.push_back(new VulkanDescriptorSet {sets[i], layout, pool_id}); - } - return new VulkanDescriptorSet {sets.back(), layout, pool_id}; - } - m_first_free_pool = pool.next_free_pool; - break; - } - CreatePool(); - } - return nullptr; -} - -void DescriptorCache::Recycle(VulkanDescriptorSet& set) { - EXIT_IF(set.set == nullptr || set.layout == nullptr); - Common::LockGuard lock(m_mutex); - m_free_sets_by_layout[set.layout].push_back(&set); -} - -VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage stage, - const ShaderRecompiler::IR::Program& program, - const NativeDescriptors& data) { - KYTY_PROFILER_FUNCTION(); - EXIT_IF(data.buffers.size() != program.info.buffers.size() || - data.images.size() != program.info.images.size() || - data.samplers.size() != program.info.samplers.size() || - data.addresses.size() != program.info.addresses.size()); - auto* set = Allocate(stage, program); - EXIT_NOT_IMPLEMENTED(set == nullptr); - - size_t descriptor_count = 0; - for (const auto& binding: program.bindings.descriptors) { - descriptor_count += DescriptorCount(binding); - } - std::vector buffer_infos; - std::vector image_infos; - std::vector writes; - buffer_infos.reserve(descriptor_count); - image_infos.reserve(descriptor_count); - writes.reserve(program.bindings.descriptors.size()); - std::vector image_occurrences(data.images.size()); - - for (const auto& binding: program.bindings.descriptors) { - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = set->set; - write.dstBinding = binding.binding; - write.descriptorType = DescriptorType(binding.kind); - write.descriptorCount = DescriptorCount(binding); - const auto buffer_start = buffer_infos.size(); - const auto image_start = image_infos.size(); - switch (binding.kind) { - case BindingKind::Buffers: - for (const auto resource: binding.resources) { - buffer_infos.push_back(BufferInfo(data.buffers.at(resource))); - } - break; - case BindingKind::AddressMemory: - for (const auto resource: binding.resources) { - buffer_infos.push_back(BufferInfo(data.addresses.at(resource))); - } - break; - case BindingKind::FlattenedSrt: - buffer_infos.push_back(BufferInfo(data.flattened_srt)); - break; - case BindingKind::UserData: buffer_infos.push_back(BufferInfo(data.user_data)); break; - case BindingKind::Gds: buffer_infos.push_back(BufferInfo(data.gds)); break; - case BindingKind::Samplers: - for (const auto resource: binding.resources) { - const auto sampler = data.samplers.at(resource); - EXIT_IF(sampler == nullptr); - image_infos.push_back({sampler, nullptr, vk::ImageLayout::eUndefined}); - } - break; - default: { - for (const auto resource: binding.resources) { - const auto& texture = data.images.at(resource); - image_infos.push_back(MakeImageInfo(texture, image_occurrences.at(resource)++)); - } - break; - } - } - if (buffer_infos.size() != buffer_start) { - write.pBufferInfo = buffer_infos.data() + buffer_start; - } - if (image_infos.size() != image_start) { - write.pImageInfo = image_infos.data() + image_start; - } - writes.push_back(write); - } - for (uint32_t i = 0; i < data.images.size(); i++) { - const auto expected = data.images[i].mip_views.empty() - ? 1u - : static_cast(data.images[i].mip_views.size()); - EXIT_IF(image_occurrences[i] != expected); - } - m_graphics.device.updateDescriptorSets(static_cast(writes.size()), writes.data(), 0, - nullptr); - return *set; -} - -vk::DescriptorSetLayout -DescriptorCache::GetDescriptorSetLayout(Stage stage, const ShaderRecompiler::IR::Program& program) { - Common::LockGuard lock(m_mutex); - return GetDescriptorSetLayoutInternal(stage, program); -} - -} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h b/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h deleted file mode 100644 index ff35bd1ec..000000000 --- a/src/graphics/host_gpu/renderer/pipeline/descriptorCache.h +++ /dev/null @@ -1,118 +0,0 @@ -#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ -#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ - -#include "common/abi.h" -#include "common/assert.h" -#include "common/common.h" -#include "common/threads.h" -#include "graphics/host_gpu/graphicContext.h" -#include "graphics/host_gpu/renderer/cache/bufferCache.h" -#include "graphics/host_gpu/renderer/cache/textureCache.h" -#include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/shader/shaderBindings.h" - -#include -#include -#include -#include -#include - -namespace Libs::Graphics { - -namespace ShaderRecompiler::IR { -struct DescriptorValue; -struct ImageResource; -struct Program; -struct ResourceSnapshot; -} // namespace ShaderRecompiler::IR - -class CommandBuffer; -struct DescriptorCacheTestAccess; -struct ShaderStageRuntime; - -struct VulkanDescriptorSet { - vk::DescriptorSet set = nullptr; - vk::DescriptorSetLayout layout = nullptr; - int pool_id = -1; -}; - -struct BufferView { - vk::Buffer buffer = nullptr; - vk::DeviceSize offset = 0; - vk::DeviceSize range = VK_WHOLE_SIZE; -}; - -class DescriptorCache { -public: - enum class Stage { Unknown, Vertex, Pixel, Compute }; - - struct TextureBinding { - ImageId image_id; - vk::ImageView image_view = nullptr; - TextureCache::ImageDesc desc; - vk::ImageLayout layout = vk::ImageLayout::eUndefined; - std::vector mip_views; - }; - - struct NativeDescriptors { - std::vector buffers; - std::vector images; - std::vector samplers; - std::vector addresses; - BufferView gds; - BufferView flattened_srt; - BufferView user_data; - }; - - struct PreparedBindings { - std::shared_ptr program; - std::shared_ptr snapshot; - NativeDescriptors resources; - std::vector buffer_ids; - std::vector address_ids; - std::vector flattened_srt; - std::vector user_data; - vk::ShaderStageFlags shader_stage; - Stage stage = Stage::Unknown; - bool committed = false; - }; - - explicit DescriptorCache(GraphicContext& graphics): m_graphics(graphics) { - EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); - } - ~DescriptorCache(); - KYTY_CLASS_NO_COPY(DescriptorCache); - - vk::DescriptorSetLayout GetDescriptorSetLayout(Stage stage, - const ShaderRecompiler::IR::Program& program); - void Recycle(VulkanDescriptorSet& set); - VulkanDescriptorSet& GetDescriptor(Stage stage, const ShaderRecompiler::IR::Program& program, - const NativeDescriptors& descriptors); - -private: - friend struct DescriptorCacheTestAccess; - - struct Pool { - vk::DescriptorPool pool = nullptr; - int next_free_pool = -1; - }; - - static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, - uint32_t element = 0); - void CreatePool(); - VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program); - vk::DescriptorSetLayout - GetDescriptorSetLayoutInternal(Stage stage, const ShaderRecompiler::IR::Program& program); - - GraphicContext& m_graphics; - Common::Mutex m_mutex; - std::vector m_pools; - int m_first_free_pool = -1; - std::unordered_map> - m_free_sets_by_layout; - std::map, vk::DescriptorSetLayout> m_descriptor_set_layouts; -}; - -} // namespace Libs::Graphics - -#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_ diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp new file mode 100644 index 000000000..20dd92bdc --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp @@ -0,0 +1,97 @@ +#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h" + +#include "common/assert.h" +#include "common/profiler.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/masterSemaphore.h" + +namespace Libs::Graphics { +namespace { + +constexpr uint32_t DescriptorHeapCount = 1024; +constexpr std::array DescriptorPoolSizes = { + vk::DescriptorPoolSize {vk::DescriptorType::eStorageBuffer, 8192}, + vk::DescriptorPoolSize {vk::DescriptorType::eSampledImage, 8192}, + vk::DescriptorPoolSize {vk::DescriptorType::eStorageImage, 1024}, + vk::DescriptorPoolSize {vk::DescriptorType::eSampler, 1024}, +}; + +} // namespace + +DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore) + : m_graphics(graphics), m_master_semaphore(master_semaphore) { + CreatePool(); +} + +DescriptorHeap::~DescriptorHeap() { + m_graphics.device.destroyDescriptorPool(m_current_pool, nullptr); + for (const auto& [pool, tick]: m_pending_pools) { + m_master_semaphore.Wait(tick); + m_graphics.device.destroyDescriptorPool(pool, nullptr); + } +} + +vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) { + KYTY_PROFILER_FUNCTION(); + EXIT_IF(layout == nullptr); + + auto& batch = m_sets[layout]; + if (batch.size != 0) { + return batch.sets[--batch.size]; + } + if (Allocate(layout, batch)) { + return batch.sets[--batch.size]; + } + + m_pending_pools.emplace_back(m_current_pool, m_master_semaphore.CurrentTick()); + if (const auto& [pool, tick] = m_pending_pools.front(); m_master_semaphore.IsFree(tick)) { + m_current_pool = pool; + m_pending_pools.pop_front(); + EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess); + } else { + CreatePool(); + } + + m_sets.clear(); + auto& fresh_batch = m_sets[layout]; + EXIT_IF(!Allocate(layout, fresh_batch)); + return fresh_batch.sets[--fresh_batch.size]; +} + +bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) { + std::array layouts; + layouts.fill(layout); + + vk::DescriptorSetAllocateInfo allocate {}; + allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo; + allocate.descriptorPool = m_current_pool; + allocate.pSetLayouts = layouts.data(); + + for (;;) { + allocate.descriptorSetCount = batch.allocation; + const auto result = m_graphics.device.allocateDescriptorSets(&allocate, batch.sets.data()); + if (result == vk::Result::eSuccess) { + batch.size = batch.allocation; + return true; + } + EXIT_IF(result != vk::Result::eErrorOutOfPoolMemory && + result != vk::Result::eErrorFragmentedPool); + if (batch.allocation == 1) { + return false; + } + batch.allocation /= 2; + } +} + +void DescriptorHeap::CreatePool() { + vk::DescriptorPoolCreateInfo create {}; + create.sType = vk::StructureType::eDescriptorPoolCreateInfo; + create.flags = {}; + create.maxSets = DescriptorHeapCount; + create.poolSizeCount = static_cast(DescriptorPoolSizes.size()); + create.pPoolSizes = DescriptorPoolSizes.data(); + EXIT_IF(m_graphics.device.createDescriptorPool(&create, nullptr, &m_current_pool) != + vk::Result::eSuccess); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h b/src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h new file mode 100644 index 000000000..86cad34d1 --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h @@ -0,0 +1,45 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_ + +#include "common/common.h" +#include "graphics/host_gpu/vulkanCommon.h" + +#include +#include +#include + +namespace Libs::Graphics { + +struct GraphicContext; +class MasterSemaphore; + +class DescriptorHeap { +public: + DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore); + ~DescriptorHeap(); + KYTY_CLASS_NO_COPY(DescriptorHeap); + + [[nodiscard]] vk::DescriptorSet Commit(vk::DescriptorSetLayout layout); + +private: + static constexpr uint32_t DescriptorSetBatch = 32; + + struct Batch { + std::array sets {}; + uint32_t size = 0; + uint32_t allocation = DescriptorSetBatch; + }; + + [[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch); + void CreatePool(); + + GraphicContext& m_graphics; + MasterSemaphore& m_master_semaphore; + vk::DescriptorPool m_current_pool = nullptr; + std::deque> m_pending_pools; + std::unordered_map m_sets; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_ diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index 11ea9736d..627eb9258 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -17,7 +17,6 @@ #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/image/textureCommon.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" @@ -45,14 +44,95 @@ namespace Libs::Graphics { -static const char* ShaderStageResourceName(DescriptorCache::Stage stage) { - switch (stage) { - case DescriptorCache::Stage::Vertex: return "Vertex"; - case DescriptorCache::Stage::Pixel: return "Pixel"; - case DescriptorCache::Stage::Compute: return "Compute"; - case DescriptorCache::Stage::Unknown: return "Unknown"; +namespace { + +using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; + +bool IsSampledImage(BindingKind kind) { + switch (kind) { + case BindingKind::Sampled1D: + case BindingKind::Sampled1DArray: + case BindingKind::Sampled2D: + case BindingKind::Sampled2DArray: + case BindingKind::Sampled2DMsaa: + case BindingKind::Sampled2DMsaaArray: + case BindingKind::Sampled3D: + case BindingKind::SampledUint1D: + case BindingKind::SampledUint1DArray: + case BindingKind::SampledUint2D: + case BindingKind::SampledUint2DArray: + case BindingKind::SampledUint2DMsaa: + case BindingKind::SampledUint2DMsaaArray: + case BindingKind::SampledUint3D: return true; + default: return false; + } +} + +bool IsStorageImage(BindingKind kind) { + switch (kind) { + case BindingKind::Storage1D: + case BindingKind::Storage1DArray: + case BindingKind::Storage2D: + case BindingKind::Storage2DArray: + case BindingKind::Storage3D: + case BindingKind::StorageUint1D: + case BindingKind::StorageUint1DArray: + case BindingKind::StorageUint2D: + case BindingKind::StorageUint2DArray: + case BindingKind::StorageUint3D: + case BindingKind::StorageAtomic1D: + case BindingKind::StorageAtomic1DArray: + case BindingKind::StorageAtomic2D: + case BindingKind::StorageAtomic2DArray: + case BindingKind::StorageAtomic3D: return true; + default: return false; + } +} + +} // namespace + +vk::DescriptorType NativeDescriptorType(BindingKind kind) { + if (kind == BindingKind::Samplers) { + return vk::DescriptorType::eSampler; + } + if (IsSampledImage(kind)) { + return vk::DescriptorType::eSampledImage; + } + if (IsStorageImage(kind)) { + return vk::DescriptorType::eStorageImage; + } + return vk::DescriptorType::eStorageBuffer; +} + +uint32_t NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) { + return binding.resources.empty() ? 1u : static_cast(binding.resources.size()); +} + +vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element) { + const auto view = + texture.mip_views.empty() + ? (element == 0u ? texture.image_view : vk::ImageView {}) + : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {}); + EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined); + return {nullptr, view, texture.layout}; +} + +static const char* ShaderStageResourceName(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "Vertex"; + case ShaderType::Pixel: return "Pixel"; + case ShaderType::Compute: return "Compute"; + default: return "Unknown"; + } +} + +static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex; + case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment; + case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute; + default: EXIT("unknown native shader stage\n"); } - return "Unknown"; } static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) { @@ -87,11 +167,11 @@ static bool IsMultisampledTexture(Prospero::ImageType type) { type == Prospero::ImageType::kColor2DMsaaArray; } -static BufferView -NativeStorageBuffer(RenderContext& context, const ShaderBufferResource& descriptor, - const ShaderRecompiler::IR::BufferResource& resource, - DescriptorCache::Stage stage, uint32_t slot, uint32_t& buffer_offset, - BufferId id) { +static BufferView NativeStorageBuffer(RenderContext& context, + const ShaderBufferResource& descriptor, + const ShaderRecompiler::IR::BufferResource& resource, + ShaderType stage, uint32_t slot, uint32_t& buffer_offset, + BufferId id) { BufferView result; buffer_offset = 0; @@ -113,8 +193,8 @@ NativeStorageBuffer(RenderContext& context, const ShaderBufferResource& descript size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange) { EXIT("storage buffer range or device alignment is unsupported\n"); } - auto [buffer, offset] = context.GetBufferCache().ObtainBuffer( - address, size, resource.written, resource.formatted, id); + auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written, + resource.formatted, id); const auto aligned_offset = offset - offset % alignment; const auto adjustment = offset - aligned_offset; const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange; @@ -136,10 +216,10 @@ NativeStorageBuffer(RenderContext& context, const ShaderBufferResource& descript return result; } -static BufferView NativeAddressBuffer( - RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource, - const ShaderRecompiler::IR::ResourceSnapshot::Address& address, - DescriptorCache::Stage stage, uint32_t slot, uint32_t& address_offset, BufferId id) { +static BufferView +NativeAddressBuffer(RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource, + const ShaderRecompiler::IR::ResourceSnapshot::Address& address, + ShaderType stage, uint32_t slot, uint32_t& address_offset, BufferId id) { BufferView result; address_offset = 0; if (address.binding_base == 0) { @@ -640,9 +720,8 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& return view; } -DescriptorCache::TextureBinding -RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderRecompiler::IR::DescriptorValue& value) { +TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderRecompiler::IR::DescriptorValue& value) { ShaderTextureResource descriptor; CopyNativeDescriptor(value, descriptor.fields); const bool storage = resource.written; @@ -813,7 +892,7 @@ static BufferView NativeUpload(RenderContext& context, std::span EXIT_IF(data.empty()); auto& command_buffer = context.GetCommandScheduler().Current(); EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute()); - auto& buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream); + auto& buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream); const auto offset = buffer.Copy(data.data(), data.size_bytes(), 256); return {.buffer = buffer.Handle(), .offset = offset, .range = data.size_bytes()}; } @@ -853,9 +932,7 @@ void RenderExecutor::ResetBindings() { m_bound_images.clear(); } -DescriptorCache::PreparedBindings -RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, - vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage) { +PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime) { KYTY_PROFILER_FUNCTION(); EXIT_IF(!runtime); const auto& program = *runtime.program; @@ -865,12 +942,10 @@ RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, EXIT("invalid native shader runtime snapshot: %s\n", error.c_str()); } - DescriptorCache::PreparedBindings prepared; - prepared.program = runtime.program; - prepared.snapshot = runtime.resources; - prepared.shader_stage = shader_stage; - prepared.stage = stage; - auto& descriptors = prepared.resources; + PreparedBindings prepared; + prepared.program = runtime.program.get(); + prepared.snapshot = runtime.resources.get(); + auto& descriptors = prepared.resources; descriptors.buffers.reserve(program.info.buffers.size()); descriptors.images.reserve(program.info.images.size()); for (uint32_t i = 0; i < program.info.images.size(); i++) { @@ -900,7 +975,7 @@ RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, return prepared; } -void RenderExecutor::FindBuffers(DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::FindBuffers(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); const auto& program = *prepared.program; @@ -916,8 +991,7 @@ void RenderExecutor::FindBuffers(DescriptorCache::PreparedBindings& prepared) { const auto stride = descriptor.Stride(); const auto records = descriptor.NumRecords(); EXIT_IF(stride != 0 && records > UINT64_MAX / stride); - const auto requested_size = - stride != 0 ? static_cast(stride) * records : records; + const auto requested_size = stride != 0 ? static_cast(stride) * records : records; if (address == 0 || requested_size == 0) { prepared.buffer_ids.emplace_back(); continue; @@ -936,19 +1010,18 @@ void RenderExecutor::FindBuffers(DescriptorCache::PreparedBindings& prepared) { continue; } EXIT_IF(resource.written); - const auto limit = - resource.kind == ShaderRecompiler::IR::ResourceKind::Flat - ? ShaderRecompiler::IR::FlatAddressWindowSize - : static_cast(m_context.GetGraphics() - .GetPhysicalDeviceProperties() - .limits.maxStorageBufferRange); - uint64_t size = 0; + const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat + ? ShaderRecompiler::IR::FlatAddressWindowSize + : static_cast(m_context.GetGraphics() + .GetPhysicalDeviceProperties() + .limits.maxStorageBufferRange); + uint64_t size = 0; EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size)); prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size)); } } -void RenderExecutor::RebindBuffers(DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); const auto& program = *prepared.program; @@ -972,7 +1045,7 @@ void RenderExecutor::RebindBuffers(DescriptorCache::PreparedBindings& prepared) CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); uint32_t buffer_offset = 0; resources.buffers.push_back(NativeStorageBuffer(m_context, descriptor, - program.info.buffers[i], prepared.stage, i, + program.info.buffers[i], program.stage, i, buffer_offset, prepared.buffer_ids[i])); pack_memory_offset(i, buffer_offset); } @@ -980,9 +1053,9 @@ void RenderExecutor::RebindBuffers(DescriptorCache::PreparedBindings& prepared) resources.addresses.reserve(program.info.addresses.size()); for (uint32_t i = 0; i < program.info.addresses.size(); i++) { uint32_t address_offset = 0; - resources.addresses.push_back( - NativeAddressBuffer(m_context, program.info.addresses[i], snapshot.addresses[i], - prepared.stage, i, address_offset, prepared.address_ids[i])); + resources.addresses.push_back(NativeAddressBuffer(m_context, program.info.addresses[i], + snapshot.addresses[i], program.stage, i, + address_offset, prepared.address_ids[i])); pack_memory_offset(static_cast(program.info.buffers.size()) + i, address_offset); } if (!prepared.flattened_srt.empty()) { @@ -994,7 +1067,7 @@ void RenderExecutor::RebindBuffers(DescriptorCache::PreparedBindings& prepared) } } -void RenderExecutor::RebindImages(DescriptorCache::PreparedBindings& prepared) { +void RenderExecutor::RebindImages(PreparedBindings& prepared) { KYTY_PROFILER_FUNCTION(); EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr); const auto& program = *prepared.program; @@ -1047,12 +1120,10 @@ RenderExecutor::GraphicsBindings RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex, const ShaderStageRuntime& pixel, bool pixel_active) { GraphicsBindings bindings { - .vertex = PrepareBindings(vertex, vk::ShaderStageFlagBits::eVertex, - DescriptorCache::Stage::Vertex), + .vertex = PrepareBindings(vertex), }; if (pixel_active) { - bindings.pixel.emplace(PrepareBindings(pixel, vk::ShaderStageFlagBits::eFragment, - DescriptorCache::Stage::Pixel)); + bindings.pixel.emplace(PrepareBindings(pixel)); } FindBuffers(bindings.vertex); if (bindings.pixel) { @@ -1071,74 +1142,190 @@ RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex, void RenderExecutor::CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, - vk::PipelineLayout layout, - DescriptorCache::PreparedBindings& prepared) { + const PipelineCache::Pipeline& pipeline, + std::span prepared_bindings) { KYTY_PROFILER_FUNCTION(); - EXIT_IF(prepared.program == nullptr || prepared.stage == DescriptorCache::Stage::Unknown || - prepared.committed); - const auto& program = *prepared.program; - auto& descriptors = prepared.resources; - auto vk_buffer = buffer.Handle(); - const auto shader_stages = ShaderPipelineStages(prepared.shader_stage); - if (descriptors.gds.buffer != nullptr) { - buffer.EndRendering(); - const auto barrier = MakeGdsDependency(descriptors.gds.buffer); - vk_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer | - vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader, - shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr); - } - - for (uint32_t i = 0; i < program.info.images.size(); i++) { - auto& image = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id); - auto& binding = descriptors.images[i]; - const auto& view = binding.desc.view_info; - const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer, - view.layer_count}; - const bool storage = binding.desc.type == TextureCache::BindingType::Storage; - if (image.info.data.Empty()) { - image.Transit(vk::ImageLayout::eGeneral, - storage - ? vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite - : vk::AccessFlagBits2::eShaderRead, - range, vk_buffer); - } else if ((image.binding.force_general || image.binding.is_target) && - !image.info.IsDepth()) { - const vk::AccessFlags2 storage_access = image.binding.shader_write - ? vk::AccessFlagBits2::eShaderWrite - : vk::AccessFlags2 {}; - image.Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | storage_access | - vk::AccessFlagBits2::eColorAttachmentRead | - vk::AccessFlagBits2::eColorAttachmentWrite, - {}, vk_buffer); - } else if (storage) { - image.Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite, - range, vk_buffer); - } else { - image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal - : vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, range, vk_buffer); + auto vk_buffer = buffer.Handle(); + size_t descriptor_count = 0; + size_t write_count = 0; + constexpr auto GraphicsStages = + vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment; + const auto push_constant_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics + ? vk::ShaderStageFlags {GraphicsStages} + : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}; + for (const auto* prepared: prepared_bindings) { + EXIT_IF(prepared == nullptr || prepared->program == nullptr || + prepared->snapshot == nullptr || prepared->committed); + write_count += prepared->program->bindings.descriptors.size(); + for (const auto& binding: prepared->program->bindings.descriptors) { + descriptor_count += NativeDescriptorCount(binding); } - binding.layout = image.backing.state.layout; + const auto shader_stage = NativeShaderStage(prepared->program->stage); + EXIT_IF((pipeline_bind_point == vk::PipelineBindPoint::eGraphics && + (shader_stage & GraphicsStages) == vk::ShaderStageFlags {}) || + (pipeline_bind_point == vk::PipelineBindPoint::eCompute && + shader_stage != vk::ShaderStageFlagBits::eCompute)); } + m_descriptor_buffers.clear(); + m_descriptor_images.clear(); + m_descriptor_writes.clear(); + m_push_constants.fill(0); + m_descriptor_buffers.reserve(descriptor_count); + m_descriptor_images.reserve(descriptor_count); + m_descriptor_writes.reserve(write_count); - if (!program.bindings.descriptors.empty()) { - auto& set = - m_context.GetDescriptorCache().GetDescriptor(prepared.stage, program, descriptors); - vk_buffer.bindDescriptorSets(pipeline_bind_point, layout, program.bindings.descriptor_set, - 1, &set.set, 0, nullptr); - buffer.RecycleDescriptorAfterFence(set); + for (auto* prepared: prepared_bindings) { + const auto& program = *prepared->program; + auto& descriptors = prepared->resources; + const auto shader_stage = NativeShaderStage(program.stage); + const auto shader_stages = ShaderPipelineStages(shader_stage); + if (descriptors.gds.buffer != nullptr) { + buffer.EndRendering(); + const auto barrier = MakeGdsDependency(descriptors.gds.buffer); + vk_buffer.pipelineBarrier( + vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer | + vk::PipelineStageFlagBits::eAllGraphics | + vk::PipelineStageFlagBits::eComputeShader, + shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr); + } + + for (uint32_t i = 0; i < program.info.images.size(); i++) { + auto& image = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id); + auto& binding = descriptors.images[i]; + const auto& view = binding.desc.view_info; + const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer, + view.layer_count}; + const bool storage = binding.desc.type == TextureCache::BindingType::Storage; + if (image.info.data.Empty()) { + image.Transit(vk::ImageLayout::eGeneral, + storage ? vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eShaderWrite + : vk::AccessFlagBits2::eShaderRead, + range, vk_buffer); + } else if ((image.binding.force_general || image.binding.is_target) && + !image.info.IsDepth()) { + const vk::AccessFlags2 storage_access = image.binding.shader_write + ? vk::AccessFlagBits2::eShaderWrite + : vk::AccessFlags2 {}; + image.Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | storage_access | + vk::AccessFlagBits2::eColorAttachmentRead | + vk::AccessFlagBits2::eColorAttachmentWrite, + {}, vk_buffer); + } else if (storage) { + image.Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite, + range, vk_buffer); + } else { + image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal + : vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, range, vk_buffer); + } + binding.layout = image.backing.state.layout; + } + + m_image_occurrences.assign(descriptors.images.size(), 0); + for (const auto& binding: program.bindings.descriptors) { + vk::WriteDescriptorSet write {}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstBinding = ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind); + write.descriptorType = NativeDescriptorType(binding.kind); + write.descriptorCount = NativeDescriptorCount(binding); + const auto buffer_start = m_descriptor_buffers.size(); + const auto image_start = m_descriptor_images.size(); + switch (binding.kind) { + case BindingKind::Buffers: + for (const auto resource: binding.resources) { + const auto& view = descriptors.buffers.at(resource); + EXIT_IF(view.buffer == nullptr); + m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range); + } + break; + case BindingKind::AddressMemory: + for (const auto resource: binding.resources) { + const auto& view = descriptors.addresses.at(resource); + EXIT_IF(view.buffer == nullptr); + m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range); + } + break; + case BindingKind::FlattenedSrt: + case BindingKind::UserData: + case BindingKind::Gds: { + const auto& view = + binding.kind == BindingKind::FlattenedSrt ? descriptors.flattened_srt + : binding.kind == BindingKind::UserData ? descriptors.user_data + : descriptors.gds; + EXIT_IF(view.buffer == nullptr); + m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range); + break; + } + case BindingKind::Samplers: + for (const auto resource: binding.resources) { + const auto sampler = descriptors.samplers.at(resource); + EXIT_IF(sampler == nullptr); + m_descriptor_images.emplace_back(sampler, nullptr, + vk::ImageLayout::eUndefined); + } + break; + default: + for (const auto resource: binding.resources) { + m_descriptor_images.push_back(MakeImageInfo( + descriptors.images.at(resource), m_image_occurrences.at(resource)++)); + } + break; + } + if (m_descriptor_buffers.size() != buffer_start) { + write.pBufferInfo = m_descriptor_buffers.data() + buffer_start; + } + if (m_descriptor_images.size() != image_start) { + write.pImageInfo = m_descriptor_images.data() + image_start; + } + m_descriptor_writes.push_back(write); + } + for (uint32_t i = 0; i < descriptors.images.size(); i++) { + const auto expected = + descriptors.images[i].mip_views.empty() + ? 1u + : static_cast(descriptors.images[i].mip_views.size()); + EXIT_IF(m_image_occurrences[i] != expected); + } + + if (program.bindings.push_constant_size != 0) { + const auto offset = program.bindings.push_constant_offset; + EXIT_IF(offset % sizeof(uint32_t) != 0 || + program.bindings.push_constant_size != + prepared->user_data.size() * sizeof(uint32_t) || + offset + program.bindings.push_constant_size > + ShaderRecompiler::IR::NativePushConstantSize); + std::copy(prepared->user_data.begin(), prepared->user_data.end(), + m_push_constants.begin() + offset / sizeof(uint32_t)); + } } - if (program.bindings.push_constant_size != 0) { - EXIT_IF(program.bindings.push_constant_size != - prepared.user_data.size() * sizeof(uint32_t)); - vk_buffer.pushConstants(layout, prepared.shader_stage, - program.bindings.push_constant_offset, - program.bindings.push_constant_size, prepared.user_data.data()); + vk_buffer.pushConstants(pipeline.pipeline_layout, push_constant_stages, 0, + ShaderRecompiler::IR::NativePushConstantSize, + m_push_constants.data()); + + if (!m_descriptor_writes.empty()) { + EXIT_IF(pipeline.descriptor_set_layout == nullptr); + if (pipeline.uses_push_descriptors) { + vk_buffer.pushDescriptorSetKHR(pipeline_bind_point, pipeline.pipeline_layout, 0, + static_cast(m_descriptor_writes.size()), + m_descriptor_writes.data()); + } else { + const auto set = m_context.GetDescriptorHeap().Commit(pipeline.descriptor_set_layout); + for (auto& write: m_descriptor_writes) { + write.dstSet = set; + } + m_context.GetGraphics().device.updateDescriptorSets( + static_cast(m_descriptor_writes.size()), m_descriptor_writes.data(), 0, + nullptr); + vk_buffer.bindDescriptorSets(pipeline_bind_point, pipeline.pipeline_layout, 0, 1, &set, + 0, nullptr); + } + } + for (auto* prepared: prepared_bindings) { + prepared->committed = true; } - prepared.committed = true; } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.h b/src/graphics/host_gpu/renderer/pipeline/descriptors.h index 0fe9353a4..d857d6885 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.h +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.h @@ -2,6 +2,8 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_ #include "common/assert.h" +#include "graphics/host_gpu/renderer/cache/bufferCache.h" +#include "graphics/host_gpu/renderer/cache/textureCache.h" #include "graphics/host_gpu/renderer/image/image.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" @@ -10,9 +12,56 @@ #include #include #include +#include namespace Libs::Graphics { +namespace ShaderRecompiler::IR { +struct ResourceSnapshot; +} + +struct BufferView { + vk::Buffer buffer = nullptr; + vk::DeviceSize offset = 0; + vk::DeviceSize range = VK_WHOLE_SIZE; +}; + +struct TextureBinding { + ImageId image_id; + vk::ImageView image_view = nullptr; + TextureCache::ImageDesc desc; + vk::ImageLayout layout = vk::ImageLayout::eUndefined; + std::vector mip_views; +}; + +struct NativeDescriptors { + std::vector buffers; + std::vector images; + std::vector samplers; + std::vector addresses; + BufferView gds; + BufferView flattened_srt; + BufferView user_data; +}; + +struct PreparedBindings { + const ShaderRecompiler::IR::Program* program = nullptr; + const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr; + NativeDescriptors resources; + std::vector buffer_ids; + std::vector address_ids; + std::vector flattened_srt; + std::vector user_data; + bool committed = false; +}; + +[[nodiscard]] vk::DescriptorType +NativeDescriptorType(ShaderRecompiler::IR::DescriptorBindingKind kind); +[[nodiscard]] uint32_t +NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding); +[[nodiscard]] vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, + uint32_t element = 0); + template [[nodiscard]] T DecodeNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& value) { static_assert(std::is_trivially_copyable_v); @@ -34,11 +83,9 @@ ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource, Prospero::ImageType type, uint32_t base_layer, uint32_t image_layers); [[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, - const Image& image, - bool r128 = false); + const Image& image, bool r128 = false); [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, - const Image& image, - bool r128 = false); + const Image& image, bool r128 = false); [[nodiscard]] bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource, const ShaderTextureResource& descriptor, const Image& image); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 93731ba4a..40c745637 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -42,6 +42,7 @@ PipelineCache::~PipelineCache() { (void)key; m_graphics.device.destroyPipeline(pipeline->pipeline, nullptr); m_graphics.device.destroyPipelineLayout(pipeline->pipeline_layout, nullptr); + m_graphics.device.destroyDescriptorSetLayout(pipeline->descriptor_set_layout, nullptr); } }; destroy(m_graphics_pipelines); @@ -210,9 +211,9 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( auto cached = std::make_unique(p); LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0, ps_id.crc32); - CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info, - vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0, - vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active); + CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vs_spirv, ps_input_info, + ps_spirv, static_params, vs_id.hash0, vs_id.crc32, ps_id.hash0, + ps_id.crc32, ps_active); LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0, ps_id.crc32); @@ -252,7 +253,7 @@ PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info, } auto cached = std::make_unique(p); - CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, input_info, cs_spirv); + CreatePipelineInternal(m_graphics, *cached, input_info, cs_spirv); EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index e4bff1031..b4c60f0bc 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -21,7 +21,6 @@ struct GraphicContext; struct RenderColorInfo; struct RenderDepthInfo; class RenderCommandBuffer; -class DescriptorCache; namespace HW { class Context; @@ -98,16 +97,17 @@ struct PipelineRenderingState { class PipelineCache { public: - PipelineCache(GraphicContext& graphics, DescriptorCache& descriptor_cache) - : m_graphics(graphics), m_descriptor_cache(descriptor_cache) { + explicit PipelineCache(GraphicContext& graphics): m_graphics(graphics) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } ~PipelineCache(); KYTY_CLASS_NO_COPY(PipelineCache); struct Pipeline { - vk::PipelineLayout pipeline_layout = nullptr; - vk::Pipeline pipeline = nullptr; + vk::PipelineLayout pipeline_layout = nullptr; + vk::Pipeline pipeline = nullptr; + vk::DescriptorSetLayout descriptor_set_layout = nullptr; + bool uses_push_descriptors = false; }; struct GraphicsPipeline: Pipeline { @@ -201,8 +201,7 @@ private: } }; - GraphicContext& m_graphics; - DescriptorCache& m_descriptor_cache; + GraphicContext& m_graphics; std::unordered_map, GraphicsPipelineKeyHash> m_graphics_pipelines; @@ -214,16 +213,14 @@ private: void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32); void CreatePipelineInternal( - GraphicContext& graphics, DescriptorCache& descriptor_cache, - PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering, - const ShaderVertexInputInfo& vs_input_info, std::span vs_shader, - const ShaderPixelInputInfo* ps_input_info, std::span ps_shader, - const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32, - uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active); -void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache, - PipelineCache::ComputePipeline& pipeline, - const ShaderComputeInputInfo& input_info, - std::span cs_shader); + GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline, + const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, + std::span vs_shader, const ShaderPixelInputInfo* ps_input_info, + std::span ps_shader, const PipelineStaticParameters& static_params, + uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active); +void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline, + const ShaderComputeInputInfo& input_info, + std::span cs_shader); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 89b8c10a7..5f559db05 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -6,7 +6,7 @@ #include "graphics/guest_gpu/gpu_defs.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/debug.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" +#include "graphics/host_gpu/renderer/pipeline/descriptors.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" @@ -19,6 +19,7 @@ #include #include #include +#include namespace Libs::Graphics { @@ -388,40 +389,42 @@ static vk::BlendOp GetBlendOp(uint32_t op) { return vk::BlendOp::eAdd; } -static void CreateLayout(DescriptorCache& descriptor_cache, - std::span set_layouts, uint32_t& set_layouts_num, - std::span push_constant_info, - uint32_t& push_constant_info_num, - const ShaderRecompiler::IR::Program& program, - vk::ShaderStageFlags vk_stage, DescriptorCache::Stage stage) { - const auto& bindings = program.bindings; - const bool need_descriptor = !bindings.descriptors.empty(); - - if (bindings.push_constant_size != 0) { - auto index = push_constant_info_num; - - push_constant_info[index].stageFlags = vk_stage; - push_constant_info[index].offset = bindings.push_constant_offset; - push_constant_info[index].size = bindings.push_constant_size; - push_constant_info_num++; +static void AddLayoutBindings(std::vector& descriptor_bindings, + const ShaderRecompiler::IR::Program& program, + vk::ShaderStageFlagBits stage) { + for (const auto& binding: program.bindings.descriptors) { + descriptor_bindings.push_back( + {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind), + NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), stage, nullptr}); } +} - if (need_descriptor) { - EXIT_IF(bindings.descriptor_set != set_layouts_num); - - set_layouts[set_layouts_num] = descriptor_cache.GetDescriptorSetLayout(stage, program); - set_layouts_num++; +static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipeline& pipeline, + std::span bindings) { + uint32_t descriptor_count = 0; + for (const auto& binding: bindings) { + descriptor_count += binding.descriptorCount; } + pipeline.uses_push_descriptors = descriptor_count <= graphics.max_push_descriptors; + + vk::DescriptorSetLayoutCreateInfo create {}; + create.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + create.flags = pipeline.uses_push_descriptors + ? vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR + : vk::DescriptorSetLayoutCreateFlags {}; + create.bindingCount = static_cast(bindings.size()); + create.pBindings = bindings.data(); + EXIT_IF(graphics.device.createDescriptorSetLayout( + &create, nullptr, &pipeline.descriptor_set_layout) != vk::Result::eSuccess); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) void CreatePipelineInternal( - GraphicContext& graphics, DescriptorCache& descriptor_cache, - PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering, - const ShaderVertexInputInfo& vs_input_info, std::span vs_shader, - const ShaderPixelInputInfo* ps_input_info, std::span ps_shader, - const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32, - uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) { + GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline, + const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, + std::span vs_shader, const ShaderPixelInputInfo* ps_input_info, + std::span ps_shader, const PipelineStaticParameters& static_params, + uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) { EXIT_IF(ps_active && ps_input_info == nullptr); const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList; @@ -823,40 +826,36 @@ void CreatePipelineInternal( color_blending.blendConstants[2] = static_params.blend_color_blue; color_blending.blendConstants[3] = static_params.blend_color_alpha; - vk::DescriptorSetLayout set_layouts[2] = {}; - uint32_t set_layouts_num = 0; - - vk::PushConstantRange push_constant_info[2]; - uint32_t push_constant_info_num = 0; - EXIT_IF(!vs_input_info.stage); - CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, - push_constant_info_num, *vs_input_info.stage.program, - vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex); + std::vector descriptor_bindings; + AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program, + vk::ShaderStageFlagBits::eVertex); if (ps_active) { EXIT_IF(!ps_input_info->stage); - CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, - push_constant_info_num, *ps_input_info->stage.program, - vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel); + AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program, + vk::ShaderStageFlagBits::eFragment); } + CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); + constexpr auto GraphicsStages = + vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment; + const vk::PushConstantRange push_constants { + GraphicsStages, 0, ShaderRecompiler::IR::NativePushConstantSize}; vk::PipelineLayoutCreateInfo pipeline_layout_info {}; pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; pipeline_layout_info.pNext = nullptr; pipeline_layout_info.flags = {}; - pipeline_layout_info.setLayoutCount = set_layouts_num; - pipeline_layout_info.pSetLayouts = (set_layouts_num > 0 ? set_layouts : nullptr); - pipeline_layout_info.pushConstantRangeCount = push_constant_info_num; - pipeline_layout_info.pPushConstantRanges = - push_constant_info_num > 0 ? push_constant_info : nullptr; + pipeline_layout_info.setLayoutCount = 1; + pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout; + pipeline_layout_info.pushConstantRangeCount = 1; + pipeline_layout_info.pPushConstantRanges = &push_constants; EXIT_IF(pipeline.pipeline_layout != nullptr); if (graphics_debug_dump_enabled()) { LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=0x%08" PRIx32 "/0x%08" PRIx32 - " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=%" PRIu32 " push_constants=%" PRIu32 - "\n", - vs_hash0, vs_crc32, ps_hash0, ps_crc32, set_layouts_num, push_constant_info_num); + " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=1 push_constants=%" PRIu32 "\n", + vs_hash0, vs_crc32, ps_hash0, ps_crc32, 1u); } result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline.pipeline_layout); @@ -988,10 +987,9 @@ void CreatePipelineInternal( } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache, - PipelineCache::ComputePipeline& pipeline, - const ShaderComputeInputInfo& input_info, - std::span cs_shader) { +void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline, + const ShaderComputeInputInfo& input_info, + std::span cs_shader) { vk::ShaderModule comp_shader_module = nullptr; vk::ShaderModuleCreateInfo create_info {}; @@ -1030,30 +1028,26 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto comp_shader_stage_info.pNext = &comp_subgroup_size; } - vk::DescriptorSetLayout set_layouts[1] = {}; - uint32_t set_layouts_num = 0; - - vk::PushConstantRange push_constant_info[1]; - uint32_t push_constant_info_num = 0; - - CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info, - push_constant_info_num, *input_info.stage.program, - vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + std::vector descriptor_bindings; + AddLayoutBindings(descriptor_bindings, *input_info.stage.program, + vk::ShaderStageFlagBits::eCompute); + CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); + const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0, + ShaderRecompiler::IR::NativePushConstantSize}; vk::PipelineLayoutCreateInfo pipeline_layout_info {}; pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; pipeline_layout_info.pNext = nullptr; pipeline_layout_info.flags = {}; - pipeline_layout_info.setLayoutCount = set_layouts_num; - pipeline_layout_info.pSetLayouts = (set_layouts_num > 0 ? set_layouts : nullptr); - pipeline_layout_info.pushConstantRangeCount = push_constant_info_num; - pipeline_layout_info.pPushConstantRanges = - push_constant_info_num > 0 ? push_constant_info : nullptr; + pipeline_layout_info.setLayoutCount = 1; + pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout; + pipeline_layout_info.pushConstantRangeCount = 1; + pipeline_layout_info.pPushConstantRanges = &push_constants; EXIT_IF(pipeline.pipeline_layout != nullptr); - LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=%u push_constants=%u\n", - set_layouts_num, push_constant_info_num); + LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n", + 1u); result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline.pipeline_layout); LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n", diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index bf54c8e88..7a42cda16 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -4,11 +4,14 @@ #include "common/abi.h" #include "common/assert.h" #include "common/common.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" +#include "graphics/host_gpu/renderer/pipeline/descriptors.h" +#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/renderTarget.h" #include "graphics/host_gpu/vulkanCommon.h" #include +#include +#include #include namespace Libs::Graphics { @@ -23,7 +26,6 @@ struct GraphicContext; struct ShaderBufferResource; struct ShaderComputeInputInfo; struct CommandSlot; -struct VulkanDescriptorSet; struct RenderDepthInfo; struct RenderColorInfo; struct DrawCallInfo; @@ -92,7 +94,6 @@ public: void WaitForFenceOnly(); void WaitForFence(); void WaitForFenceAndReset(); - void RecycleDescriptorAfterFence(VulkanDescriptorSet& set); [[nodiscard]] vk::CommandBuffer Handle() const; [[nodiscard]] GraphicContext& GetGraphics() const noexcept { return m_graphics; } @@ -102,25 +103,23 @@ public: private: void Release(); void FinalizeFence(bool reset_recording); - void RecycleDescriptorsAfterFence(); - RenderContext& m_context; - CommandScheduler& m_scheduler; - GraphicContext& m_graphics; - CommandSlot* m_slot = nullptr; - bool m_execute = false; - bool m_fence_waited = false; - uint64_t m_submit_seq = 0; - uint32_t m_debug_op = 0; - uint64_t m_debug_submit_id = 0; - uint32_t m_debug_arg0 = 0; - uint32_t m_debug_arg1 = 0; - uint32_t m_debug_arg2 = 0; - uint32_t m_debug_arg3 = 0; - uint64_t m_debug_arg4 = 0; - std::vector m_descriptor_sets_after_fence; - mutable RenderState m_render_state; - mutable bool m_rendering = false; + RenderContext& m_context; + CommandScheduler& m_scheduler; + GraphicContext& m_graphics; + CommandSlot* m_slot = nullptr; + bool m_execute = false; + bool m_fence_waited = false; + uint64_t m_submit_seq = 0; + uint32_t m_debug_op = 0; + uint64_t m_debug_submit_id = 0; + uint32_t m_debug_arg0 = 0; + uint32_t m_debug_arg1 = 0; + uint32_t m_debug_arg2 = 0; + uint32_t m_debug_arg3 = 0; + uint64_t m_debug_arg4 = 0; + mutable RenderState m_render_state; + mutable bool m_rendering = false; }; class RenderCommandBuffer final: public CommandBuffer { @@ -159,24 +158,22 @@ public: void DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode); - [[nodiscard]] DescriptorCache::PreparedBindings - PrepareBindings(const ShaderStageRuntime& runtime, vk::ShaderStageFlags shader_stage, - DescriptorCache::Stage stage); - void FindBuffers(DescriptorCache::PreparedBindings& bindings); - void RebindBuffers(DescriptorCache::PreparedBindings& bindings); - void RebindImages(DescriptorCache::PreparedBindings& bindings); + [[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime); + void FindBuffers(PreparedBindings& bindings); + void RebindBuffers(PreparedBindings& bindings); + void RebindImages(PreparedBindings& bindings); void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, - vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings); + const PipelineCache::Pipeline& pipeline, + std::span bindings); private: struct GraphicsBindings { - DescriptorCache::PreparedBindings vertex; - std::optional pixel; + PreparedBindings vertex; + std::optional pixel; }; - [[nodiscard]] DescriptorCache::TextureBinding - ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderRecompiler::IR::DescriptorValue& value); + [[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderRecompiler::IR::DescriptorValue& value); [[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex, const ShaderStageRuntime& pixel, bool pixel_active); @@ -207,8 +204,14 @@ private: [[nodiscard]] bool TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, const RenderCommandBuffer& buffer); - RenderContext& m_context; - std::vector m_bound_images; + RenderContext& m_context; + std::vector m_bound_images; + std::vector m_descriptor_buffers; + std::vector m_descriptor_images; + std::vector m_descriptor_writes; + std::vector m_image_occurrences; + std::array + m_push_constants {}; friend struct RenderExecutorTestAccess; }; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 4e9f59d05..c9486bdc3 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -11,7 +11,6 @@ #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/image/imageInfo.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/pipeline/descriptors.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" @@ -334,14 +333,15 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf buffer.EndRendering(); auto& pipeline = m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader); - auto bindings = PrepareBindings(input_info.stage, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto bindings = PrepareBindings(input_info.stage); FindBuffers(bindings); RebindBuffers(bindings); RebindImages(bindings); - auto vk_buffer = buffer.Handle(); - CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline.pipeline_layout, bindings); + auto vk_buffer = buffer.Handle(); + PreparedBindings* descriptor_stage = &bindings; + CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline, + std::span {&descriptor_stage, 1u}); bool has_storage_writes = HasShaderBufferWrites(input_info.stage); has_storage_writes = std::any_of(program.info.images.begin(), program.info.images.end(), diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp index be6d4f8cd..5ddf30f1f 100644 --- a/src/graphics/host_gpu/renderer/renderContext.cpp +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -13,8 +13,9 @@ namespace Libs::Graphics { RenderContext::RenderContext(GraphicContext& graphics) : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics), - m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache), - m_sampler_cache(graphics), m_gpu_resources(graphics, m_command_scheduler) { + m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()), + m_pipeline_cache(graphics), m_sampler_cache(graphics), + m_gpu_resources(graphics, m_command_scheduler) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } diff --git a/src/graphics/host_gpu/renderer/renderContext.h b/src/graphics/host_gpu/renderer/renderContext.h index 11848f285..ab56c2ca4 100644 --- a/src/graphics/host_gpu/renderer/renderContext.h +++ b/src/graphics/host_gpu/renderer/renderContext.h @@ -10,7 +10,7 @@ #include "graphics/host_gpu/renderer/cache/samplerCache.h" #include "graphics/host_gpu/renderer/cache/textureCache.h" #include "graphics/host_gpu/renderer/commandScheduler.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" +#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "kernel/eventQueue.h" @@ -41,7 +41,7 @@ public: Common::Mutex& GetMutex() { return m_mutex; } CommandScheduler& GetCommandScheduler() { return m_command_scheduler; } PipelineCache& GetPipelineCache() { return m_pipeline_cache; } - DescriptorCache& GetDescriptorCache() { return m_descriptor_cache; } + DescriptorHeap& GetDescriptorHeap() { return m_descriptor_heap; } SamplerCache& GetSamplerCache() { return m_sampler_cache; } GpuResourceManager& GetGpuResources() { return m_gpu_resources; } BufferCache& GetBufferCache() { return m_gpu_resources.GetBufferCache(); } @@ -63,7 +63,7 @@ private: Common::Mutex m_mutex; RenderExecutor m_render_executor; CommandScheduler m_command_scheduler; - DescriptorCache m_descriptor_cache; + DescriptorHeap m_descriptor_heap; PipelineCache m_pipeline_cache; SamplerCache m_sampler_cache; GpuResourceManager m_gpu_resources; diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index fbd4d4bdd..ee3de48aa 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -15,7 +15,6 @@ #include "graphics/host_gpu/renderer/colorRenderTarget.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/depthRenderTarget.h" -#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h" #include "graphics/host_gpu/renderer/render.h" @@ -798,9 +797,9 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer } struct VertexBufferRange { - uint64_t base_address = 0; - uint64_t requested_end = 0; - uint64_t acquired_end = 0; + uint64_t base_address = 0; + uint64_t requested_end = 0; + uint64_t acquired_end = 0; std::pair binding; [[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; } @@ -862,15 +861,14 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer& b Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize()); range.acquired_end = range.base_address + size; range.binding = cache.ObtainBuffer(range.base_address, size, false); - SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, - range.binding.first->Handle(), - "Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", - range.base_address, size); + SetVulkanObjectNameF( + buffer.GetContext().GetGraphics().device, range.binding.first->Handle(), + "Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", range.base_address, size); } // Rebuild slot bindings, offsetting non-empty slots into their acquired merged range. PreparedVertexBuffers prepared; - prepared.count = static_cast(vs_input_info.buffers_num); + prepared.count = static_cast(vs_input_info.buffers_num); vk::Buffer null_buffer = nullptr; for (int i = 0; i < vs_input_info.buffers_num; i++) { const auto& vertex = vs_input_info.buffers[i]; @@ -1062,7 +1060,8 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw LogDrawPhase(draw.name, "ShaderCompileInfoPS"); } if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, state.vs_input_info, - target_export_mapping, state.ps_input_info, state.ps_shader)) { + target_export_mapping, + state.ps_input_info, state.ps_shader)) { EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name); } } @@ -1091,8 +1090,8 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffe prepared.offset = stream.Copy(source.host_data, source.size, 16); prepared.buffer = stream.Handle(); } else { - auto [buffer_ptr, offset] = buffer.GetContext().GetBufferCache().ObtainBuffer( - source.address, source.size, false); + auto [buffer_ptr, offset] = + buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false); prepared.buffer = buffer_ptr->Handle(); prepared.offset = offset; } @@ -1108,7 +1107,8 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer& buffe return prepared; } -static void CommitVertexBuffers(vk::CommandBuffer vk_buffer, const PreparedVertexBuffers& prepared) { +static void CommitVertexBuffers(vk::CommandBuffer vk_buffer, + const PreparedVertexBuffers& prepared) { for (uint32_t i = 0; i < prepared.count; i++) { EXIT_IF(prepared.buffers[i] == nullptr); } @@ -1230,15 +1230,18 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer SetDrawDebugPhase(buffer, submit_id, draw, 0x200u); } CommitVertexBuffers(vk_buffer, vertex_bindings); - CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout, - bindings.vertex); if (bindings.pixel.has_value()) { if (set_auto_debug) { SetDrawDebugPhase(buffer, submit_id, draw, 0x300u); } - CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout, - *bindings.pixel); } + std::array descriptor_stages {&bindings.vertex, nullptr}; + const size_t descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u; + if (bindings.pixel) { + descriptor_stages[1] = &*bindings.pixel; + } + CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline, + std::span {descriptor_stages.data(), descriptor_stage_count}); CommitIndexBuffer(vk_buffer, index_binding); const auto dynamic_params = diff --git a/src/graphics/host_gpu/vulkanInstance.h b/src/graphics/host_gpu/vulkanInstance.h index 01c5657f3..94c16a6f9 100644 --- a/src/graphics/host_gpu/vulkanInstance.h +++ b/src/graphics/host_gpu/vulkanInstance.h @@ -30,6 +30,7 @@ struct VulkanInstance { uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; + uint32_t max_push_descriptors = 0; vk::ShaderStageFlags required_subgroup_size_stages = {}; Common::Mutex queue_mutex; uint32_t queue_family = static_cast(-1); diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index b81c27b18..bfe8e7808 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -583,13 +583,13 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V robustness2.nullDescriptor = supported_robustness2.nullDescriptor; } - const bool subgroup_size_control_enabled = - graphics.subgroup_size_control_enabled && supported_features13.subgroupSizeControl == VK_TRUE; + const bool subgroup_size_control_enabled = graphics.subgroup_size_control_enabled && + supported_features13.subgroupSizeControl == VK_TRUE; - auto features13 = required_features13; - features13.pNext = robustness2_ext_enabled ? static_cast(&robustness2) - : static_cast(&features12); - features13.robustImageAccess = supported_features13.robustImageAccess; + auto features13 = required_features13; + features13.pNext = robustness2_ext_enabled ? static_cast(&robustness2) + : static_cast(&features12); + features13.robustImageAccess = supported_features13.robustImageAccess; features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE; LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n", @@ -834,7 +834,7 @@ void WindowContext::CreateVulkan() { EXIT("--spirv-debug-printf and --gpu-assisted-validation are mutually exclusive\n"); } - vk::ValidationFeatureEnableEXT enabled_features[3] = {}; + vk::ValidationFeatureEnableEXT enabled_features[3] = {}; uint32_t enabled_features_count = 0; #ifdef KYTY_ENABLE_BEST_PRACTICES enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eBestPractices; @@ -939,7 +939,15 @@ void WindowContext::CreateVulkan() { EXIT("Could not find suitable device"); } - graphic_ctx.physical_device.getProperties(&graphic_ctx.physical_device_properties); + vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {}; + vk::PhysicalDeviceProperties2 physical_device_properties {}; + push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties; + push_descriptor_properties.pNext = nullptr; + physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2; + physical_device_properties.pNext = &push_descriptor_properties; + graphic_ctx.physical_device.getProperties2(&physical_device_properties); + graphic_ctx.physical_device_properties = physical_device_properties.properties; + graphic_ctx.max_push_descriptors = push_descriptor_properties.maxPushDescriptors; graphic_ctx.physical_device.getMemoryProperties(&graphic_ctx.physical_device_memory_properties); const auto& device_properties = graphic_ctx.GetPhysicalDeviceProperties(); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index f5b1a4381..fa781713a 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -798,10 +798,7 @@ bool TryRecompile(std::span code, const CompileOptions& options, if (!IR::CollectShaderInfo(ir, info_options, error)) { return false; } - IR::BindingLayoutOptions layout_options; - layout_options.descriptor_set = options.descriptor_set; - layout_options.push_constant_offset = options.push_constant_offset; - if (!IR::AllocateBindings(ir, layout_options, error)) { + if (!IR::AllocateBindings(ir, options.push_constant_offset, error)) { return false; } if (!Spirv::AnalyzeProgramRequirements(ir, error)) { diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index d66322cc7..fc4975143 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -21,8 +21,7 @@ struct CompileOptions { uint64_t shader_hash = 0; uint64_t shader_base = 0; std::optional flat_memory_base; - uint32_t descriptor_set = 0; - uint32_t push_constant_offset = 0; + uint32_t push_constant_offset = 0; bool dump_ir = true; bool early_dump = false; const char* dump_label = nullptr; diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 262c2387f..bf0cf4721 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -89,18 +89,12 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { } std::array seen {}; - for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) { - const auto& binding = program.bindings.descriptors[i]; - const auto kind = static_cast(binding.kind); + for (const auto& binding: program.bindings.descriptors) { + const auto kind = static_cast(binding.kind); if (kind >= KindCount || seen[kind] || !present[kind] || binding.resources != expected[kind]) { return Fail(error, "native descriptor groups do not match shader topology"); } - for (uint32_t j = 0; j < i; j++) { - if (program.bindings.descriptors[j].binding == binding.binding) { - return Fail(error, "native descriptor binding numbers are not unique"); - } - } seen[kind] = true; } for (size_t i = 0; i < KindCount; i++) { @@ -109,7 +103,10 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { } } const auto has_user_storage = present[static_cast(Kind::UserData)]; - if (program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() || + if (program.bindings.push_constant_offset % sizeof(uint32_t) != 0 || + program.bindings.push_constant_offset + program.bindings.push_constant_size > + IR::NativePushConstantSize || + program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() || program.bindings.memory_offset_count != program.info.buffers.size() + program.info.addresses.size() || (!has_user_storage && diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 5db3bc8dd..961840486 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -667,8 +667,8 @@ void AddInputAnnotationsAndNames(EmitterState& state); void AddOutputAnnotationsAndNames(EmitterState& state); -void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, uint32_t set, - uint32_t binding); +void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, + IR::DescriptorBindingKind kind); void AddDescriptorAnnotationsAndNames(EmitterState& state); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 0ccef0c59..69d7096da 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -510,23 +510,20 @@ void AddOutputAnnotationsAndNames(EmitterState& state) { } } -void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, uint32_t set, - uint32_t binding) { +void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, + IR::DescriptorBindingKind kind) { if (variable == 0) { return; } state.builder.AddName(variable, name); - state.builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, set}); - state.builder.AddAnnotation({OpDecorate, variable, DecorationBinding, binding}); + state.builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, 0}); + state.builder.AddAnnotation( + {OpDecorate, variable, DecorationBinding, IR::NativeBinding(state.program.stage, kind)}); } void AddDescriptorAnnotationsAndNames(EmitterState& state) { - const auto set = state.program.bindings.descriptor_set; - auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) { - const auto* binding = DescriptorBinding(state, kind); - if (binding != nullptr) { - DecorateDescriptor(state, variable, name, set, binding->binding); - } + auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) { + DecorateDescriptor(state, variable, name, kind); }; if (state.storage_buffer_variable != 0) { Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers); @@ -570,7 +567,7 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) { "storage_atomic_2d_array", "storage_atomic_3d"}; for (uint32_t i = 0; i < state.storage_image_variables.size(); i++) { - const auto view = static_cast(i % StorageImageViewKindCount); + const auto view = static_cast(i % StorageImageViewKindCount); const auto image_class = static_cast(i / StorageImageViewKindCount); Decorate(state.storage_image_variables[i], StorageNames[i], StorageBindingKind(image_class, view)); @@ -588,17 +585,13 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) { } void AddVsharpAnnotationsAndNames(EmitterState& state) { - const auto& bind = state.program.bindings; if (state.push_constant_variable != 0) { state.builder.AddName(PushConstantBlockType(state), "BufferResource"); state.builder.AddName(state.push_constant_variable, "vsharp"); } if (state.vsharp_storage_variable != 0) { - const auto* storage = DescriptorBinding(state, IR::DescriptorBindingKind::UserData); - if (storage != nullptr) { - DecorateDescriptor(state, state.vsharp_storage_variable, "user_data", - bind.descriptor_set, storage->binding); - } + DecorateDescriptor(state, state.vsharp_storage_variable, "user_data", + IR::DescriptorBindingKind::UserData); } } diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 17688c83c..60e8d3a60 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -338,6 +338,13 @@ enum class DescriptorBindingKind { Count, }; +constexpr uint32_t NativePushConstantSize = 128; + +[[nodiscard]] constexpr uint32_t NativeBinding(ShaderType stage, DescriptorBindingKind kind) { + return static_cast(kind) + + (stage == ShaderType::Pixel ? static_cast(DescriptorBindingKind::Count) : 0u); +} + [[nodiscard]] inline std::optional DescriptorBindingForImage(const ImageResource& image) { using Dimension = Decoder::ImageDimension; @@ -412,19 +419,17 @@ DescriptorBindingForImage(const ImageResource& image) { } struct DescriptorBinding { - DescriptorBindingKind kind = DescriptorBindingKind::Buffers; - uint32_t binding = 0; + DescriptorBindingKind kind = DescriptorBindingKind::Buffers; std::vector resources; bool operator==(const DescriptorBinding& other) const = default; }; struct BindingLayout { - uint32_t descriptor_set = 0; uint32_t push_constant_offset = 0; - uint32_t push_constant_size = 0; - uint32_t memory_offset_dword = 0; - uint32_t memory_offset_count = 0; + uint32_t push_constant_size = 0; + uint32_t memory_offset_dword = 0; + uint32_t memory_offset_count = 0; std::vector user_data_registers; std::vector descriptors; diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index 0f8c03e1d..9680eaea9 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -4,48 +4,18 @@ #include #include -#include #include namespace Libs::Graphics::ShaderRecompiler::IR { namespace { -constexpr uint32_t MaxPushConstantBytes = 128; - -constexpr std::array ImageBindingKinds = { - DescriptorBindingKind::Sampled1D, - DescriptorBindingKind::Sampled1DArray, - DescriptorBindingKind::Sampled2D, - DescriptorBindingKind::Sampled2DArray, - DescriptorBindingKind::Sampled2DMsaa, - DescriptorBindingKind::Sampled2DMsaaArray, - DescriptorBindingKind::Sampled3D, - DescriptorBindingKind::SampledUint1D, - DescriptorBindingKind::SampledUint1DArray, - DescriptorBindingKind::SampledUint2D, - DescriptorBindingKind::SampledUint2DArray, - DescriptorBindingKind::SampledUint2DMsaa, - DescriptorBindingKind::SampledUint2DMsaaArray, - DescriptorBindingKind::SampledUint3D, - DescriptorBindingKind::Storage1D, - DescriptorBindingKind::Storage1DArray, - DescriptorBindingKind::Storage2D, - DescriptorBindingKind::Storage2DArray, - DescriptorBindingKind::Storage3D, - DescriptorBindingKind::StorageUint1D, - DescriptorBindingKind::StorageUint1DArray, - DescriptorBindingKind::StorageUint2D, - DescriptorBindingKind::StorageUint2DArray, - DescriptorBindingKind::StorageUint3D, - DescriptorBindingKind::StorageAtomic1D, - DescriptorBindingKind::StorageAtomic1DArray, - DescriptorBindingKind::StorageAtomic2D, - DescriptorBindingKind::StorageAtomic2DArray, - DescriptorBindingKind::StorageAtomic3D, -}; +constexpr auto FirstImageBinding = DescriptorBindingKind::Sampled1D; +constexpr uint32_t ImageBindingCount = + static_cast(DescriptorBindingKind::StorageAtomic3D) - + static_cast(FirstImageBinding) + 1u; bool CollectUserData(const Program& program, std::vector& result) { - std::set registers; + std::array registers {}; for (const auto* block: program.values->blocks) { for (const auto& inst: *block) { if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) { @@ -58,17 +28,21 @@ bool CollectUserData(const Program& program, std::vector& result) { if (index >= NumScalarRegs) { return false; } - registers.insert(index); + registers[index] = true; + } + } + result.clear(); + for (uint32_t index = 0; index < registers.size(); index++) { + if (registers[index]) { + result.push_back(index); } } - result.assign(registers.begin(), registers.end()); return true; } void AddBinding(BindingLayout& layout, DescriptorBindingKind kind, std::vector resources = {}) { - layout.descriptors.push_back( - {kind, static_cast(layout.descriptors.size()), std::move(resources)}); + layout.descriptors.push_back({kind, std::move(resources)}); } bool UsesGds(const Program& program, bool& uses_gds) { @@ -94,7 +68,7 @@ bool UsesGds(const Program& program, bool& uses_gds) { } // namespace -bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std::string* error) { +bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::string* error) { if (!program.shader_info_complete || program.binding_layout_complete) { if (error != nullptr) { *error = !program.shader_info_complete ? "shader info is not ready" @@ -102,28 +76,22 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std } return false; } - if (options.push_constant_offset > MaxPushConstantBytes) { - if (error != nullptr) { - *error = "push-constant offset exceeds the Vulkan minimum guarantee"; - } - return false; - } - if (options.push_constant_offset % 4u != 0) { - if (error != nullptr) { - *error = "push-constant offset is not dword aligned"; - } - return false; - } if (program.values == nullptr) { if (error != nullptr) { *error = "typed value program is not available"; } return false; } + if (push_constant_offset > NativePushConstantSize || + push_constant_offset % sizeof(uint32_t) != 0) { + if (error != nullptr) { + *error = "push-constant offset exceeds the Vulkan minimum guarantee or is unaligned"; + } + return false; + } BindingLayout next; - next.descriptor_set = options.descriptor_set; - next.push_constant_offset = options.push_constant_offset; + next.push_constant_offset = push_constant_offset; if (!CollectUserData(program, next.user_data_registers)) { if (error != nullptr) { *error = "typed shader contains an invalid user-data register"; @@ -142,7 +110,7 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std AddBinding(next, DescriptorBindingKind::Buffers, std::move(resources)); } - std::array, ImageBindingKinds.size()> image_groups; + std::array, ImageBindingCount> image_groups; for (uint32_t i = 0; i < program.info.images.size(); i++) { const auto kind = DescriptorBindingForImage(program.info.images[i]); if (!kind.has_value()) { @@ -151,14 +119,14 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std } return false; } - const auto group = std::find(ImageBindingKinds.begin(), ImageBindingKinds.end(), *kind); - if (group == ImageBindingKinds.end()) { + const auto group = static_cast(*kind) - static_cast(FirstImageBinding); + if (group >= image_groups.size()) { if (error != nullptr) { *error = "shader info contains an unmapped image binding class"; } return false; } - auto& resources = image_groups[static_cast(group - ImageBindingKinds.begin())]; + auto& resources = image_groups[group]; const auto dynamic = program.info.images[i].mip_mode == ImageMipMode::DynamicStorage; const auto count = dynamic ? program.info.images[i].mip_count : 1u; if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) { @@ -171,7 +139,10 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std } for (uint32_t i = 0; i < image_groups.size(); i++) { if (!image_groups[i].empty()) { - AddBinding(next, ImageBindingKinds[i], std::move(image_groups[i])); + AddBinding( + next, + static_cast(static_cast(FirstImageBinding) + i), + std::move(image_groups[i])); } } @@ -208,9 +179,8 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std AddBinding(next, DescriptorBindingKind::FlattenedSrt); } - const auto available_push_dwords = (MaxPushConstantBytes - options.push_constant_offset) / 4u; - const auto push_limit = std::min(options.max_push_dwords, available_push_dwords); - if (next.ShaderDataDwords() <= push_limit) { + const auto push_dwords = (NativePushConstantSize - push_constant_offset) / sizeof(uint32_t); + if (next.ShaderDataDwords() <= push_dwords) { next.push_constant_size = next.ShaderDataDwords() * sizeof(uint32_t); } else { AddBinding(next, DescriptorBindingKind::UserData); diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h index 9a0e0a08c..6069e8a3a 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h @@ -5,13 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -struct BindingLayoutOptions { - uint32_t descriptor_set = 0; - uint32_t push_constant_offset = 0; - uint32_t max_push_dwords = 32; -}; - -bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std::string* error); +bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::string* error); const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind); diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 01dc6fa02..f488253ce 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -846,6 +846,11 @@ static void ShaderGetStaticInputInfoPS( regs.ps_regs.data_addr); } ps_info.scratch_size_dwords = data.scratch_size_dwords; + ps_info.push_constant_offset = + vs_info.stage.program != nullptr + ? vs_info.stage.program->bindings.push_constant_offset + + vs_info.stage.program->bindings.push_constant_size + : 0; // SPI_PS_IN_CONTROL.NUM_INTERP occupies bits 5:0. Keep the remaining control // flags in the hardware state and extract only the input count here. @@ -873,15 +878,6 @@ static void ShaderGetStaticInputInfoPS( ps_info.interpolator_settings[i] = sh.ps_interpolator_settings[i]; } - ps_info.descriptor_set = - vs_info.stage.program != nullptr && !vs_info.stage.program->bindings.descriptors.empty() - ? 1 - : 0; - ps_info.push_constant_offset = vs_info.stage.program != nullptr - ? vs_info.stage.program->bindings.push_constant_offset + - vs_info.stage.program->bindings.push_constant_size - : 0; - for (int i = 0; i < 8; i++) { ps_info.target_output_mode[i] = sh.target_output_mode[i]; ps_info.target_export_mapping[i] = sh.target_output_mode[i] != 0 @@ -1020,11 +1016,10 @@ static void LogShaderProgramCacheHit(const char* stage, uint64_t shader_hash, ui static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program) { std::string ret = fmt::format( - "set={} push={} groups={} user={} buffers={} images={} samplers={} addresses={} srt={}", - program.bindings.descriptor_set, program.bindings.push_constant_size, - program.bindings.descriptors.size(), program.bindings.user_data_registers.size(), - program.info.buffers.size(), program.info.images.size(), program.info.samplers.size(), - program.info.addresses.size(), + "push={} groups={} user={} buffers={} images={} samplers={} addresses={} srt={}", + program.bindings.push_constant_size, program.bindings.descriptors.size(), + program.bindings.user_data_registers.size(), program.info.buffers.size(), + program.info.images.size(), program.info.samplers.size(), program.info.addresses.size(), program.values != nullptr ? program.values->srt_reads.size() : 0u); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { const auto& buffer = program.info.buffers[i]; @@ -1050,8 +1045,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog static void ShaderAppendNativeSpecialization(std::vector& ids, const ShaderRecompiler::IR::Program& program) { EXIT_IF(!program.binding_layout_complete); - ids.push_back(program.bindings.descriptor_set); - ids.push_back(program.bindings.push_constant_offset); ids.push_back(program.bindings.push_constant_size); ids.push_back(program.bindings.memory_offset_dword); ids.push_back(program.bindings.memory_offset_count); @@ -1061,7 +1054,6 @@ static void ShaderAppendNativeSpecialization(std::vector& ids.push_back(static_cast(program.bindings.descriptors.size())); for (const auto& binding: program.bindings.descriptors) { ids.push_back(static_cast(binding.kind)); - ids.push_back(binding.binding); ids.push_back(static_cast(binding.resources.size())); ids.insert(ids.end(), binding.resources.begin(), binding.resources.end()); } @@ -1456,8 +1448,6 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi options.scratch_dwords = input_info.scratch_size_dwords; options.user_data = regs.gs_user_sgpr.value; options.read_specialization_memory = ReadShaderGuestMemory; - options.descriptor_set = 0; - options.push_constant_offset = 0; options.input_info.vertex = &input_info; options.dump_ir = ShaderRecompilerTextDumpEnabled(); options.early_dump = options.dump_ir; @@ -1507,10 +1497,9 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis options.shader_base = shader_addr; options.user_data_count = regs.ps_regs.rsrc2.user_sgpr; options.scratch_dwords = input_info.scratch_size_dwords; + options.push_constant_offset = input_info.push_constant_offset; options.user_data = regs.ps_user_sgpr.value; options.read_specialization_memory = ReadShaderGuestMemory; - options.descriptor_set = input_info.descriptor_set; - options.push_constant_offset = input_info.push_constant_offset; options.input_info.pixel = &input_info; options.dump_ir = ShaderRecompilerTextDumpEnabled(); options.early_dump = options.dump_ir; @@ -1560,8 +1549,6 @@ bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderReg options.scratch_dwords = input_info.scratch_size_dwords; options.user_data = regs.cs_user_sgpr.value; options.read_specialization_memory = ReadShaderGuestMemory; - options.descriptor_set = 0; - options.push_constant_offset = 0; options.input_info.compute = &input_info; options.wave_size = input_info.wave_size; options.dump_ir = ShaderRecompilerTextDumpEnabled(); @@ -1666,7 +1653,6 @@ ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputIn ret.hash0 = (regs.ps_regs.chksum >> 32u) & 0xffffffffu; ret.crc32 = regs.ps_regs.chksum & 0xffffffffu; - ret.ids.push_back(input_info.descriptor_set); ret.ids.push_back(input_info.push_constant_offset); ret.ids.push_back(input_info.scratch_size_dwords); ret.ids.push_back(input_info.input_num); @@ -1678,6 +1664,7 @@ ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputIn ret.ids.push_back(static_cast(input_info.ps_front_face)); ret.ids.push_back(static_cast(input_info.ps_no_perspective)); ret.ids.push_back(static_cast(input_info.ps_pixel_kill_enable)); + ret.ids.push_back(static_cast(input_info.ps_depth_export_enable)); ret.ids.push_back(static_cast(input_info.ps_sample_mask_export_enable)); ret.ids.push_back(static_cast(input_info.ps_early_z)); ret.ids.push_back(static_cast(input_info.ps_execute_on_noop)); @@ -1738,10 +1725,7 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp ret.ids.push_back(input_info.threads_num[i]); ret.ids.push_back(static_cast(input_info.group_id[i])); } - ret.ids.push_back(static_cast(input_info.dispatch_thread_dimensions)); - for (uint32_t dim: input_info.dispatch_threads_num) { - ret.ids.push_back(dim); - } + ret.ids.push_back(static_cast(input_info.tg_size_en)); if (include_bind_specialization) { EXIT_IF(!input_info.stage); diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 7b9506161..dbd033878 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -120,7 +120,6 @@ struct ShaderPixelInputInfo { uint8_t target_output_mode[8] = {}; std::array target_export_mapping = {}; uint32_t mrt_output_mask = 0; - uint32_t descriptor_set = 0; uint32_t push_constant_offset = 0; uint32_t scratch_size_dwords = 0; bool ps_pos_x = false; diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index bb8890842..aa56a021e 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -636,24 +636,22 @@ void TestImagesSamplersAndAliases() { void TestSampleAdjustSamplerScratch() { Fixture fixture(ShaderType::Pixel); const auto active = fixture.Emit(ValueOpcode::WqmMask, {Value(true)}); - const auto lane = fixture.Emit(ValueOpcode::SelectU32, - {active, Value(1u), Value(0u)}); - const auto low = fixture.Emit(ValueOpcode::BitwiseAnd32, - {lane, Value(0xffu)}); - const auto high = fixture.Emit(ValueOpcode::BitwiseAnd32, - {lane, Value(0xffu)}); + const auto lane = + fixture.Emit(ValueOpcode::SelectU32, {active, Value(1u), Value(0u)}); + const auto low = + fixture.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(0xffu)}); + const auto high = + fixture.Emit(ValueOpcode::BitwiseAnd32, {lane, Value(0xffu)}); const auto quads = fixture.Emit( ValueOpcode::BitwiseOr32, - {low, fixture.Emit(ValueOpcode::ShiftLeftLogical32, - {high, Value(8u)})}); - const auto scratch = fixture.Emit(ValueOpcode::ShiftLeftLogical32, - {quads, Value(12u)}); - const auto word3 = fixture.Emit(ValueOpcode::BitwiseOr32, - {fixture.UserData(3), scratch}); - const auto image = fixture.Image( - {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), - Value(0u), Value(0u)}, - 0x1ec); + {low, fixture.Emit(ValueOpcode::ShiftLeftLogical32, {high, Value(8u)})}); + const auto scratch = + fixture.Emit(ValueOpcode::ShiftLeftLogical32, {quads, Value(12u)}); + const auto word3 = + fixture.Emit(ValueOpcode::BitwiseOr32, {fixture.UserData(3), scratch}); + const auto image = fixture.Image({Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u), Value(0u)}, + 0x1ec); const auto sampler = fixture.Sampler( {fixture.UserData(0), fixture.UserData(1), fixture.UserData(2), word3}, 0x1ec); @@ -685,22 +683,21 @@ void TestSampleAdjustSamplerScratch() { const auto CheckRejected = [](uint32_t flags, uint32_t shift, const char *message) { Fixture rejected(ShaderType::Pixel); - const auto condition = - rejected.Emit(ValueOpcode::WqmMask, {Value(true)}); + const auto condition = rejected.Emit(ValueOpcode::WqmMask, {Value(true)}); const auto bit = rejected.Emit(ValueOpcode::SelectU32, {condition, Value(1u), Value(0u)}); - const auto dynamic = rejected.Emit(ValueOpcode::ShiftLeftLogical32, - {bit, Value(shift)}); - const auto dynamic_word3 = rejected.Emit( - ValueOpcode::BitwiseOr32, {rejected.UserData(3), dynamic}); - const auto rejected_image = rejected.Image( - {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), - Value(0u), Value(0u)}, - 0x200); - const auto rejected_sampler = rejected.Sampler( - {rejected.UserData(0), rejected.UserData(1), rejected.UserData(2), - dynamic_word3}, - 0x200); + const auto dynamic = + rejected.Emit(ValueOpcode::ShiftLeftLogical32, {bit, Value(shift)}); + const auto dynamic_word3 = rejected.Emit(ValueOpcode::BitwiseOr32, + {rejected.UserData(3), dynamic}); + const auto rejected_image = + rejected.Image({Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u)}, + 0x200); + const auto rejected_sampler = + rejected.Sampler({rejected.UserData(0), rejected.UserData(1), + rejected.UserData(2), dynamic_word3}, + 0x200); MemoryInfo rejected_memory; rejected_memory.kind = ResourceKind::Image; rejected_memory.image_dimension = Decoder::ImageDimension::Dim2D; @@ -789,7 +786,7 @@ void TestDynamicStorageMipTracking() { "base-1 through last-3 dynamic storage range was not specialized"); ShaderComputeInputInfo compute{}; Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && - AllocateBindings(fixture.program, {}, &error), + AllocateBindings(fixture.program, 0, &error), "dynamic storage mip bindings were not allocated"); const auto *storage_binding = FindBinding(fixture.program.bindings, DescriptorBindingKind::Storage2D); @@ -888,7 +885,7 @@ void TestSrtFlatteningAndRuntimeMemoization() { ShaderComputeInputInfo compute{}; Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && - AllocateBindings(fixture.program, {}, &error) && + AllocateBindings(fixture.program, 0, &error) && FindBinding(fixture.program.bindings, DescriptorBindingKind::FlattenedSrt) != nullptr, "flattened typed SRT reads did not receive a binding"); @@ -933,7 +930,7 @@ void TestDynamicSrtReadRemainsExplicit() { ShaderComputeInputInfo compute{}; Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && - AllocateBindings(fixture.program, {}, &error) && + AllocateBindings(fixture.program, 0, &error) && FindBinding(fixture.program.bindings, DescriptorBindingKind::FlattenedSrt) == nullptr && FindBinding(fixture.program.bindings, @@ -1198,24 +1195,87 @@ void TestShaderInfoAndBindingLayout() { StageInputKind::GlobalInvocationId, "typed shader values were not reflected in shader info"); - BindingLayoutOptions options; - options.descriptor_set = 2; - options.max_push_dwords = 1; - Check(AllocateBindings(fixture.program, options, &error), + Check(AllocateBindings(fixture.program, 0, &error), "typed binding allocation failed"); - Check(fixture.program.bindings.descriptor_set == 2 && - FindBinding(fixture.program.bindings, - DescriptorBindingKind::Buffers) != nullptr && + Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::Buffers) != + nullptr && FindBinding(fixture.program.bindings, DescriptorBindingKind::Gds) != nullptr && FindBinding(fixture.program.bindings, - DescriptorBindingKind::UserData) != nullptr, + DescriptorBindingKind::UserData) == nullptr && + fixture.program.bindings.push_constant_size == + fixture.program.bindings.ShaderDataDwords() * sizeof(uint32_t), "typed resources were not assigned native bindings"); + Check(NativeBinding(ShaderType::Compute, DescriptorBindingKind::Buffers) == + static_cast(DescriptorBindingKind::Buffers) && + NativeBinding(ShaderType::Vertex, DescriptorBindingKind::Buffers) == + static_cast(DescriptorBindingKind::Buffers) && + NativeBinding(ShaderType::Pixel, DescriptorBindingKind::Buffers) == + static_cast(DescriptorBindingKind::Count) + + static_cast(DescriptorBindingKind::Buffers), + "fixed stage binding ranges are inconsistent"); Check(fixture.program.bindings.user_data_registers == std::vector({3u, 4u}), "binding layout did not collect live typed user-data values"); } +void TestGraphicsPushConstantLayout() { + const auto AddUserData = [](Fixture &fixture, uint32_t count) { + for (uint32_t index = 0; index < count; index++) { + fixture.Emit(ValueOpcode::ReferenceU32, {fixture.UserData(index)}); + } + fixture.program.shader_info_complete = true; + }; + + std::string error; + Fixture vertex(ShaderType::Vertex); + AddUserData(vertex, 9); + Check(AllocateBindings(vertex.program, 0, &error) && + vertex.program.bindings.push_constant_offset == 0 && + vertex.program.bindings.push_constant_size == 9 * sizeof(uint32_t), + "vertex shader was not placed at the start of the graphics push bank"); + + const auto pixel_offset = vertex.program.bindings.push_constant_size; + Fixture pixel(ShaderType::Pixel); + AddUserData(pixel, 4); + Check(AllocateBindings(pixel.program, pixel_offset, &error) && + pixel.program.bindings.push_constant_offset == pixel_offset && + pixel.program.bindings.push_constant_size == 4 * sizeof(uint32_t) && + FindBinding(pixel.program.bindings, DescriptorBindingKind::UserData) == + nullptr, + "pixel shader did not follow the vertex data in the graphics push bank"); + + Fixture edge(ShaderType::Pixel); + AddUserData(edge, 1); + Check(AllocateBindings(edge.program, NativePushConstantSize - sizeof(uint32_t), + &error) && + edge.program.bindings.push_constant_size == sizeof(uint32_t), + "last aligned push-constant dword did not fit in the graphics bank"); + + Fixture spill(ShaderType::Pixel); + AddUserData(spill, 32); + Check(AllocateBindings(spill.program, pixel_offset, &error) && + spill.program.bindings.push_constant_size == 0 && + FindBinding(spill.program.bindings, DescriptorBindingKind::UserData) != + nullptr, + "pixel shader overlapping the vertex push data did not spill to storage"); + + Fixture full(ShaderType::Pixel); + AddUserData(full, 1); + Check(AllocateBindings(full.program, NativePushConstantSize, &error) && + full.program.bindings.push_constant_size == 0 && + FindBinding(full.program.bindings, DescriptorBindingKind::UserData) != + nullptr, + "full graphics push bank did not spill pixel user data to storage"); + + Fixture invalid(ShaderType::Pixel); + AddUserData(invalid, 1); + Check(!AllocateBindings(invalid.program, + NativePushConstantSize + sizeof(uint32_t), &error) && + !invalid.program.binding_layout_complete, + "push-constant placement beyond the graphics bank was accepted"); +} + void TestResourceLimitIsTransactional() { Fixture fixture; MemoryInfo memory; @@ -1339,6 +1399,7 @@ int main() { Run("exec-masked FLAT address", TestExecMaskedFlatAddressProvenance); Run("buffer swizzle specialization", TestBufferSwizzleSpecialization); Run("shader info and bindings", TestShaderInfoAndBindingLayout); + Run("graphics push constants", TestGraphicsPushConstantLayout); Run("resource limit", TestResourceLimitIsTransactional); Run("malformed memory kinds", TestMalformedMemoryKindsRejected); } catch (const std::exception &exception) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index d94fd3020..54255c626 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -342,7 +342,7 @@ struct TextureCacheTestAccess { }; struct RenderExecutorTestAccess { - static DescriptorCache::TextureBinding + static TextureBinding ResolveTexture(RenderExecutor &executor, const ShaderRecompiler::IR::ImageResource &resource, const ShaderRecompiler::IR::DescriptorValue &value) { @@ -356,10 +356,93 @@ struct RenderExecutorTestAccess { return executor.PrepareGraphicsBindings(vertex, pixel, pixel_active); } - static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer, - DescriptorCache::PreparedBindings &bindings) { - executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, nullptr, - bindings); + static PipelineCache::Pipeline + CreateDescriptorPipeline(RenderExecutor &executor, + std::span stages) { + std::vector layout_bindings; + bool compute = false; + for (const auto *prepared : stages) { + const auto &program = *prepared->program; + compute |= program.stage == ShaderType::Compute; + const auto shader_stage = program.stage == ShaderType::Vertex + ? vk::ShaderStageFlagBits::eVertex + : program.stage == ShaderType::Pixel + ? vk::ShaderStageFlagBits::eFragment + : vk::ShaderStageFlagBits::eCompute; + for (const auto &binding : program.bindings.descriptors) { + layout_bindings.push_back( + {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind), + NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), + shader_stage}); + } + } + vk::DescriptorSetLayoutCreateInfo descriptor_info{}; + descriptor_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + descriptor_info.flags = + vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR; + descriptor_info.bindingCount = + static_cast(layout_bindings.size()); + descriptor_info.pBindings = layout_bindings.data(); + PipelineCache::Pipeline pipeline{}; + auto &device = executor.m_context.GetGraphics().device; + EXIT_IF(device.createDescriptorSetLayout(&descriptor_info, nullptr, + &pipeline.descriptor_set_layout) != + vk::Result::eSuccess); + + vk::PipelineLayoutCreateInfo pipeline_info{}; + pipeline_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + pipeline_info.setLayoutCount = 1; + pipeline_info.pSetLayouts = &pipeline.descriptor_set_layout; + const vk::PushConstantRange push_range { + compute ? vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute} + : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eVertex | + vk::ShaderStageFlagBits::eFragment}, + 0, ShaderRecompiler::IR::NativePushConstantSize}; + pipeline_info.pushConstantRangeCount = 1; + pipeline_info.pPushConstantRanges = &push_range; + EXIT_IF(device.createPipelineLayout(&pipeline_info, nullptr, + &pipeline.pipeline_layout) != + vk::Result::eSuccess); + pipeline.uses_push_descriptors = true; + return pipeline; + } + + static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor, + CommandBuffer &buffer, + PreparedBindings &bindings) { + std::array stages{&bindings}; + auto pipeline = CreateDescriptorPipeline(executor, stages); + const auto bind_point = bindings.program->stage == ShaderType::Compute + ? vk::PipelineBindPoint::eCompute + : vk::PipelineBindPoint::eGraphics; + executor.CommitBindings(buffer, bind_point, pipeline, stages); + return pipeline; + } + + static PipelineCache::Pipeline CommitBindings(RenderExecutor &executor, + CommandBuffer &buffer, + PreparedBindings &vertex, + PreparedBindings &pixel) { + std::array stages{&vertex, &pixel}; + auto pipeline = CreateDescriptorPipeline(executor, stages); + executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline, + stages); + return pipeline; + } + + static void DestroyDescriptorPipelines( + RenderExecutor &executor, + std::span pipelines) { + auto &device = executor.m_context.GetGraphics().device; + for (const auto &pipeline : pipelines) { + device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr); + device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout, + nullptr); + } + } + + static const auto &PushConstants(const RenderExecutor &executor) { + return executor.m_push_constants; } static void ResolveRenderDepthTarget(RenderExecutor &executor, @@ -392,22 +475,14 @@ struct RenderExecutorTestAccess { executor.ResetBindings(); } - static bool BoundImagesInOrder(const RenderExecutor &executor, - ImageId first, ImageId second) { + static bool BoundImagesInOrder(const RenderExecutor &executor, ImageId first, + ImageId second) { return executor.m_bound_images.size() == 2 && executor.m_bound_images[0] == first && executor.m_bound_images[1] == second; } }; -struct DescriptorCacheTestAccess { - static vk::DescriptorImageInfo - MakeImageInfo(const DescriptorCache::TextureBinding &binding, - uint32_t element = 0) { - return DescriptorCache::MakeImageInfo(binding, element); - } -}; - namespace { using u32 = uint32_t; @@ -873,7 +948,7 @@ struct TestCase { size_t storage_buffer_range_dwords = 0; std::vector storage_buffer_offsets; std::vector address_memory_offsets; - bool force_shader_data_storage = false; + bool expand_shader_data_storage = false; bool expected_force_point_sampler = false; std::optional flat_memory_base; std::vector gds_initial; @@ -987,7 +1062,7 @@ void CheckRectListShaders() { shaders.control.size() > 1 && shaders.control[1] == 0x00010500u && shaders.evaluation.size() > 1 && shaders.evaluation[1] == 0x00010500u, - "vector selection requires SPIR-V 1.5"); + "vector selection requires SPIR-V 1.5"); ValidateSpirv(name, shaders.control); ValidateSpirv(name, shaders.evaluation); @@ -1154,21 +1229,29 @@ CompiledShader CompileCase(const TestCase &test) { test.expected_storage_mip_descriptors, "dynamic storage image did not receive one descriptor per mip"); } - if (test.force_shader_data_storage) { + if (test.expand_shader_data_storage) { + auto &block = *result.program.values->blocks.front(); + for (u32 reg = 0; reg < 33; reg++) { + auto &user_data = block.AppendNewInst( + ShaderRecompiler::IR::ValueOpcode::GetUserData, + {ShaderRecompiler::IR::Value( + static_cast(reg))}); + block.AppendNewInst(ShaderRecompiler::IR::ValueOpcode::ReferenceU32, + {ShaderRecompiler::IR::Value(&user_data)}); + } result.program.bindings = {}; result.program.binding_layout_complete = false; - if (!ShaderRecompiler::IR::AllocateBindings( - result.program, {.max_push_dwords = 0}, &error)) { + if (!ShaderRecompiler::IR::AllocateBindings(result.program, 0, &error)) { Fail(test.name, "binding layout", error.c_str()); } const auto *shader_data = ShaderRecompiler::IR::FindBinding( result.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData); Require(test.name, "binding layout", - result.program.bindings.user_data_registers.empty() && + result.program.bindings.user_data_registers.size() == 33 && result.program.bindings.push_constant_size == 0 && shader_data != nullptr, - "offset-only shader data did not use its storage fallback"); + "oversized shader data did not use its storage fallback"); std::vector storage_spirv; if (!ShaderRecompiler::Spirv::EmitProgram(result.program, result.resources, options.input_info, storage_spirv, @@ -1448,6 +1531,114 @@ public: std::printf("[host] %-32s ok\n", "CommandPoolGrowth"); } + void CheckDescriptorHeapLargeSet() { + EnsureRuntimeContext(); + std::array bindings{}; + constexpr std::array stages{ + vk::ShaderStageFlagBits::eVertex, + vk::ShaderStageFlagBits::eFragment, + vk::ShaderStageFlagBits::eCompute, + }; + for (uint32_t i = 0; i < bindings.size(); i++) { + bindings[i].binding = i; + bindings[i].descriptorType = vk::DescriptorType::eSampler; + bindings[i].descriptorCount = 16; + bindings[i].stageFlags = stages[i]; + } + vk::DescriptorSetLayoutCreateInfo create{}; + create.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + create.bindingCount = static_cast(bindings.size()); + create.pBindings = bindings.data(); + vk::DescriptorSetLayout layout = nullptr; + RequireVk("DescriptorHeapLargeSet", "layout", + m_device.createDescriptorSetLayout(&create, nullptr, &layout), + "vkCreateDescriptorSetLayout"); + + vk::PipelineLayoutCreateInfo pipeline_create{}; + pipeline_create.sType = vk::StructureType::ePipelineLayoutCreateInfo; + pipeline_create.setLayoutCount = 1; + pipeline_create.pSetLayouts = &layout; + vk::PipelineLayout pipeline_layout = nullptr; + RequireVk("DescriptorHeapLargeSet", "pipeline layout", + m_device.createPipelineLayout(&pipeline_create, nullptr, + &pipeline_layout), + "vkCreatePipelineLayout"); + + { + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + + const auto commit = [&] { + const auto set = context.GetDescriptorHeap().Commit(layout); + Require("DescriptorHeapLargeSet", "allocation", set != nullptr, + "ordinary descriptor heap could not allocate an oversized sampler set"); + scheduler.Current().Handle().bindDescriptorSets( + vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, &set, 0, + nullptr); + }; + + for (uint32_t i = 0; i < 22; i++) { + commit(); + } + scheduler.FinishCurrent(); + for (uint32_t i = 0; i < 21; i++) { + commit(); + } + } + + m_device.destroyPipelineLayout(pipeline_layout, nullptr); + m_device.destroyDescriptorSetLayout(layout, nullptr); + std::printf("[host] %-32s ok\n", "DescriptorHeapLargeSet"); + } + + void CheckGraphicsPushConstantBank() { + constexpr const char *name = "GraphicsPushConstantBank"; + EnsureRuntimeContext(); + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + + ShaderRecompiler::IR::Program vertex_program{}; + vertex_program.stage = ShaderType::Vertex; + vertex_program.bindings.push_constant_size = 2 * sizeof(uint32_t); + ShaderRecompiler::IR::Program pixel_program{}; + pixel_program.stage = ShaderType::Pixel; + pixel_program.bindings.push_constant_offset = 2 * sizeof(uint32_t); + pixel_program.bindings.push_constant_size = 2 * sizeof(uint32_t); + ShaderRecompiler::IR::ResourceSnapshot snapshot{}; + PreparedBindings vertex{}; + vertex.program = &vertex_program; + vertex.snapshot = &snapshot; + vertex.user_data = {0x11111111u, 0x22222222u}; + PreparedBindings pixel{}; + pixel.program = &pixel_program; + pixel.snapshot = &snapshot; + pixel.user_data = {0x33333333u, 0x44444444u}; + + const auto pipeline = RenderExecutorTestAccess::CommitBindings( + context.GetRenderExecutor(), scheduler.Current(), vertex, pixel); + const auto &bank = + RenderExecutorTestAccess::PushConstants(context.GetRenderExecutor()); + Require(name, "packing", + bank[0] == 0x11111111u && bank[1] == 0x22222222u && + bank[2] == 0x33333333u && bank[3] == 0x44444444u && + std::ranges::all_of(bank.begin() + 4, bank.end(), + [](uint32_t value) { return value == 0; }) && + vertex.committed && pixel.committed, + "graphics stages were not packed into one zero-filled push bank"); + scheduler.Finish(); + RenderExecutorTestAccess::DestroyDescriptorPipelines( + context.GetRenderExecutor(), std::span {&pipeline, 1u}); + std::printf("[host] %-32s ok\n", name); + } + void CheckSchedulerTimeline() { EnsureRuntimeContext(); CommandScheduler scheduler(Renderer(), m_runtime_context); @@ -1706,12 +1897,12 @@ public: auto &upload_stream = cache.GetUtilityBuffer(MemoryUsage::Stream); const auto stream_offset = upload_stream.Copy(overflow_data.data(), overflow_data.size(), 16); - Require( - "StreamBufferRing", "cache-owned stream upload", - upload_stream.Handle() != nullptr && stream_offset < upload_stream.Size() && - std::memcmp(upload_stream.Mapped().data() + stream_offset, overflow_data.data(), - overflow_data.size()) == 0, - "cache-owned stream upload lost bytes"); + Require("StreamBufferRing", "cache-owned stream upload", + upload_stream.Handle() != nullptr && + stream_offset < upload_stream.Size() && + std::memcmp(upload_stream.Mapped().data() + stream_offset, + overflow_data.data(), overflow_data.size()) == 0, + "cache-owned stream upload lost bytes"); const auto atom = m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize; @@ -2253,10 +2444,10 @@ public: "DMA prefetch modified ownership or did not establish a dirty " "cached page"); - buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), - clean_fill_value, false); - buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, - sizeof(clean_copy_words), false, false); + buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), + clean_fill_value, false); + buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, + sizeof(clean_copy_words), false, false); Require("GpuCommandLane", "clean cached DMA ownership", !buffer_cache.HasGpuDirtyBytes(clean_cached_fill, sizeof(source_words)) && @@ -2364,8 +2555,8 @@ public: Require("GpuCommandLane", "dirty tracked fault setup", dirty.first != nullptr, "dirty write-fault test could not create a cached Buffer"); - buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), - dirty_fault_value, false); + buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), + dirty_fault_value, false); }); Require("GpuCommandLane", "dirty tracked fault ownership", buffer_cache.HasGpuDirtyBytes(dirty_fault_address, @@ -2426,14 +2617,15 @@ public: constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000; constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu; gpu.SendCommandSync([&] { - auto dirty = buffer_cache.ObtainBuffer( - empty_copy_fault_address, sizeof(empty_copy_fault_value), true, false); + auto dirty = buffer_cache.ObtainBuffer(empty_copy_fault_address, + sizeof(empty_copy_fault_value), + true, false); Require("GpuCommandLane", "empty-copy write setup", dirty.first != nullptr, "empty-copy write test could not create a cached Buffer"); - buffer_cache.FillBuffer(empty_copy_fault_address, - sizeof(empty_copy_fault_value), - empty_copy_fault_value, false); + buffer_cache.FillBuffer(empty_copy_fault_address, + sizeof(empty_copy_fault_value), + empty_copy_fault_value, false); buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value)); const bool gpu_owned = buffer_cache.IsRegionGpuModified( @@ -2776,8 +2968,7 @@ public: cache.FindBuffer(index_begin + 0x100, sizeof(uint32_t)); const auto index_right = cache.FindBuffer( index_begin + 2 * index_page + 0x100, sizeof(uint32_t)); - Require(name, "registered-range owners", - index_left && index_right, + Require(name, "registered-range owners", index_left && index_right, "failed to create disjoint Buffer index owners"); Require( name, "registered-range boundaries", @@ -2793,16 +2984,15 @@ public: "overlap"); const auto index_bridge = cache.FindBuffer(index_begin + index_page - 1, index_page + 2); - Require( - name, "registered-range merge", - index_bridge && - cache.IsRegionRegistered(index_begin + index_page, index_page), - "bridging acquisition did not publish its merged Buffer range"); + Require(name, "registered-range merge", + index_bridge && cache.IsRegionRegistered(index_begin + index_page, + index_page), + "bridging acquisition did not publish its merged Buffer range"); const auto [resolved_left, resolved_left_offset] = cache.ObtainBuffer( index_begin + 0x100, sizeof(uint32_t), true, false, index_left); - const auto [resolved_right, resolved_right_offset] = cache.ObtainBuffer( - index_begin + 2 * index_page + 0x100, sizeof(uint32_t), true, false, - index_right); + const auto [resolved_right, resolved_right_offset] = + cache.ObtainBuffer(index_begin + 2 * index_page + 0x100, + sizeof(uint32_t), true, false, index_right); Require(name, "two-pass stale id resolution", resolved_left == &cache.GetBuffer(index_bridge) && resolved_right == resolved_left && @@ -2815,9 +3005,10 @@ public: MarkGpuWrite(base + first_offset, sizeof(first_value)); MarkGpuWrite(base + second_offset, sizeof(second_value)); - cache.FillBuffer(base + first_offset, sizeof(first_value), first_value, false); - cache.FillBuffer(base + second_offset, sizeof(second_value), - second_value, false); + cache.FillBuffer(base + first_offset, sizeof(first_value), first_value, + false); + cache.FillBuffer(base + second_offset, sizeof(second_value), second_value, + false); auto &download = BufferCacheTestAccess::DownloadBuffer(cache); auto *fixed_download = &download; const auto fixed_download_handle = download.Handle(); @@ -2831,11 +3022,12 @@ public: sizeof(ring_fault_first_value)); MarkGpuWrite(base + ring_fault_second_offset, sizeof(ring_fault_second_value)); - cache.FillBuffer(base + ring_fault_first_offset, - sizeof(ring_fault_first_value), ring_fault_first_value, false); - cache.FillBuffer(base + ring_fault_second_offset, - sizeof(ring_fault_second_value), - ring_fault_second_value, false); + cache.FillBuffer(base + ring_fault_first_offset, + sizeof(ring_fault_first_value), ring_fault_first_value, + false); + cache.FillBuffer(base + ring_fault_second_offset, + sizeof(ring_fault_second_value), ring_fault_second_value, + false); Require(name, "fault-ring wrapped batch", resources.HandleFault(PageFaultAccess::Read, base + ring_fault_first_offset), @@ -2937,7 +3129,8 @@ public: Require(name, "direct-unmap allocation", unmap_allocation.first != nullptr, "direct-unmap buffer allocation failed"); - cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value, false); + cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value, + false); cache.UnmapMemory(base + 0x4000, 0x4000); uint32_t unmap_backing = 0; std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing)); @@ -2947,20 +3140,20 @@ public: "direct dirty unmap cleared tracking before " "publishing bytes"); - auto partial_unmap_allocation = cache.ObtainBuffer( - base + 0x8000, 0x8000, true, false); + auto partial_unmap_allocation = + cache.ObtainBuffer(base + 0x8000, 0x8000, true, false); Require(name, "partial-unmap allocation", partial_unmap_allocation.first != nullptr, "cross-page cached buffer allocation failed"); - cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), - partial_unmap_value, false); - cache.FillBuffer(base + partial_unmap_survivor_offset, - sizeof(partial_unmap_survivor_value), - partial_unmap_survivor_value, false); + cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), + partial_unmap_value, false); + cache.FillBuffer(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), + partial_unmap_survivor_value, false); cache.UnmapMemory(base + 0x8000, 0x4000); - auto survivor = cache.ObtainBuffer( - base + partial_unmap_survivor_offset, - sizeof(partial_unmap_survivor_value), false); + auto survivor = + cache.ObtainBuffer(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), false); Require(name, "partial-unmap survivor", survivor.first != nullptr, "still-mapped cached-buffer remainder could not be recreated"); auto partial_unmap_readback = @@ -2968,8 +3161,9 @@ public: vk::BufferUsageFlagBits::eTransferDst, {0}); const vk::BufferCopy survivor_copy{survivor.second, 0, sizeof(partial_unmap_survivor_value)}; - scheduler.Current().Handle().copyBuffer( - survivor.first->Handle(), partial_unmap_readback.buffer, 1, &survivor_copy); + scheduler.Current().Handle().copyBuffer(survivor.first->Handle(), + partial_unmap_readback.buffer, 1, + &survivor_copy); vk::BufferMemoryBarrier survivor_barrier{}; survivor_barrier.sType = vk::StructureType::eBufferMemoryBarrier; survivor_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; @@ -2997,12 +3191,12 @@ public: std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale)); std::memcpy(memory + large_offset + large_size - sizeof(large_stale), &large_stale, sizeof(large_stale)); - auto large_allocation = cache.ObtainBuffer( - base + large_offset, large_size, true, false); + auto large_allocation = + cache.ObtainBuffer(base + large_offset, large_size, true, false); Require(name, "near-capacity dirty allocation", large_allocation.first != nullptr, "failed to allocate the near-capacity dirty native buffer"); - cache.FillBuffer(base + large_offset, large_size, large_value, false); + cache.FillBuffer(base + large_offset, large_size, large_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3049,18 +3243,19 @@ public: base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale)); Libs::LibKernel::Memory::WriteBacking( base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale)); - auto grouped_first = cache.ObtainBuffer( - base + grouped_first_offset, grouped_owner_size, true, false); - auto grouped_second = cache.ObtainBuffer( - base + grouped_second_offset, grouped_owner_size, true, false); + auto grouped_first = cache.ObtainBuffer(base + grouped_first_offset, + grouped_owner_size, true, false); + auto grouped_second = cache.ObtainBuffer(base + grouped_second_offset, + grouped_owner_size, true, false); Require(name, "per-owner GC allocation", - grouped_first.first != nullptr && grouped_second.first != nullptr && + grouped_first.first != nullptr && + grouped_second.first != nullptr && grouped_first.first != grouped_second.first, "disjoint GC candidates merged into one Buffer owner"); - cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, - grouped_first_value, false); - cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, - grouped_second_value, false); + cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, + grouped_first_value, false); + cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, + grouped_second_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3102,8 +3297,8 @@ public: "failed to create the clean cross-page retirement owner"); (void)cache.ObtainBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), true, false); - cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), - disjoint_value, false); + cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), + disjoint_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3130,9 +3325,9 @@ public: !cache.HasGpuDirtyBytes(base + disjoint_dirty_offset, sizeof(disjoint_value)), "disjoint unmap lost the completed whole-owner publication"); - auto disjoint_new_owner = cache.ObtainBuffer( - base + disjoint_new_owner_offset, - sizeof(disjoint_value), true, false); + auto disjoint_new_owner = + cache.ObtainBuffer(base + disjoint_new_owner_offset, + sizeof(disjoint_value), true, false); Require(name, "disjoint post-publication reacquire", disjoint_new_owner.first != nullptr, "disjoint acquisition failed after whole-owner publication"); @@ -3164,8 +3359,8 @@ public: "failed to create a fresh cross-page retirement owner"); (void)cache.ObtainBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), true, false); - cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), - reacquire_value, false); + cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), + reacquire_value, false); for (uint32_t tick = 0; tick <= 160; tick++) { cache.RunGarbageCollector(); } @@ -3181,9 +3376,9 @@ public: reacquire_before == reacquire_stale, "fresh whole-owner publication completed before reacquisition"); scheduler.FinishCurrent(); - auto reacquired = cache.ObtainBuffer( - base + reacquire_disjoint_offset, - sizeof(reacquire_value), true, false); + auto reacquired = + cache.ObtainBuffer(base + reacquire_disjoint_offset, + sizeof(reacquire_value), true, false); Require(name, "independent disjoint post-publication reacquire", reacquired.first != nullptr, "clean disjoint-half acquisition failed after publication"); @@ -3631,7 +3826,7 @@ public: }; // A formatted Buffer read must use the private - // Exercise the cache-native image-copy path. Use a request larger + // Exercise the cache-native image-copy path. Use a request larger // than the stream shortcut and poison guest backing // after upload so stale CPU staging cannot accidentally // satisfy the content check. @@ -3700,7 +3895,7 @@ public: "prefix, or transferred ownership"); auto &buffer_cache = resources.GetBufferCache(); - std::pair mip_formatted; + std::pair mip_formatted; bool formatted_owner_was_absent = false; gpu.SendCommandSync([&] { formatted_owner_was_absent = !buffer_cache.IsRegionRegistered( @@ -4108,8 +4303,8 @@ public: Require(name, "MS stencil buffer allocation", ms_stencil_owner.first != nullptr, "failed to create an unequal-sample stencil source"); - resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, - ms_stencil_size, 0x41414141u, false); + resources.GetBufferCache().FillBuffer( + base + ms_stencil_offset, ms_stencil_size, 0x41414141u, false); auto ms_depth_desc = color_desc; ms_depth_desc.type = BindingType::DepthTarget; ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size}; @@ -4456,12 +4651,12 @@ public: Require(name, "partial-page buffer allocation", partial_write.first != nullptr, "partial-page GPU write did not create a native buffer"); - resources.GetBufferCache().FillBuffer(base + partial_image_offset, - sizeof(partial_image_value), - partial_image_value, false); - resources.GetBufferCache().FillBuffer(base + partial_buffer_offset, - sizeof(partial_buffer_value), - partial_buffer_value, false); + resources.GetBufferCache().FillBuffer(base + partial_image_offset, + sizeof(partial_image_value), + partial_image_value, false); + resources.GetBufferCache().FillBuffer(base + partial_buffer_offset, + sizeof(partial_buffer_value), + partial_buffer_value, false); auto partial_desc = MakeLinearDesc( base + partial_image_offset, sizeof(partial_image_value), vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, @@ -4471,7 +4666,8 @@ public: !texture_cache.GetImage(partial_image).IsGpuModified(), "image upload incorrectly consumed buffer dirty ownership"); auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer( - base + partial_image_offset, sizeof(partial_image_value), false, true); + base + partial_image_offset, sizeof(partial_image_value), false, + true); Require(name, "partial-page image mirror", partial_image_mirror.first != nullptr && !texture_cache.GetImage(partial_image).IsGpuModified(), @@ -4624,9 +4820,9 @@ public: const auto publish_image = texture_cache.FindImage(publish_image_desc); (void)texture_cache.FindTexture(publish_image, publish_image_desc); texture_cache.MarkGpuWritten(publish_image); - resources.GetBufferCache().FillBuffer(base + publish_buffer_offset, - sizeof(publish_buffer_value), - publish_buffer_value, false); + resources.GetBufferCache().FillBuffer(base + publish_buffer_offset, + sizeof(publish_buffer_value), + publish_buffer_value, false); auto publish_replacement_desc = publish_image_desc; publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat; publish_replacement_desc.info.guest_format = @@ -4753,8 +4949,8 @@ public: texture_cache.TouchMeta(base + metadata_b, 0, false) && !texture_cache.IsMetaCleared(base + metadata_b, 0), "per-slice metadata update incorrectly required prior GPU ownership"); - resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), - 0, false); + resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), + 0, false); Require( name, "metadata buffer fill state", texture_cache.IsMetaCleared(base + metadata_b, 0), @@ -4975,9 +5171,9 @@ public: "mixed image source could not dirty its first page"); std::memcpy(memory + mixed_source_offset, &mixed_cpu_value, sizeof(mixed_cpu_value)); - resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, - sizeof(mixed_gpu_value), - mixed_gpu_value, false); + resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, + sizeof(mixed_gpu_value), + mixed_gpu_value, false); auto mixed_source_desc = MakeLinearDesc( base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D, @@ -5029,8 +5225,8 @@ public: Require(name, "bridged CPU owner allocation", bridged_cpu_owner.first != nullptr, "bridged image source did not create its CPU-side cache owner"); - resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset, - bridged_texel_size, 0, false); + resources.GetBufferCache().FillBuffer(base + bridged_cpu_offset, + bridged_texel_size, 0, false); Require( name, "bridged CPU write fault", resources.HandleFault(PageFaultAccess::Write, @@ -5051,8 +5247,9 @@ public: bridged_gpu_owner.first != nullptr && bridged_gpu_owner.first != bridged_cpu_owner.first, "bridged image source did not retain two disjoint cache owners"); - resources.GetBufferCache().FillBuffer( - base + bridged_gpu_offset, bridged_texel_size, bridged_gpu_value, false); + resources.GetBufferCache().FillBuffer(base + bridged_gpu_offset, + bridged_texel_size, + bridged_gpu_value, false); Require(name, "bridged ownership split", resources.GetBufferCache().HasGpuDirtyBytes( base + bridged_gpu_offset, bridged_texel_size) && @@ -5458,9 +5655,9 @@ public: Require(name, "same-page dirty sibling allocation", dirty_sibling.first != nullptr, "failed to create the disjoint same-page Buffer owner"); - resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, - sizeof(dirty_sibling_value), - dirty_sibling_value, false); + resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, + sizeof(dirty_sibling_value), + dirty_sibling_value, false); auto exact_image_desc = MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t), vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, @@ -5495,7 +5692,8 @@ public: auto gc_image_desc_b = gc_image_desc_a; gc_image_desc_b.info.data.address = base + gc_image_offsets[1]; auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer( - gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size, true); + gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size, + true); Require(name, "clean pre-image buffer alias", clean_buffer_alias.first != nullptr, "failed to create the clean cached Buffer alias"); @@ -5590,7 +5788,8 @@ public: const vk::BufferCopy alias_copy{refreshed_buffer_alias.second, 0, sizeof(uint32_t)}; scheduler.Current().Handle().copyBuffer( - refreshed_buffer_alias.first->Handle(), alias_readback.buffer, 1, &alias_copy); + refreshed_buffer_alias.first->Handle(), alias_readback.buffer, 1, + &alias_copy); HostReadBarrier(alias_readback.buffer, alias_readback.size, vk::PipelineStageFlagBits::eTransfer, vk::AccessFlagBits::eTransferWrite); @@ -6369,10 +6568,9 @@ public: Require(name, "guest readback queue", TextureCacheTestAccess::TryDownload(cache, id), "tiled BGRA16 guest readback was rejected"); - auto mirror = resources.GetBufferCache().ObtainBuffer( - base, total.size, false, true); - Require(name, "mirror owner", - mirror.first != nullptr, + auto mirror = resources.GetBufferCache().ObtainBuffer(base, total.size, + false, true); + Require(name, "mirror owner", mirror.first != nullptr, "tiled BGRA16 mirror has no BufferCache owner"); auto mirror_readback = CreateHostBuffer( name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0}); @@ -6571,14 +6769,13 @@ public: "the 3D render target could not be queued for guest-layout readback"); auto mirror = resources.GetBufferCache().ObtainBuffer( base, allocation_size, false, true); - Require(name, "volume mirror", - mirror.first != nullptr, + Require(name, "volume mirror", mirror.first != nullptr, "the 3D render-target readback has no BufferCache owner"); auto slice_probe = CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0}); const vk::BufferCopy copy{mirror.second + 31 * slice_size, 0, 4}; - scheduler.Current().Handle().copyBuffer(mirror.first->Handle(), slice_probe.buffer, - 1, ©); + scheduler.Current().Handle().copyBuffer(mirror.first->Handle(), + slice_probe.buffer, 1, ©); vk::BufferMemoryBarrier barrier{}; barrier.sType = vk::StructureType::eBufferMemoryBarrier; barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; @@ -6883,6 +7080,15 @@ public: auto &texture_cache = resources.GetTextureCache(); auto &executor = context.GetRenderExecutor(); resources.MapMemory(base, allocation_size); + std::vector descriptor_pipelines; + const auto allocate_bindings = + [&](ShaderRecompiler::IR::Program &program) { + program.shader_info_complete = true; + std::string error; + Require(name, "binding layout", + ShaderRecompiler::IR::AllocateBindings(program, 0, &error), + error.c_str()); + }; constexpr auto stencil_format = Prospero::BufferFormat::k8UInt; constexpr auto linear = Prospero::TileMode::kLinear; @@ -6922,14 +7128,13 @@ public: ShaderRecompiler::IR::SpecializeResources( null_program, null_snapshot, &null_error), null_error.c_str()); + allocate_bindings(null_program); ShaderStageRuntime null_runtime{ std::make_shared( std::move(null_program)), std::make_shared( std::move(null_snapshot))}; - auto null_bindings = executor.PrepareBindings( - null_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto null_bindings = executor.PrepareBindings(null_runtime); executor.RebindImages(null_bindings); Require(name, "null descriptor count", null_bindings.resources.images.size() == 3, @@ -6951,18 +7156,18 @@ public: null_image_id), "the shared null image did not preserve texture/storage " "acquisition without guest readback or RenderExecutor ownership"); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - null_bindings); + descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings( + executor, scheduler.Current(), null_bindings)); Require(name, "null descriptor general layouts", - std::ranges::all_of( - null_bindings.resources.images, - [](const auto &binding) { - const auto info = - DescriptorCacheTestAccess::MakeImageInfo(binding); - return binding.layout == vk::ImageLayout::eGeneral && - info.imageView == binding.image_view && - info.imageLayout == binding.layout; - }) && + std::ranges::all_of(null_bindings.resources.images, + [](const auto &binding) { + const auto info = MakeImageInfo(binding); + return binding.layout == + vk::ImageLayout::eGeneral && + info.imageView == + binding.image_view && + info.imageLayout == binding.layout; + }) && null_image.backing.state.layout == vk::ImageLayout::eGeneral, "shared sampled/storage null descriptors did not retain " "the general layout through descriptor generation"); @@ -7018,6 +7223,7 @@ public: ShaderRecompiler::Decoder::ImageDimension::Dim2D; storage_resource.written = true; storage_program.info.images.push_back(storage_resource); + allocate_bindings(storage_program); ShaderRecompiler::IR::DescriptorValue storage_descriptor{}; std::copy(std::begin(storage.fields), std::end(storage.fields), storage_descriptor.dwords.begin()); @@ -7070,7 +7276,7 @@ public: auto sampled_overwide_resolved = RenderExecutorTestAccess::ResolveTexture( executor, sampled_overwide_resource, overwide_mipped_storage_descriptor); - DescriptorCache::PreparedBindings mipped_prepared{}; + PreparedBindings mipped_prepared{}; auto mipped_program = std::make_shared(); mipped_program->info.images.push_back(storage_resource); mipped_program->info.images.push_back(mipped_storage_resource); @@ -7082,8 +7288,8 @@ public: mipped_storage_descriptor, overwide_mipped_storage_descriptor, overwide_mipped_storage_descriptor}; - mipped_prepared.program = std::move(mipped_program); - mipped_prepared.snapshot = std::move(mipped_snapshot); + mipped_prepared.program = mipped_program.get(); + mipped_prepared.snapshot = mipped_snapshot.get(); mipped_prepared.resources.images.push_back( std::move(plain_mipped_storage_binding)); mipped_prepared.resources.images.push_back( @@ -7119,12 +7325,12 @@ public: mipped_binding.mip_views[2] != nullptr && mipped_binding.mip_views[0] != mipped_binding.mip_views[1] && mipped_binding.mip_views[1] != mipped_binding.mip_views[2] && - DescriptorCacheTestAccess::MakeImageInfo(mipped_binding, 0) - .imageView == mipped_binding.mip_views[0] && - DescriptorCacheTestAccess::MakeImageInfo(mipped_binding, 1) - .imageView == mipped_binding.mip_views[1] && - DescriptorCacheTestAccess::MakeImageInfo(mipped_binding, 2) - .imageView == mipped_binding.mip_views[2], + MakeImageInfo(mipped_binding, 0).imageView == + mipped_binding.mip_views[0] && + MakeImageInfo(mipped_binding, 1).imageView == + mipped_binding.mip_views[1] && + MakeImageInfo(mipped_binding, 2).imageView == + mipped_binding.mip_views[2], "base-1 through last-3 storage view did not create three " "ordered one-level descriptors"); Require(name, "over-wide fixed storage mip view", @@ -7275,9 +7481,7 @@ public: std::move(storage_program)), std::make_shared( std::move(storage_snapshot))}; - auto storage_discovery = executor.PrepareBindings( - storage_runtime, vk::ShaderStageFlagBits::eVertex, - DescriptorCache::Stage::Vertex); + auto storage_discovery = executor.PrepareBindings(storage_runtime); const auto storage_id = storage_discovery.resources.images[0].image_id; Require(name, "storage prefetch purity", storage_discovery.resources.images[0].image_view == nullptr && @@ -7299,6 +7503,7 @@ public: ShaderRecompiler::Decoder::ImageDimension::Dim2D; sampled_resource.read = true; sampled_program.info.images.push_back(sampled_resource); + allocate_bindings(sampled_program); ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{}; sampled_snapshot.images.push_back(storage_descriptor); ShaderStageRuntime sampled_runtime{ @@ -7333,14 +7538,14 @@ public: executor, storage_runtime, ordered_sampled_runtime, true); const auto ordered_sampled_id = ordered_bindings.pixel->resources.images[0].image_id; - Require(name, "VS-before-PS retained-owner order", - ordered_bindings.vertex.resources.images[0].image_id == - storage_id && - ordered_sampled_id != storage_id && - RenderExecutorTestAccess::BoundImagesInOrder( - executor, storage_id, ordered_sampled_id), - "production graphics binding did not retain vertex resources " - "before pixel resources"); + Require( + name, "VS-before-PS retained-owner order", + ordered_bindings.vertex.resources.images[0].image_id == storage_id && + ordered_sampled_id != storage_id && + RenderExecutorTestAccess::BoundImagesInOrder(executor, storage_id, + ordered_sampled_id), + "production graphics binding did not retain vertex resources " + "before pixel resources"); RenderExecutorTestAccess::ResetBindings(executor); auto graphics_bindings = @@ -7368,10 +7573,9 @@ public: texture_cache.GetImage(storage_id).usage.texture, "the production graphics binding path did not complete vertex " "storage acquisition before pixel sampling"); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - graphics_bindings.vertex); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - *graphics_bindings.pixel); + descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings( + executor, scheduler.Current(), graphics_bindings.vertex, + *graphics_bindings.pixel)); Require(name, "early writable alias retention", texture_cache.GetImage(storage_id).backing.state.access_mask == (vk::AccessFlagBits2::eShaderRead | @@ -7400,21 +7604,18 @@ public: texture_cache.GetImage(storage_id).binding.force_general, "an already sampled image was not promoted when a later " "storage alias bound the same backing"); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - writable_alias_bindings.vertex); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - *writable_alias_bindings.pixel); + descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings( + executor, scheduler.Current(), writable_alias_bindings.vertex, + *writable_alias_bindings.pixel)); Require( name, "forced-general descriptor capture", writable_alias_bindings.vertex.resources.images[0].layout == vk::ImageLayout::eGeneral && writable_alias_bindings.pixel->resources.images[0].layout == vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - writable_alias_bindings.vertex.resources.images[0]) + MakeImageInfo(writable_alias_bindings.vertex.resources.images[0]) .imageLayout == vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - writable_alias_bindings.pixel->resources.images[0]) + MakeImageInfo(writable_alias_bindings.pixel->resources.images[0]) .imageLayout == vk::ImageLayout::eGeneral && texture_cache.GetImage(storage_id).backing.state.layout == vk::ImageLayout::eGeneral && @@ -7469,40 +7670,38 @@ public: std::make_shared(); split_program->resource_tracking_complete = true; split_program->info.images = {storage_resource, sampled_resource}; - DescriptorCache::PreparedBindings split_bindings{}; - split_bindings.program = split_program; - split_bindings.snapshot = + allocate_bindings(*split_program); + PreparedBindings split_bindings{}; + auto split_snapshot = std::make_shared(); - split_bindings.shader_stage = vk::ShaderStageFlagBits::eVertex; - split_bindings.stage = DescriptorCache::Stage::Vertex; + split_bindings.program = split_program.get(); + split_bindings.snapshot = split_snapshot.get(); split_bindings.resources.images.push_back( {split_id, texture_cache.FindTexture(split_id, split_storage_desc), split_storage_desc}); split_bindings.resources.images.push_back( {split_id, texture_cache.FindTexture(split_id, split_sampled_desc), split_sampled_desc}); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - split_bindings); + descriptor_pipelines.push_back(RenderExecutorTestAccess::CommitBindings( + executor, scheduler.Current(), split_bindings)); const auto &split_image = texture_cache.GetImage(split_id); - Require(name, "per-binding subresource layouts", - split_bindings.resources.images[0].layout == - vk::ImageLayout::eGeneral && - split_bindings.resources.images[1].layout == - vk::ImageLayout::eShaderReadOnlyOptimal && - DescriptorCacheTestAccess::MakeImageInfo( - split_bindings.resources.images[0]) - .imageLayout == vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - split_bindings.resources.images[1]) - .imageLayout == - vk::ImageLayout::eShaderReadOnlyOptimal && - split_image.backing.subresource_states.size() == 2 && - split_image.backing.subresource_states[0].layout == - vk::ImageLayout::eGeneral && - split_image.backing.subresource_states[1].layout == - vk::ImageLayout::eShaderReadOnlyOptimal, - "descriptor layouts were queried after a later mip transition " - "instead of being captured at each binding"); + Require( + name, "per-binding subresource layouts", + split_bindings.resources.images[0].layout == + vk::ImageLayout::eGeneral && + split_bindings.resources.images[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + MakeImageInfo(split_bindings.resources.images[0]).imageLayout == + vk::ImageLayout::eGeneral && + MakeImageInfo(split_bindings.resources.images[1]).imageLayout == + vk::ImageLayout::eShaderReadOnlyOptimal && + split_image.backing.subresource_states.size() == 2 && + split_image.backing.subresource_states[0].layout == + vk::ImageLayout::eGeneral && + split_image.backing.subresource_states[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "descriptor layouts were queried after a later mip transition " + "instead of being captured at each binding"); Libs::LibKernel::Memory::WriteBacking( storage_address, &storage_stale_value, sizeof(storage_stale_value)); @@ -7865,9 +8064,7 @@ public: std::make_shared( std::move(array_snapshot))}; - auto array_binding = executor.PrepareBindings( - array_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto array_binding = executor.PrepareBindings(array_runtime); executor.RebindImages(array_binding); const auto expanded_array_id = array_binding.resources.images[0].image_id; const auto &expanded_array = texture_cache.GetImage(expanded_array_id); @@ -7940,9 +8137,8 @@ public: ShaderStageRuntime colliding_msaa_runtime{ std::move(colliding_msaa_program), std::move(colliding_msaa_snapshot)}; - auto colliding_msaa_binding = executor.PrepareBindings( - colliding_msaa_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto colliding_msaa_binding = + executor.PrepareBindings(colliding_msaa_runtime); executor.RebindImages(colliding_msaa_binding); const auto &resolved_colliding_msaa = colliding_msaa_binding.resources.images[0]; @@ -8000,9 +8196,7 @@ public: msaa_snapshot->images.push_back(msaa_descriptor); ShaderStageRuntime msaa_runtime{std::move(msaa_program), std::move(msaa_snapshot)}; - auto msaa_binding = executor.PrepareBindings( - msaa_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto msaa_binding = executor.PrepareBindings(msaa_runtime); executor.RebindImages(msaa_binding); const auto &resolved_msaa = msaa_binding.resources.images[0]; Require( @@ -8041,9 +8235,7 @@ public: msaa_array_snapshot->images.push_back(msaa_array_descriptor); ShaderStageRuntime msaa_array_runtime{std::move(msaa_array_program), std::move(msaa_array_snapshot)}; - auto msaa_array_binding = executor.PrepareBindings( - msaa_array_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto msaa_array_binding = executor.PrepareBindings(msaa_array_runtime); executor.RebindImages(msaa_array_binding); const auto &resolved_msaa_array = msaa_array_binding.resources.images[0]; Require(name, "MSAA array backing expansion", @@ -8163,11 +8355,10 @@ public: rebound_depth.samples = 1; rebound_depth.image_id = depth_id; scheduler.FinishCurrent(); - Require( - name, "deferred target slot erasure", - TextureCacheTestAccess::Owner(texture_cache, target_subresource_id) == - nullptr, - "the displaced target slot was not erased after GPU completion"); + Require(name, "deferred target slot erasure", + TextureCacheTestAccess::Owner(texture_cache, + target_subresource_id) == nullptr, + "the displaced target slot was not erased after GPU completion"); auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets( executor, scheduler.Current(), &rebound_color, 1, rebound_depth); const auto stencil_proxy_id = texture_cache.FindImageFromRange( @@ -8281,9 +8472,7 @@ public: std::make_shared( std::move(snapshot))}; - auto prepared = context.GetRenderExecutor().PrepareBindings( - runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto prepared = context.GetRenderExecutor().PrepareBindings(runtime); const auto sampled_stencil_id = prepared.resources.images[0].image_id; Require(name, "first stencil discovery", prepared.resources.images.size() == 1 && @@ -8316,9 +8505,7 @@ public: "at the stencil guest address"); RenderExecutorTestAccess::ResetBindings(executor); - auto redirected = context.GetRenderExecutor().PrepareBindings( - runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto redirected = context.GetRenderExecutor().PrepareBindings(runtime); Require(name, "redirected owner discovery", redirected.resources.images.size() == 1 && redirected.resources.images[0].image_id == depth_id && @@ -8354,9 +8541,8 @@ public: std::move(stencil_storage_program), std::make_shared( std::move(stencil_storage_snapshot))}; - auto storage_redirected = executor.PrepareBindings( - stencil_storage_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); + auto storage_redirected = + executor.PrepareBindings(stencil_storage_runtime); executor.RebindImages(storage_redirected); Require( name, "storage stencil acquisition", @@ -8367,6 +8553,8 @@ public: RenderExecutorTestAccess::ResetBindings(executor); resources.UnmapMemory(base, allocation_size); scheduler.Finish(); + RenderExecutorTestAccess::DestroyDescriptorPipelines( + executor, descriptor_pipelines); } Require(name, "unmap direct backing", @@ -8681,6 +8869,9 @@ public: auto Binding = [&](Kind kind) { return ShaderRecompiler::IR::FindBinding(layout, kind); }; + auto Native = [](Kind kind) { + return ShaderRecompiler::IR::NativeBinding(ShaderType::Compute, kind); + }; auto Count = [&](Kind kind) { const auto *binding = Binding(kind); if (binding == nullptr) { @@ -8733,50 +8924,10 @@ public: layout_bindings.push_back(item); }; for (const auto &binding : layout.descriptors) { - vk::DescriptorType type = vk::DescriptorType::eStorageBuffer; - u32 count = static_cast(binding.resources.size()); - switch (binding.kind) { - case Kind::Sampled1D: - case Kind::Sampled1DArray: - case Kind::Sampled2D: - case Kind::Sampled2DArray: - case Kind::Sampled3D: - case Kind::SampledUint1D: - case Kind::SampledUint1DArray: - case Kind::SampledUint2D: - case Kind::SampledUint2DArray: - case Kind::SampledUint3D: - type = vk::DescriptorType::eSampledImage; - break; - case Kind::Storage1D: - case Kind::Storage1DArray: - case Kind::Storage2D: - case Kind::Storage2DArray: - case Kind::Storage3D: - case Kind::StorageUint1D: - case Kind::StorageUint1DArray: - case Kind::StorageUint2D: - case Kind::StorageUint2DArray: - case Kind::StorageUint3D: - case Kind::StorageAtomic1D: - case Kind::StorageAtomic1DArray: - case Kind::StorageAtomic2D: - case Kind::StorageAtomic2DArray: - case Kind::StorageAtomic3D: - type = vk::DescriptorType::eStorageImage; - break; - case Kind::Samplers: - type = vk::DescriptorType::eSampler; - break; - case Kind::Gds: - case Kind::FlattenedSrt: - case Kind::UserData: - count = 1; - break; - default: - break; - } - add_layout_binding(binding.binding, type, count); + add_layout_binding(ShaderRecompiler::IR::NativeBinding( + ShaderType::Compute, binding.kind), + NativeDescriptorType(binding.kind), + NativeDescriptorCount(binding)); } vk::DescriptorSetLayoutCreateInfo layout_info{}; @@ -8797,7 +8948,7 @@ public: vk::PushConstantRange push_range{}; if (layout.push_constant_size != 0) { push_range.stageFlags = vk::ShaderStageFlagBits::eCompute; - push_range.offset = layout.push_constant_offset; + push_range.offset = 0; push_range.size = layout.push_constant_size; pipeline_layout_info.pushConstantRangeCount = 1; pipeline_layout_info.pPushConstantRanges = &push_range; @@ -8916,7 +9067,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = buffers->binding; + write.dstBinding = Native(Kind::Buffers); write.descriptorCount = static_cast(buffer_infos.size()); write.descriptorType = vk::DescriptorType::eStorageBuffer; write.pBufferInfo = buffer_infos.data(); @@ -8931,7 +9082,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = address->binding; + write.dstBinding = Native(Kind::AddressMemory); write.descriptorCount = static_cast(address_memory_infos.size()); write.descriptorType = vk::DescriptorType::eStorageBuffer; write.pBufferInfo = address_memory_infos.data(); @@ -8945,7 +9096,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = flattened->binding; + write.dstBinding = Native(Kind::FlattenedSrt); write.descriptorCount = 1; write.descriptorType = vk::DescriptorType::eStorageBuffer; write.pBufferInfo = &flattened_info; @@ -8959,7 +9110,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = user->binding; + write.dstBinding = Native(Kind::UserData); write.descriptorCount = 1; write.descriptorType = vk::DescriptorType::eStorageBuffer; write.pBufferInfo = &user_data_info; @@ -8972,7 +9123,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = gds->binding; + write.dstBinding = Native(Kind::Gds); write.descriptorCount = 1; write.descriptorType = vk::DescriptorType::eStorageBuffer; write.pBufferInfo = &gds_info; @@ -9003,7 +9154,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = sampled->binding; + write.dstBinding = Native(sampled->kind); write.descriptorCount = static_cast(sampled_infos.size()); write.descriptorType = vk::DescriptorType::eSampledImage; write.pImageInfo = sampled_infos.data(); @@ -9030,7 +9181,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = binding->binding; + write.dstBinding = Native(binding->kind); write.descriptorCount = static_cast(infos->size()); write.descriptorType = vk::DescriptorType::eStorageImage; write.pImageInfo = infos->data(); @@ -9050,7 +9201,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = samplers->binding; + write.dstBinding = Native(Kind::Samplers); write.descriptorCount = static_cast(sampler_infos.size()); write.descriptorType = vk::DescriptorType::eSampler; write.pImageInfo = sampler_infos.data(); @@ -9070,8 +9221,8 @@ public: compiled.packed_user_data.size() * sizeof(u32) == layout.push_constant_size, "native user-data size does not match push-constant range"); - cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, - layout.push_constant_offset, layout.push_constant_size, + cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, + layout.push_constant_size, compiled.packed_user_data.data()); } cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z); @@ -9165,7 +9316,7 @@ public: fragment_bind.push_constant_size, "fragment push constant data size does not match reflection"); push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment; - push_constant_range.offset = fragment_bind.push_constant_offset; + push_constant_range.offset = 0; push_constant_range.size = fragment_bind.push_constant_size; } @@ -9292,8 +9443,7 @@ public: cmd.beginRendering(rendering); cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); if (push_constant_range.size != 0) { - cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, - fragment_bind.push_constant_offset, + cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, 0, fragment_bind.push_constant_size, test.push_constants.data()); } @@ -15285,7 +15435,7 @@ TestCase BufferOffsetsUsePackedLaneAndStorageFallback() { test.expected = {0x12345678u, 0, 0, 0xabcdef01u}; test.storage_buffer_range_dwords = 1; test.storage_buffer_offsets = {0, 12}; - test.force_shader_data_storage = true; + test.expand_shader_data_storage = true; test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; return test; @@ -18509,7 +18659,7 @@ void CheckIndirectImageKeySwitch() { program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u}); std::string error; - Require(name, "binding layout", AllocateBindings(program, {}, &error), + Require(name, "binding layout", AllocateBindings(program, 0, &error), error.c_str()); ResourceSnapshot snapshot{}; DescriptorValue descriptor{}; @@ -22431,8 +22581,8 @@ void CheckSlotVectorLifetime() { const auto second = slots.insert(&destroyed); Require("SlotVectorLifetime", "generation", first_destroyed && !destroyed && first.index == second.index && - first != second && - !slots.is_allocated(first) && slots.is_allocated(second), + first != second && !slots.is_allocated(first) && + slots.is_allocated(second), "reused slot accepted a stale cache resource id"); std::printf("[host] %-32s ok\n", "SlotVectorLifetime"); } @@ -23259,6 +23409,21 @@ int main(int argc, char **argv) { vulkan.CheckSchedulerTimeline(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--descriptor-heap-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckDescriptorHeapLargeSet(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--push-constant-bank-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGraphicsPushConstantBank(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--shader-data-storage-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, BufferOffsetsUsePackedLaneAndStorageFallback()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) { VulkanHarness vulkan; vulkan.CheckGpuMappedRangeLifecycle(); @@ -23503,6 +23668,8 @@ int main(int argc, char **argv) { CheckIndirectImageKeySwitch(); CheckPs5GameExampleImageClearRuntimeShape(); vulkan.CheckSchedulerTimeline(); + vulkan.CheckDescriptorHeapLargeSet(); + vulkan.CheckGraphicsPushConstantBank(); vulkan.CheckGpuMappedRangeLifecycle(); vulkan.CheckStreamBufferRing(); vulkan.CheckCommandPoolGrowth(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 0cf891567..1c0a94241 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -690,6 +690,24 @@ bool SpirvHasDecorationValue(const std::vector &binary, return SpirvDecorationValueCount(binary, decoration, value) != 0; } +bool SpirvHasMemberDecorationValue(std::span binary, + uint32_t decoration, uint32_t value) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 72u && word_count >= 5u && binary[i + 3] == decoration && + binary[i + 4] == value) { + return true; + } + i += word_count; + } + return false; +} + std::vector SpirvDecorationValueTargets(const std::vector &binary, uint32_t decoration, uint32_t value) { @@ -9868,22 +9886,23 @@ void TestRenderTargetReverseExportMapping() { mapped.code_size_bytes = sizeof(shader); ShaderMapUserData(regs.ps_regs.data_addr, mapped); HW::ShaderRegisters sh{}; - ShaderVertexInputInfo vs_info{}; - vs_info.stage.program = std::make_shared(); std::array mappings{}; mappings[0] = gr32.export_mapping; ShaderPixelInputInfo compiled_info{}; + ShaderVertexInputInfo vertex_info{}; std::span compiled_spirv; - Check(ShaderCompileInfoPS(regs, sh, vs_info, mappings, compiled_info, - compiled_spirv) && - compiled_info.target_export_mapping[0].IsIdentity(), - "inactive reverse MRT mapping was not normalized out of the shader " - "cache key"); + Check( + ShaderCompileInfoPS(regs, sh, vertex_info, mappings, compiled_info, + compiled_spirv) && + compiled_info.target_export_mapping[0].IsIdentity(), + "inactive reverse MRT mapping was not normalized out of the shader " + "cache key"); sh.target_output_mode[0] = 4; - Check(ShaderCompileInfoPS(regs, sh, vs_info, mappings, compiled_info, - compiled_spirv) && - compiled_info.target_export_mapping[0] == gr32.export_mapping, - "active reverse MRT mapping was lost before shader specialization"); + Check( + ShaderCompileInfoPS(regs, sh, vertex_info, mappings, compiled_info, + compiled_spirv) && + compiled_info.target_export_mapping[0] == gr32.export_mapping, + "active reverse MRT mapping was lost before shader specialization"); } void TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled() { @@ -9988,15 +10007,21 @@ void TestNewShaderRecompilerNativeBindingPlan() { "native binding plan did not allocate the sampler"); Check(SpirvContainsOpcode(result.spirv, 86), "SPIR-V binary does not combine separate image/sampler descriptors"); - Check(SpirvHasDecorationValue(result.spirv, 33u, buffers->binding), + Check(SpirvHasDecorationValue(result.spirv, 33u, + ShaderRecompiler::IR::NativeBinding( + result.program.stage, buffers->kind)), "SPIR-V lacks storage-buffer Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 33u, sampled->binding), + Check(SpirvHasDecorationValue(result.spirv, 33u, + ShaderRecompiler::IR::NativeBinding( + result.program.stage, sampled->kind)), "SPIR-V lacks sampled-image Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 33u, samplers->binding), + Check(SpirvHasDecorationValue(result.spirv, 33u, + ShaderRecompiler::IR::NativeBinding( + result.program.stage, samplers->kind)), "SPIR-V lacks sampler Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 34u, - result.program.bindings.descriptor_set), - "SPIR-V lacks DescriptorSet decoration"); + Check(SpirvDecorationValueCount(result.spirv, 34u, 0u) == + result.program.bindings.descriptors.size(), + "SPIR-V resources do not all use descriptor set zero"); CheckSpirvBinaryValidates(result.spirv); auto malformed = result.program; @@ -10945,6 +10970,31 @@ void TestComputeLdsAllocationIdentity() { ShaderGetIdCS(regs, lds_896, true), "compute pipeline identity omitted the LDS allocation"); + auto tg_size_disabled = lds_896; + auto tg_size_enabled = lds_896; + tg_size_enabled.tg_size_en = true; + Check(ShaderGetIdCS(regs, tg_size_disabled, false) != + ShaderGetIdCS(regs, tg_size_enabled, false), + "compute shader identity omitted TG_SIZE semantics"); + + auto dispatch_disabled = lds_896; + dispatch_disabled.dispatch_threads_num[0] = 64; + dispatch_disabled.dispatch_threads_num[1] = 32; + dispatch_disabled.dispatch_threads_num[2] = 1; + auto dispatch_enabled = dispatch_disabled; + dispatch_enabled.dispatch_thread_dimensions = true; + Check(ShaderGetIdCS(regs, dispatch_disabled, false) == + ShaderGetIdCS(regs, dispatch_enabled, false), + "runtime dispatch enable created a compute shader variant"); + + auto second_extent = dispatch_enabled; + second_extent.dispatch_threads_num[0] = 1920; + second_extent.dispatch_threads_num[1] = 1080; + second_extent.dispatch_threads_num[2] = 4; + Check(ShaderGetIdCS(regs, dispatch_enabled, false) == + ShaderGetIdCS(regs, second_extent, false), + "runtime dispatch extent created a compute shader variant"); + mapped.scratch_size_dwords = 7; ShaderMapUserData(regs.cs_regs.data_addr, mapped); regs.cs_regs.lds_size = decode_lds_field(lds_896_rsrc2); @@ -10981,14 +11031,25 @@ void TestComputeLdsAllocationIdentity() { CheckSpirvBinaryValidates(append_result.spirv); } -void TestPixelProgramCacheDescriptorSetIdentity() { +void TestPixelProgramCacheBindingIdentity() { const uint32_t shader_01[] = {0xbf810000u}; const uint32_t shader_10[] = {0xbf810000u}; HW::ShaderRegisters sh{}; - auto check_transition = [&](const uint32_t *shader, uint64_t checksum, - bool first_has_vs_descriptors, - bool second_has_vs_descriptors) { + HW::PixelShaderInfo identity_regs{}; + ShaderPixelInputInfo no_depth_export{}; + auto depth_export = no_depth_export; + depth_export.ps_depth_export_enable = true; + Check(ShaderGetIdPS(identity_regs, no_depth_export, false) != + ShaderGetIdPS(identity_regs, depth_export, false), + "pixel shader identity omitted depth-export semantics"); + auto shifted_push = no_depth_export; + shifted_push.push_constant_offset = 36; + Check(ShaderGetIdPS(identity_regs, no_depth_export, false) != + ShaderGetIdPS(identity_regs, shifted_push, false), + "pixel shader identity omitted graphics push-bank placement"); + + auto check_cache_identity = [&](const uint32_t *shader, uint64_t checksum) { HW::PixelShaderInfo regs{}; regs.ps_regs.data_addr = reinterpret_cast(shader); regs.ps_regs.chksum = checksum; @@ -10996,33 +11057,120 @@ void TestPixelProgramCacheDescriptorSetIdentity() { mapped.code_size_bytes = sizeof(uint32_t); ShaderMapUserData(regs.ps_regs.data_addr, mapped); - auto compile = [&](bool has_vs_descriptors) { - auto vs_program = std::make_shared(); - if (has_vs_descriptors) { - vs_program->bindings.descriptors.emplace_back(); - } - ShaderVertexInputInfo vs_info{}; - vs_info.stage.program = std::move(vs_program); - const std::array identity_mappings{}; - ShaderPixelInputInfo ps_info{}; - std::span spirv; - Check(ShaderCompileInfoPS(regs, sh, vs_info, identity_mappings, ps_info, - spirv), - "pixel program-cache transition failed to compile"); - const auto expected_set = has_vs_descriptors ? 1u : 0u; - Check(ps_info.descriptor_set == expected_set && - ps_info.stage.program != nullptr && - ps_info.stage.program->bindings.descriptor_set == expected_set, - "pixel program cache reused SPIR-V with the paired VS descriptor " - "set"); - }; + const std::array identity_mappings{}; + ShaderVertexInputInfo vertex_info{}; + ShaderPixelInputInfo first_info{}; + std::span first_spirv; + Check(ShaderCompileInfoPS(regs, sh, vertex_info, identity_mappings, first_info, + first_spirv), + "pixel program-cache fixture failed to compile"); - compile(first_has_vs_descriptors); - compile(second_has_vs_descriptors); + ShaderPixelInputInfo second_info{}; + std::span second_spirv; + Check(ShaderCompileInfoPS(regs, sh, vertex_info, identity_mappings, second_info, + second_spirv), + "pixel program-cache lookup failed"); + Check( + first_info.stage.program != nullptr && + second_info.stage.program == first_info.stage.program && + second_spirv.data() == first_spirv.data() && + second_spirv.size() == first_spirv.size(), + "pixel program cache did not reuse an identical graphics placement"); }; - check_transition(shader_01, 0x91a27e6300000001ull, false, true); - check_transition(shader_10, 0x91a27e6300000002ull, true, false); + check_cache_identity(shader_01, 0x91a27e6300000001ull); + check_cache_identity(shader_10, 0x91a27e6300000002ull); + + const uint32_t push_shader[] = { + EncodeVop1(0x01, 0, 0), EncodeVop1(0x01, 1, 1), + EncodeVop1(0x01, 2, 2), EncodeVop1(0x01, 3, 3), + EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 2, 3), 0xbf810000u, + }; + HW::PixelShaderInfo push_regs{}; + push_regs.ps_regs.data_addr = reinterpret_cast(push_shader); + push_regs.ps_regs.chksum = 0x91a27e6300000003ull; + push_regs.ps_regs.rsrc2.user_sgpr = 4; + ShaderMappedData push_mapped{}; + push_mapped.code_size_bytes = sizeof(push_shader); + ShaderMapUserData(push_regs.ps_regs.data_addr, push_mapped); + HW::ShaderRegisters push_sh{}; + push_sh.target_output_mode[0] = 4; + const std::array mappings{}; + const auto MakeVertex = [](uint32_t push_size) { + ShaderVertexInputInfo info{}; + auto program = std::make_shared(); + program->stage = ShaderType::Vertex; + program->bindings.push_constant_size = push_size; + info.stage.program = std::move(program); + return info; + }; + const auto vertex_at_zero = MakeVertex(0); + const auto vertex_at_36 = MakeVertex(36); + ShaderPixelInputInfo at_zero{}; + ShaderPixelInputInfo at_36{}; + ShaderPixelInputInfo at_zero_again{}; + std::span spirv_at_zero; + std::span spirv_at_36; + std::span spirv_at_zero_again; + Check(ShaderCompileInfoPS(push_regs, push_sh, vertex_at_zero, mappings, + at_zero, spirv_at_zero) && + ShaderCompileInfoPS(push_regs, push_sh, vertex_at_36, mappings, + at_36, spirv_at_36) && + ShaderCompileInfoPS(push_regs, push_sh, vertex_at_zero, mappings, + at_zero_again, spirv_at_zero_again), + "pixel program-cache placement fixture failed to compile"); + Check(at_zero.push_constant_offset == 0 && + at_36.push_constant_offset == 36 && + at_zero.stage.program != at_36.stage.program && + spirv_at_zero.data() != spirv_at_36.data() && + at_zero_again.stage.program == at_zero.stage.program && + spirv_at_zero_again.data() == spirv_at_zero.data() && + SpirvHasMemberDecorationValue(spirv_at_36, 35u, 36u), + "pixel program cache did not partition and reuse graphics push placement"); +} + +void TestGraphicsPushConstantPlacement() { + using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; + constexpr uint32_t OffsetDecoration = 35; + const uint32_t shader[] = { + EncodeVop1(0x01, 0, 0), EncodeVop1(0x01, 1, 1), + EncodeVop1(0x01, 2, 2), EncodeVop1(0x01, 3, 3), + EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 2, 3), 0xbf810000u, + }; + const std::array user_data = { + 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u}; + ShaderPixelInputInfo pixel_info{}; + + auto options = MakeCompileOptions(ShaderType::Pixel); + options.input_info.pixel = &pixel_info; + options.user_data = user_data.data(); + options.user_data_count = static_cast(user_data.size()); + options.push_constant_offset = 36; + + ShaderRecompiler::CompileResult placed; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, placed, &error), + error.c_str()); + Check(placed.program.bindings.push_constant_offset == 36 && + placed.program.bindings.push_constant_size == sizeof(user_data) && + ShaderRecompiler::IR::FindBinding(placed.program.bindings, + BindingKind::UserData) == nullptr, + "pixel push constants did not follow the vertex payload"); + Check(SpirvHasMemberDecorationValue(placed.spirv, OffsetDecoration, 36), + "SPIR-V push block omitted the graphics-bank member offset"); + CheckSpirvBinaryValidates(placed.spirv); + + options.push_constant_offset = 124; + ShaderRecompiler::CompileResult spilled; + error.clear(); + Check(ShaderRecompiler::TryRecompile(shader, options, spilled, &error), + error.c_str()); + Check(spilled.program.bindings.push_constant_offset == 124 && + spilled.program.bindings.push_constant_size == 0 && + ShaderRecompiler::IR::FindBinding(spilled.program.bindings, + BindingKind::UserData) != nullptr, + "pixel data that crossed the graphics push bank did not spill"); + CheckSpirvBinaryValidates(spilled.spirv); } void TestNewShaderRecompilerUnsupportedMemoryDecode() { @@ -11478,7 +11626,8 @@ int main() { TestGraphicsCreateInterpolantMapping(); TestNewShaderRecompilerPixelPipelineEntry(); TestComputeLdsAllocationIdentity(); - TestPixelProgramCacheDescriptorSetIdentity(); + TestPixelProgramCacheBindingIdentity(); + TestGraphicsPushConstantPlacement(); TestNewShaderRecompilerUnsupportedMemoryDecode(); return 0;