From 27dc7c281ff081ff3568ad47c98b141804c49860 Mon Sep 17 00:00:00 2001 From: nmzik Date: Thu, 22 Oct 2026 22:15:56 +0200 Subject: [PATCH] graphics: keep bounded image heaps resident in a bindless descriptor set --- src/graphics/guest_gpu/graphicsRun.cpp | 2 +- src/graphics/host_gpu/graphicContext.h | 1 + .../host_gpu/renderer/cache/textureCache.cpp | 11 +- .../host_gpu/renderer/cache/textureCache.h | 1 + .../host_gpu/renderer/image/image.cpp | 7 + src/graphics/host_gpu/renderer/image/image.h | 23 +- .../renderer/pipeline/descriptors.cpp | 19 +- .../host_gpu/renderer/pipeline/imageHeap.cpp | 508 ++++++++++++++++++ .../host_gpu/renderer/pipeline/imageHeap.h | 132 +++++ .../renderer/pipeline/pipelineCache.cpp | 11 +- .../renderer/pipeline/pipelineCache.h | 7 +- .../host_gpu/renderer/pipeline/shaders.cpp | 13 +- src/graphics/host_gpu/renderer/render.h | 4 +- .../host_gpu/renderer/renderCompute.cpp | 8 + .../host_gpu/renderer/renderContext.cpp | 8 +- .../host_gpu/renderer/renderContext.h | 3 + src/graphics/host_gpu/renderer/renderDraw.cpp | 1 + .../presentation/window/vulkanWindow.cpp | 11 + .../recompiler/backend/spirv/SpirvEmitter.cpp | 1 + .../backend/spirv/spirvEmitterAnalysis.cpp | 30 +- .../backend/spirv/spirvEmitterImage.cpp | 36 +- .../backend/spirv/spirvEmitterInternal.h | 3 + .../spirv/spirvEmitterMemoryHelpers.cpp | 51 +- .../backend/spirv/spirvEmitterModule.cpp | 41 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 7 + .../ir/passes/ResourceMaterialization.cpp | 71 ++- .../ir/passes/ResourceMaterialization.h | 8 + .../ir/passes/ShaderInfoCollection.cpp | 1 + .../shader/recompiler/ir/passes/SrtWalker.h | 28 + tests/ResourceTrackingTests.cpp | 124 +++++ tests/ShaderRecompilerComputeTests.cpp | 281 +++++++++- 31 files changed, 1377 insertions(+), 75 deletions(-) create mode 100644 src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp create mode 100644 src/graphics/host_gpu/renderer/pipeline/imageHeap.h diff --git a/src/graphics/guest_gpu/graphicsRun.cpp b/src/graphics/guest_gpu/graphicsRun.cpp index 36522bbff..1cc557ef2 100644 --- a/src/graphics/guest_gpu/graphicsRun.cpp +++ b/src/graphics/guest_gpu/graphicsRun.cpp @@ -1008,7 +1008,7 @@ void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) { } void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) { - // The SDK stalls only this command processor, not the host command lane or other queues. + // Stall only this command processor, not the host command lane or other queues. if (!m_renderer.GetVideoOut().IsSafeForRendering(static_cast(video_out_handle), static_cast(display_buffer_index))) { SuspendPm4(); diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index 9f999c188..52e085393 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -43,6 +43,7 @@ struct GraphicContext { uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; uint32_t max_push_descriptors = 0; + vk::DescriptorSetLayout image_heap_layout = nullptr; vk::ShaderStageFlags required_subgroup_size_stages = {}; Common::Mutex queue_mutex; uint32_t queue_family = static_cast(-1); diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 3d75ac511..80b42d15c 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -229,6 +229,7 @@ void TextureCache::UnregisterImage(ImageId id) { if (!image.registered) { return; } + image.HeapChanged(); UntrackImage(id); ImagePageTable::PageRange pages {}; if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) { @@ -378,6 +379,7 @@ void TextureCache::UntrackImageHead(ImageId id) { const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE); const auto size = address - begin; image.track_addr = address; + image.HeapChanged(); if (image.track_addr == image.track_addr_end) { MarkAsMaybeDirty(id, image); } @@ -395,6 +397,7 @@ void TextureCache::UntrackImageTail(ImageId id) { const auto address = Common::AlignDown(end, TRACKER_PAGE_SIZE); const auto size = end - address; image.track_addr_end = address; + image.HeapChanged(); if (image.track_addr == image.track_addr_end) { MarkAsMaybeDirty(id, image); } @@ -795,7 +798,8 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques auto& cached = *owner; const auto current_tick = m_scheduler.CurrentTick(); const bool safe_to_delete = - current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval; + current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval && + cached.heap_refs.load(std::memory_order_relaxed) == 0u; const uint32_t requested_block = requested.bytes_per_block * requested.samples; const uint32_t cached_block = cached.info.bytes_per_block * cached.info.samples; @@ -2015,6 +2019,11 @@ void TextureCache::RunGarbageCollector() { if (owner == nullptr || !owner->registered || owner->depth_id) { continue; } + if (owner->heap_refs.load(std::memory_order_relaxed) != 0u) { + // Resident heaps sample it without touching it. + TouchImage(*owner); + continue; + } if (owner->IsGpuModified()) { const bool safe = owner->SafeToDownload(); if (safe && owner->info.IsTiled()) { diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 300cf436e..9050421e2 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -191,6 +191,7 @@ private: friend struct TextureCacheTestAccess; friend class BufferCache; friend class RenderExecutor; + friend class ImageHeap; friend bool BcAlias::Matches(const ImageInfo&, const ImageInfo&); friend bool BcAlias::Publish(TextureCache&, Image&); friend void BcAlias::Resolve(TextureCache&, ImageId, ImageId); diff --git a/src/graphics/host_gpu/renderer/image/image.cpp b/src/graphics/host_gpu/renderer/image/image.cpp index 240fe0f14..fc0cebbc8 100644 --- a/src/graphics/host_gpu/renderer/image/image.cpp +++ b/src/graphics/host_gpu/renderer/image/image.cpp @@ -212,6 +212,13 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destinat barriers.push_back(barrier); } + // Resident heaps sample their images in the read state; only leaving it invalidates them. + if (std::ranges::any_of(barriers, [&](const vk::ImageMemoryBarrier2& barrier) { + return barrier.oldLayout == ReadLayout() && + barrier.srcAccessMask == vk::AccessFlagBits2::eShaderRead; + })) { + HeapChanged(); + } state = {destination_stage, destination_access, destination_layout}; return barriers; } diff --git a/src/graphics/host_gpu/renderer/image/image.h b/src/graphics/host_gpu/renderer/image/image.h index e46966203..b3c5ae3d3 100644 --- a/src/graphics/host_gpu/renderer/image/image.h +++ b/src/graphics/host_gpu/renderer/image/image.h @@ -7,6 +7,7 @@ #include "graphics/host_gpu/graphicContext.h" #include "graphics/host_gpu/renderer/image/imageInfo.h" +#include #include #include #include @@ -71,6 +72,7 @@ public: void MarkCpuDirty() { m_cpu_dirty = true; m_maybe_cpu_dirty = false; + HeapChanged(); } [[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; } @@ -80,6 +82,7 @@ public: if (!IsCpuDirty()) { m_maybe_cpu_hash = HashGuestEdges(); m_maybe_cpu_dirty = true; + HeapChanged(); } } [[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) { @@ -104,7 +107,10 @@ public: void ClearGpuModified() noexcept { m_gpu_modified = false; } [[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; } - void MarkBufferModified() noexcept { m_buffer_modified = true; } + void MarkBufferModified() noexcept { + m_buffer_modified = true; + HeapChanged(); + } void ClearBufferModified() noexcept { m_buffer_modified = false; } [[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, @@ -120,6 +126,19 @@ public: return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024); } [[nodiscard]] uint64_t HashGuestEdges() const; + [[nodiscard]] vk::ImageLayout ReadLayout() const noexcept { + return info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal + : vk::ImageLayout::eShaderReadOnlyOptimal; + } + // Resident image heaps sample an image without rediscovering it until one changes its + // contents, tracking, identity or layout. + void HeapChanged() const noexcept { + if (heap_refs.load(std::memory_order_relaxed) != 0u) { + heap_epoch.fetch_add(1u, std::memory_order_relaxed); + } + } + + static inline std::atomic heap_epoch {0}; ImageInfo info; VulkanImage backing; @@ -137,6 +156,8 @@ public: ImageSubresourceRange stencil_subresources; uint64_t tick_accessed_last = 0; size_t lru_id = 0; + // Resident image heap elements that select this image. + std::atomic heap_refs {0}; private: friend struct ImageTestAccess; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index fa91258d5..cde93c6ef 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -690,6 +690,10 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime, prepared.samplers.clear(); prepared.shader_data.clear(); for (uint32_t i = 0; i < program.info.images.size(); i++) { + if (program.info.images[i].heap) { + prepared.images[i] = {}; + continue; + } auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]); BindImage(binding.image_id, binding.desc.type == TextureCache::BindingType::Storage); binding.mip_views.swap(prepared.images[i].mip_views); @@ -852,14 +856,19 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) { void RenderExecutor::PrepareGraphicsBindings(std::span stages, std::span colors) { - bool uses_dma = false; + bool uses_dma = false; + bool uses_image_heap = false; FindBuffers(stages); for (auto* stage: stages) { uses_dma |= stage->runtime->program->info.uses_dma; + uses_image_heap |= stage->runtime->program->info.uses_image_heap; } if (uses_dma) { m_context.PrepareBda(); } + if (uses_image_heap) { + m_context.GetImageHeap().Rebind(*this); + } for (auto* stage: stages) { RebindImages(*stage); } @@ -959,6 +968,9 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, } } } + if (pipeline.uses_image_heap) { + m_context.GetImageHeap().Commit(buffer, pipeline_bind_point, pipeline.pipeline_layout); + } m_descriptor_images.resize(image_descriptor_count); m_descriptor_writes.clear(); m_descriptor_writes.reserve(write_count); @@ -1016,9 +1028,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite, range, vk_buffer); } else { - image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal - : vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, range, vk_buffer); + image.Transit(image.ReadLayout(), vk::AccessFlagBits2::eShaderRead, range, vk_buffer); } binding.layout = image.backing.state.layout; } @@ -1089,6 +1099,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, for (uint32_t i = 0; i < program.info.images.size(); ++i) { const auto& resource = program.info.images[i]; const auto& texture = descriptors.images[i]; + if (resource.heap) continue; EXIT_IF(texture.image_id && resource.mip_count != (texture.mip_views.empty() ? 1u : texture.mip_views.size())); const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(resource); diff --git a/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp b/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp new file mode 100644 index 000000000..b89c09b17 --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipeline/imageHeap.cpp @@ -0,0 +1,508 @@ +#include "graphics/host_gpu/renderer/pipeline/imageHeap.h" + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/profiler.h" +#include "graphics/host_gpu/pageManager.h" +#include "graphics/host_gpu/regionDefinitions.h" +#include "graphics/host_gpu/renderer/cache/streamBuffer.h" +#include "graphics/host_gpu/renderer/render.h" +#include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "kernel/memory.h" + +#include +#include + +namespace Libs::Graphics { +namespace { + +namespace IR = ShaderRecompiler::IR; + +// Smaller tables are cheap to materialize on each draw and are often transient. +constexpr uint64_t MinTableBytes = 64ull * 1024ull; +constexpr uint32_t ElementMask = (1u << IR::ImageHeapKindShift) - 1u; + +template +void ForEachRun(size_t begin, size_t end, Pred&& pred, Fn&& fn) { + while (begin < end) { + if (!pred(begin)) { + ++begin; + continue; + } + auto run = begin + 1u; + while (run < end && pred(run)) ++run; + fn(begin, run); + begin = run; + } +} + +// Grows [begin, end) to cover [first, last). +void Widen(size_t& begin, size_t& end, size_t first, size_t last) { + const bool empty = begin == end; + begin = empty ? first : std::min(begin, first); + end = empty ? last : std::max(end, last); +} + +// Guest bytes of a table that its tracker pages [begin, end) cover. +std::pair PageRange(uint64_t base, uint64_t size, size_t begin, size_t end) { + const auto origin = Common::AlignDown(base, TRACKER_PAGE_SIZE); + return {std::max(base, origin + begin * TRACKER_PAGE_SIZE), + std::min(base + size, origin + end * TRACKER_PAGE_SIZE)}; +} + +} // namespace + +size_t ImageHeap::DescriptorHash::operator()(const DescriptorValue& value) const noexcept { + return XXH3_64bits(value.dwords.data(), sizeof(value.dwords)); +} + +ImageHeap::ImageHeap(RenderContext& context, PageManager& page_manager) + : m_context(context), m_page_manager(page_manager) { + auto& graphics = context.GetGraphics(); + vk::PhysicalDeviceVulkan12Properties limits {}; + vk::PhysicalDeviceProperties2 properties {}; + properties.pNext = &limits; + graphics.physical_device.getProperties2(&properties); + EXIT_NOT_IMPLEMENTED(std::min({limits.maxDescriptorSetUpdateAfterBindSampledImages, + limits.maxPerStageDescriptorUpdateAfterBindSampledImages, + limits.maxPerStageUpdateAfterBindResources}) < + IR::ImageHeapCapacity); + + const vk::DescriptorBindingFlags binding_flags = + vk::DescriptorBindingFlagBits::ePartiallyBound | + vk::DescriptorBindingFlagBits::eUpdateAfterBind | + vk::DescriptorBindingFlagBits::eUpdateUnusedWhilePending; + vk::DescriptorSetLayoutBindingFlagsCreateInfo flags_info {}; + flags_info.bindingCount = 1; + flags_info.pBindingFlags = &binding_flags; + const vk::DescriptorSetLayoutBinding binding {0, vk::DescriptorType::eSampledImage, + IR::ImageHeapCapacity, + vk::ShaderStageFlagBits::eAll, nullptr}; + vk::DescriptorSetLayoutCreateInfo layout_info {}; + layout_info.pNext = &flags_info; + layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::eUpdateAfterBindPool; + layout_info.bindingCount = 1; + layout_info.pBindings = &binding; + EXIT_IF(graphics.device.createDescriptorSetLayout(&layout_info, nullptr, &m_layout) != + vk::Result::eSuccess); + const vk::DescriptorPoolSize pool_size {vk::DescriptorType::eSampledImage, + IR::ImageHeapCapacity}; + vk::DescriptorPoolCreateInfo pool_info {}; + pool_info.flags = vk::DescriptorPoolCreateFlagBits::eUpdateAfterBind; + pool_info.maxSets = 1; + pool_info.poolSizeCount = 1; + pool_info.pPoolSizes = &pool_size; + EXIT_IF(graphics.device.createDescriptorPool(&pool_info, nullptr, &m_pool) != + vk::Result::eSuccess); + vk::DescriptorSetAllocateInfo allocate {}; + allocate.descriptorPool = m_pool; + allocate.descriptorSetCount = 1; + allocate.pSetLayouts = &m_layout; + EXIT_IF(graphics.device.allocateDescriptorSets(&allocate, &m_set) != vk::Result::eSuccess); + graphics.image_heap_layout = m_layout; + // Slot word 0 selects element 0, which stays a null image. + m_elements.push_back({.binding = {.layout = vk::ImageLayout::eShaderReadOnlyOptimal}}); + m_writes.push_back(0u); + WriteDescriptors(); +} + +ImageHeap::~ImageHeap() { + for (const auto& memory: m_memories) Unwatch(*memory); + auto& graphics = m_context.GetGraphics(); + graphics.image_heap_layout = nullptr; + graphics.device.destroyDescriptorPool(m_pool, nullptr); + graphics.device.destroyDescriptorSetLayout(m_layout, nullptr); +} + +bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) { + KYTY_PROFILER_FUNCTION(); + const auto size = request.size & ~uint64_t {3}; + if (size < MinTableBytes || request.count == 0u || request.step == 0u || + ((request.base | request.first | request.step) & 3u) != 0u) { + return false; + } + auto memory = std::ranges::find_if(m_memories, [&](const auto& candidate) { + return candidate->base == request.base && candidate->size == size; + }); + if (memory == m_memories.end()) { + if (!m_context.IsMapped(request.base, size)) return false; + auto created = std::make_unique(); + created->base = request.base; + created->size = size; + created->words.resize(size / 4u); + created->dirty_pages.assign((Common::AlignUp(request.base + size, TRACKER_PAGE_SIZE) - + Common::AlignDown(request.base, TRACKER_PAGE_SIZE)) / + TRACKER_PAGE_SIZE, + uint8_t {1}); + std::scoped_lock lock {m_mutex}; + memory = m_memories.insert(m_memories.end(), std::move(created)); + } + // A draw refreshes a table and a view takes its changes once, so all programs of the draw see + // one snapshot and one set of kinds. + if ((*memory)->draw != m_draw) { + if (!Refresh(**memory)) return false; + (*memory)->draw = m_draw; + } + auto& views = (*memory)->views; + const auto profile = IR::ImageHeapResource(*request.root, 0u); + auto view = std::ranges::find_if(views, [&](const auto& candidate) { + return candidate->step == request.step && candidate->first == request.first && + candidate->count == request.count && candidate->profile == profile; + }); + if (view == views.end()) { + const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed); + auto created = std::make_unique(View {.memory = **memory, + .step = request.step, + .first = request.first, + .count = request.count, + .profile = profile, + .resolve_epoch = epoch, + .layout_epoch = epoch}); + created->slots.resize(size / 4u + 1u); + created->buffer = std::make_unique( + m_context.GetGraphics(), m_context.GetCommandScheduler(), MemoryUsage::DeviceLocal, 0, + vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer | + vk::BufferUsageFlagBits::eShaderDeviceAddress, + size + 4u); + UpdateProbes(*created, 0u, size); + // The first upload also clears the slots no probe selects. + created->dirty_begin = 0; + created->dirty_end = created->slots.size(); + view = views.insert(views.end(), std::move(created)); + } else { + // Empty after the first acquire in a draw, since the table refreshes once per draw. + UpdateProbes(**view, (*view)->changed_begin, (*view)->changed_end); + (*view)->changed_begin = (*view)->changed_end = 0; + } + if (std::ranges::find(m_active, view->get()) == m_active.end()) m_active.push_back(view->get()); + const auto& selected = **view; + result = {.slots = selected.buffer->BufferDeviceAddress(), + .slot_count = static_cast(selected.slots.size() - 1u), + .kinds = selected.kinds}; + return true; +} + +bool ImageHeap::Refresh(Memory& memory) { + // GPU writes leave no fault; reading the table downloads them. + const bool gpu_dirty = m_context.GetBufferCache().HasGpuDirtyBytes(memory.base, memory.size); + const auto pages = memory.dirty_pages.size(); + m_runs.clear(); + { + std::scoped_lock lock {m_mutex}; + // Watch before reading, so a write during the read faults the page again. + ForEachRun( + 0, pages, [&](size_t page) { return memory.dirty_pages[page] != 0u; }, + [&](size_t begin, size_t end) { + std::fill(memory.dirty_pages.begin() + begin, memory.dirty_pages.begin() + end, + uint8_t {0}); + Watch(memory, begin, end); + m_runs.emplace_back(begin, end); + }); + } + if (gpu_dirty) m_runs.assign(1u, {0u, pages}); + for (size_t run = 0; run < m_runs.size(); ++run) { + const auto [address, end] = PageRange(memory.base, memory.size, m_runs[run].first, + m_runs[run].second); + m_scratch.resize((end - address) / 4u); + if (!LibKernel::Memory::TryReadBufferBacking(address, m_scratch.data(), end - address)) { + std::scoped_lock lock {m_mutex}; + for (; run < m_runs.size(); ++run) { + MarkDirty(memory, m_runs[run].first, m_runs[run].second); + } + return false; + } + const auto words = std::span(memory.words).subspan((address - memory.base) / 4u, + m_scratch.size()); + size_t first = 0; + size_t last = words.size(); + while (first < last && words[first] == m_scratch[first]) ++first; + while (first < last && words[last - 1u] == m_scratch[last - 1u]) --last; + if (first == last) continue; + std::copy(m_scratch.begin() + first, m_scratch.begin() + last, words.begin() + first); + const auto offset = address - memory.base; + for (auto& view: memory.views) { + Widen(view->changed_begin, view->changed_end, offset + first * 4u, offset + last * 4u); + } + } + return true; +} + +void ImageHeap::MarkDirty(Memory& memory, size_t begin, size_t end) { + ForEachRun( + begin, end, [&](size_t page) { return memory.dirty_pages[page] == 0u; }, + [&](size_t first, size_t last) { + std::fill(memory.dirty_pages.begin() + first, memory.dirty_pages.begin() + last, + uint8_t {1}); + Watch(memory, first, last); + }); +} + +template +void ImageHeap::Watch(const Memory& memory, size_t begin, size_t end) { + const auto [address, limit] = PageRange(memory.base, memory.size, begin, end); + m_page_manager.UpdatePageWatchers(address, limit - address); +} + +void ImageHeap::Unwatch(const Memory& memory) { + ForEachRun( + 0, memory.dirty_pages.size(), [&](size_t page) { return memory.dirty_pages[page] == 0u; }, + [&](size_t begin, size_t end) { Watch(memory, begin, end); }); +} + +void ImageHeap::InvalidateMemory(uint64_t address, uint64_t size) { + std::scoped_lock lock {m_mutex}; + for (auto& memory: m_memories) { + const auto begin = std::max(address, memory->base); + const auto end = std::min(address + size, memory->base + memory->size); + if (begin >= end) continue; + const auto origin = Common::AlignDown(memory->base, TRACKER_PAGE_SIZE); + MarkDirty(*memory, (begin - origin) / TRACKER_PAGE_SIZE, + (end - 1u - origin) / TRACKER_PAGE_SIZE + 1u); + } +} + +void ImageHeap::UpdateProbes(View& view, uint64_t begin, uint64_t end) { + // A probe reads its descriptor from first + probe * step. + const uint64_t bytes = view.profile.r128 ? 16u : 32u; + auto probe = begin < view.first + bytes ? 0u : (begin - view.first - bytes) / view.step + 1u; + for (; probe < view.count && view.first + probe * view.step < end; ++probe) { + UpdateProbe(view, probe); + } +} + +void ImageHeap::UpdateProbe(View& view, uint64_t probe) { + const auto& words = view.memory.words; + const auto offset = (view.first + probe * view.step) / 4u; + DescriptorValue value {.dword_count = 8u}; + std::copy_n(words.begin() + offset, + std::min(view.profile.r128 ? 4u : 8u, words.size() - offset), + value.dwords.begin()); + const auto kind = IR::ImageHeapKind(view.profile, value); + Assign(view, offset, kind == 0u ? 0u : Intern(view, kind, value)); +} + +void ImageHeap::Assign(View& view, size_t slot, uint32_t entry) { + const auto previous = std::exchange(view.slots[slot], entry); + Release(view, previous); + if (previous != entry) Widen(view.dirty_begin, view.dirty_end, slot, slot + 1u); +} + +uint32_t ImageHeap::Intern(View& view, uint32_t kind, const DescriptorValue& value) { + const auto arm = static_cast(std::ranges::find(view.kind_keys, kind) - + view.kind_keys.begin()); + if (arm == view.kind_keys.size()) { + view.kind_keys.push_back(kind); + view.kinds.push_back(value); + } + auto [found, inserted] = view.elements.try_emplace(value, 0u); + if (inserted) { + found->second = Allocate(); + auto& element = m_elements[found->second]; + element.value = value; + element.kind = kind; + view.pending.push_back(found->second); + } + ++m_elements[found->second].refs; + return arm << IR::ImageHeapKindShift | found->second; +} + +void ImageHeap::Release(View& view, uint32_t entry) { + const auto index = entry & ElementMask; + if (index == 0u || --m_elements[index].refs != 0u) return; + auto& element = m_elements[index]; + if (const auto found = view.elements.find(element.value); + found != view.elements.end() && found->second == index) { + view.elements.erase(found); + } + Unreference(element); + view.released.push_back(index); +} + +void ImageHeap::Unreference(Element& element) { + if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(element.binding.image_id)) { + image->heap_refs.fetch_sub(1u, std::memory_order_relaxed); + image->tick_accessed_last = m_context.GetCommandScheduler().CurrentTick(); + } +} + +uint32_t ImageHeap::Allocate() { + auto& scheduler = m_context.GetCommandScheduler(); + while (!m_retired.empty() && scheduler.IsFree(m_retired.front().first)) { + m_free.push_back(m_retired.front().second); + m_retired.pop_front(); + } + if (m_free.empty()) { + if (m_elements.size() == IR::ImageHeapCapacity) { + EXIT("image heaps exceed %u resident images\n", IR::ImageHeapCapacity); + } + m_elements.emplace_back(); + return static_cast(m_elements.size() - 1u); + } + const auto index = m_free.back(); + m_free.pop_back(); + m_elements[index] = {}; + return index; +} + +void ImageHeap::Retire(View& view) { + const auto tick = m_context.GetCommandScheduler().CurrentTick(); + for (const auto index: view.released) m_retired.emplace_back(tick, index); + view.released.clear(); +} + +void ImageHeap::Rebind(RenderExecutor& executor) { + KYTY_PROFILER_FUNCTION(); + auto& textures = m_context.GetTextureCache(); + const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed); + for (auto* view: m_active) { + bool stale = false; + const auto mark = [&](uint32_t index) { m_elements[index].stale = stale = true; }; + if (view->resolve_epoch != epoch) { + view->resolve_epoch = epoch; + for (const auto& [value, index]: view->elements) { + const auto& binding = m_elements[index].binding; + if (!binding.image_id) continue; + const auto* image = textures.m_slot_images.try_get(binding.image_id); + if (image == nullptr || !image->registered || image->depth_id || + image->binding.needs_rebind) { + mark(index); + } else { + (void)textures.FindTexture(binding.image_id, binding.desc); + } + } + } + // Discovery applies the color metadata clears written since the last draw. + std::erase_if(view->dcc, [&](uint32_t index) { + const auto& element = m_elements[index]; + if (element.refs == 0u || element.stale) return true; + if (executor.ResolveTexture(IR::ImageHeapResource(view->profile, element.kind), + element.value).image_id == element.binding.image_id) { + return false; + } + mark(index); + return true; + }); + if (stale) { + // A changed image needs a new element; pending draws still read the old one. The new + // element keeps the old value, since a view takes table changes only when acquired. + std::erase_if(view->elements, + [&](const auto& item) { return m_elements[item.second].stale; }); + for (uint64_t probe = 0; probe < view->count; ++probe) { + const auto slot = (view->first + probe * view->step) / 4u; + const auto& element = m_elements[view->slots[slot] & ElementMask]; + if (!element.stale) continue; + const auto value = element.value; // Intern may grow m_elements. + Assign(*view, slot, Intern(*view, element.kind, value)); + } + } + for (const auto index: view->pending) { + if (m_elements[index].refs == 0u) continue; + auto binding = executor.ResolveTexture( + IR::ImageHeapResource(view->profile, m_elements[index].kind), + m_elements[index].value); + if (binding.image_id) { + auto& image = textures.GetImage(binding.image_id); + image.heap_refs.fetch_add(1u, std::memory_order_relaxed); + binding.layout = image.ReadLayout(); + binding.image_view = textures.FindTexture(binding.image_id, binding.desc); + if (binding.desc.info.metadata.kind == ImageMetadataKind::Dcc) { + view->dcc.push_back(index); + } + view->away.push_back(index); + } + m_elements[index].binding = std::move(binding); + m_writes.push_back(index); + } + view->pending.clear(); + } + WriteDescriptors(); +} + +void ImageHeap::WriteDescriptors() { + if (m_writes.empty()) return; + std::vector infos; + std::vector writes(m_writes.size()); + infos.reserve(m_writes.size()); + for (size_t write = 0; write < m_writes.size(); ++write) { + const auto& binding = m_elements[m_writes[write]].binding; + infos.push_back({nullptr, binding.image_view, binding.layout}); + writes[write].dstSet = m_set; + writes[write].dstArrayElement = m_writes[write]; + writes[write].descriptorCount = 1; + writes[write].descriptorType = vk::DescriptorType::eSampledImage; + writes[write].pImageInfo = &infos.back(); + } + m_context.GetGraphics().device.updateDescriptorSets(static_cast(writes.size()), + writes.data(), 0, nullptr); + m_writes.clear(); +} + +void ImageHeap::Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point, + vk::PipelineLayout layout) { + KYTY_PROFILER_FUNCTION(); + auto& textures = m_context.GetTextureCache(); + const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed); + for (auto* view: m_active) { + if (view->dirty_begin != view->dirty_end) { + const auto slots = std::span(view->slots).subspan(view->dirty_begin, + view->dirty_end - view->dirty_begin); + auto& stream = m_context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream); + const auto offset = stream.Copy(slots.data(), slots.size_bytes(), sizeof(uint32_t)); + view->buffer->CopyFrom(buffer, stream, offset, view->dirty_begin * sizeof(uint32_t), + slots.size_bytes()); + view->dirty_begin = view->dirty_end = 0; + } + Retire(*view); + if (view->layout_epoch != epoch) { + view->layout_epoch = epoch; + view->away.clear(); + for (const auto& [value, index]: view->elements) view->away.push_back(index); + } + // This draw's own bindings own the layouts of its attachments and storage images. An image + // this draw's discovery replaced lives until the tick ends, and its slots still select it. + std::erase_if(view->away, [&](uint32_t index) { + const auto& element = m_elements[index]; + auto* image = textures.m_slot_images.try_get(element.binding.image_id); + if (element.refs == 0u || image == nullptr) return true; + if (image->binding.is_target || image->binding.shader_write || + image->binding.force_general) { + return false; + } + const auto& view_info = element.binding.desc.view_info; + image->Transit(image->ReadLayout(), vk::AccessFlagBits2::eShaderRead, + ImageSubresourceRange {view_info.base_level, view_info.level_count, + view_info.base_layer, view_info.layer_count}, + buffer.Handle()); + return true; + }); + } + m_active.clear(); + buffer.Handle().bindDescriptorSets(bind_point, layout, IR::ImageHeapSet, 1, &m_set, 0, + nullptr); +} + +void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) { + auto& scheduler = m_context.GetCommandScheduler(); + std::scoped_lock lock {m_mutex}; + std::erase_if(m_memories, [&](const std::unique_ptr& memory) { + if (address >= memory->base + memory->size || memory->base >= address + size) { + return false; + } + Unwatch(*memory); + for (auto& view: memory->views) { + std::erase(m_active, view.get()); + for (const auto& [value, index]: view->elements) { + Unreference(m_elements[index]); + view->released.push_back(index); + } + Retire(*view); + if (scheduler.Active()) { + scheduler.DeferOperation([buffer = std::move(view->buffer)] {}); + } + } + return true; + }); +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/imageHeap.h b/src/graphics/host_gpu/renderer/pipeline/imageHeap.h new file mode 100644 index 000000000..aa5b3fb63 --- /dev/null +++ b/src/graphics/host_gpu/renderer/pipeline/imageHeap.h @@ -0,0 +1,132 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_ + +#include "common/common.h" +#include "graphics/host_gpu/renderer/pipeline/descriptors.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics { + +class Buffer; +class CommandBuffer; +class PageManager; +class RenderContext; +class RenderExecutor; +struct ImageHeapTestAccess; + +// Guest image tables that shaders index by GPU-computed offsets, kept resident in one bindless +// sampled-image array. Each view of a table owns device slot words, one per table dword, that +// select an array element. CPU writes to a table fault in its snapshot, which each draw refreshes +// once; a view takes the changes when a draw first acquires it. Changes to the images behind +// elements bump Image::heap_epoch. +class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource { +public: + ImageHeap(RenderContext& context, PageManager& page_manager); + ~ImageHeap(); + KYTY_CLASS_NO_COPY(ImageHeap); + + // Starts a draw or dispatch, before its programs acquire views. + void BeginDraw() { ++m_draw; } + bool Acquire(const ShaderRecompiler::IR::ImageHeapRequest& request, + ShaderRecompiler::IR::ImageHeapView& result) override; + // Image discovery: resolves new elements and rediscovers changed images. + void Rebind(RenderExecutor& executor); + // Recording: uploads slots, moves resident images to their read layout and binds the array. + void Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point, vk::PipelineLayout layout); + // CPU write fault, on any thread. + void InvalidateMemory(uint64_t address, uint64_t size); + void UnmapMemory(uint64_t address, uint64_t size); + +private: + using DescriptorValue = ShaderRecompiler::IR::DescriptorValue; + + struct DescriptorHash { + size_t operator()(const DescriptorValue& value) const noexcept; + }; + + struct Element { + DescriptorValue value; + TextureBinding binding; + uint32_t kind = 0; + uint32_t refs = 0; // Slots of the owning view that select it. + bool stale = false; + }; + + struct Memory; + + struct View { + Memory& memory; + uint64_t step = 0; + uint64_t first = 0; + uint64_t count = 0; + ShaderRecompiler::IR::ImageResource profile; + std::vector slots; // Plus a trailing null slot. + std::vector kind_keys; + std::vector kinds; // One representative per kind_keys entry. + std::unordered_map elements; + std::vector pending; // Not yet resolved. + std::vector released; // No slot selects them after the next upload. + std::vector away; // Not yet moved to the read layout. + std::vector dcc; // Color metadata clears apply on each discovery. + std::unique_ptr buffer; + size_t dirty_begin = 0; // Slots to upload. + size_t dirty_end = 0; + size_t changed_begin = 0; // Table bytes changed since it was probed. + size_t changed_end = 0; + uint64_t resolve_epoch = 0; + uint64_t layout_epoch = 0; + }; + + struct Memory { + uint64_t base = 0; + uint64_t size = 0; + std::vector words; + std::vector dirty_pages; // Unwatched, so unread since a write. + uint64_t draw = 0; // Last draw that refreshed it. + std::vector> views; + }; + + [[nodiscard]] bool Refresh(Memory& memory); + void MarkDirty(Memory& memory, size_t begin, size_t end); + template + void Watch(const Memory& memory, size_t begin, size_t end); + void Unwatch(const Memory& memory); + void UpdateProbes(View& view, uint64_t begin, uint64_t end); + void UpdateProbe(View& view, uint64_t probe); + void Assign(View& view, size_t slot, uint32_t entry); + uint32_t Intern(View& view, uint32_t kind, const DescriptorValue& value); + void Release(View& view, uint32_t entry); + void Unreference(Element& element); + uint32_t Allocate(); + void Retire(View& view); + void WriteDescriptors(); + + RenderContext& m_context; + PageManager& m_page_manager; + std::mutex m_mutex; // Fault-visible Memory state and m_memories. + std::vector> m_memories; + std::vector m_active; // Acquired since the last commit. + uint64_t m_draw = 1; + std::vector m_elements; + std::vector m_free; + std::deque> m_retired; // Tick, element. + std::vector> m_runs; + std::vector m_scratch; + std::vector m_writes; // Elements whose descriptors changed. + vk::DescriptorSetLayout m_layout = nullptr; + vk::DescriptorPool m_pool = nullptr; + vk::DescriptorSet m_set = nullptr; + + friend struct ImageHeapTestAccess; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_ diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index c1f520642..fd2e331bc 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -353,6 +353,7 @@ struct PipelineCache::ProgramCache { .user_data = user_data, .shader_base = params.Base(), .read_specialization_memory = ReadShaderGuestMemory, + .image_heap = image_heap, }; if constexpr (std::is_same_v) { runtime.workgroup_counts = input_info.workgroup_counts; @@ -480,8 +481,8 @@ struct PipelineCache::ProgramCache { return permutation.handle; } - explicit ProgramCache(GraphicContext& graphics) - : graphics(graphics), + ProgramCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap) + : graphics(graphics), image_heap(image_heap), // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and // signed-zero results; GLSL.std.450 alone does not order -0 below +0. profile {.maintenance8 = graphics.maintenance8_enabled, @@ -501,12 +502,14 @@ struct PipelineCache::ProgramCache { std::unordered_map programs; ProgramKey lookup_key; GraphicContext& graphics; + ShaderRecompiler::IR::ImageHeapSource* image_heap; const ShaderRecompiler::Spirv::Profile profile; uint64_t next_shader_id = 0; }; -PipelineCache::PipelineCache(GraphicContext& graphics) - : m_graphics(graphics), m_program_cache(std::make_unique(graphics)), +PipelineCache::PipelineCache(GraphicContext& graphics, + ShaderRecompiler::IR::ImageHeapSource* image_heap) + : m_graphics(graphics), m_program_cache(std::make_unique(graphics, image_heap)), m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); InitializeDriverCache(); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index 1f8e217f9..f49f88200 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -30,6 +30,10 @@ class UserConfig; struct ComputeShaderInfo; } // namespace HW +namespace ShaderRecompiler::IR { +class ImageHeapSource; +} // namespace ShaderRecompiler::IR + #pragma pack(push, 1) struct PipelineStaticParameters { @@ -104,7 +108,7 @@ struct ShaderProgram { // The owning renderer serializes access, including saves while the GPU is running. class PipelineCache { public: - explicit PipelineCache(GraphicContext& graphics); + PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap); ~PipelineCache(); KYTY_CLASS_NO_COPY(PipelineCache); void Save(); @@ -114,6 +118,7 @@ public: vk::Pipeline pipeline = nullptr; vk::DescriptorSetLayout descriptor_set_layout = nullptr; bool uses_push_descriptors = false; + bool uses_image_heap = false; }; struct GraphicsPrograms { diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 0f413713f..9a2863bc4 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -473,20 +473,23 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p for (const auto& stage: vertex_info) { const auto native_stage = NativeShaderStage(stage.logical_stage); AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage); + pipeline.uses_image_heap |= stage.stage.program->info.uses_image_heap; graphics_stages |= native_stage; } if (ps_active) { EXIT_IF(!ps_input_info->stage); AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program, vk::ShaderStageFlagBits::eFragment); + pipeline.uses_image_heap |= ps_input_info->stage.program->info.uses_image_heap; } CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); const vk::PushConstantRange push_constants {graphics_stages, 0, ShaderRecompiler::IR::NativePushConstantSize}; + const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout}; vk::PipelineLayoutCreateInfo pipeline_layout_info {}; - pipeline_layout_info.setLayoutCount = 1; - pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout; + pipeline_layout_info.setLayoutCount = pipeline.uses_image_heap ? 2u : 1u; + pipeline_layout_info.pSetLayouts = set_layouts.data(); pipeline_layout_info.pushConstantRangeCount = 1; pipeline_layout_info.pPushConstantRanges = &push_constants; @@ -615,13 +618,15 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p std::vector descriptor_bindings; AddLayoutBindings(descriptor_bindings, *input_info.stage.program, vk::ShaderStageFlagBits::eCompute); + pipeline.uses_image_heap = input_info.stage.program->info.uses_image_heap; CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0, ShaderRecompiler::IR::NativePushConstantSize}; + const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout}; vk::PipelineLayoutCreateInfo pipeline_layout_info {}; - pipeline_layout_info.setLayoutCount = 1; - pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout; + pipeline_layout_info.setLayoutCount = pipeline.uses_image_heap ? 2u : 1u; + pipeline_layout_info.pSetLayouts = set_layouts.data(); pipeline_layout_info.pushConstantRangeCount = 1; pipeline_layout_info.pPushConstantRanges = &push_constants; diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index f7d58a979..9a43337d4 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -171,6 +171,8 @@ public: void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point, const PipelineCache::Pipeline& pipeline, std::span bindings); + [[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, + const ShaderRecompiler::IR::DescriptorValue& value); private: // False, with nothing drawn, when the draw's indirect records need CPU counts. @@ -182,8 +184,6 @@ private: std::optional pixel; }; - [[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, - const ShaderRecompiler::IR::DescriptorValue& value); void PrepareGraphicsBindings(std::span stages, std::span colors); void ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& target, diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index adbda3494..93e225149 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -311,6 +311,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, input_info.workgroup_counts[axis] = threads / size + (threads % size != 0u); } } + m_context.GetImageHeap().BeginDraw(); const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info); if (!compute_program) return; @@ -363,6 +364,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, if (program.info.uses_dma) { m_context.PrepareBda(); } + if (program.info.uses_image_heap) { + m_context.GetImageHeap().Rebind(*this); + } RebindImages(bindings); BindSharedMemory(m_context, input_info, bindings); RebindBuffers(bindings); @@ -497,6 +501,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, ShaderComputeInputInfo input_info {}; input_info.async_compute = async_compute; input_info.indirect_args_address = args_addr; + m_context.GetImageHeap().BeginDraw(); const auto compute_program = m_context.GetPipelineCache().GetComputeProgram( cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info); if (!compute_program) return; @@ -511,6 +516,9 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer, m_context.PrepareBda(); } BindSharedMemory(m_context, input_info, bindings, args_addr); + if (program.info.uses_image_heap) { + m_context.GetImageHeap().Rebind(*this); + } RebindImages(bindings); // Acquiring arguments can merge cache buffers; finalize shader bindings afterward. const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer( diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp index e256b8ae4..2cb9e75b0 100644 --- a/src/graphics/host_gpu/renderer/renderContext.cpp +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -13,9 +13,10 @@ namespace Libs::Graphics { RenderContext::RenderContext(GraphicContext& graphics) : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics), m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()), - m_pipeline_cache(graphics), m_sampler_cache(graphics), + m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics), m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache), - m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) { + m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache), + m_image_heap(*this, m_page_manager) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } @@ -64,6 +65,7 @@ bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) no if (access == PageFaultAccess::Write) { m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size); m_texture_cache.InvalidateMemory(fault_vaddr, fault_size); + m_image_heap.InvalidateMemory(fault_vaddr, fault_size); } else { m_buffer_cache.ReadMemory(fault_vaddr, fault_size); } @@ -76,6 +78,7 @@ bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) { } m_buffer_cache.InvalidateMemory(vaddr, size); m_texture_cache.InvalidateMemory(vaddr, size); + m_image_heap.InvalidateMemory(vaddr, size); return true; } @@ -112,6 +115,7 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) { } m_buffer_cache.InvalidateMemory(vaddr, size); m_texture_cache.UnmapMemory(vaddr, size); + m_image_heap.UnmapMemory(vaddr, size); std::lock_guard lock(m_mapped_ranges_mutex); m_mapped_ranges.Subtract(vaddr, size); }; diff --git a/src/graphics/host_gpu/renderer/renderContext.h b/src/graphics/host_gpu/renderer/renderContext.h index e264213f5..665e4516a 100644 --- a/src/graphics/host_gpu/renderer/renderContext.h +++ b/src/graphics/host_gpu/renderer/renderContext.h @@ -12,6 +12,7 @@ #include "graphics/host_gpu/renderer/cache/textureCache.h" #include "graphics/host_gpu/renderer/commandScheduler.h" #include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h" +#include "graphics/host_gpu/renderer/pipeline/imageHeap.h" #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "kernel/eventQueue.h" @@ -43,6 +44,7 @@ public: CommandScheduler& GetCommandScheduler() { return m_command_scheduler; } PipelineCache& GetPipelineCache() { return m_pipeline_cache; } DescriptorHeap& GetDescriptorHeap() { return m_descriptor_heap; } + ImageHeap& GetImageHeap() { return m_image_heap; } SamplerCache& GetSamplerCache() { return m_sampler_cache; } BufferCache& GetBufferCache() { return m_buffer_cache; } TextureCache& GetTextureCache() { return m_texture_cache; } @@ -76,6 +78,7 @@ private: PageManager m_page_manager; BufferCache m_buffer_cache; TextureCache m_texture_cache; + ImageHeap m_image_heap; mutable std::shared_mutex m_mapped_ranges_mutex; RangeSet m_mapped_ranges; std::unique_ptr m_gpu; diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index 8a6f181d7..6f980aee3 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -893,6 +893,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, } } auto& pipeline_cache = buffer.GetContext().GetPipelineCache(); + buffer.GetContext().GetImageHeap().BeginDraw(); if (draw.IsIndexed()) { LogDrawPhase(draw.Name(), "GetGraphicsPrograms"); } diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index f3a6a0ef8..fab1c337b 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -65,6 +65,9 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE; features.shaderStorageImageArrayNonUniformIndexing = VK_TRUE; features.shaderStorageBufferArrayNonUniformIndexing = VK_TRUE; + features.descriptorBindingSampledImageUpdateAfterBind = VK_TRUE; + features.descriptorBindingUpdateUnusedWhilePending = VK_TRUE; + features.descriptorBindingPartiallyBound = VK_TRUE; return features; } @@ -295,6 +298,14 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa check_feature(features12.shaderStorageBufferArrayNonUniformIndexing, "shaderStorageBufferArrayNonUniformIndexing", required_features12.shaderStorageBufferArrayNonUniformIndexing); + check_feature(features12.descriptorBindingSampledImageUpdateAfterBind, + "descriptorBindingSampledImageUpdateAfterBind", + required_features12.descriptorBindingSampledImageUpdateAfterBind); + check_feature(features12.descriptorBindingUpdateUnusedWhilePending, + "descriptorBindingUpdateUnusedWhilePending", + required_features12.descriptorBindingUpdateUnusedWhilePending); + check_feature(features12.descriptorBindingPartiallyBound, "descriptorBindingPartiallyBound", + required_features12.descriptorBindingPartiallyBound); check_feature(features13.robustImageAccess, "robustImageAccess"); check_feature(robustness2.nullDescriptor, "nullDescriptor"); check_feature(features13.dynamicRendering, "dynamicRendering", diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 0f6552727..6f6028a02 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -161,6 +161,7 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) { if (!kind.has_value()) { Fail(program, "native shader plan has an invalid image class"); } + if (image.heap) continue; if (image.mip_count == 0u || (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) { Fail(program, "native shader plan has an invalid image mip descriptor count"); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 74196f266..d47e6b631 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -120,6 +120,10 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) { } else { EXIT("invalid image resource class"); } + if (image.dimension == ImageDimension::Dim1D || image.dimension == ImageDimension::Dim1DArray) { + state.builder.RequireCapability(sampled == 1 ? spv::CapabilitySampled1D + : spv::CapabilityImage1D); + } const auto& info = ImageDimensionInfoFor(image.dimension); if (sampled == 2 && info.multisampled != 0u) { state.builder.RequireCapability(spv::CapabilityStorageImageMultisample); @@ -147,14 +151,20 @@ void DecorateNonUniformDescriptor(EmitterState& state, uint32_t id, spv::Capabil state.builder.RequireCapability(capability); state.builder.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); } + +ImageDefinition& ImageDescriptorDefinition(EmitterState& state, const IR::ImageResource& image, + IR::DescriptorBindingKind kind) { + return (image.heap ? state.heap_images : state.images)[IR::ImageBindingIndex(kind)]; +} } // namespace uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip, uint32_t array_index) { const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index); const auto image = state.builder.AllocateId(); - const auto kind = *IR::DescriptorBindingForImage(state.program.info.images[resource]); - state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type, + const auto& resource_info = state.program.info.images[resource]; + const auto kind = *IR::DescriptorBindingForImage(resource_info); + state.builder.AddFunction(spv::OpLoad, ImageDescriptorDefinition(state, resource_info, kind).type, image, pointer); if (array_index != 0u) { DecorateNonUniformDescriptor(state, image, @@ -187,13 +197,13 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled); const auto image = LoadImageDescriptor(state, resource, mip, array_index); const auto sampled_image = state.builder.AllocateId(); - const auto kind = *IR::DescriptorBindingForImage(image_resource); - auto& sampled_type = state.images[IR::ImageBindingIndex(kind)].sampled_type; - if (sampled_type == 0) { - sampled_type = state.builder.Type( - spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type); + auto& definition = ImageDescriptorDefinition(state, image_resource, + *IR::DescriptorBindingForImage(image_resource)); + if (definition.sampled_type == 0) { + definition.sampled_type = state.builder.Type(spv::OpTypeSampledImage, definition.type); } - state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id); + state.builder.AddFunction(spv::OpSampledImage, definition.sampled_type, sampled_image, image, + sampler_id); if (array_index != 0u || sampler_dynamic) { DecorateNonUniformDescriptor(state, sampled_image, spv::CapabilitySampledImageArrayNonUniformIndexing); } @@ -205,11 +215,11 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t const auto& image = state.program.info.images.at(resource); EXIT_IF(mip >= image.mip_count); const auto kind = IR::DescriptorBindingForImage(image); - EXIT_IF(!kind.has_value()); + EXIT_IF(!kind.has_value() || (image.heap && array_index == 0u)); if (array_index == 0u) { array_index = ConstantU32(state, image.descriptor_index + mip); } - const auto& definition = state.images[IR::ImageBindingIndex(*kind)]; + const auto& definition = ImageDescriptorDefinition(state, image, *kind); return DescriptorElementPointer(state, definition.pointer_type, definition.variable, array_index, *kind, resource, "image descriptor array was not emitted"); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index af6fc72c8..a4d895b8b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -700,6 +700,21 @@ auto EmitImageAccess(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t resul image.indirect_resources.size() < 2u) { ctx.Fail(inst, "has no indirect image runtime mapping"); } + if (image.heap) { + const auto entry = EmitImageHeapEntry(state, key, image.indirect_mapping_offset); + const auto kinds = static_cast(image.indirect_resources.size()) - 1u; + // One kind leaves every slot word a bare element. + if (kinds == 1u) return emit(image.indirect_resources[1], entry); + const auto element = Binary(state, spv::OpBitwiseAnd, TypeU32(state), entry, + ConstantU32(state, (1u << IR::ImageHeapKindShift) - 1u)); + const auto EmitKind = [&](uint32_t kind) { + return emit(image.indirect_resources[kind + 1u], element); + }; + return EmitIndexSwitch(state, + Binary(state, spv::OpShiftRightLogical, TypeU32(state), entry, + ConstantU32(state, IR::ImageHeapKindShift)), + kinds, result_type, EmitKind); + } const auto selected = EmitIndirectResourceIndex( state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u); const auto SameType = [&](uint32_t lhs, uint32_t rhs) { @@ -879,14 +894,19 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { } if (op == IR::ValueOpcode::ImageQueryLod) { state.builder.RequireCapability(spv::CapabilityImageQuery); - const auto dimension = image.dimension; - const auto sampled = MakeSampledImage(state, mem.resource, - LoadSamplerDescriptor(state, mem.sampler)); - const auto lod = state.builder.AllocateId(); - state.builder.AddFunction( - spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled, - CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components, - image.cube)); + const auto sampler = LoadSamplerDescriptor(state, mem.sampler); + const auto lod = EmitImageAccess(ctx, inst, TypeF32Vector(state, 2), + [&](uint32_t resource, uint32_t array_index) { + const auto& candidate = state.program.info.images[resource]; + const auto query = state.builder.AllocateId(); + state.builder.AddFunction( + spv::OpImageQueryLod, TypeF32Vector(state, 2), query, + MakeSampledImage(state, resource, sampler, 0u, array_index), + CoordF32(ctx, mem, *address, 0, + ImageDimensionInfoFor(candidate.dimension).spatial_components, + candidate.cube)); + return query; + }); uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0)}; for (uint32_t index = 0; index < 2u; index++) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 15fd29009..87c069bbf 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -130,6 +130,7 @@ struct EmitterState { uint32_t lds_u64_variable = 0; std::array scratch_variable {}; std::array images {}; + std::array heap_images {}; uint32_t sampler_variable = 0; uint32_t sampler_type = 0; uint32_t sampler_pointer_type = 0; @@ -431,6 +432,8 @@ struct MemoryResourceAccess { uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset, uint32_t search_iterations, uint32_t default_resource); +// Slot word of an image heap byte offset; offsets past the heap select the null element. +uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset); MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, uint32_t array_index = 0u); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index eebb8137b..223ff9eaf 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -38,19 +38,39 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32 return ret; } +static uint32_t LoadFlattenedSrt(EmitterState& state, uint32_t index) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, + state.flattened_srt_variable, ConstantU32(state, 0), index); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; +} + +uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset) { + const auto Mapping = [&](uint32_t word) { + return LoadFlattenedSrt(state, ConstantU32(state, mapping_offset + word)); + }; + // Keys past the table select its trailing null slot. + const auto index = EmitUMin32(state, Binary(state, spv::OpShiftRightLogical, TypeU32(state), key, + ConstantU32(state, 2u)), + Mapping(0u)); + const auto offset = Unary(state, spv::OpUConvert, TypeU64(state), + Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index, + ConstantU32(state, 2u))); + const auto pointer = Unary(state, spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), + Binary(state, spv::OpIAdd, TypeU64(state), + PackU64(state, Mapping(1u), Mapping(2u)), offset)); + const auto entry = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpLoad, TypeU32(state), entry, pointer, + spv::MemoryAccessAlignedMask, 4u); + return entry; +} + uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset, uint32_t search_iterations, uint32_t default_resource) { - const auto LoadMapping = [&](uint32_t index) { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), - pointer, state.flattened_srt_variable, ConstantU32(state, 0), - index); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); - return value; - }; const auto mapping = ConstantU32(state, mapping_offset); - const auto count = LoadMapping(mapping); + const auto count = LoadFlattenedSrt(state, mapping); if (search_iterations == 0u) { const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key, ConstantU32(state, 2u)); @@ -58,7 +78,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u)); const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe, ConstantU32(state, mapping_offset + 1u)); - return Select(state, TypeU32(state), valid, LoadMapping(entry), + return Select(state, TypeU32(state), valid, LoadFlattenedSrt(state, entry), ConstantU32(state, default_resource)); } const auto entry_at = [&](uint32_t index) { @@ -76,7 +96,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m ConstantU32(state, 1u)); const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u)); const auto less = Binary(state, spv::OpULessThan, TypeBool(state), - LoadMapping(entry_at(probe)), key); + LoadFlattenedSrt(state, entry_at(probe)), key); low = Select(state, TypeU32(state), Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less), Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low); @@ -87,9 +107,10 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count); const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u))); const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range, - Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key)); - const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry, - ConstantU32(state, 1u))); + Binary(state, spv::OpIEqual, TypeBool(state), + LoadFlattenedSrt(state, entry), key)); + const auto resource = LoadFlattenedSrt( + state, Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u))); return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource)); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 29b379c7b..99128270c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -206,6 +206,17 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) { return state.builder.Type(spv::OpTypeArray, TypeF32(state), ConstantU32(state, count)); } +void DefineImageArray(EmitterState& state, ImageDefinition& definition, const IR::ImageResource& image, + uint32_t count, uint32_t set, uint32_t binding) { + definition.type = ImageType(state, image); + const auto array = state.builder.Type(spv::OpTypeArray, definition.type, ConstantU32(state, count)); + definition.variable = state.builder.DefineGlobalVariable( + TypePointer(state, spv::StorageClassUniformConstant, array), spv::StorageClassUniformConstant); + state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationDescriptorSet, set); + state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationBinding, binding); + definition.pointer_type = TypePointer(state, spv::StorageClassUniformConstant, definition.type); +} + void DefineDescriptors(EmitterState& state) { using Kind = IR::DescriptorBindingKind; auto& info = state.program.info; @@ -222,6 +233,14 @@ void DefineDescriptors(EmitterState& state) { (image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) { EXIT("shader image has an invalid descriptor type or mip count"); } + if (image.heap) { + // Heap kinds alias one array; the root itself is never accessed. + auto& definition = state.heap_images[IR::ImageBindingIndex(*kind)]; + if (image.indirect_resources.empty() && definition.variable == 0) { + DefineImageArray(state, definition, image, IR::ImageHeapCapacity, IR::ImageHeapSet, 0u); + } + continue; + } auto& count = counts[static_cast(*kind)]; image.descriptor_index = count; if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = ℑ @@ -331,21 +350,11 @@ void DefineDescriptors(EmitterState& state) { default: { EXIT_IF(IR::ImageBindingResourceClass(kind) == IR::ImageResourceClass::None); - const auto& image = *image_types[IR::ImageBindingIndex(kind)]; - const auto name = "image_" + std::to_string(static_cast(kind)); auto& definition = state.images[IR::ImageBindingIndex(kind)]; - definition.type = ImageType(state, image); - definition.variable = Define(ArrayType(definition.type), name.c_str(), - spv::StorageClassUniformConstant); - definition.pointer_type = TypePointer( - state, spv::StorageClassUniformConstant, definition.type); - if (image.dimension == ImageDimension::Dim1D || - image.dimension == ImageDimension::Dim1DArray) { - state.builder.RequireCapability(image.resource_class == - IR::ImageResourceClass::Sampled - ? spv::CapabilitySampled1D - : spv::CapabilityImage1D); - } + DefineImageArray(state, definition, *image_types[IR::ImageBindingIndex(kind)], + counts[index], 0u, IR::NativeBinding(state.program.stage, kind)); + state.builder.AddName(definition.variable, + ("image_" + std::to_string(static_cast(kind))).c_str()); break; } } @@ -752,7 +761,7 @@ void DefineModule(EmitterState& state) { if (state.program.info.buffer_u16) { state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess); } - if (state.program.info.uses_dma) { + if (state.program.info.uses_dma || state.program.info.uses_image_heap) { state.builder.RequireCapability(spv::CapabilityInt64); state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses); state.builder.RequireExtension("SPV_KHR_physical_storage_buffer"); @@ -820,7 +829,7 @@ void DefineModule(EmitterState& state) { state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric"); } state.builder.RequireExtension("SPV_KHR_float_controls"); - state.builder.AddMemoryModel(state.program.info.uses_dma + state.builder.AddMemoryModel(state.program.info.uses_dma || state.program.info.uses_image_heap ? spv::AddressingModelPhysicalStorageBuffer64 : spv::AddressingModelLogical, spv::MemoryModelGLSL450); diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index a65ab353d..353f7f10e 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -169,6 +169,7 @@ struct ImageResource { bool depth_compare = false; bool cube = false; bool r128 = false; + bool heap = false; // Resident image heap root or kind. uint32_t indirect_root = NoIndirectImage; uint32_t indirect_mapping_offset = 0; uint32_t indirect_search_iterations = 0; @@ -344,6 +345,11 @@ inline constexpr uint32_t FirstImageBinding = 1u; inline constexpr uint32_t FirstComparisonImageBinding = 22u; inline constexpr uint32_t FirstStorageImageBinding = 29u; inline constexpr uint32_t ImageBindingCount = 50u; +// Resident image heaps share one sampled-image array at binding 0 of this set. A heap slot +// word is kind << ImageHeapKindShift | element; element 0 is the null descriptor. +inline constexpr uint32_t ImageHeapSet = 1u; +inline constexpr uint32_t ImageHeapCapacity = 65536u; +inline constexpr uint32_t ImageHeapKindShift = 20u; enum class DescriptorBindingKind : uint32_t { Buffers = 0u, @@ -529,6 +535,7 @@ struct ShaderInfo { bool uses_lds = false; bool uses_gds = false; bool uses_flattened_srt = false; + bool uses_image_heap = false; bool uses_dispatch_threads = false; bool bvh = false; bool subgroup_ballot = false; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 7b7156bad..bdd20b8c1 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -681,6 +681,32 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p const auto count = table_size >= first + 4u ? std::min((table_size - first - 4u) / step + 1u, domain) : 0u; + descriptors[resource_index] = {.dword_count = dword_count}; + if constexpr (std::is_same_v) { + const auto& image = program.info.images[resource_index]; + // A resident heap maps slot words instead of materializing its entries. + ImageHeapView view; + if (image_heap && runtime.image_heap != nullptr && + image.resource_class == ImageResourceClass::Sampled && + image.mip_mode != ImageMipMode::Dynamic && + runtime.image_heap->Acquire({table_base, table_size, step, first, count, &image}, + view)) { + snapshot.specialization_reads.emplace_back(table_base, table_size); + if (view.kinds.empty()) return true; + auto root = root_resource; + root.indirect_root = resource_index; + root.indirect_mapping_offset = static_cast(snapshot.flattened_srt.size()); + root.heap = true; + specializations[resource_index] = root; + snapshot.flattened_srt.insert(snapshot.flattened_srt.end(), + {view.slot_count, static_cast(view.slots), static_cast(view.slots >> 32u)}); + for (const auto& kind: view.kinds) { + descriptors.push_back(kind); + specializations.push_back(root); + } + return true; + } + } if (child_count == 0u || (!image_heap && count * child_count > MaxIndirectDescriptorProbes)) return failed("candidate count", fmt::format("rows={} limit={}", count, MaxIndirectDescriptorProbes)); // Affine SMEM offsets wrap in U32 before adding the aligned immediate. @@ -692,7 +718,6 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p return read_failed("image heap read", 0u, 0u, table_base, 0u, heap_words.size() * sizeof(uint32_t)); } const auto mapping_offset = snapshot.flattened_srt.size(); - descriptors[resource_index] = {.dword_count = dword_count}; snapshot.flattened_srt.resize(mapping_offset + 2u, 0u); for (uint32_t word = 0; word < count; ++word) { const auto table_offset = affine ? first + word * step : uint64_t {word} * 4u; @@ -1010,10 +1035,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size() ? snapshot.flattened_srt[root.indirect_mapping_offset] : 0u; + // A heap maps its slot count and slot address; tables map each key. + size_t mapping_words = 3u; + if (!root.heap) + mapping_words = 1u + size_t {key_count} * (root.indirect_search_iterations == 0u ? 1u : 2u); if (key_count == 0u || - static_cast(root.indirect_mapping_offset) + 1u + - static_cast(key_count) * (root.indirect_search_iterations == 0u ? 1u : 2u) > - snapshot.flattened_srt.size()) { + root.indirect_mapping_offset + mapping_words > snapshot.flattened_srt.size()) { return SpecializationFail("indirect image specialization has an invalid key mapping"); } uint32_t exemplar = ImageResource::NoIndirectImage; @@ -1749,6 +1776,41 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime return BuildResourceSpecialization(program, snapshot, specialization); } +uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value) { + // Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, and + // entries that need shader-side conversion, read as null instead of failing the draw. + const auto format = static_cast((value.dwords[1] >> 20u) & 0x1ffu); + const auto tile = static_cast((value.dwords[3] >> 20u) & 0x1fu); + const auto numeric_class = + Prospero::SampledTextureNumericClass(Prospero::RemapTextureFormat(format)); + if (NullImageDescriptor(value) || !ValidImageDescriptor(value, root.r128) || + DescriptorDimension(value, root.dimension) != root.dimension || + Prospero::IsFmaskTextureFormat(format) || format == Prospero::BufferFormat::k11_11_10UNorm || + ImageConversionFormat(format) != Prospero::BufferFormat::kInvalid || + (root.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt && + tile == Prospero::TileMode::kDepth) || + numeric_class == Prospero::TextureNumericClass::Unsupported || + (root.depth_compare && numeric_class != Prospero::TextureNumericClass::Float)) { + value.dwords.fill(0u); + return 0u; + } + return 1u + static_cast(numeric_class) * 2u + (DescriptorIsCube(value) ? 1u : 0u); +} + +ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind) { + ImageResource resource; + resource.resource_class = ImageResourceClass::Sampled; + resource.dimension = root.dimension; + resource.unfiltered_access_only = root.unfiltered_access_only; + resource.depth_compare = root.depth_compare; + resource.r128 = root.r128; + if (kind != 0u) { + resource.numeric_class = static_cast((kind - 1u) / 2u); + resource.cube = ((kind - 1u) & 1u) != 0u; + } + return resource; +} + void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) { EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete); EXIT_IF(program.info.buffers.size() > specialization.buffers.size() || @@ -1799,6 +1861,7 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& image.indirect_mapping_offset = source.indirect_mapping_offset; image.indirect_search_iterations = source.indirect_search_iterations; image.cube = source.cube; + image.heap = source.heap; image.indirect_resources.clear(); } for (uint32_t index = 0; index < images.size(); index++) { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index 78a3b5f1b..b4829e97d 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -31,6 +31,7 @@ struct ResourceSpecialization { uint32_t indirect_search_iterations = 0; bool cube = false; bool fmask = false; + bool heap = false; bool operator==(const Image&) const = default; }; @@ -59,6 +60,13 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime // Applies an already-derived specialization to native IR before layout and emission. void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization); +// Kind of an image heap entry as root sees it, or 0 (and a zeroed value) when root reads it as +// null. Entries of one kind specialize identically. +uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value); +// The sampled resource that root resolves entries of kind as; kind 0 gives the identity of +// root's view of a heap. +ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind); + } // namespace Libs::Graphics::ShaderRecompiler::IR #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */ diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp index 7d91d05ac..744d84c56 100644 --- a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -594,6 +594,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) { }; info.uses_flattened_srt |= std::ranges::any_of(info.buffers, uses_mapping) || std::ranges::any_of(info.images, uses_mapping); + info.uses_image_heap = std::ranges::any_of(info.images, &ImageResource::heap); switch (program.stage) { case ShaderType::Vertex: case ShaderType::Local: CollectVertexInputs(input_info.vertex, info, inputs.components); break; diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index e530b770f..f8b99c092 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -12,6 +12,33 @@ class Value; using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span values); +// Image descriptors at base + first + k * step, k < count, within size bytes that a sampled +// root selects by GPU-computed byte offset. The host keeps them resident instead of +// materializing them. +struct ImageHeapRequest { + uint64_t base = 0; + uint64_t size = 0; + uint64_t step = 0; + uint64_t first = 0; + uint64_t count = 0; + const ImageResource* root = nullptr; +}; + +struct ImageHeapView { + uint64_t slots = 0; // Device address of one slot per heap dword. + uint32_t slot_count = 0; // Keys past it select a trailing null slot. + std::span kinds; // One representative descriptor per kind. +}; + +class ImageHeapSource { +public: + // False when the host does not keep the table resident. + virtual bool Acquire(const ImageHeapRequest& request, ImageHeapView& view) = 0; + +protected: + ~ImageHeapSource() = default; +}; + struct SrtRuntime { std::span user_data; uint64_t shader_base = 0; @@ -20,6 +47,7 @@ struct SrtRuntime { SrtMemoryReader read_specialization_memory = nullptr; std::span workgroup_counts; uint64_t indirect_args_address = 0; + ImageHeapSource* image_heap = nullptr; }; class SrtReadCapture { diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index c8de765a9..72b7c69e3 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -436,6 +436,128 @@ void TestInvariantIndirectImageMaterialization(bool nested) { "malformed image table tracking was not transactional"); } +void TestResidentImageHeap() { + struct FakeImageHeap final : ImageHeapSource { + bool accept = true; + std::vector kinds; + std::vector requests; + bool Acquire(const ImageHeapRequest &request, ImageHeapView &view) override { + requests.push_back(request); + if (!accept) return false; + view = {.slots = 0x123456789abcull, .slot_count = 36u, .kinds = kinds}; + return true; + } + }; + auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u); + fixture->PlanAndTrack(); + const auto plan = ExtractResourcePlan(fixture->program); + std::array descriptor{}; + descriptor[0] = 0x20u; + descriptor[1] = static_cast(Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u; + descriptor[2] = 3u | (3u << 14u); + descriptor[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); + auto cube = descriptor; + cube[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | + (static_cast(Libs::Graphics::Prospero::ImageType::kCube) << 28u); + std::array data{0x3000u, 4u << 16u, 0x100000u, 0u, + 0x2000u, 48u << 16u, 3u, 0u}; + std::copy_n(data.begin(), 4u, data.begin() + 12u); + LinearTestMemory memory; + memory.fail_address = 0x3000u; + memory.watched_address = 0x2000u; + std::copy(descriptor.begin(), descriptor.end(), memory.words.begin() + 0x1010u / 4u); + FakeImageHeap heap; + heap.kinds = {{descriptor, 8u}, {cube, 8u}}; + SrtRuntime runtime{.user_data = data, .userdata = &memory, + .read_specialization_memory = ReadLinearTestMemory, .image_heap = &heap}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + heap.requests.size() == 1u && heap.requests[0].base == 0x2000u && + heap.requests[0].size == 144u && heap.requests[0].step == 16u && + heap.requests[0].first == 0u && heap.requests[0].count == 9u && + heap.requests[0].root == &plan.info.images[0] && + memory.watched_reads == 0u && snapshot.images.size() == 3u, + "resident image heap was materialized entry by entry"); + const auto &root = specialization.images[0]; + const auto mapping = root.indirect_mapping_offset; + Check(root.heap && root.indirect_root == 0u && root.indirect_search_iterations == 0u && + snapshot.flattened_srt.size() == mapping + 3u && + snapshot.flattened_srt[mapping] == 36u && + snapshot.flattened_srt[mapping + 1u] == 0x56789abcu && + snapshot.flattened_srt[mapping + 2u] == 0x1234u && + specialization.images[1].heap && specialization.images[1].indirect_root == 0u && + !specialization.images[1].cube && specialization.images[2].heap && + specialization.images[2].cube && snapshot.images[2].dwords == cube && + std::ranges::find(snapshot.specialization_reads, + std::pair{0x2000u, 144u}) != + snapshot.specialization_reads.end(), + "resident image heap lost its slot mapping, typed kinds or read dependency"); + auto &program = fixture->program; + ApplyResourceSpecialization(program, specialization); + ShaderComputeInputInfo compute{}; + CollectShaderInfo(program, {.compute = &compute}); + Check(program.info.uses_image_heap && program.info.images.size() == 3u && + program.info.images[0].indirect_resources == std::vector({0u, 1u, 2u}) && + std::ranges::all_of(program.info.images, &ImageResource::heap), + "applied heap kinds lost their root or heap flag"); + heap.kinds.clear(); + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + snapshot.images.size() == 1u && !specialization.images[0].heap && + specialization.images[0].indirect_root == ImageResource::NoIndirectImage, + "a resident heap without kinds did not read as one null image"); + heap.accept = false; + Check(MaterializeResources(plan, runtime, snapshot, specialization) && + heap.requests.size() == 3u && memory.watched_reads == 1u && + snapshot.images.size() == 2u && !specialization.images[0].heap && + snapshot.images[1].dwords == descriptor, + "a declined heap was not materialized entry by entry"); +} + +void TestImageHeapKinds() { + using Format = Libs::Graphics::Prospero::BufferFormat; + using Type = Libs::Graphics::Prospero::ImageType; + ImageResource root; + root.resource_class = ImageResourceClass::Sampled; + root.dimension = Decoder::ImageDimension::Dim2DArray; + root.source = 3u; + const auto descriptor = [](Format format, Type type) { + DescriptorValue value{.dword_count = 8u}; + value.dwords[0] = 0x20u; + value.dwords[1] = static_cast(format) << 20u; + value.dwords[2] = 3u | (3u << 14u); + value.dwords[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | (static_cast(type) << 28u); + return value; + }; + const auto kind_of = [&](Format format, Type type) { + auto value = descriptor(format, type); + return ImageHeapKind(root, value); + }; + const auto float_kind = kind_of(Format::k32_32_32_32Float, Type::kColor2DArray); + const auto uint_kind = kind_of(Format::k32UInt, Type::kColor2DArray); + const auto cube_kind = kind_of(Format::k32_32_32_32Float, Type::kCube); + auto volume = descriptor(Format::k32_32_32_32Float, Type::kColor3D); + Check(float_kind != 0u && uint_kind != 0u && cube_kind != 0u && float_kind != uint_kind && + float_kind != cube_kind && kind_of(Format::kInvalid, Type::kColor2DArray) == 0u && + ImageHeapKind(root, volume) == 0u && + std::ranges::all_of(volume.dwords, [](uint32_t word) { return word == 0u; }), + "heap kinds did not separate typed entries or null mismatched ones"); + const auto resource = ImageHeapResource(root, uint_kind); + const auto cube_resource = ImageHeapResource(root, cube_kind); + auto other = root; + other.source = 9u; + Check(resource.numeric_class == Libs::Graphics::Prospero::TextureNumericClass::Uint && + !resource.cube && cube_resource.cube && + cube_resource.numeric_class == Libs::Graphics::Prospero::TextureNumericClass::Float && + resource.resource_class == ImageResourceClass::Sampled && + ImageHeapResource(root, 0u) == ImageHeapResource(other, 0u), + "heap kind resources lost their type or kept root identity"); + root.depth_compare = true; + Check(kind_of(Format::k32UInt, Type::kColor2DArray) == 0u, + "depth comparison accepted an integer heap entry"); +} + void TestBoundedImageViewEligibility() { using Type = Libs::Graphics::Prospero::ImageType; auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u); @@ -8796,6 +8918,8 @@ int main() { Run("dynamic storage mips", TestDynamicStorageMipTracking); Run("invariant indirect images", [] { TestInvariantIndirectImageMaterialization(true); }); Run("unbounded material image keys", [] { TestInvariantIndirectImageMaterialization(false); }); + Run("resident image heap", TestResidentImageHeap); + Run("image heap kinds", TestImageHeapKinds); Run("bounded image view eligibility", TestBoundedImageViewEligibility); Run("exact unfiltered image candidates", TestExactUnfilteredImageCandidates); Run("waterfall image table", TestWaterfallImageTable); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index c7606febd..1de673799 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -228,6 +228,18 @@ struct TileManagerTestAccess { } }; +struct ImageHeapTestAccess { + static size_t ElementCount(const ImageHeap &heap) { return heap.m_elements.size(); } + static size_t MemoryCount(const ImageHeap &heap) { return heap.m_memories.size(); } + static const auto &Views(const ImageHeap &heap) { return heap.m_memories.at(0)->views; } + static const auto &Element(const ImageHeap &heap, uint32_t entry) { + return heap.m_elements[entry & ((1u << ShaderRecompiler::IR::ImageHeapKindShift) - 1u)]; + } + static ImageId ImageOf(const ImageHeap &heap, uint32_t entry) { + return Element(heap, entry).binding.image_id; + } +}; + struct TextureCacheTestAccess { static_assert(std::same_as>); @@ -5150,6 +5162,234 @@ public: std::printf("[host] %-32s ok\n", name); } + void CheckResidentImageHeap() { + constexpr const char *name = "ResidentImageHeap"; + constexpr uintptr_t base = 0x0000000206800000ull; + constexpr uint64_t allocation_size = 0x200000; + constexpr uint64_t table = base + 0x100000; + constexpr uint64_t idle_table = base + 0x180000; + constexpr uint64_t table_size = 0x10000; + namespace IR = ShaderRecompiler::IR; + using Format = Prospero::BufferFormat; + EnsureRuntimeContext(); + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_size, 0, &direct_offset) == 0, + "image heap allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_size) == 0 && mapped == reinterpret_cast(base), + "image heap mapping failed"); + std::memset(mapped, 0, allocation_size); + // A 4x4 linear 2D texture per index. + const auto descriptor = [](uint32_t texture, Format format) { + const uint64_t address = base + 0x10000u + texture * 0x1000u; + std::array words{}; + words[0] = static_cast(address >> 8u); + words[1] = static_cast(address >> 40u) | (static_cast(format) << 20u); + words[2] = 3u | (3u << 14u); + words[3] = DstSel(4, 5, 6, 7) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + return words; + }; + const auto put = [&](uint64_t probe, const std::array &words) { + std::memcpy(reinterpret_cast(table + probe * 32u), words.data(), sizeof(words)); + }; + put(0, descriptor(0, Format::k8_8_8_8UNorm)); + put(1, descriptor(0, Format::k8_8_8_8UNorm)); + put(5, descriptor(1, Format::k8_8_8_8UNorm)); + { + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + context.MapMemory(base, allocation_size); + auto &heap = context.GetImageHeap(); + auto &executor = context.GetRenderExecutor(); + auto &textures = context.GetTextureCache(); + auto &device = context.GetGraphics().device; + const vk::DescriptorSetLayoutCreateInfo empty_info{}; + vk::DescriptorSetLayout empty = nullptr; + vk::PipelineLayout layout = nullptr; + Require(name, "set 0 layout", + device.createDescriptorSetLayout(&empty_info, nullptr, &empty) == + vk::Result::eSuccess, + "could not create the set 0 layout"); + const std::array set_layouts{empty, context.GetGraphics().image_heap_layout}; + vk::PipelineLayoutCreateInfo layout_info{}; + layout_info.setLayoutCount = 2; + layout_info.pSetLayouts = set_layouts.data(); + Require(name, "pipeline layout", + device.createPipelineLayout(&layout_info, nullptr, &layout) == vk::Result::eSuccess, + "could not create a pipeline layout with the heap set"); + + IR::ImageResource sampled{}; + sampled.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + sampled.unfiltered_access_only = false; + auto unfiltered = sampled; + unfiltered.unfiltered_access_only = true; + IR::ImageHeapView view{}; + const auto acquire = [&](const IR::ImageResource &root, uint64_t address = table, + uint64_t count = table_size / 32u) { + return heap.Acquire({address, table_size, 32u, 0u, count, &root}, view); + }; + const auto draw = [&] { + heap.Rebind(executor); + heap.Commit(scheduler.Current(), vk::PipelineBindPoint::eCompute, layout); + scheduler.Finish(); + heap.BeginDraw(); + }; + const auto write = [&](uint64_t probe, const std::array &words) { + Libs::LibKernel::Memory::WriteBacking(table + probe * 32u, words.data(), sizeof(words)); + Require(name, "table write", context.InvalidateMemory(table + probe * 32u, sizeof(words)), + "the table was not mapped"); + }; + + Require(name, "first view", + acquire(sampled) && view.kinds.size() == 1u && view.slot_count == table_size / 4u, + "a 64-KiB table was not kept resident"); + const auto &views = ImageHeapTestAccess::Views(heap); + const auto &filtered = *views[0]; + Require(name, "slot words", + filtered.slots[0] != 0u && filtered.slots[8] == filtered.slots[0] && + filtered.slots[40] != 0u && filtered.slots[40] != filtered.slots[0] && + filtered.slots[1] == 0u && filtered.slots.back() == 0u && + ImageHeapTestAccess::ElementCount(heap) == 3u && + ImageHeapTestAccess::Element(heap, 0u).binding.layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "probes did not share equal descriptors, left a non-null gap or wrote the null " + "element without a sampled layout"); + Require(name, "second view", + acquire(unfiltered) && views.size() == 2u && + ImageHeapTestAccess::ElementCount(heap) == 5u, + "a second root profile did not get its own view"); + const auto &idle = *views[1]; + draw(); + const auto resident = ImageHeapTestAccess::ImageOf(heap, filtered.slots[0]); + Require(name, "resident images", + resident && textures.GetImage(resident).heap_refs.load() == 2u, + "resolved elements did not pin their image"); + + // Only acquired views take table changes; the idle one must not allocate. + const auto idle_slot = idle.slots[40]; + for (uint32_t texture = 2; texture < 4u; ++texture) { + write(5, descriptor(texture, Format::k8_8_8_8UNorm)); + Require(name, "changed table", acquire(sampled), "a changed table was declined"); + draw(); + } + Require(name, "idle view", + idle.slots[40] == idle_slot && ImageHeapTestAccess::ElementCount(heap) <= 6u, + "an idle view took table changes or leaked released elements"); + Require(name, "reactivated view", acquire(unfiltered), "the idle view was declined"); + draw(); + Require(name, "deferred probes", + ImageHeapTestAccess::ImageOf(heap, idle.slots[40]) == + ImageHeapTestAccess::ImageOf(heap, filtered.slots[40]) && + ImageHeapTestAccess::ElementCount(heap) <= 7u, + "a reactivated view missed the changes made while it was idle"); + + // All programs of one draw see the table and kinds of its first acquire. + Require(name, "draw kinds", acquire(sampled), "the view was declined"); + write(7, descriptor(12, Format::k8_8_8_8UInt)); + Require(name, "stable kinds", + acquire(sampled) && view.kinds.size() == 1u && filtered.slots[56] == 0u, + "a later program of a draw saw a kind the first one lacked"); + draw(); + Require(name, "next draw kinds", + acquire(sampled) && view.kinds.size() == 2u && + filtered.slots[56] >> IR::ImageHeapKindShift == 1u, + "the next draw did not take the new kind"); + draw(); + + Require(name, "bounded view", + acquire(sampled, table, 4u) && views.size() == 3u && views[2]->slots[0] != 0u && + views[2]->slots[40] == 0u, + "a view probed descriptors past the guest table bound"); + draw(); + + // Only leaving the read state invalidates resident images. + auto &image = textures.GetImage(resident); + const auto epoch = Libs::Graphics::Image::heap_epoch.load(); + for (uint32_t pass = 0; pass < 2u; ++pass) { + image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite, {}, + scheduler.Current().Handle()); + } + Require(name, "write epoch", Libs::Graphics::Image::heap_epoch.load() == epoch + 1u, + "repeated writes after leaving the read state bumped the heap epoch again"); + Require(name, "read again", acquire(sampled), "the view was declined"); + draw(); + Require(name, "read layout", + image.backing.state.layout == vk::ImageLayout::eShaderReadOnlyOptimal && + Libs::Graphics::Image::heap_epoch.load() == epoch + 1u, + "commit did not return a resident image to its read layout"); + + // An image freed during a draw's discovery lives until the tick ends, and the draw's + // slots still select it. + write(9, descriptor(13, Format::k8_8_8_8UNorm)); + Require(name, "freed image", acquire(sampled), "the view was declined"); + heap.Rebind(executor); + const auto freed = ImageHeapTestAccess::ImageOf(heap, filtered.slots[72]); + textures.UnmapMemory(base + 0x10000u + 13u * 0x1000u, 0x1000u); + heap.Commit(scheduler.Current(), vk::PipelineBindPoint::eCompute, layout); + Require(name, "freed image layout", + !textures.GetImage(freed).registered && + textures.GetImage(freed).backing.state.layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "commit skipped the read transition of a freed but sampled image"); + scheduler.Finish(); + heap.BeginDraw(); + + // A draw skipped after its programs acquired views still leaves the next draw a fresh + // table. + Require(name, "skipped draw", acquire(sampled), "the view was declined"); + heap.BeginDraw(); + write(11, descriptor(16, Format::k8_8_8_8UNorm)); + Require(name, "draw after a skipped one", acquire(sampled) && filtered.slots[88] != 0u, + "a draw sampled the table of a skipped draw"); + // The skipped draw's view stays acquired. Another view refreshing the table must not let + // a stale element take a newer kind from it. + heap.BeginDraw(); + write(5, descriptor(15, Format::k8_8_8_8SInt)); + Require(name, "other view", acquire(unfiltered), "the view was declined"); + textures.UnmapMemory(base + 0x10000u + 3u * 0x1000u, 0x1000u); + draw(); + Require(name, "stale rescan", + filtered.kinds.size() == 2u && + ImageHeapTestAccess::Element(heap, filtered.slots[40]).value.dwords == + descriptor(3, Format::k8_8_8_8UNorm), + "a stale element's rescan took a kind its view's programs lack"); + + heap.UnmapMemory(table, table_size); + Require(name, "unmapped table", + ImageHeapTestAccess::MemoryCount(heap) == 0u && + textures.GetImage(resident).heap_refs.load() == 0u, + "unmapping the table kept its images pinned"); + // The idle table stays watched; destroying the heap must unwatch it before the page + // manager fails on live watches. + Require(name, "idle table", + acquire(sampled, idle_table) && view.kinds.empty() && + ImageHeapTestAccess::MemoryCount(heap) == 1u, + "an all-null table was not kept resident"); + draw(); + context.UnmapMemory(base, idle_table - base); + scheduler.Finish(); + device.destroyPipelineLayout(layout, nullptr); + device.destroyDescriptorSetLayout(empty, nullptr); + } + Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "image heap unmap failed"); + Require(name, "release", Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "image heap release failed"); + std::printf("[host] %-32s ok\n", name); + } + void CheckComputeMetaClearClassification() { constexpr const char *name = "ComputeMetaClearClassification"; constexpr uint64_t read_only_meta = 0x0000000204201f00ull; @@ -41289,6 +41529,41 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { Require(name, "alternating image emitted size", spirv.size() <= 8192u, "alternating descriptors exceeded the 32 KiB SPIR-V regression budget"); + // A resident heap selects its typed kind and element from a device slot word. + auto heap_root = root; + heap_root.heap = true; + heap_root.indirect_search_iterations = 0; + heap_root.indirect_resources = {0u, 1u, 2u}; + auto heap_kind = candidate; + heap_kind.heap = true; + auto heap_uint = heap_kind; + heap_uint.numeric_class = Prospero::TextureNumericClass::Uint; + for (const bool mixed : {true, false}) { + program.info.images = {heap_root, heap_kind}; + if (mixed) program.info.images.push_back(heap_uint); + program.info.images[0].indirect_resources.resize(program.info.images.size()); + // ApplyResourceSpecialization adds the integer sampler variant of a Uint kind. + program.info.samplers.resize(1); + if (mixed) { + program.info.samplers.push_back(program.info.samplers[0]); + program.info.samplers[1].integer_border = true; + } + program.shader_info_complete = false; + CollectShaderInfo(program, {.compute = &compute}); + spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); + ValidateSpirv("ResidentImageHeap", spirv); + Require(name, "image heap disassembly", tools.Disassemble(spirv, &text), + "failed to disassemble the image heap shader"); + Require(name, "image heap selection", + program.info.uses_image_heap && CountText(text, "OpConvertUToPtr") == 1 && + CountText(text, "UMin") == 1 && + CountText(text, "OpSwitch") == (mixed ? 1 : 0) && + CountText(text, "OpImageSampleExplicitLod") == (mixed ? 2 : 1) && + CountText(text, "DescriptorSet 1") == (mixed ? 2 : 1) && + text.find("PhysicalStorageBuffer64") != std::string::npos, + "image heap kinds did not sample the slot-selected heap element"); + } + program.memory_info[0].image_dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; program.memory_info[0].image_address_components = 4; @@ -44054,7 +44329,7 @@ void CheckComputeLdsLimit(VulkanHarness &vulkan) { regs.cs_regs.num_thread_y = 1; regs.cs_regs.num_thread_z = 1; regs.cs_regs.wave_size = 32; - PipelineCache cache(graphics); + PipelineCache cache(graphics, nullptr); ShaderProgram previous; uint16_t previous_units = 0; for (const auto units : {static_cast(limit_units - 1u), limit_units, @@ -49350,7 +49625,7 @@ void CheckPm4IndirectControlFlow(RenderContext &renderer) { std::printf("[host] %-32s ok\n", name); } -// waitUntilSafeForRendering stalls only its own command processor (SDK), and a display buffer +// waitUntilSafeForRendering stalls only its own command processor, and a display buffer // stays busy while its flip is pending or while it is on screen. void CheckWaitUntilSafeForRendering() { constexpr const char *name = "WaitUntilSafeForRendering"; @@ -50653,6 +50928,7 @@ int main(int argc, char **argv) { CheckNestedPointerImageTable(vulkan); CheckGuardedRecordImageTable(vulkan); CheckIndirectImageKeySwitch(vulkan); + vulkan.CheckResidentImageHeap(); CheckFiniteFirstLaneImageChoice(vulkan); CheckFiniteInlineSamplerPhi(vulkan); RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives()); @@ -50831,6 +51107,7 @@ int main(int argc, char **argv) { CheckNestedPointerImageTable(vulkan); CheckGuardedRecordImageTable(vulkan); CheckIndirectImageKeySwitch(vulkan); + vulkan.CheckResidentImageHeap(); CheckDynamicSampleOffsets(vulkan, 32u); CheckDynamicSampleOffsets(vulkan, 64u); CheckFiniteFirstLaneImageChoice(vulkan);