diff --git a/CMakeLists.txt b/CMakeLists.txt index 60721050a..938f6a114 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,6 +263,22 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names) ) list(APPEND gpu_tiler_shader_headers "${shader_header}") endforeach() + +set(fault_buffer_shader_source "${gpu_tiler_shader_dir}/fault_buffer_process.comp") +set(fault_buffer_shader_spv "${gpu_tiler_generated_dir}/fault_buffer_process.spv") +set(fault_buffer_shader_header "${gpu_tiler_generated_dir}/fault_buffer_process_spv.h") +add_custom_command( + OUTPUT "${fault_buffer_shader_header}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}" + COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os + -DCACHING_PAGEBITS=14 -DMAX_PAGE_FAULTS=1024 + -o "${fault_buffer_shader_spv}" "${fault_buffer_shader_source}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${fault_buffer_shader_spv} -DOUTPUT=${fault_buffer_shader_header} + -DSYMBOL=FAULT_BUFFER_PROCESS_SPV -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${fault_buffer_shader_source}" + VERBATIM +) +list(APPEND kyty_emulator_src "${fault_buffer_shader_header}") list(APPEND kyty_emulator_src ${gpu_tiler_shader_headers}) set(gpu_blit_generated_dir "${PROJECT_BINARY_DIR}/gpu_blit_shaders") diff --git a/src/graphics/host_gpu/rangeSet.h b/src/graphics/host_gpu/rangeSet.h index e22ade453..089c8c44f 100644 --- a/src/graphics/host_gpu/rangeSet.h +++ b/src/graphics/host_gpu/rangeSet.h @@ -53,6 +53,15 @@ public: } } + void Clear() { m_ranges.clear(); } + + template + void ForEach(Func&& func) const { + for (const auto& [begin, end]: m_ranges) { + func(begin, end); + } + } + [[nodiscard]] std::vector Intersections(uint64_t address, uint64_t size) const { std::vector result; ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); }); @@ -98,7 +107,7 @@ public: private: static uint64_t End(uint64_t address, uint64_t size) { - if (address == 0 || size == 0 || size > UINT64_MAX - address) { + if (size == 0 || size > UINT64_MAX - address) { EXIT("invalid range-set address or size\n"); } return address + size; diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 4fbcb1c86..4e689c71a 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -51,41 +51,50 @@ struct BufferCache::DownloadCopy { }; void BufferCache::Register(BufferId id) { - auto& buffer = m_slot_buffers[id]; - PageTable::PageRange pages {}; - EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages)); - for (size_t page = pages.first; page < pages.last_exclusive; ++page) { - m_page_table[page] = id; - } - const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id); - (void)it; - EXIT_IF(!inserted); - buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick); - m_total_used_memory += buffer.Size(); + ChangeRegister(id); } void BufferCache::Unregister(BufferId id) { + ChangeRegister(id); +} + +template +void BufferCache::ChangeRegister(BufferId id) { auto& buffer = m_slot_buffers[id]; - if (buffer.is_deleted) { - return; - } PageTable::PageRange pages {}; EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages)); for (size_t page = pages.first; page < pages.last_exclusive; ++page) { - auto* owner = m_page_table.Find(page); - if (owner != nullptr && *owner == id) { - *owner = {}; + if constexpr (insert) { + m_page_table[page] = id; + } else { + m_page_table[page] = {}; } } - const auto found = m_buffers.find(buffer.CpuAddress()); - EXIT_IF(found == m_buffers.end() || found->second != id); - m_buffers.erase(found); - if (buffer.Size() > m_total_used_memory) { - EXIT("BufferCache: allocation accounting underflow\n"); + const auto size_pages = pages.last_exclusive - pages.first; + if constexpr (insert) { + const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id); + (void)it; + EXIT_IF(!inserted); + m_total_used_memory += buffer.Size(); + buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick); + std::vector addresses; + addresses.reserve(size_pages); + for (uint64_t i = 0; i < size_pages; ++i) { + addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS)); + } + WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress), + addresses.data(), addresses.size() * sizeof(vk::DeviceAddress)); + } else { + const auto found = m_buffers.find(buffer.CpuAddress()); + EXIT_IF(found == m_buffers.end() || found->second != id); + m_buffers.erase(found); + EXIT_IF(buffer.Size() > m_total_used_memory); + m_total_used_memory -= buffer.Size(); + m_lru_cache.Free(buffer.lru_id); + m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress), + size_pages * sizeof(vk::DeviceAddress), 0); + buffer.is_deleted = true; } - m_total_used_memory -= buffer.Size(); - m_lru_cache.Free(buffer.lru_id); - buffer.is_deleted = true; } void BufferCache::TouchBuffer(const Buffer& buffer) { @@ -184,16 +193,21 @@ void BufferCache::DownloadBufferMemory(std::span copies) { BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, TextureCache& texture_cache) - : m_graphics(graphics), m_scheduler(scheduler), - m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize), - m_memory_tracker(page_manager), - m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB), - m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), - m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), - m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB), - m_texture_cache(texture_cache) { + : m_graphics(graphics), m_scheduler(scheduler), + m_fault_manager(graphics, scheduler, *this, CACHING_PAGEBITS, CACHING_NUMPAGES), + m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize), + m_bda_pagetable_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, + BDA_PAGETABLE_SIZE), + m_memory_tracker(page_manager), + m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB), + m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), + m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), + m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB), + m_texture_cache(texture_cache) { std::memset(m_gds_buffer.Mapped().data(), 0, static_cast(m_gds_buffer.Size())); m_gds_buffer.Flush(0, m_gds_buffer.Size()); + SetVulkanObjectNameF(m_graphics.device, m_bda_pagetable_buffer.Handle(), + "BDA Page Table Buffer"); const auto null_id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16); EXIT_IF(null_id != NULL_BUFFER_ID); @@ -316,8 +330,8 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) { BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) { auto& command = m_scheduler.Current(); EXIT_IF(command.IsInvalid()); - auto begin = vaddr & ~(CACHING_PAGE_SIZE - 1); - auto end = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1); + auto begin = vaddr & ~(CACHING_PAGESIZE - 1); + auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1); auto first = m_buffers.lower_bound(begin); if (first != m_buffers.begin()) { const auto previous = std::prev(first); @@ -333,8 +347,9 @@ BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) { end = std::max(end, buffer.CpuAddress() + buffer.Size()); } - const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin, - AllFlags, end - begin); + const auto id = m_slot_buffers.insert( + m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin, + AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin); auto& buffer = m_slot_buffers[id]; SetVulkanObjectNameF(m_graphics.device, buffer.Handle(), "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin); @@ -433,7 +448,7 @@ std::pair BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t EXIT("BufferCache: buffer request requires a recording command buffer\n"); } - if (!is_written && size <= CACHING_PAGE_SIZE && + if (!is_written && size <= CACHING_PAGESIZE && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && m_memory_tracker.IsRegionCpuModified(vaddr, size)) { const auto alignment = std::max( @@ -737,4 +752,24 @@ void BufferCache::RunGarbageCollector() { } } +void BufferCache::ProcessFaultBuffer() { + m_fault_manager.ProcessFaultBuffer(); +} + +void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) { + const auto end = vaddr + size; + auto it = m_buffers.upper_bound(vaddr); + if (it != m_buffers.begin()) { + --it; + } + for (; it != m_buffers.end() && it->first < end; ++it) { + auto& buffer = m_slot_buffers[it->second]; + const auto start = std::max(buffer.CpuAddress(), vaddr); + const auto finish = std::min(buffer.CpuAddress() + buffer.Size(), end); + if (start < finish) { + (void)SynchronizeBuffer(buffer, start, finish - start, false, false); + } + } +} + } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index b9ae65f3b..73ec809b9 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -7,6 +7,7 @@ #include "common/slotVector.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/rangeSet.h" +#include "graphics/host_gpu/renderer/cache/faultManager.h" #include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h" #include "graphics/host_gpu/renderer/cache/streamBuffer.h" @@ -26,7 +27,11 @@ inline constexpr BufferId NULL_BUFFER_ID {0}; class BufferCache { public: - static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull; + static constexpr uint32_t CACHING_PAGEBITS = 14; + static constexpr uint64_t CACHING_PAGESIZE = uint64_t {1} << CACHING_PAGEBITS; + static constexpr uint64_t CACHING_NUMPAGES = uint64_t {1} << (40 - CACHING_PAGEBITS); + static constexpr uint64_t BDA_PAGETABLE_SIZE = + CACHING_NUMPAGES * sizeof(vk::DeviceAddress); BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager, TextureCache& texture_cache); @@ -51,6 +56,8 @@ public: EXIT("BufferCache: invalid utility-buffer usage\n"); } [[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; } + [[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept { return &m_bda_pagetable_buffer; } + [[nodiscard]] Buffer* GetFaultBuffer() noexcept { return m_fault_manager.GetFaultBuffer(); } [[nodiscard]] std::pair ObtainBufferForImage(uint64_t vaddr, uint64_t size); void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds); void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds, @@ -60,14 +67,16 @@ public: [[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); + void ProcessFaultBuffer(); + void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size); void RunGarbageCollector(); private: friend struct BufferCacheTestAccess; struct DownloadCopy; - using PageTable = MultiLevelPageTable; - static_assert(CACHING_PAGE_SIZE == (uint64_t {1} << PageTable::kPageBits)); + using PageTable = MultiLevelPageTable; + static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits)); static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; [[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept { return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1); @@ -78,6 +87,8 @@ private: [[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size); void Register(BufferId id); void Unregister(BufferId id); + template + void ChangeRegister(BufferId id); void DeleteBuffer(BufferId id); [[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, bool is_written, bool is_texel_buffer); @@ -90,7 +101,9 @@ private: GraphicContext& m_graphics; CommandScheduler& m_scheduler; + FaultManager m_fault_manager; Buffer m_gds_buffer; + Buffer m_bda_pagetable_buffer; Common::SlotVector m_slot_buffers; Common::LeastRecentlyUsedCache m_lru_cache; std::map m_buffers; diff --git a/src/graphics/host_gpu/renderer/cache/faultManager.cpp b/src/graphics/host_gpu/renderer/cache/faultManager.cpp new file mode 100644 index 000000000..2f85b3955 --- /dev/null +++ b/src/graphics/host_gpu/renderer/cache/faultManager.cpp @@ -0,0 +1,167 @@ +#include "graphics/host_gpu/renderer/cache/faultManager.h" + +#include "common/assert.h" +#include "common/logging/log.h" +#include "gpu_tiler_shaders/fault_buffer_process_spv.h" +#include "graphics/host_gpu/graphicContext.h" +#include "graphics/host_gpu/renderer/cache/bufferCache.h" +#include "graphics/host_gpu/renderer/commandScheduler.h" +#include "graphics/host_gpu/vulkanCommon.h" + +#include +#include +#include +#include + +namespace Libs::Graphics { + +namespace { + +constexpr size_t MaxPageFaults = 1024; +constexpr size_t PageFaultAreaSize = MaxPageFaults * sizeof(uint64_t); + +} // namespace + +FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, + BufferCache& buffer_cache, uint32_t caching_pagebits, + uint64_t caching_num_pages) + : m_graphics(graphics), m_scheduler(scheduler), m_buffer_cache(buffer_cache), + m_caching_pagesize(uint64_t {1} << caching_pagebits), + m_caching_num_pages(caching_num_pages), m_fault_buffer_size(caching_num_pages / 8), + m_fault_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, + m_fault_buffer_size), + m_download_buffer(graphics, scheduler, MemoryUsage::Download, 0, AllFlags, + MaxPendingFaults * PageFaultAreaSize) { + SetVulkanObjectNameF(m_graphics.device, m_fault_buffer.Handle(), "Fault Buffer"); + + const vk::DescriptorSetLayoutBinding bindings[] { + {0, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr}, + {1, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr}, + }; + vk::DescriptorSetLayoutCreateInfo layout_info {}; + layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR; + layout_info.bindingCount = std::size(bindings); + layout_info.pBindings = bindings; + RequireVulkanSuccess( + m_graphics.device.createDescriptorSetLayout(&layout_info, nullptr, + &m_fault_process_desc_layout), + "create fault-buffer descriptor layout"); + + vk::ShaderModuleCreateInfo module_info {}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t); + module_info.pCode = FAULT_BUFFER_PROCESS_SPV; + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module), + "create fault-buffer shader module"); + + vk::PipelineLayoutCreateInfo pipeline_layout_info {}; + pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + pipeline_layout_info.setLayoutCount = 1; + pipeline_layout_info.pSetLayouts = &m_fault_process_desc_layout; + RequireVulkanSuccess( + m_graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr, + &m_fault_process_pipeline_layout), + "create fault-buffer pipeline layout"); + + vk::PipelineShaderStageCreateInfo stage {}; + stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stage.stage = vk::ShaderStageFlagBits::eCompute; + stage.module = module; + stage.pName = "main"; + vk::ComputePipelineCreateInfo pipeline_info {}; + pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo; + pipeline_info.stage = stage; + pipeline_info.layout = m_fault_process_pipeline_layout; + const auto result = m_graphics.device.createComputePipelines( + nullptr, 1, &pipeline_info, nullptr, &m_fault_process_pipeline); + m_graphics.device.destroyShaderModule(module, nullptr); + RequireVulkanSuccess(result, "create fault-buffer pipeline"); + SetVulkanObjectNameF(m_graphics.device, m_fault_process_pipeline, "Fault Buffer Parser"); +} + +FaultManager::~FaultManager() { + m_graphics.device.destroyPipeline(m_fault_process_pipeline, nullptr); + m_graphics.device.destroyPipelineLayout(m_fault_process_pipeline_layout, nullptr); + m_graphics.device.destroyDescriptorSetLayout(m_fault_process_desc_layout, nullptr); +} + +void FaultManager::ProcessFaultBuffer() { + if (const auto wait_tick = m_fault_areas[m_current_area]; wait_tick != 0) { + m_scheduler.Wait(wait_tick); + m_scheduler.PopPendingOperations(); + } + + const auto offset = m_current_area * PageFaultAreaSize; + auto* mapped = m_download_buffer.Mapped().data() + offset; + std::memset(mapped, 0, PageFaultAreaSize); + m_download_buffer.Flush(offset, PageFaultAreaSize); + + vk::BufferMemoryBarrier2 pre_barrier {}; + pre_barrier.sType = vk::StructureType::eBufferMemoryBarrier2; + pre_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; + pre_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite; + pre_barrier.dstStageMask = vk::PipelineStageFlagBits2::eComputeShader; + pre_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderRead; + pre_barrier.buffer = m_fault_buffer.Handle(); + pre_barrier.offset = 0; + pre_barrier.size = m_fault_buffer_size; + auto post_barrier = pre_barrier; + post_barrier.srcStageMask = vk::PipelineStageFlagBits2::eComputeShader; + post_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite; + post_barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands; + post_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderWrite; + + const vk::DescriptorBufferInfo infos[] { + {m_fault_buffer.Handle(), 0, m_fault_buffer_size}, + {m_download_buffer.Handle(), offset, PageFaultAreaSize}, + }; + std::array writes {}; + for (uint32_t index = 0; index < writes.size(); ++index) { + writes[index].sType = vk::StructureType::eWriteDescriptorSet; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = vk::DescriptorType::eStorageBuffer; + writes[index].pBufferInfo = &infos[index]; + } + + m_scheduler.EndRendering(); + auto command = m_scheduler.Current().Handle(); + vk::DependencyInfo dependency {}; + dependency.sType = vk::StructureType::eDependencyInfo; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.bufferMemoryBarrierCount = 1; + dependency.pBufferMemoryBarriers = &pre_barrier; + command.pipelineBarrier2(dependency); + command.bindPipeline(vk::PipelineBindPoint::eCompute, m_fault_process_pipeline); + command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, + m_fault_process_pipeline_layout, 0, writes); + const auto num_threads = m_caching_num_pages / 32; + const auto num_workgroups = (num_threads + 63) / 64; + command.dispatch(static_cast(num_workgroups), 1, 1); + dependency.pBufferMemoryBarriers = &post_barrier; + command.pipelineBarrier2(dependency); + + const auto area = m_current_area; + m_scheduler.DeferOperation([this, mapped, offset, area] { + m_download_buffer.Invalidate(offset, PageFaultAreaSize); + m_fault_ranges.Clear(); + const auto* faults = std::bit_cast(mapped); + const auto count = static_cast(faults[0]); + for (uint32_t index = 1; index <= count; ++index) { + m_fault_ranges.Add(faults[index], m_caching_pagesize); + LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]); + } + m_fault_ranges.ForEach([this](uint64_t start, uint64_t end) { + EXIT_IF(end - start > std::numeric_limits::max()); + (void)m_buffer_cache.FindBuffer(start, end - start); + }); + m_fault_areas[area] = 0; + }); + + m_fault_areas[m_current_area++] = m_scheduler.CurrentTick(); + m_current_area %= MaxPendingFaults; +} + +} // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/faultManager.h b/src/graphics/host_gpu/renderer/cache/faultManager.h new file mode 100644 index 000000000..73c2fe70c --- /dev/null +++ b/src/graphics/host_gpu/renderer/cache/faultManager.h @@ -0,0 +1,46 @@ +#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_ +#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_ + +#include "common/abi.h" +#include "graphics/host_gpu/rangeSet.h" +#include "graphics/host_gpu/renderer/cache/streamBuffer.h" + +#include +#include + +namespace Libs::Graphics { + +class BufferCache; + +class FaultManager { + static constexpr size_t MaxPendingFaults = 8; + +public: + FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, BufferCache& buffer_cache, + uint32_t caching_pagebits, uint64_t caching_num_pages); + ~FaultManager(); + KYTY_CLASS_NO_COPY(FaultManager); + + [[nodiscard]] Buffer* GetFaultBuffer() noexcept { return &m_fault_buffer; } + void ProcessFaultBuffer(); + +private: + GraphicContext& m_graphics; + CommandScheduler& m_scheduler; + BufferCache& m_buffer_cache; + RangeSet m_fault_ranges; + uint64_t m_caching_pagesize; + uint64_t m_caching_num_pages; + uint64_t m_fault_buffer_size; + Buffer m_fault_buffer; + Buffer m_download_buffer; + std::array m_fault_areas {}; + uint32_t m_current_area = 0; + vk::DescriptorSetLayout m_fault_process_desc_layout = nullptr; + vk::Pipeline m_fault_process_pipeline = nullptr; + vk::PipelineLayout m_fault_process_pipeline_layout = nullptr; +}; + +} // namespace Libs::Graphics + +#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_ diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index 35caadf81..0af87b190 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -76,7 +76,19 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) { m_gpu->SendCommandSync(unmap); } +void GpuResourceManager::PrepareBda() { + std::shared_lock lock(m_mapped_ranges_mutex); + m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) { + m_buffer_cache.SynchronizeBuffersInRange(start, end - start); + }); + m_fault_process_pending = true; +} + void GpuResourceManager::RunGarbageCollector() { + if (m_fault_process_pending) { + m_fault_process_pending = false; + m_buffer_cache.ProcessFaultBuffer(); + } m_texture_cache.ProcessDownloadImages(); m_texture_cache.RunGarbageCollector(); m_buffer_cache.RunGarbageCollector(); diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h index 1c3f4ce45..44cf35d1a 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h @@ -30,6 +30,7 @@ public: [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; void MapMemory(uint64_t vaddr, uint64_t size); void UnmapMemory(uint64_t vaddr, uint64_t size); + void PrepareBda(); void RunGarbageCollector(); private: @@ -40,6 +41,7 @@ private: mutable std::shared_mutex m_mapped_ranges_mutex; RangeSet m_mapped_ranges; GuestGpu* m_gpu = nullptr; + bool m_fault_process_pending = false; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp b/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp index f6a0e2228..a82828660 100644 --- a/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp +++ b/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp @@ -73,8 +73,12 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag buffer_info.usage = flags; buffer_info.sharingMode = vk::SharingMode::eExclusive; + const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress); + const VmaAllocationCreateFlags bda_flag = + with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; VmaAllocationCreateInfo allocation_info {}; - allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage); + allocation_info.flags = + VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage); allocation_info.usage = AllocationUsage(usage); allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal ? VkMemoryPropertyFlags {} @@ -99,6 +103,13 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag m_buffer->memory.type = allocation_result.memoryType; m_buffer->memory.unique_id = VulkanNextMemoryUniqueId(); graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements); + if (static_cast(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) { + vk::BufferDeviceAddressInfo address_info {}; + address_info.sType = vk::StructureType::eBufferDeviceAddressInfo; + address_info.buffer = m_buffer->buffer; + m_device_address = graphics.device.getBufferAddress(address_info); + EXIT_IF(m_device_address == 0); + } VkMemoryPropertyFlags properties = 0; vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties); @@ -122,6 +133,11 @@ vk::Buffer Buffer::Handle() const noexcept { return m_buffer->buffer; } +vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept { + EXIT_IF(m_device_address == 0); + return m_device_address; +} + bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept { return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size; } @@ -141,6 +157,15 @@ void Buffer::Flush(uint64_t offset, uint64_t size) { } } +void Buffer::Invalidate(uint64_t offset, uint64_t size) { + EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset); + if (!m_is_coherent && size != 0) { + const auto result = + vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size); + EXIT_NOT_IMPLEMENTED(static_cast(result) != vk::Result::eSuccess); + } +} + vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source, vk::AccessFlags destination) const { if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) { @@ -310,16 +335,6 @@ void StreamBuffer::Commit() { watch.tick = tick; } -void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) { - EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset); - if (IsCoherent() || size == 0) { - return; - } - const auto result = vmaInvalidateAllocation(Graphics().allocator, - NativeBuffer().memory.allocation, offset, size); - EXIT_NOT_IMPLEMENTED(static_cast(result) != vk::Result::eSuccess); -} - uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) { EXIT_IF(source == nullptr); const auto [data, offset] = Map(size, alignment); diff --git a/src/graphics/host_gpu/renderer/cache/streamBuffer.h b/src/graphics/host_gpu/renderer/cache/streamBuffer.h index f8072c200..3615fd700 100644 --- a/src/graphics/host_gpu/renderer/cache/streamBuffer.h +++ b/src/graphics/host_gpu/renderer/cache/streamBuffer.h @@ -48,12 +48,14 @@ public: [[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; } [[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; } [[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; } + [[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept; [[nodiscard]] uint64_t Offset(uint64_t address) const noexcept { return address - m_cpu_address; } [[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept; void Write(uint64_t offset, const void* source, uint64_t size); void Flush(uint64_t offset, uint64_t size); + void Invalidate(uint64_t offset, uint64_t size); void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset, uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite, @@ -84,6 +86,7 @@ private: MemoryUsage m_usage = MemoryUsage::DeviceLocal; uint64_t m_cpu_address = 0; uint64_t m_size = 0; + vk::DeviceAddress m_device_address = 0; std::unique_ptr m_buffer; std::span m_mapped; bool m_is_coherent = false; @@ -97,9 +100,6 @@ public: [[nodiscard]] std::pair Map(uint64_t size, uint64_t alignment = 0, bool allow_wait = true); void Commit(); - // Download mappings become visible to the CPU only after their GPU completion tick is free. - // Call this from the scheduler's deferred completion operation before reading Mapped(). - void Invalidate(uint64_t offset, uint64_t size); [[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0); private: diff --git a/src/graphics/host_gpu/renderer/commandScheduler.cpp b/src/graphics/host_gpu/renderer/commandScheduler.cpp index 2abf691b2..566220385 100644 --- a/src/graphics/host_gpu/renderer/commandScheduler.cpp +++ b/src/graphics/host_gpu/renderer/commandScheduler.cpp @@ -196,8 +196,8 @@ void CommandScheduler::Finish() { Submit(); } m_master.Wait(CurrentTick() - 1); - PopPendingOperations(); BeginNext(); + PopPendingOperations(); } void CommandScheduler::Wait(uint64_t tick) { diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index bbf05b851..53d7f7e98 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -216,48 +216,6 @@ static BufferView NativeStorageBuffer(RenderContext& return result; } -static BufferView -NativeAddressBuffer(RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource, - const ShaderRecompiler::IR::ResourceSnapshot::Address& address, - ShaderType stage, uint32_t slot, uint32_t& address_offset, BufferId id) { - BufferView result; - address_offset = 0; - if (address.binding_base == 0) { - BindNullStorageBuffer(context, result); - return result; - } - if (resource.written) { - EXIT("writable address resources are unsupported\n"); - } - const auto limit = - resource.kind == ShaderRecompiler::IR::ResourceKind::Flat - ? ShaderRecompiler::IR::FlatAddressWindowSize - : static_cast( - context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange); - uint64_t size = 0; - const auto access = HostMemoryAccess::Mapped; - if (!HostMemoryQueryRange(address.binding_base, limit, access, size)) { - EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n", - address.binding_base); - } - const auto& graphics = context.GetGraphics(); - const auto alignment = graphics.StorageMinAlignment(); - auto [buffer, offset] = - context.GetBufferCache().ObtainBuffer(address.binding_base, size, false, false, id); - const auto aligned_offset = offset - offset % alignment; - const auto adjustment = offset - aligned_offset; - const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange; - const auto visible_size = std::min(size, static_cast(max_range - adjustment)); - address_offset = static_cast(adjustment); - result.buffer = buffer->Handle(); - result.offset = aligned_offset; - result.range = static_cast(visible_size + adjustment); - SetVulkanObjectNameF(graphics.device, result.buffer, - "Kyty.{}.AddressBuffer[slot={} guest=0x{:016x} size=0x{:x}]", - ShaderStageResourceName(stage), slot, address.binding_base, visible_size); - return result; -} - static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) { bool supported_dimension = false; switch (resource.dimension) { @@ -1000,25 +958,6 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) { prepared.buffer_ids.push_back(cache.FindBuffer(address, size)); } - prepared.address_ids.clear(); - prepared.address_ids.reserve(program.info.addresses.size()); - for (uint32_t i = 0; i < program.info.addresses.size(); i++) { - const auto& resource = program.info.addresses[i]; - const auto& address = snapshot.addresses[i]; - if (address.binding_base == 0) { - prepared.address_ids.emplace_back(); - continue; - } - EXIT_IF(resource.written); - const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat - ? ShaderRecompiler::IR::FlatAddressWindowSize - : static_cast(m_context.GetGraphics() - .GetPhysicalDeviceProperties() - .limits.maxStorageBufferRange); - uint64_t size = 0; - EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size)); - prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size)); - } } void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { @@ -1028,8 +967,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { const auto& snapshot = *prepared.snapshot; auto& resources = prepared.resources; const auto& layout = program.bindings; - EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size() || - prepared.address_ids.size() != program.info.addresses.size()); + EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size()); resources.buffers.clear(); resources.buffers.reserve(program.info.buffers.size()); @@ -1049,15 +987,6 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { buffer_offset, prepared.buffer_ids[i])); pack_memory_offset(i, buffer_offset); } - resources.addresses.clear(); - resources.addresses.reserve(program.info.addresses.size()); - for (uint32_t i = 0; i < program.info.addresses.size(); i++) { - uint32_t address_offset = 0; - resources.addresses.push_back(NativeAddressBuffer(m_context, program.info.addresses[i], - snapshot.addresses[i], program.stage, i, - address_offset, prepared.address_ids[i])); - pack_memory_offset(static_cast(program.info.buffers.size()) + i, address_offset); - } if (!prepared.flattened_srt.empty()) { resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt); } @@ -1129,6 +1058,10 @@ RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex, if (bindings.pixel) { FindBuffers(*bindings.pixel); } + if (bindings.vertex.program->info.uses_dma || + (bindings.pixel && bindings.pixel->program->info.uses_dma)) { + m_context.GetGpuResources().PrepareBda(); + } RebindBuffers(bindings.vertex); if (bindings.pixel) { RebindBuffers(*bindings.pixel); @@ -1241,13 +1174,15 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range); } break; - case BindingKind::AddressMemory: - for (const auto resource: binding.resources) { - const auto& view = descriptors.addresses.at(resource); - EXIT_IF(view.buffer == nullptr); - m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range); - } + case BindingKind::BdaPagetable: + case BindingKind::FaultBuffer: { + auto& cache = m_context.GetBufferCache(); + const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable + ? cache.GetBdaPageTableBuffer() + : cache.GetFaultBuffer(); + m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size()); break; + } case BindingKind::FlattenedSrt: case BindingKind::UserData: case BindingKind::Gds: { diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.h b/src/graphics/host_gpu/renderer/pipeline/descriptors.h index d857d6885..3c8849a89 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.h +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.h @@ -38,7 +38,6 @@ struct NativeDescriptors { std::vector buffers; std::vector images; std::vector samplers; - std::vector addresses; BufferView gds; BufferView flattened_srt; BufferView user_data; @@ -49,7 +48,6 @@ struct PreparedBindings { const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr; NativeDescriptors resources; std::vector buffer_ids; - std::vector address_ids; std::vector flattened_srt; std::vector user_data; bool committed = false; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index f780249f3..d763d11a4 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -82,8 +82,8 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou const auto& resources = *input.stage.resources; if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 || !program.info.images.empty() || !program.info.samplers.empty() || - !program.info.addresses.empty() || !resources.images.empty() || - !resources.samplers.empty() || !resources.addresses.empty()) { + program.info.uses_dma || !resources.images.empty() || + !resources.samplers.empty()) { return false; } const auto& resource = program.info.buffers.front(); @@ -337,6 +337,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader); auto bindings = PrepareBindings(input_info.stage); FindBuffers(bindings); + if (program.info.uses_dma) { + m_context.GetGpuResources().PrepareBda(); + } RebindBuffers(bindings); RebindImages(bindings); diff --git a/src/graphics/host_gpu/shaders/fault_buffer_process.comp b/src/graphics/host_gpu/shaders/fault_buffer_process.comp new file mode 100644 index 000000000..04a86bad3 --- /dev/null +++ b/src/graphics/host_gpu/shaders/fault_buffer_process.comp @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#version 450 +#extension GL_ARB_gpu_shader_int64 : enable + +layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in; + +layout(std430, binding = 0) buffer input_buf { + uint fault_buffer[]; +}; + +layout(std430, binding = 1) buffer output_buf { + uint64_t download_buffer[]; +}; +layout(std430, binding = 1) buffer output_buf32 { + uint download_buffer32[]; +}; + +void main() { + const uint id = gl_GlobalInvocationID.x; + uint word = fault_buffer[id]; + fault_buffer[id] = 0u; + const uint base_bit = id * 32u; + while (word != 0u) { + const uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u; + if (store_index >= MAX_PAGE_FAULTS) { + return; + } + const uint bit = findLSB(word); + word &= word - 1; + const uint page = base_bit + bit; + download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS; + } +} diff --git a/src/graphics/host_gpu/vma.cpp b/src/graphics/host_gpu/vma.cpp index 6d2d29fee..ec5d5fec4 100644 --- a/src/graphics/host_gpu/vma.cpp +++ b/src/graphics/host_gpu/vma.cpp @@ -72,7 +72,10 @@ bool GraphicContext::CreateAllocator() { info.device = device; info.pVulkanFunctions = &functions; info.vulkanApiVersion = VULKAN_TARGET_API_VERSION; - info.flags = memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0; + info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT; + if (memory_budget_ext_enabled) { + info.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; + } const auto result = static_cast(vmaCreateAllocator(&info, &allocator)); if (result != vk::Result::eSuccess) { diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index b4ff8f37e..97d8e3b44 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -594,20 +594,16 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE); EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE); EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE); + EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE); + EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE); EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE); + EXIT_NOT_IMPLEMENTED(supported_features12.bufferDeviceAddress != VK_TRUE); #if !defined(__APPLE__) EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE); #endif features12.timelineSemaphore = VK_TRUE; features12.shaderOutputLayer = VK_TRUE; - - if (Config::GpuAssistedValidationEnabled()) { - if (supported_features12.bufferDeviceAddress != VK_TRUE) { - EXIT("--gpu-assisted-validation requires bufferDeviceAddress, unsupported by this " - "device\n"); - } - features12.bufferDeviceAddress = VK_TRUE; - } + features12.bufferDeviceAddress = VK_TRUE; vk::PhysicalDeviceFeatures device_features {}; device_features.fragmentStoresAndAtomics = VK_TRUE; @@ -625,12 +621,9 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V device_features.shaderClipDistance = VK_TRUE; device_features.shaderCullDistance = VK_TRUE; device_features.largePoints = VK_TRUE; + device_features.vertexPipelineStoresAndAtomics = VK_TRUE; graphics.sample_rate_shading_enabled = true; - device_features.vertexPipelineStoresAndAtomics = - supported_features2.features.vertexPipelineStoresAndAtomics; - if (Config::GpuAssistedValidationEnabled()) { - device_features.shaderInt64 = supported_features2.features.shaderInt64; - } + device_features.shaderInt64 = VK_TRUE; vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {}; robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT; diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 7f80291ca..fff5bc0db 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -665,7 +665,6 @@ bool TryRecompile(std::span code, const CompileOptions& options, runtime.read_memory = ReadZeroMemory; } runtime.userdata = options.read_memory_data; - runtime.flat_memory_base = options.flat_memory_base; if (!IR::MaterializeResources(ir, runtime, resources, error)) { return false; } diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index fc4975143..755c27842 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -6,7 +6,6 @@ #include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" -#include #include #include @@ -20,7 +19,6 @@ struct CompileOptions { uint32_t scratch_dwords = 0; uint64_t shader_hash = 0; uint64_t shader_base = 0; - std::optional flat_memory_base; uint32_t push_constant_offset = 0; bool dump_ir = true; bool early_dump = false; diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 9aaf3975e..d8ab138f2 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -70,8 +70,9 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { if (uses_gds) { Expect(Kind::Gds); } - if (!program.info.addresses.empty()) { - Expect(Kind::AddressMemory, Dense(program.info.addresses.size())); + if (program.info.uses_dma) { + Expect(Kind::BdaPagetable); + Expect(Kind::FaultBuffer); } const bool uses_flattened_runtime = !program.srt_reads.empty() || @@ -104,8 +105,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { program.bindings.push_constant_offset + program.bindings.push_constant_size > IR::NativePushConstantSize || program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() || - program.bindings.memory_offset_count != - program.info.buffers.size() + program.info.addresses.size() || + program.bindings.memory_offset_count != program.info.buffers.size() || (!has_user_storage && program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) || (has_user_storage && program.bindings.push_constant_size != 0) || @@ -146,8 +146,8 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { if (planning_only_handle(inst)) { break; } - if (dense >= program.info.addresses.size()) { - return Fail(error, "typed address handle has an invalid dense resource"); + if (inst.NumArgs() != 2 || !program.info.uses_dma) { + return Fail(error, "typed address handle has invalid DMA metadata"); } break; case IR::ValueOpcode::GetScratchResource: @@ -188,7 +188,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { const auto MarkBallot = [&] { requirements.subgroup_ballot = true; }; for (const auto* block: program.blocks) { for (const auto& inst: *block) { - if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) { + const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access; + if (address_access != IR::AddressAccess::None) { const auto memory_index = inst.Flags().index; if (memory_index >= program.memory_info.size()) { return Fail(error, "address operation has invalid memory metadata"); @@ -198,6 +199,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { return Fail(error, "scratch operation has no per-thread storage"); } requirements.function_scratch = true; + } else if (address_access == IR::AddressAccess::Write) { + return Fail(error, "writable FLAT/GLOBAL addresses require GPU ownership tracking"); } } if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 03f02f813..7a41c2042 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -36,8 +36,10 @@ enum : uint32_t { ExecutionModeSignedZeroInfNanPreserve = 4461, ExecutionModeDerivativeGroupQuadsKHR = 5289, AddressingModelLogical = 0, + AddressingModelPhysicalStorageBuffer64 = 5348, MemoryModelGLSL450 = 1, CapabilityShader = 1, + CapabilityInt64 = 11, CapabilityImageGatherExtended = 25, CapabilityClipDistance = 32, CapabilityCullDistance = 33, @@ -52,6 +54,7 @@ enum : uint32_t { CapabilityShaderViewportIndexLayerEXT = 5254, CapabilityFragmentBarycentricKHR = 5284, CapabilityComputeDerivativeGroupQuadsKHR = 5288, + CapabilityPhysicalStorageBufferAddresses = 5347, StorageClassUniformConstant = 0, StorageClassInput = 1, StorageClassOutput = 3, @@ -60,6 +63,7 @@ enum : uint32_t { StorageClassPushConstant = 9, StorageClassImage = 11, StorageClassStorageBuffer = 12, + StorageClassPhysicalStorageBuffer = 5349, FunctionControlNone = 0, SelectionControlNone = 0, LoopControlNone = 0, @@ -148,7 +152,9 @@ enum : uint32_t { OpConstantComposite = 44, OpUndef = 1, OpFunction = 54, + OpFunctionParameter = 55, OpFunctionEnd = 56, + OpFunctionCall = 57, OpVariable = 59, OpImageTexelPointer = 60, OpLoad = 61, @@ -178,6 +184,8 @@ enum : uint32_t { OpConvertFToS = 110, OpConvertSToF = 111, OpConvertUToF = 112, + OpUConvert = 113, + OpConvertUToPtr = 120, OpBitcast = 124, OpSNegate = 126, OpFNegate = 127, @@ -256,11 +264,16 @@ enum : uint32_t { OpSwitch = 251, OpKill = 252, OpReturn = 253, + OpReturnValue = 254, OpGroupNonUniformBallot = 339, OpGroupNonUniformBallotFindLSB = 343, OpGroupNonUniformShuffle = 345, }; +enum : uint32_t { + MemoryAccessAlignedMask = 0x2, +}; + enum : uint32_t { GlslRoundEven = 2, GlslTrunc = 3, @@ -345,9 +358,10 @@ struct EmitterState { ShaderType stage = ShaderType::Unknown; uint32_t wave_size = 64; uint32_t storage_buffer_variable = 0; - std::array - memory_byte_offsets {}; - uint32_t address_memory_variable = 0; + std::array memory_byte_offsets {}; + uint32_t bda_pagetable_variable = 0; + uint32_t fault_buffer_variable = 0; + uint32_t bda_pointer_function = 0; uint32_t gds_variable = 0; uint32_t gds_length = 0; uint32_t push_constant_variable = 0; @@ -384,6 +398,7 @@ uint32_t TypeBool(EmitterState& state); uint32_t TypeBoolVector(EmitterState& state, uint32_t components); uint32_t TypeU32(EmitterState& state); uint32_t TypeU64(EmitterState& state); +uint32_t TypeDeviceAddress(EmitterState& state); uint32_t TypeU32Pair(EmitterState& state); uint32_t TypeI32(EmitterState& state); uint32_t TypeI32Pair(EmitterState& state); @@ -397,6 +412,8 @@ uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t point uint32_t TypeFunction(EmitterState& state); uint32_t TypeStorageBufferPointer(EmitterState& state); uint32_t TypeStorageBufferElementPointer(EmitterState& state); +uint32_t TypeDeviceAddressStoragePointer(EmitterState& state); +uint32_t TypePhysicalU32Pointer(EmitterState& state); uint32_t TypePushConstantElementPointer(EmitterState& state); uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords); uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class); @@ -849,6 +866,8 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst); bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error); +void DefineGetBdaPointer(EmitterState& state); + // These templates accept local lambdas from several emitter translation units. template void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 10baff4e1..6085c03b6 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1,5 +1,7 @@ #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include "graphics/host_gpu/renderer/cache/bufferCache.h" + #include namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { @@ -95,60 +97,146 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad return result; } -uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, - uint32_t low, uint32_t high) { +uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low, + uint32_t high) { auto& state = ctx.state; auto immediate = static_cast(mem.offset); - if (mem.kind == IR::ResourceKind::ScalarAddress) { - immediate = static_cast(static_cast(immediate) & ~3u); - low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u)); - } const auto immediate_low = ConstantU32(state, static_cast(immediate)); const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u); - if (mem.kind == IR::ResourceKind::Scratch) { - low = AddU64Low(state, low, high, immediate_low, immediate_high, high); - const auto valid = Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0)); - return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX)); - } - const auto base = state.program.info.addresses[mem.resource].specialized_base; - if (mem.kind == IR::ResourceKind::Flat || - (mem.kind == IR::ResourceKind::Global && - state.program.info.addresses[mem.resource].unbased)) { - low = AddU64Low(state, low, high, immediate_low, immediate_high, high); - const auto base_low = ConstantU32(state, static_cast(base)); - const auto base_high = ConstantU32(state, static_cast(base >> 32u)); - const auto relative = Binary(state, OpISub, TypeU32(state), low, base_low); - const auto borrow = Binary(state, OpULessThan, TypeBool(state), low, base_low); - const auto relative_high = Binary( - state, OpISub, TypeU32(state), Binary(state, OpISub, TypeU32(state), high, base_high), - Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0))); - const auto valid = - Binary(state, OpIEqual, TypeBool(state), relative_high, ConstantU32(state, 0)); - return Select(state, TypeU32(state), valid, relative, ConstantU32(state, UINT32_MAX)); - } - const auto initial = base + static_cast(static_cast(immediate)); - auto relative_high = ConstantU32(state, static_cast(initial >> 32u)); - const auto relative = AddU64Low(state, ConstantU32(state, static_cast(initial)), - relative_high, low, ConstantU32(state, 0), relative_high); + low = AddU64Low(state, low, high, immediate_low, immediate_high, high); const auto valid = - Binary(state, OpIEqual, TypeBool(state), relative_high, ConstantU32(state, 0)); - return Select(state, TypeU32(state), valid, relative, ConstantU32(state, UINT32_MAX)); + Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0)); + return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX)); } -uint32_t AddAddressMemoryOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, - uint32_t address) { - auto& state = ctx.state; - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); - const auto offset = - state.memory_byte_offsets[state.program.info.buffers.size() + binding.array_index]; - const auto corrected = Binary(state, OpIAdd, TypeU32(state), address, offset); - const auto not_invalid = - Binary(state, OpINotEqual, TypeBool(state), address, ConstantU32(state, UINT32_MAX)); - const auto no_overflow = - Binary(state, OpUGreaterThanEqual, TypeBool(state), corrected, address); - return Select(state, TypeU32(state), AndCondition(state, not_invalid, no_overflow), corrected, - ConstantU32(state, UINT32_MAX)); +uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) { + return state.builder.Constant(OpConstant, TypeDeviceAddress(state), + {static_cast(value), + static_cast(value >> 32u)}); +} + +uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) { + const auto low64 = Unary(state, OpUConvert, TypeDeviceAddress(state), low); + const auto high64 = Binary(state, OpShiftLeftLogical, TypeDeviceAddress(state), + Unary(state, OpUConvert, TypeDeviceAddress(state), high), + ConstantDeviceAddress(state, 32)); + return Binary(state, OpBitwiseOr, TypeDeviceAddress(state), low64, high64); +} + +uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + auto& state = ctx.state; + auto low = ctx.Arg(inst, 1); + if (mem.kind == IR::ResourceKind::ScalarAddress) { + low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u)); + } + uint32_t address = 0; + if (mem.address_is_full) { + address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2)); + } else { + const auto* handle = inst.Arg(0).Resolve().TryInstruction(); + if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetAddressResource || + handle->NumArgs() != 2) { + ctx.Fail(inst, "has no address base pair"); + return ConstantDeviceAddress(state, 0); + } + const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1)); + address = Binary(state, OpIAdd, TypeDeviceAddress(state), base, + Unary(state, OpUConvert, TypeDeviceAddress(state), low)); + } + auto immediate = static_cast(mem.offset); + if (mem.kind == IR::ResourceKind::ScalarAddress) { + immediate = static_cast(static_cast(immediate) & ~3u); + } + return immediate == 0 + ? address + : Binary(state, OpIAdd, TypeDeviceAddress(state), address, + ConstantDeviceAddress(state, + static_cast(static_cast(immediate)))); +} + +uint32_t FaultElementPointer(EmitterState& state, uint32_t index) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, + state.fault_buffer_variable, ConstantU32(state, 0), index}); + return pointer; +} + +void RecordBdaFault(EmitterState& state, uint32_t page) { + const auto word = Binary(state, OpShiftRightLogical, TypeU32(state), page, + ConstantU32(state, 5)); + const auto bit = Binary( + state, OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1), + Binary(state, OpBitwiseAnd, TypeU32(state), page, ConstantU32(state, 31))); + const auto pointer = FaultElementPointer(state, word); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); + state.builder.AddFunction( + {OpStore, pointer, Binary(state, OpBitwiseOr, TypeU32(state), value, bit)}); +} + +uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) { + auto& state = ctx.state; + const auto result = state.builder.AllocateId(); + state.builder.AddFunction( + {OpFunctionCall, TypeDeviceAddress(state), result, state.bda_pointer_function, address}); + return result; +} + +uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) { + auto& state = ctx.state; + const auto bda = GetBdaPointer(ctx, address); + const auto present = Binary(state, OpINotEqual, TypeBool(state), bda, + ConstantDeviceAddress(state, 0)); + return EmitValueOrZeroIfCondition(state, present, [&]() { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction( + {OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, bda}); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpLoad, TypeU32(state), value, pointer, MemoryAccessAlignedMask, sizeof(uint32_t)}); + return value; + }); +} + +uint32_t LoadBda(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + uint32_t bits) { + auto& state = ctx.state; + const auto address = GuestAddress(ctx, inst, mem); + const auto active = ctx.Arg(inst, inst.NumArgs() - 1); + return EmitValueOrZeroIfCondition(state, active, [&]() { + const auto aligned = Binary(state, OpBitwiseAnd, TypeDeviceAddress(state), address, + ConstantDeviceAddress(state, ~uint64_t {3})); + const auto first = LoadBdaDword(ctx, aligned); + const auto byte = Binary(state, OpBitwiseAnd, TypeU32(state), + Unary(state, OpUConvert, TypeU32(state), address), + ConstantU32(state, 3)); + const auto crosses = bits == 8u + ? ConstantBool(state, false) + : Binary(state, bits == 16u ? OpUGreaterThan : OpINotEqual, + TypeBool(state), byte, + ConstantU32(state, bits == 16u ? 2u : 0u)); + const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() { + return LoadBdaDword( + ctx, Binary(state, OpIAdd, TypeDeviceAddress(state), aligned, + ConstantDeviceAddress(state, sizeof(uint32_t)))); + }); + const auto shift = Binary(state, OpShiftLeftLogical, TypeU32(state), byte, + ConstantU32(state, 3)); + const auto upper_shift = Binary( + state, OpShiftLeftLogical, TypeU32(state), + Binary(state, OpBitwiseAnd, TypeU32(state), + Binary(state, OpISub, TypeU32(state), ConstantU32(state, 4), byte), + ConstantU32(state, 3)), + ConstantU32(state, 3)); + const auto merged = Binary( + state, OpBitwiseOr, TypeU32(state), + Binary(state, OpShiftRightLogical, TypeU32(state), first, shift), + Binary(state, OpShiftLeftLogical, TypeU32(state), second, upper_shift)); + return bits == 32u + ? merged + : Binary(state, OpBitwiseAnd, TypeU32(state), merged, + ConstantU32(state, bits == 8u ? 0xffu : 0xffffu)); + }); } uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { @@ -163,11 +251,10 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0), ConstantU32(ctx.state, mem.offset)); } - const auto address = AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2)); - if (mem.kind == IR::ResourceKind::Scratch) { - return address; + if (mem.kind != IR::ResourceKind::Scratch) { + EXIT("physical address memory must use the BDA emitter\n"); } - return AddAddressMemoryOffset(ctx, mem, address); + return ScratchByteAddress(ctx, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2)); } uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { @@ -923,6 +1010,58 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo } // namespace +void DefineGetBdaPointer(EmitterState& state) { + if (!state.program.info.uses_dma) { + return; + } + const auto type = TypeDeviceAddress(state); + const auto function_type = state.builder.Type(OpTypeFunction, {type, type}); + state.bda_pointer_function = state.builder.AllocateId(); + const auto address = state.builder.AllocateId(); + const auto entry_label = state.builder.AllocateId(); + state.builder.AddName(state.bda_pointer_function, "get_bda_pointer"); + state.builder.AddFunction( + {OpFunction, type, state.bda_pointer_function, FunctionControlNone, function_type}); + state.builder.AddFunction({OpFunctionParameter, type, address}); + EmitLabel(state, entry_label); + + const auto page64 = Binary( + state, OpShiftRightLogical, type, address, + ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS)); + const auto page = Unary(state, OpUConvert, TypeU32(state), page64); + const auto entry_pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, TypeDeviceAddressStoragePointer(state), entry_pointer, + state.bda_pagetable_variable, ConstantU32(state, 0), page}); + const auto base = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, type, base, entry_pointer}); + const auto missing = + Binary(state, OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0)); + const auto fault_label = state.builder.AllocateId(); + const auto available_label = state.builder.AllocateId(); + const auto merge_label = state.builder.AllocateId(); + state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); + state.builder.AddFunction( + {OpBranchConditional, missing, fault_label, available_label}); + + EmitLabel(state, fault_label); + RecordBdaFault(state, page); + state.builder.AddFunction({OpBranch, merge_label}); + + EmitLabel(state, available_label); + const auto offset = Binary( + state, OpBitwiseAnd, type, address, + ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1)); + const auto available = Binary(state, OpIAdd, type, base, offset); + state.builder.AddFunction({OpBranch, merge_label}); + + EmitLabel(state, merge_label); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpPhi, type, result, ConstantDeviceAddress(state, 0), fault_label, + available, available_label}); + state.builder.AddFunction({OpReturnValue, result}); + state.builder.AddFunction({OpFunctionEnd}); +} + bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; const auto op = inst.GetOpcode(); @@ -981,6 +1120,10 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { } const auto address_info = IR::AddressOpcodeInfoOf(op); const bool load_address = address_info.access == IR::AddressAccess::Read; + if (load_address && ctx.Memory(inst).kind != IR::ResourceKind::Scratch) { + ctx.Define(inst, LoadBda(ctx, inst, ctx.Memory(inst), address_info.data_bits)); + return true; + } const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 || op == IR::ValueOpcode::LoadBufferU16 || op == IR::ValueOpcode::LoadBufferU32; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 1fee95ef0..a877d59db 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -108,20 +108,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: return access; case IR::ResourceKind::ScalarAddress: case IR::ResourceKind::Flat: - case IR::ResourceKind::Global: { - if (state.address_memory_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - mem.resource, - "address memory binding was not emitted"); - } - const auto binding = ResourceForDescriptor( - state, IR::DescriptorBindingKind::AddressMemory, mem.resource); - access.object_pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), - access.object_pointer, state.address_memory_variable, - ConstantU32(state, binding.array_index)}); - break; - } + case IR::ResourceKind::Global: + EXIT("physical address memory must use the BDA emitter\n"); case IR::ResourceKind::ScalarBuffer: case IR::ResourceKind::Buffer: { if (state.storage_buffer_variable == 0) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 263179d88..342d03b8f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -28,6 +28,10 @@ uint32_t TypeU64(EmitterState& state) { return TypeU32Vector(state, 2); } +uint32_t TypeDeviceAddress(EmitterState& state) { + return state.builder.Type(OpTypeInt, {64, 0}); +} + uint32_t TypeU32Pair(EmitterState& state) { const auto element = TypeU32(state); return state.builder.Type(OpTypeStruct, {element, element}); @@ -90,6 +94,14 @@ uint32_t TypeStorageBufferElementPointer(EmitterState& state) { return TypePointer(state, StorageClassStorageBuffer, TypeU32(state)); } +uint32_t TypeDeviceAddressStoragePointer(EmitterState& state) { + return TypePointer(state, StorageClassStorageBuffer, TypeDeviceAddress(state)); +} + +uint32_t TypePhysicalU32Pointer(EmitterState& state) { + return TypePointer(state, StorageClassPhysicalStorageBuffer, TypeU32(state)); +} + uint32_t TypePushConstantElementPointer(EmitterState& state) { return TypePointer(state, StorageClassPushConstant, TypeU32(state)); } @@ -130,6 +142,15 @@ uint32_t SampleMaskArrayType(EmitterState& state) { return state.builder.Type(OpTypeArray, {TypeI32(state), ConstantU32(state, 1)}); } +uint32_t BdaPagetableType(EmitterState& state) { + const auto array = state.builder.DecoratedType( + OpTypeRuntimeArray, {TypeDeviceAddress(state)}, + {{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}}); + return state.builder.DecoratedType( + OpTypeStruct, {array}, + {{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}}); +} + uint32_t F32ArrayType(EmitterState& state, uint32_t count) { return state.builder.Type(OpTypeArray, {TypeF32(state), ConstantU32(state, count)}); } @@ -143,13 +164,14 @@ void DefineDescriptorVariables(EmitterState& state) { state.storage_buffer_variable = state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer); } - if (DescriptorBinding(state, IR::DescriptorBindingKind::AddressMemory) != nullptr) { - const auto count = - ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::AddressMemory)); - const auto array_type = state.builder.Type(OpTypeArray, {StorageBufferType(state), count}); - const auto pointer_type = TypePointer(state, StorageClassStorageBuffer, array_type); - state.address_memory_variable = - state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer); + if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) { + state.bda_pagetable_variable = state.builder.DefineGlobalVariable( + TypePointer(state, StorageClassStorageBuffer, BdaPagetableType(state)), + StorageClassStorageBuffer); + } + if (DescriptorBinding(state, IR::DescriptorBindingKind::FaultBuffer) != nullptr) { + state.fault_buffer_variable = state.builder.DefineGlobalVariable( + TypeStorageBufferPointer(state), StorageClassStorageBuffer); } if (state.program.bindings.push_constant_size != 0) { const auto pointer_type = @@ -564,9 +586,13 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) { if (state.storage_buffer_variable != 0) { Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers); } - if (state.address_memory_variable != 0) { - Decorate(state.address_memory_variable, "address_memory", - IR::DescriptorBindingKind::AddressMemory); + if (state.bda_pagetable_variable != 0) { + Decorate(state.bda_pagetable_variable, "bda_pagetable", + IR::DescriptorBindingKind::BdaPagetable); + } + if (state.fault_buffer_variable != 0) { + Decorate(state.fault_buffer_variable, "fault_buffer", + IR::DescriptorBindingKind::FaultBuffer); } constexpr const char* SampledNames[] = {"sampled_1d", "sampled_1d_array", @@ -644,6 +670,11 @@ void DefineModule(EmitterState& state) { state.builder.RequireCapability(CapabilityShader); state.builder.RequireCapability(CapabilitySignedZeroInfNanPreserve); + if (state.program.info.uses_dma) { + state.builder.RequireCapability(CapabilityInt64); + state.builder.RequireCapability(CapabilityPhysicalStorageBufferAddresses); + state.builder.RequireExtension("SPV_KHR_physical_storage_buffer"); + } if (state.clip_distance_variable != 0) { state.builder.RequireCapability(CapabilityClipDistance); } @@ -682,7 +713,10 @@ void DefineModule(EmitterState& state) { state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric"); } state.builder.RequireExtension("SPV_KHR_float_controls"); - state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); + state.builder.AddMemoryModel( + {state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64 + : AddressingModelLogical, + MemoryModelGLSL450}); state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main", state.interface_variables); // GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp index dd182acb8..ea134abaa 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp @@ -566,6 +566,7 @@ bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* e SetError(error, ctx.error.c_str()); return false; } + DefineGetBdaPointer(state); for (const auto* block: program.blocks) { if (std::ranges::any_of(*block, [](const IR::Inst& inst) { return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 || diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index 487202aed..ed9b66fe8 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -136,12 +136,10 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) { memory.offen = decoded.offen; memory.resource = ResourceIndexFromOperand(decoded.src1); memory.sampler = ResourceIndexFromOperand(decoded.src2); - if (memory.kind == ResourceKind::ScalarAddress) { - memory.resource = RawScalarLoadBase(decoded.src0); - } else if (memory.kind == ResourceKind::ScalarBuffer) { + if (memory.kind == ResourceKind::ScalarBuffer) { memory.resource = ResourceIndexFromOperand(decoded.src0); - } else if (memory.kind == ResourceKind::Lds || memory.kind == ResourceKind::Gds || - memory.kind == ResourceKind::Scratch) { + } else if (IsAddressResourceKind(memory.kind) || memory.kind == ResourceKind::Lds || + memory.kind == ResourceKind::Gds) { memory.resource = 0; memory.sampler = 0; } @@ -421,7 +419,7 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) { bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) { const auto memory = MemoryInfoFromDecoded(inst); const auto resource = - raw ? GetScalarAddressResource(memory.resource) : GetBufferResource(memory); + raw ? GetScalarAddressResource(RawScalarLoadBase(inst.src0)) : GetBufferResource(memory); const auto offset = ReadU32(inst.src1); std::array loaded {}; for (uint32_t component = 0; component < memory.data_dwords; component++) { diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index fb6f63968..a7b0546a3 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -176,20 +176,6 @@ struct SampledResourcePair { bool operator==(const SampledResourcePair& other) const = default; }; -struct AddressResource { - uint32_t source = UINT32_MAX; - uint32_t first_use_pc = 0; - ResourceKind kind = ResourceKind::Flat; - int32_t min_offset = 0; - uint64_t specialized_base = 0; - bool unbased = true; - bool read = false; - bool written = false; - bool atomic = false; - - bool operator==(const AddressResource& other) const = default; -}; - enum class StageInputKind { VertexIndex, InstanceIndex, @@ -321,7 +307,8 @@ enum class DescriptorBindingKind { StorageAtomic3D, Samplers, Gds, - AddressMemory, + BdaPagetable, + FaultBuffer, FlattenedSrt, UserData, Count, @@ -431,13 +418,11 @@ struct BindingLayout { struct ShaderInfo { static constexpr uint32_t MaxBuffers = 32; - static constexpr uint32_t MaxAddresses = 32; static constexpr uint32_t MaxImages = 32; static constexpr uint32_t MaxSamplers = 32; static constexpr uint32_t MaxSampledPairs = 64; std::vector buffers; - std::vector addresses; std::vector images; std::vector samplers; std::vector sampled_pairs; @@ -446,6 +431,7 @@ struct ShaderInfo { std::array vertex_fetch_components {}; int32_t vertex_offset_sgpr = -1; bool has_bitwise_xor = false; + bool uses_dma = false; bool operator==(const ShaderInfo& other) const = default; }; diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index d0f2a2192..e3d8d9475 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -93,8 +93,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri return false; } next.memory_offset_dword = static_cast(next.user_data_registers.size()); - next.memory_offset_count = - static_cast(program.info.buffers.size() + program.info.addresses.size()); + next.memory_offset_count = static_cast(program.info.buffers.size()); if (!program.info.buffers.empty()) { std::vector resources(program.info.buffers.size()); @@ -157,12 +156,9 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri if (uses_gds) { AddBinding(next, DescriptorBindingKind::Gds); } - if (!program.info.addresses.empty()) { - std::vector resources(program.info.addresses.size()); - for (uint32_t i = 0; i < resources.size(); i++) { - resources[i] = i; - } - AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources)); + if (program.info.uses_dma) { + AddBinding(next, DescriptorBindingKind::BdaPagetable); + AddBinding(next, DescriptorBindingKind::FaultBuffer); } const bool uses_flattened_runtime = !program.srt_reads.empty() || diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 7248281d4..57358f69a 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -267,13 +267,6 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, return true; } -uint64_t AddressSpecialization(const AddressResource& resource, - const ResourceSnapshot::Address& snapshot) { - return resource.kind == ResourceKind::Flat || resource.unbased - ? snapshot.binding_base - : snapshot.guest_base - snapshot.binding_base; -} - size_t FlattenedRuntimeDwords(const Program& program) { size_t size = program.srt_reads.size(); for (uint32_t resource = 0; resource < program.info.images.size(); resource++) { @@ -298,8 +291,7 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn } if (snapshot.buffers.size() != program.info.buffers.size() || snapshot.images.size() != program.info.images.size() || - snapshot.samplers.size() != program.info.samplers.size() || - snapshot.addresses.size() != program.info.addresses.size()) { + snapshot.samplers.size() != program.info.samplers.size()) { if (error != nullptr) { *error = "resource snapshot does not match dense shader topology"; } @@ -343,14 +335,6 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn } return true; }; - for (uint32_t i = 0; i < snapshot.addresses.size(); i++) { - if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) { - if (error != nullptr) { - *error = fmt::format("address resource {} binds above its guest base", i); - } - return false; - } - } std::unordered_set indirect_resources; for (const auto& table: snapshot.indirect_images) { const auto* source = table.resource < program.info.images.size() @@ -526,15 +510,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh } } } - for (uint32_t i = 0; i < program.info.addresses.size(); i++) { - if (program.info.addresses[i].specialized_base != - AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) { - if (error != nullptr) { - *error = fmt::format("address resource {} no longer matches specialization", i); - } - return false; - } - } return true; } @@ -550,7 +525,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, std::vector requests; std::vector clean_flat_slots(program.srt_reads.size()); requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u + - program.info.samplers.size() + program.info.addresses.size()); + program.info.samplers.size()); for (const auto& buffer: program.info.buffers) { requests.push_back({buffer.source, buffer.first_use_pc}); } @@ -576,12 +551,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, for (const auto& sampler: program.info.samplers) { requests.push_back({sampler.source, sampler.first_use_pc}); } - for (const auto& address: program.info.addresses) { - if (!address.unbased) { - requests.push_back({address.source, address.first_use_pc}); - } - } - std::vector values; std::vector flattened_srt; if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, clean_flat_slots, @@ -686,37 +655,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, return false; } next.samplers.assign(cursor, cursor + program.info.samplers.size()); - cursor += program.info.samplers.size(); - for (const auto& address: program.info.addresses) { - if (!address.unbased) { - const auto value = *cursor++; - auto base = (static_cast(value.dwords[0]) | - static_cast(value.dwords[1]) << 32u) & - AddressMask; - if (address.kind == ResourceKind::ScalarAddress) { - base &= ~uint64_t {3}; - } - const auto before = static_cast(-static_cast(address.min_offset)); - uint64_t binding_base = 0; - if (address.kind == ResourceKind::Flat) { - binding_base = base & ~(FlatAddressWindowSize - 1u); - } else if (base >= before) { - binding_base = base - before; - } - next.addresses.push_back({base, binding_base}); - } else { - if (!runtime.flat_memory_base.has_value()) { - if (error != nullptr) { - *error = fmt::format("unbased {} address at pc 0x{:08x} requires runtime " - "guest-address translation", - address.kind == ResourceKind::Flat ? "FLAT" : "global", - address.first_use_pc); - } - return false; - } - next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base}); - } - } next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end()); if (!ValidateResourceSnapshot(program, next, error)) { return false; @@ -791,10 +729,6 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri next.buffers[i].descriptor_format = descriptor.Format(); next.buffers[i].descriptor_swizzle = descriptor.DstSelXYZW(); } - for (uint32_t i = 0; i < next.addresses.size(); i++) { - next.addresses[i].specialized_base = - AddressSpecialization(next.addresses[i], next_snapshot.addresses[i]); - } for (uint32_t i = 0; i < next.images.size(); i++) { const auto& descriptor = next_snapshot.images[i]; auto& image = next.images[i]; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index 8b89968f3..a6772ab1a 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -5,15 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -constexpr uint64_t FlatAddressWindowSize = 0x10000; - struct ResourceSnapshot { - struct Address { - uint64_t guest_base = 0; - uint64_t binding_base = 0; - - bool operator==(const Address& other) const = default; - }; struct IndirectImage { uint32_t resource = 0; uint32_t capacity = 0; @@ -25,7 +17,6 @@ struct ResourceSnapshot { std::vector buffers; std::vector images; std::vector samplers; - std::vector
addresses; std::vector flattened_srt; std::vector user_data; std::vector indirect_images; diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 390c2b3c4..d58c9f6e4 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -96,10 +95,10 @@ class Tracker { public: explicit Tracker(Program& program): m_program(program), m_info(program.info) { m_info.buffers.clear(); - m_info.addresses.clear(); m_info.images.clear(); m_info.samplers.clear(); m_info.sampled_pairs.clear(); + m_info.uses_dma = false; } bool Run(std::string* error) { @@ -180,90 +179,6 @@ private: m_program.shader_hash, StageName(m_program.stage), pc, reason)); } - struct AddressPart { - Value value; - bool rooted = false; - }; - - AddressPart FindAddressPart(Value value, Value active, - std::unordered_set& visiting) const { - value = value.Resolve(); - if (value.IsImmediate()) { - return {value, false}; - } - const auto* inst = value.TryInstruction(); - if (inst == nullptr || !visiting.insert(inst).second) { - return {}; - } - const auto finish = [&](AddressPart part) { - visiting.erase(inst); - return part; - }; - std::string reason; - if (ValidateRuntimeValue(m_program, value, reason)) { - return finish({value, true}); - } - const auto merge = [&](AddressPart left, AddressPart right, bool require_both) { - if (left.value.IsEmpty() || right.value.IsEmpty()) { - return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left); - } - return EquivalentValue(m_program, left.value, right.value) - ? AddressPart {left.value, left.rooted || right.rooted} - : AddressPart {}; - }; - switch (inst->GetOpcode()) { - case ValueOpcode::SelectU32: - if (EquivalentValue(m_program, inst->Arg(0), active)) { - return finish(FindAddressPart(inst->Arg(1), active, visiting)); - } - return finish(merge(FindAddressPart(inst->Arg(1), active, visiting), - FindAddressPart(inst->Arg(2), active, visiting), true)); - case ValueOpcode::Phi: { - const auto invariant = ResolveInvariantPhi(m_program, value); - return finish(invariant.IsEmpty() ? AddressPart {} - : FindAddressPart(invariant, active, visiting)); - } - case ValueOpcode::IAdd32: - case ValueOpcode::ISub32: { - const auto left = FindAddressPart(inst->Arg(0), active, visiting); - const auto right = FindAddressPart(inst->Arg(1), active, visiting); - if (left.rooted == right.rooted || - (inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) { - return finish({}); - } - return finish(left.rooted ? left : right); - } - case ValueOpcode::CompositeExtractU32x2: { - const auto* source = inst->Arg(0).ResolveInstruction(); - if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) { - return finish({}); - } - const auto left = FindAddressPart(source->Arg(0), active, visiting); - const auto right = FindAddressPart(source->Arg(1), active, visiting); - if (left.rooted == right.rooted) { - return finish({}); - } - return finish(left.rooted ? left : right); - } - default: return finish({}); - } - } - - bool MakeFlatAddressSource(const Inst& handle, Value active, - DescriptorSource& descriptor) const { - std::unordered_set visiting; - auto low = FindAddressPart(handle.Arg(0), active, visiting); - visiting.clear(); - auto high = FindAddressPart(handle.Arg(1), active, visiting); - if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) { - return false; - } - descriptor.dword_count = 2; - descriptor.dwords[0] = low.value; - descriptor.dwords[1] = high.value; - return true; - } - bool MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust, DescriptorSource& descriptor, uint32_t pc, std::string* error) const { if (handle.NumArgs() != width) { @@ -577,42 +492,14 @@ private: return true; } - bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle, - uint32_t& source, bool& unbased, std::string* error) { - handle = value.Resolve().TryInstruction(); + bool ValidateAddressHandle(Value value, uint32_t pc, std::string* error) const { + const auto* handle = value.Resolve().TryInstruction(); if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) { return Fail(pc, error, "address operation requires GetAddressResource"); } if (handle->NumArgs() != 2) { return Fail(pc, error, "GetAddressResource must have two address dwords"); } - DescriptorSource descriptor; - const auto& memory = m_program.memory_info[memory_inst.Flags().index]; - if (memory.address_is_full) { - const auto active = memory_inst.Arg(memory_inst.NumArgs() - 1u); - if (memory.kind != ResourceKind::Flat || - !MakeFlatAddressSource(*handle, active, descriptor)) { - unbased = true; - source = UINT32_MAX; - return true; - } - } else if (!MakeSource(*handle, 2, false, false, descriptor, pc, error)) { - return false; - } - std::string reason; - uint32_t bad_dword = 0; - if (!ValidateSource(descriptor, reason, bad_dword)) { - if (memory.address_is_full) { - unbased = true; - source = UINT32_MAX; - return true; - } - return Fail( - pc, error, - fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason)); - } - unbased = false; - source = InternSource(descriptor); return true; } @@ -706,39 +593,6 @@ private: return static_cast(m_info.samplers.size() - 1); } - uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory, - AddressAccess access, uint32_t pc) { - auto immediate = static_cast(memory.offset); - if (memory.kind == ResourceKind::ScalarAddress) { - immediate = static_cast(static_cast(immediate) & ~3u); - } - const auto min_offset = unbased ? 0 : std::min(immediate, 0); - for (uint32_t i = 0; i < m_info.addresses.size(); i++) { - auto& address = m_info.addresses[i]; - if (address.source == source && address.unbased == unbased && - address.kind == memory.kind) { - address.first_use_pc = std::min(address.first_use_pc, pc); - address.min_offset = std::min(address.min_offset, min_offset); - address.read = address.read || access == AddressAccess::Read; - address.written = address.written || access == AddressAccess::Write; - return i; - } - } - if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) { - return UINT32_MAX; - } - AddressResource address; - address.source = source; - address.first_use_pc = pc; - address.kind = memory.kind; - address.min_offset = min_offset; - address.unbased = unbased; - address.read = access == AddressAccess::Read; - address.written = access == AddressAccess::Write; - m_info.addresses.push_back(address); - return static_cast(m_info.addresses.size() - 1); - } - bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) { for (auto& pair: m_info.sampled_pairs) { if (pair.image == image && pair.sampler == sampler) { @@ -839,16 +693,11 @@ private: } return true; } - bool unbased = false; - if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) { + if (!ValidateAddressHandle(inst.Arg(0), flags.pc, error)) { return false; } - resource = AddAddress(source, unbased, memory, address_info.access, flags.pc); - if (resource == UINT32_MAX) { - return Fail(flags.pc, error, "address resource limit exceeded"); - } - return AddHandlePatch(handle, resource, flags.pc, error) && - AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error); + m_info.uses_dma = true; + return true; } if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) { diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index f21223ce5..1ecdf010c 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -3,7 +3,6 @@ #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include #include namespace Libs::Graphics::ShaderRecompiler::IR { @@ -17,7 +16,6 @@ struct SrtRuntime { uint64_t shader_base = 0; SrtMemoryReader read_memory = nullptr; void* userdata = nullptr; - std::optional flat_memory_base; SrtMemoryReader read_specialization_memory = nullptr; }; diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index eefb2d822..b7b1df848 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -994,10 +994,10 @@ static void LogShaderProgramCacheHit(const char* stage, uint64_t shader_hash, ui static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program) { std::string ret = fmt::format( - "push={} groups={} user={} buffers={} images={} samplers={} addresses={} srt={}", + "push={} groups={} user={} buffers={} images={} samplers={} dma={} srt={}", program.bindings.push_constant_size, program.bindings.descriptors.size(), program.bindings.user_data_registers.size(), program.info.buffers.size(), - program.info.images.size(), program.info.samplers.size(), program.info.addresses.size(), + program.info.images.size(), program.info.samplers.size(), program.info.uses_dma, program.srt_reads.size()); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { const auto& buffer = program.info.buffers[i]; @@ -1014,9 +1014,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog static_cast(image.conversion_format), image.shader_swizzle, image.indirect_root, image.indirect_mapping_offset, image.indirect_mapping_capacity); } - for (uint32_t i = 0; i < program.info.addresses.size(); i++) { - ret += fmt::format(" a{}[base=0x{:x}]", i, program.info.addresses[i].specialized_base); - } return ret; } @@ -1056,11 +1053,6 @@ static void ShaderAppendNativeSpecialization(std::vector& ids.push_back(static_cast(image.indirect_resources.size())); ids.insert(ids.end(), image.indirect_resources.begin(), image.indirect_resources.end()); } - ids.push_back(static_cast(program.info.addresses.size())); - for (const auto& address: program.info.addresses) { - ids.push_back(static_cast(address.specialized_base)); - ids.push_back(static_cast(address.specialized_base >> 32u)); - } } static std::span AddShaderProgramPermutation(const char* stage, diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 61b94bbf8..bf2d41e71 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -295,7 +295,7 @@ void TestInvariantIndirectImageMaterialization() { const auto same_snapshot = [](const ResourceSnapshot &lhs, const ResourceSnapshot &rhs) { return lhs.buffers == rhs.buffers && lhs.images == rhs.images && - lhs.samplers == rhs.samplers && lhs.addresses == rhs.addresses && + lhs.samplers == rhs.samplers && lhs.flattened_srt == rhs.flattened_srt && lhs.user_data == rhs.user_data && lhs.indirect_images.empty() == rhs.indirect_images.empty(); @@ -848,7 +848,7 @@ void TestSrtFlatteningAndRuntimeMemoization() { Check(fixture.program.srt_reads.size() == 1, "shared typed scalar read did not receive one flat SRT slot"); Check(fixture.program.info.buffers.size() == 1 && - fixture.program.info.addresses.empty(), + !fixture.program.info.uses_dma, "planning-only scalar reads leaked into resource topology"); Check(fixture.program.memory_info[0].planning_only, "canonical runtime scalar read was not marked planning-only"); @@ -910,7 +910,7 @@ void TestDynamicSrtReadRemainsExplicit() { Check(fixture.program.srt_reads.empty() && fixture.program.dynamic_reads.size() == 1 && - fixture.program.info.addresses.size() == 1, + fixture.program.info.uses_dma, "dynamic scalar read was incorrectly flattened or lost"); std::array user_data{0x1000u, 0u, 4u}; TestMemory memory; @@ -932,11 +932,13 @@ void TestDynamicSrtReadRemainsExplicit() { FindBinding(fixture.program.bindings, DescriptorBindingKind::FlattenedSrt) == nullptr && FindBinding(fixture.program.bindings, - DescriptorBindingKind::AddressMemory) != nullptr, + DescriptorBindingKind::BdaPagetable) != nullptr && + FindBinding(fixture.program.bindings, + DescriptorBindingKind::FaultBuffer) != nullptr, "dynamic scalar read received the wrong resource bindings"); Check(fixture.program.bindings.memory_offset_dword == fixture.program.bindings.user_data_registers.size() && - fixture.program.bindings.memory_offset_count == 2u && + fixture.program.bindings.memory_offset_count == 1u && fixture.program.bindings.ShaderDataDwords() == fixture.program.bindings.memory_offset_dword + 1u, "unified memory-offset layout is inconsistent"); @@ -1029,7 +1031,7 @@ void TestInvariantLoopPhi() { "loop-invariant descriptor phi was not evaluated through typed SSA"); } -void TestAddressMaterializationAndSpecialization() { +void TestDmaAddressMaterialization() { Fixture fixture; const auto based = fixture.Address(fixture.UserData(0), fixture.UserData(1), 4); @@ -1050,29 +1052,19 @@ void TestAddressMaterializationAndSpecialization() { fixture.AddMemory(flat, 8)); fixture.PlanAndTrack(); - Check(fixture.program.info.addresses.size() == 2 && - !fixture.program.info.addresses[0].unbased && - fixture.program.info.addresses[0].min_offset == -8 && - fixture.program.info.addresses[1].unbased, - "typed based and unbased addresses were classified incorrectly"); + Check(fixture.program.info.uses_dma, + "typed address operations did not enable DMA"); std::array user_data{0x2008u, 0u}; - SrtRuntime runtime{.user_data = user_data, .flat_memory_base = 0x9000u}; + SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; std::string error; Check(MaterializeResources(fixture.program, runtime, snapshot, &error), - "address resources did not materialize"); - Check(snapshot.addresses.size() == 2 && - snapshot.addresses[0].guest_base == 0x2008u && - snapshot.addresses[0].binding_base == 0x2000u && - snapshot.addresses[1].binding_base == 0x9000u, - "materialized address windows are incorrect"); - Check(SpecializeResources(fixture.program, snapshot, &error) && - fixture.program.info.addresses[0].specialized_base == 8u && - fixture.program.info.addresses[1].specialized_base == 0x9000u, - "typed address specialization was not applied"); + "DMA shader resources did not materialize"); + Check(SpecializeResources(fixture.program, snapshot, &error), + "DMA shader resources were not specialized"); } -void TestExecMaskedFlatAddressProvenance() { +void TestDynamicFlatAddressesUseDma() { Fixture fixture; const auto low_root = fixture.UserData(0); const auto high_root = fixture.UserData(1); @@ -1092,18 +1084,14 @@ void TestExecMaskedFlatAddressProvenance() { fixture.AddMemory(flat, 0xa4)); fixture.PlanAndTrack(); - Check(fixture.program.info.addresses.size() == 1 && - !fixture.program.info.addresses[0].unbased, - "exec-masked FLAT address lost its active user-data root"); + Check(fixture.program.info.uses_dma, + "exec-masked FLAT address did not enable DMA"); std::array user_data{0x23456780u, 1u, 1u}; SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; std::string error; - Check(MaterializeResources(fixture.program, runtime, snapshot, &error) && - snapshot.addresses.size() == 1 && - snapshot.addresses[0].guest_base == 0x0000000123456780ull && - snapshot.addresses[0].binding_base == 0x0000000123450000ull, - "exec-masked FLAT address materialized the wrong user-data root"); + Check(MaterializeResources(fixture.program, runtime, snapshot, &error), + "exec-masked FLAT shader resources did not materialize"); Fixture mismatch; const auto mismatch_active = mismatch.Emit(ValueOpcode::INotEqual32, @@ -1122,9 +1110,8 @@ void TestExecMaskedFlatAddressProvenance() { {mismatch_address, mismatch_low, mismatch_high, other_active}, mismatch.AddMemory(flat, 0xa4)); mismatch.PlanAndTrack(); - Check(mismatch.program.info.addresses.size() == 1 && - mismatch.program.info.addresses[0].unbased, - "FLAT address used a select arm guarded by a different active mask"); + Check(mismatch.program.info.uses_dma, + "dynamic FLAT address did not enable DMA"); } void TestBufferSwizzleSpecialization() { @@ -1393,8 +1380,8 @@ int main() { Run("phi validation", TestPhiValidation); Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue); Run("invariant loop phi", TestInvariantLoopPhi); - Run("address materialization", TestAddressMaterializationAndSpecialization); - Run("exec-masked FLAT address", TestExecMaskedFlatAddressProvenance); + Run("DMA address materialization", TestDmaAddressMaterialization); + Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma); Run("buffer swizzle specialization", TestBufferSwizzleSpecialization); Run("shader info and bindings", TestShaderInfoAndBindingLayout); Run("graphics push constants", TestGraphicsPushConstantLayout); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index eb37799a7..fcb473d9a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -919,6 +919,11 @@ void CheckLeastRecentlyUsedCacheOrdering() { std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache"); } +struct BdaMapping { + uint64_t guest_base = 0; + u32 backing_offset = 0; +}; + struct TestCase { const char *name = ""; std::vector code; @@ -959,15 +964,15 @@ struct TestCase { bool compile_only = false; size_t storage_buffer_range_dwords = 0; std::vector storage_buffer_offsets; - std::vector address_memory_offsets; + std::vector bda_mappings; bool expand_shader_data_storage = false; bool expected_force_point_sampler = false; - std::optional flat_memory_base; std::vector gds_initial; std::vector expected_gds; std::vector> decoded_counts; std::vector> ir_counts; u32 expected_storage_mip_descriptors = 0; + std::string expected_compile_error; }; struct GraphicsCase { @@ -1168,7 +1173,6 @@ CompiledShader CompileCase(const TestCase &test) { options.user_data = user_data.data(); options.read_memory = ReadTestMemory; options.read_memory_data = const_cast *>(&test.initial); - options.flat_memory_base = test.flat_memory_base; options.scratch_dwords = test.compute_info.scratch_size_dwords; if (test.has_compute_info) { options.wave_size = test.compute_info.wave_size; @@ -1177,8 +1181,14 @@ CompiledShader CompileCase(const TestCase &test) { ShaderRecompiler::CompileResult result; std::string error; if (!ShaderRecompiler::TryRecompile(test.code, options, result, &error)) { + if (!test.expected_compile_error.empty() && + error.find(test.expected_compile_error) != std::string::npos) { + return {}; + } Fail(test.name, "decode/IR", error.c_str()); } + Require(test.name, "decode/IR", test.expected_compile_error.empty(), + "shader unexpectedly compiled despite the required rejection"); for (const auto &[text, expected] : test.decoded_counts) { const auto actual = CountText(result.decoded_dump, text); Require(test.name, "decoded RDNA2", actual == expected, @@ -1267,17 +1277,6 @@ CompiledShader CompileCase(const TestCase &test) { packed_user_data[result.program.bindings.memory_offset_dword + i / 4u] |= offset << ((i % 4u) * 8u); } - for (u32 i = 0; i < result.program.info.addresses.size(); i++) { - u32 offset = 0; - if (i < test.address_memory_offsets.size()) { - offset = test.address_memory_offsets[i]; - } - Require(test.name, "shader data", offset < 256, - "address-memory offset is not representable"); - const auto index = buffer_count + i; - packed_user_data[result.program.bindings.memory_offset_dword + - index / 4u] |= offset << ((index % 4u) * 8u); - } return {std::move(result.spirv), std::move(result.program), std::move(result.resources.flattened_srt), std::move(packed_user_data)}; @@ -1469,6 +1468,7 @@ public: struct Buffer { vk::Buffer buffer = nullptr; vk::DeviceMemory memory = nullptr; + vk::DeviceAddress device_address = 0; vk::DeviceSize size = 0; bool coherent = false; }; @@ -3157,7 +3157,7 @@ public: "chunked host copy lost bytes across its scratch boundary"); constexpr uint64_t index_offset = 0x180000; - constexpr uint64_t index_page = BufferCache::CACHING_PAGE_SIZE; + constexpr uint64_t index_page = BufferCache::CACHING_PAGESIZE; constexpr uint64_t index_span = 3 * index_page; const uint64_t index_begin = base + index_offset; Require(name, "registered-range empty lookup", @@ -3386,7 +3386,7 @@ public: "Buffer ownership"); constexpr uint64_t starvation_offset = 0x100000; - constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGE_SIZE; + constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGESIZE; constexpr uint32_t starvation_count = 33; constexpr uint32_t starvation_value = 0x31415926u; BufferCacheTestAccess::SetGarbageCollectionThresholds( @@ -3443,7 +3443,7 @@ public: constexpr uint64_t lookup_only_offset = 0x218000; constexpr uint64_t obtained_offset = 0x220000; constexpr uint64_t residency_size = - 2 * BufferCache::CACHING_PAGE_SIZE; + 2 * BufferCache::CACHING_PAGESIZE; BufferCacheTestAccess::SetGarbageCollectionThresholds( cache, std::numeric_limits::max(), std::numeric_limits::max()); @@ -4199,7 +4199,7 @@ public: const uint64_t mip_guest_size = mip_total.size + 256; Require(name, "formatted mip fixture", mip_sizes[0].offset == 0 && - mip_prefix_size > BufferCache::CACHING_PAGE_SIZE && + mip_prefix_size > BufferCache::CACHING_PAGESIZE && mip_prefix_size < mip_guest_size && mip_guest_size % sizeof(uint32_t) == 0, "single-mip fixture did not expose a cache-sized fitting backing " @@ -8946,7 +8946,7 @@ public: Buffer CreateStorageBuffer(const char *shader_name, const std::vector &initial, - size_t dword_count) { + size_t dword_count, bool device_address = false) { Buffer ret; ret.size = static_cast(std::max(dword_count, 1u) * sizeof(u32)); @@ -8955,6 +8955,9 @@ public: buffer_info.sType = vk::StructureType::eBufferCreateInfo; buffer_info.size = ret.size; buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer; + if (device_address) { + buffer_info.usage |= vk::BufferUsageFlagBits::eShaderDeviceAddress; + } buffer_info.sharingMode = vk::SharingMode::eExclusive; RequireVk(shader_name, "dispatch", m_device.createBuffer(&buffer_info, nullptr, &ret.buffer), @@ -8979,12 +8982,26 @@ public: alloc.sType = vk::StructureType::eMemoryAllocateInfo; alloc.allocationSize = req.size; alloc.memoryTypeIndex = memory_type; + vk::MemoryAllocateFlagsInfo flags{}; + if (device_address) { + flags.sType = vk::StructureType::eMemoryAllocateFlagsInfo; + flags.flags = vk::MemoryAllocateFlagBits::eDeviceAddress; + alloc.pNext = &flags; + } RequireVk(shader_name, "dispatch", m_device.allocateMemory(&alloc, nullptr, &ret.memory), "vkAllocateMemory"); RequireVk(shader_name, "dispatch", m_device.bindBufferMemory(ret.buffer, ret.memory, 0), "vkBindBufferMemory"); + if (device_address) { + vk::BufferDeviceAddressInfo address_info{}; + address_info.sType = vk::StructureType::eBufferDeviceAddressInfo; + address_info.buffer = ret.buffer; + ret.device_address = m_device.getBufferAddress(address_info); + Require(shader_name, "dispatch", ret.device_address != 0, + "storage buffer has no device address"); + } std::vector contents(dword_count, 0); for (size_t i = 0; i < initial.size() && i < contents.size(); i++) { @@ -9366,7 +9383,9 @@ public: pool_sizes.push_back({type, count}); }; add_pool_size(vk::DescriptorType::eStorageBuffer, - Count(Kind::Buffers) + Count(Kind::AddressMemory) + + Count(Kind::Buffers) + + (Binding(Kind::BdaPagetable) != nullptr ? 1u : 0u) + + (Binding(Kind::FaultBuffer) != nullptr ? 1u : 0u) + Count(Kind::Gds) + (Binding(Kind::FlattenedSrt) != nullptr ? 1u : 0u) + (Binding(Kind::UserData) != nullptr ? 1u : 0u)); @@ -9411,7 +9430,6 @@ public: std::vector writes; std::vector buffer_infos; - std::vector address_memory_infos; std::vector sampled_infos; std::vector storage_infos; std::vector storage_uint_infos; @@ -9422,6 +9440,33 @@ public: vk::DescriptorBufferInfo flattened_info{}; vk::DescriptorBufferInfo user_data_info{}; vk::DescriptorBufferInfo gds_info{}; + vk::DescriptorBufferInfo bda_pagetable_info{}; + vk::DescriptorBufferInfo fault_buffer_info{}; + + const bool uses_bda = Binding(Kind::BdaPagetable) != nullptr; + Require(test.name, "dispatch", + uses_bda == (Binding(Kind::FaultBuffer) != nullptr), + "BDA page table and fault buffer must be bound together"); + if (uses_bda) { + EnsureBdaBuffers(test.name); + bda_pagetable_info = {m_bda_pagetable_buffer.buffer, 0, + m_bda_pagetable_buffer.size}; + fault_buffer_info = {m_fault_buffer.buffer, 0, m_fault_buffer.size}; + const std::array special_buffers{ + std::pair{Kind::BdaPagetable, &bda_pagetable_info}, + std::pair{Kind::FaultBuffer, &fault_buffer_info}, + }; + for (const auto &[kind, info] : special_buffers) { + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = Native(kind); + write.descriptorCount = 1; + write.descriptorType = vk::DescriptorType::eStorageBuffer; + write.pBufferInfo = info; + writes.push_back(write); + } + } const auto *buffers = Binding(Kind::Buffers); if (buffers != nullptr) { @@ -9450,21 +9495,6 @@ public: write.pBufferInfo = buffer_infos.data(); writes.push_back(write); } - if (const auto *address = Binding(Kind::AddressMemory); - address != nullptr) { - address_memory_infos.resize(address->resources.size()); - for (auto &info : address_memory_infos) { - info = {buffer.buffer, 0, buffer.size}; - } - vk::WriteDescriptorSet write{}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = Native(Kind::AddressMemory); - write.descriptorCount = static_cast(address_memory_infos.size()); - write.descriptorType = vk::DescriptorType::eStorageBuffer; - write.pBufferInfo = address_memory_infos.data(); - writes.push_back(write); - } if (const auto *flattened = Binding(Kind::FlattenedSrt); flattened != nullptr) { flattened_buffer = CreateStorageBuffer(test.name, compiled.flattened_srt, @@ -9590,6 +9620,52 @@ public: } vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch"); + if (uses_bda) { + Require(test.name, "dispatch", buffer.device_address != 0, + "BDA test backing has no device address"); + Require(test.name, "dispatch", !test.bda_mappings.empty(), + "BDA test has no explicit guest mapping"); + cmd.fillBuffer(m_bda_pagetable_buffer.buffer, 0, + m_bda_pagetable_buffer.size, 0); + cmd.fillBuffer(m_fault_buffer.buffer, 0, m_fault_buffer.size, 0); + for (const auto &[guest_base, backing] : test.bda_mappings) { + const auto page_offset = guest_base & + (BufferCache::CACHING_PAGESIZE - 1); + Require(test.name, "dispatch", + backing >= page_offset && backing < buffer.size, + "BDA test mapping begins outside its backing buffer"); + const auto bytes = page_offset + buffer.size - backing; + const auto pages = + (bytes + BufferCache::CACHING_PAGESIZE - 1) / + BufferCache::CACHING_PAGESIZE; + auto address = buffer.device_address + backing - page_offset; + auto page = guest_base >> BufferCache::CACHING_PAGEBITS; + for (uint64_t mapped = 0; mapped < pages; mapped++) { + cmd.updateBuffer(m_bda_pagetable_buffer.buffer, + (page + mapped) * sizeof(vk::DeviceAddress), + sizeof(address), &address); + address += BufferCache::CACHING_PAGESIZE; + } + } + std::array barriers{}; + barriers[0].sType = vk::StructureType::eBufferMemoryBarrier; + barriers[0].srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead; + barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barriers[0].buffer = m_bda_pagetable_buffer.buffer; + barriers[0].offset = 0; + barriers[0].size = m_bda_pagetable_buffer.size; + barriers[1] = barriers[0]; + barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead | + vk::AccessFlagBits::eShaderWrite; + barriers[1].buffer = m_fault_buffer.buffer; + barriers[1].size = m_fault_buffer.size; + cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eComputeShader, {}, 0, + nullptr, static_cast(barriers.size()), + barriers.data(), 0, nullptr); + } cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline); cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, &descriptor_set, 0, nullptr); @@ -10951,6 +11027,7 @@ private: allocator_info.device = m_device; allocator_info.pVulkanFunctions = &functions; allocator_info.vulkanApiVersion = VK_API_VERSION_1_3; + allocator_info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT; RequireVk("VulkanHarness", "runtime context", static_cast(vmaCreateAllocator( &allocator_info, &m_runtime_context.allocator)), @@ -11001,13 +11078,18 @@ private: if ((queues[i].queueFlags & (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) == (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) { + vk::PhysicalDeviceVulkan12Features features12{}; + features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR; + barycentric.pNext = &features12; vk::PhysicalDeviceFeatures2 features{}; features.sType = vk::StructureType::ePhysicalDeviceFeatures2; features.pNext = &barycentric; physical.getFeatures2(&features); - if (barycentric.fragmentShaderBarycentric != true) { + if (barycentric.fragmentShaderBarycentric != true || + features.features.shaderInt64 != true || + features12.bufferDeviceAddress != true) { continue; } m_physical_device = physical; @@ -11051,6 +11133,11 @@ private: Require("VulkanHarness", "dispatch", available_features.sampleRateShading == true, "sample-rate shading is not supported"); + Require("VulkanHarness", "dispatch", available_features.shaderInt64 == true, + "shaderInt64 is not supported"); + Require("VulkanHarness", "dispatch", + available_features12.bufferDeviceAddress == true, + "bufferDeviceAddress is not supported"); float priority = 1.0f; vk::DeviceQueueCreateInfo queue_info{}; @@ -11067,6 +11154,7 @@ private: device_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; device_features12.timelineSemaphore = true; + device_features12.bufferDeviceAddress = true; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR; barycentric.pNext = &device_features12; @@ -11081,6 +11169,7 @@ private: vk::PhysicalDeviceFeatures device_features{}; device_features.shaderStorageImageWriteWithoutFormat = true; device_features.sampleRateShading = true; + device_features.shaderInt64 = true; device_info.pEnabledFeatures = &device_features; constexpr const char *device_extensions[] = { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, @@ -11105,6 +11194,8 @@ private: void Destroy() { if (m_device != nullptr) { RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle"); + DestroyBuffer(&m_fault_buffer); + DestroyBuffer(&m_bda_pagetable_buffer); if (m_runtime_context.allocator != nullptr) { m_renderer.reset(); vmaDestroyAllocator(m_runtime_context.allocator); @@ -11276,6 +11367,53 @@ private: image->layout = final_layout; } + Buffer CreateDeviceBuffer(const char *shader_name, vk::DeviceSize size, + vk::BufferUsageFlags usage) { + Buffer ret; + ret.size = size; + vk::BufferCreateInfo buffer_info{}; + buffer_info.sType = vk::StructureType::eBufferCreateInfo; + buffer_info.size = size; + buffer_info.usage = usage; + buffer_info.sharingMode = vk::SharingMode::eExclusive; + RequireVk(shader_name, "dispatch", + m_device.createBuffer(&buffer_info, nullptr, &ret.buffer), + "vkCreateBuffer"); + + vk::MemoryRequirements requirements{}; + m_device.getBufferMemoryRequirements(ret.buffer, &requirements); + u32 memory_type = 0; + Require(shader_name, "dispatch", + FindMemoryType(requirements.memoryTypeBits, + vk::MemoryPropertyFlagBits::eDeviceLocal, + &memory_type) || + FindMemoryType(requirements.memoryTypeBits, {}, &memory_type), + "no memory type for BDA support buffer"); + vk::MemoryAllocateInfo allocation{}; + allocation.sType = vk::StructureType::eMemoryAllocateInfo; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + m_device.allocateMemory(&allocation, nullptr, &ret.memory), + "vkAllocateMemory"); + RequireVk(shader_name, "dispatch", + m_device.bindBufferMemory(ret.buffer, ret.memory, 0), + "vkBindBufferMemory"); + return ret; + } + + void EnsureBdaBuffers(const char *shader_name) { + if (m_bda_pagetable_buffer.buffer != nullptr) { + return; + } + const auto usage = vk::BufferUsageFlagBits::eStorageBuffer | + vk::BufferUsageFlagBits::eTransferDst; + m_bda_pagetable_buffer = + CreateDeviceBuffer(shader_name, BufferCache::BDA_PAGETABLE_SIZE, usage); + m_fault_buffer = CreateDeviceBuffer( + shader_name, BufferCache::CACHING_NUMPAGES / 8, usage); + } + Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size, vk::BufferUsageFlags usage, const std::vector &contents) { @@ -11349,6 +11487,8 @@ private: vk::CommandPool m_command_pool = nullptr; u32 m_queue_family = 0; vk::PhysicalDeviceMemoryProperties m_memory_properties{}; + Buffer m_bda_pagetable_buffer; + Buffer m_fault_buffer; GraphicContext m_runtime_context{}; std::unique_ptr m_renderer; }; @@ -11392,6 +11532,10 @@ void CompareGraphicsWords(const GraphicsCase &test, void RunCase(VulkanHarness *vulkan, const TestCase &test) { auto compiled = CompileCase(test); + if (!test.expected_compile_error.empty()) { + std::printf("[compute] %-32s ok\n", test.name); + return; + } if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) { Require(test.name, "resource specialization", !compiled.program.info.images.empty() && @@ -11410,15 +11554,15 @@ void RunCase(VulkanHarness *vulkan, const TestCase &test) { std::printf("[compute] %-32s ok\n", test.name); return; } - const auto dwords = std::max( - {test.initial.size(), test.expected.size(), static_cast(1)}); - auto buffer = vulkan->CreateStorageBuffer(test.name, test.initial, dwords); - using Kind = ShaderRecompiler::IR::DescriptorBindingKind; auto Has = [&](Kind kind) { return ShaderRecompiler::IR::FindBinding(compiled.program.bindings, kind) != nullptr; }; + const auto dwords = std::max( + {test.initial.size(), test.expected.size(), static_cast(1)}); + auto buffer = vulkan->CreateStorageBuffer( + test.name, test.initial, dwords, Has(Kind::BdaPagetable)); VulkanHarness::Image sampled_image; VulkanHarness::Image storage_image; VulkanHarness::Image storage_image_uint; @@ -17341,16 +17485,16 @@ TestCase FlatLoadVariants() { O::FLAT_LOAD_USHORT, O::FLAT_LOAD_SSHORT, O::FLAT_LOAD_DWORD, O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3, O::FLAT_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; - test.flat_memory_base = 0; - test.address_memory_offsets = {static_cast(byte_offset)}; + test.bda_mappings = {{0, static_cast(byte_offset)}}; return test; } TestCase FlatSubdwordLoadsApplyByteOffset() { using O = ShaderOpcode; + constexpr size_t byte_offset = 3; std::vector code; - AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 20, static_cast(byte_offset)); AppendVMovU32(&code, 21, 0); code.push_back(EncodeFlat0(0x08, 0, 0)); code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); @@ -17360,7 +17504,6 @@ TestCase FlatSubdwordLoadsApplyByteOffset() { AppendStoreVgpr(&code, 1, 1); AppendEnd(&code); - constexpr size_t byte_offset = 3; constexpr u32 guest_word = 0x80ff7f01u; std::vector host_memory( (byte_offset + sizeof(guest_word) + sizeof(u32) - 1u) / sizeof(u32)); @@ -17372,8 +17515,7 @@ TestCase FlatSubdwordLoadsApplyByteOffset() { test.code = std::move(code); test.initial = std::move(host_memory); test.expected = {0x01u, 0x0000ff7fu}; - test.flat_memory_base = 0; - test.address_memory_offsets = {static_cast(byte_offset)}; + test.bda_mappings = {{0, 0}}; test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_USHORT, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; return test; @@ -17457,8 +17599,7 @@ TestCase FlatVirtualAddressRebasesGuestAllocation() { test.code = std::move(code); test.initial = {0xfeedfaceu, 0xcafebabeu, 0, 0x12345678u}; test.expected = {0x12345678u, 0x12345678u, 0}; - test.flat_memory_base = GuestBase; - test.address_memory_offsets = {8, 8, 8}; + test.bda_mappings = {{GuestBase, 8}}; test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; return test; @@ -17480,10 +17621,9 @@ TestCase GlobalSignedImmediateRebasesBeforeSaddr() { TestCase test; test.name = "GlobalSignedImmediateRebasesBeforeSaddr"; test.code = std::move(code); - test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x22222222u, - 0x12345678u}; + test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x12345678u}; test.expected = {0x12345678u}; - test.address_memory_offsets = {8}; + test.bda_mappings = {{GuestBase, 8}}; test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; return test; @@ -17517,7 +17657,7 @@ TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() { test.expected = {0x11111111u, 0x11111111u}; test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.flat_memory_base = 0; + test.bda_mappings = {{0, 0}}; return test; } @@ -17596,7 +17736,8 @@ TestCase FlatStoreVariants() { {O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT, O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2, O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}}; - test.flat_memory_base = 0; + test.expected_compile_error = + "writable FLAT/GLOBAL addresses require GPU ownership tracking"; return test; } diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp index 25e5fbd03..18aebc25c 100644 --- a/tests/ShaderStageRuntimeTests.cpp +++ b/tests/ShaderStageRuntimeTests.cpp @@ -73,12 +73,7 @@ UnbasedFlatProgram() { program.srt_plan_complete = true; program.resource_tracking_complete = true; AddValueBlock(program); - AddressResource address; - address.source = UINT32_MAX; - address.first_use_pc = 0x80; - address.kind = ResourceKind::Flat; - address.unbased = true; - program.info.addresses.push_back(address); + program.info.uses_dma = true; return std::make_shared(std::move(program)); } @@ -102,7 +97,7 @@ void TestMappedSrtUsesDirectReaderByDefault() { "cache rematerialization did not use the direct reader by default"); } -void TestUnbasedFlatCacheHitFailsClosed() { +void TestUnbasedFlatCacheHitMaterializes() { using namespace Libs::Graphics; auto cached_program = UnbasedFlatProgram(); auto prior_program = std::make_shared(); @@ -110,12 +105,9 @@ void TestUnbasedFlatCacheHitFailsClosed() { std::make_shared(); ShaderStageRuntime stage{prior_program, prior_resources}; std::string error; - Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error) && - error.find("requires runtime guest-address translation") != - std::string::npos, - "unbased FLAT cache hit did not fail without a runtime translator"); - Check(stage.program == prior_program && stage.resources == prior_resources, - "unbased FLAT cache rejection replaced the prior stage snapshot"); + Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error), error.c_str()); + Check(stage.program == cached_program && stage.resources != prior_resources, + "unbased FLAT cache hit did not publish its BDA stage snapshot"); } } // namespace @@ -130,7 +122,7 @@ void DbgExit(int) { std::abort(); } int main() { TestMappedSrtUsesDirectReaderByDefault(); - TestUnbasedFlatCacheHitFailsClosed(); + TestUnbasedFlatCacheHitMaterializes(); std::puts("ShaderStageRuntimeTests: all cases passed"); return 0; } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 17a09af52..19fdde892 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4319,14 +4319,14 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { error.c_str()); const auto *address_binding = ShaderRecompiler::IR::FindBinding( raw.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::AddressMemory); - Check(raw.program.info.addresses.size() == 1u && - raw.program.info.addresses[0].kind == - ShaderRecompiler::IR::ResourceKind::ScalarAddress && - raw.program.info.addresses[0].read && - !raw.program.info.addresses[0].written && - raw.program.info.buffers.empty() && address_binding != nullptr && - address_binding->resources == std::vector{0u} && + ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable); + const auto *fault_binding = ShaderRecompiler::IR::FindBinding( + raw.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer); + Check(raw.program.info.uses_dma && raw.program.info.buffers.empty() && + address_binding != nullptr && fault_binding != nullptr && + address_binding->resources.empty() && + fault_binding->resources.empty() && ShaderRecompiler::IR::FindBinding( raw.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Buffers) == @@ -4335,17 +4335,12 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { raw.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) == nullptr && - raw.program.bindings.memory_offset_count == 1u, - "raw scalar load did not use only the address-memory domain"); + raw.program.bindings.memory_offset_count == 0u, + "raw scalar load did not use only the DMA domain"); Check(count_live_memory_ops( raw.program, ShaderRecompiler::IR::ValueOpcode::LoadAddressU32, ShaderRecompiler::IR::ResourceKind::ScalarAddress) == 1u, "raw scalar load did not remain a live typed address operation"); - Check(raw.resources.addresses.size() == 1u && - raw.resources.addresses[0].guest_base == 0x1000u && - raw.resources.addresses[0].binding_base == 0x1000u && - raw.program.info.addresses[0].specialized_base == 0u, - "raw scalar address specialization was incorrect"); Check(SpirvContainsOpcode(raw.spirv, 199), "raw scalar SOFFSET alignment was not emitted"); CheckSpirvBinaryValidates(raw.spirv); @@ -4377,12 +4372,12 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { ShaderRecompiler::IR::DescriptorBindingKind::Buffers); Check(buffer.program.info.buffers.size() == 1u && buffer.program.info.buffers[0].scalar && - buffer.program.info.addresses.empty() && + !buffer.program.info.uses_dma && buffer_binding != nullptr && buffer_binding->resources == std::vector{0u} && ShaderRecompiler::IR::FindBinding( buffer.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::AddressMemory) == + ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable) == nullptr && ShaderRecompiler::IR::FindBinding( buffer.program.bindings, @@ -6201,7 +6196,6 @@ void TestNewShaderRecompilerFlatOldBackedTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - options.flat_memory_base = 0; ShaderRecompiler::CompileResult result; std::string error; @@ -6224,23 +6218,32 @@ void TestNewShaderRecompilerFlatOldBackedTranslation() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerUnbasedFlatRequiresTranslator() { +void TestNewShaderRecompilerUnbasedFlatUsesBda() { const uint32_t shader[] = { EncodeFlat0(0x0c, 0, 0), EncodeFlat1(0, 0x7d, 0, 1), - EncodeFlat0(0x1c, 0, 0), - EncodeFlat1(0, 0x7d, 0, 2), - 0xbf810000u, + EncodeExp0(0x00, 0x1), + EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), }; - auto options = MakeCompileOptions(ShaderType::Compute); + auto options = MakeCompileOptions(ShaderType::Pixel); ShaderRecompiler::CompileResult result; std::string error; - Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && - Common::ContainsStr(error, - "requires runtime guest-address translation"), - "unbased FLAT compiled without an explicit translator"); + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(result.program.info.uses_dma && + ShaderRecompiler::IR::FindBinding( + result.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable) != + nullptr && + ShaderRecompiler::IR::FindBinding( + result.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer) != + nullptr, + "unbased FLAT did not compile through BDA"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerFlatSignedLoadTranslation() { @@ -6256,7 +6259,6 @@ void TestNewShaderRecompilerFlatSignedLoadTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - options.flat_memory_base = 0; ShaderRecompiler::CompileResult result; std::string error; @@ -6311,7 +6313,6 @@ void TestNewShaderRecompilerFlatStoreTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - options.flat_memory_base = 0; ShaderRecompiler::CompileResult result; std::string error; @@ -11966,49 +11967,40 @@ void TestNewShaderRecompilerUnsupportedMemoryDecode() { "MTBUF", "opcode=0x08"); } -void TestNewShaderRecompilerFlatUserPointerProvenance() { +void TestNewShaderRecompilerFlatUserPointerUsesDma() { const uint32_t shader[] = { EncodeVop2(0x25, 0, 0, 2), // v_add_nc_u32 v0, s0, v2 EncodeVop1(0x01, 1, 1), // v_mov_b32 v1, s1 - EncodeFlat0(0x1c, 0, 0), - EncodeFlat1(0, 0x7d, 3, 0), // flat_store_dword v3, v[0:1] - 0xbf810000u, + EncodeFlat0(0x0c, 0, 0), + EncodeFlat1(3, 0x7d, 0, 0), // flat_load_dword v3, v[0:1] + EncodeExp0(0x00, 0x1), + EncodeExp1(3, 0, 0, 0), + EncodeSopp(0x01), }; const uint32_t user_data[] = {0x34567000u, 0x00000012u}; - auto options = MakeCompileOptions(ShaderType::Compute); + auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; options.user_data_count = static_cast(std::size(user_data)); - options.flat_memory_base = 0; ShaderRecompiler::CompileResult result; std::string error; const bool compiled = ShaderRecompiler::TryRecompile(shader, options, result, &error); Check(compiled, error.c_str()); - Check(result.program.info.addresses.size() == 1 && - !result.program.info.addresses[0].unbased && - result.program.info.addresses[0].source < - result.program.descriptor_sources.size(), - "FLAT user pointer provenance was not attached"); - Check(result.resources.addresses.size() == 1 && - result.resources.addresses[0].guest_base == 0x0000001234567000ull && - result.resources.addresses[0].binding_base == - 0x0000001234560000ull && - result.program.info.addresses[0].specialized_base == - 0x0000001234560000ull, - "FLAT user pointer did not materialize its guest base"); + Check(result.program.info.uses_dma, + "FLAT user pointer did not enable DMA"); CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() { +void TestNewShaderRecompilerFlatAddressDomainsUseDma() { const uint32_t segmented_shader[] = { EncodeVop2(0x25, 0, 0, 2), EncodeVop1(0x01, 1, 1), - EncodeFlat0(0x1c, 2, 0), - EncodeFlat1(0, 0x7d, 3, 0), // global_store_dword v3, v[0:1] + EncodeFlat0(0x0c, 2, 0), + EncodeFlat1(3, 0x7d, 0, 0), // global_load_dword v3, v[0:1] EncodeFlat0(0x1c, 1, 0), - EncodeFlat1(0, 0x7d, 4, 0), // scratch_store_dword v4, v[0:1] + EncodeFlat1(0, 0x7d, 3, 0), // scratch_store_dword v3, v[0:1] 0xbf810000u, }; const uint32_t user_data[] = {0x34567000u, 0u, 0x1000u}; @@ -12016,19 +12008,14 @@ void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() { auto options = MakeCompileOptions(ShaderType::Compute); options.user_data = user_data; options.user_data_count = static_cast(std::size(user_data)); - options.flat_memory_base = 0; options.scratch_dwords = 1; ShaderRecompiler::CompileResult result; std::string error; const bool compiled = ShaderRecompiler::TryRecompile(segmented_shader, options, result, &error); Check(compiled, error.c_str()); - Check(result.program.info.addresses.size() == 1 && - result.program.info.addresses[0].kind == - ShaderRecompiler::IR::ResourceKind::Global && - result.program.info.addresses[0].unbased && - result.program.info.addresses[0].source == UINT32_MAX, - "GLOBAL null-SADDR did not remain an unbased guest address"); + Check(result.program.info.uses_dma, + "GLOBAL null-SADDR did not enable DMA"); Check(Common::ContainsStr(result.ir_dump, "GetScratchResource") && result.program.scratch_dwords == 1, "SCRATCH incorrectly entered guest address tracking"); @@ -12324,9 +12311,9 @@ int main() { TestNewShaderRecompilerRejectsDppOn64BitCompares(); TestNewShaderRecompilerIrLookupMissFailsExplicitly(); TestPsInputCountRegisterDecode(); - TestNewShaderRecompilerUnbasedFlatRequiresTranslator(); - TestNewShaderRecompilerFlatUserPointerProvenance(); - TestNewShaderRecompilerFlatAddressProvenanceBoundaries(); + TestNewShaderRecompilerUnbasedFlatUsesBda(); + TestNewShaderRecompilerFlatUserPointerUsesDma(); + TestNewShaderRecompilerFlatAddressDomainsUseDma(); TestNewShaderRecompilerCfgStraightLine(); TestNewShaderRecompilerCfgIfElse(); TestNewShaderRecompilerCfgConsecutiveNativePhis();