raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-24 12:56:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

graphics: add RDNA2 FLAT BDA buffer cache and faults

Browse Source
This commit is contained in:
nmzik
2026-08-27 03:45:41 +02:00
parent 8072718462
commit e738535294
37 changed files with 1002 additions and 684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+16
View File
@@ -263,6 +263,22 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
)
list(APPEND gpu_tiler_shader_headers "${shader_header}")
endforeach()
set(fault_buffer_shader_source "${gpu_tiler_shader_dir}/fault_buffer_process.comp")
set(fault_buffer_shader_spv "${gpu_tiler_generated_dir}/fault_buffer_process.spv")
set(fault_buffer_shader_header "${gpu_tiler_generated_dir}/fault_buffer_process_spv.h")
add_custom_command(
OUTPUT "${fault_buffer_shader_header}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}"
COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
-DCACHING_PAGEBITS=14 -DMAX_PAGE_FAULTS=1024
-o "${fault_buffer_shader_spv}" "${fault_buffer_shader_source}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${fault_buffer_shader_spv} -DOUTPUT=${fault_buffer_shader_header}
-DSYMBOL=FAULT_BUFFER_PROCESS_SPV -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${fault_buffer_shader_source}"
VERBATIM
)
list(APPEND kyty_emulator_src "${fault_buffer_shader_header}")
list(APPEND kyty_emulator_src ${gpu_tiler_shader_headers})
set(gpu_blit_generated_dir "${PROJECT_BINARY_DIR}/gpu_blit_shaders")
src/graphics/host_gpu/rangeSet.h
+10 -1
View File
@@ -53,6 +53,15 @@ public:
}
}
void Clear() { m_ranges.clear(); }
template <typename Func>
void ForEach(Func&& func) const {
for (const auto& [begin, end]: m_ranges) {
func(begin, end);
}
}
[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
std::vector<Range> result;
ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
@@ -98,7 +107,7 @@ public:
private:
static uint64_t End(uint64_t address, uint64_t size) {
if (address == 0 || size == 0 || size > UINT64_MAX - address) {
if (size == 0 || size > UINT64_MAX - address) {
EXIT("invalid range-set address or size\n");
}
return address + size;
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+73 -38
View File
@@ -51,41 +51,50 @@ struct BufferCache::DownloadCopy {
};
void BufferCache::Register(BufferId id) {
auto& buffer = m_slot_buffers[id];
PageTable::PageRange pages {};
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
m_page_table[page] = id;
}
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
(void)it;
EXIT_IF(!inserted);
buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
m_total_used_memory += buffer.Size();
ChangeRegister<true>(id);
}
void BufferCache::Unregister(BufferId id) {
ChangeRegister<false>(id);
}
template <bool insert>
void BufferCache::ChangeRegister(BufferId id) {
auto& buffer = m_slot_buffers[id];
if (buffer.is_deleted) {
return;
}
PageTable::PageRange pages {};
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
auto* owner = m_page_table.Find(page);
if (owner != nullptr && *owner == id) {
*owner = {};
if constexpr (insert) {
m_page_table[page] = id;
} else {
m_page_table[page] = {};
}
}
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
if (buffer.Size() > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
const auto size_pages = pages.last_exclusive - pages.first;
if constexpr (insert) {
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
(void)it;
EXIT_IF(!inserted);
m_total_used_memory += buffer.Size();
buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
std::vector<vk::DeviceAddress> addresses;
addresses.reserve(size_pages);
for (uint64_t i = 0; i < size_pages; ++i) {
addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS));
}
WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress),
addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
} else {
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
EXIT_IF(buffer.Size() > m_total_used_memory);
m_total_used_memory -= buffer.Size();
m_lru_cache.Free(buffer.lru_id);
m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress),
size_pages * sizeof(vk::DeviceAddress), 0);
buffer.is_deleted = true;
}
m_total_used_memory -= buffer.Size();
m_lru_cache.Free(buffer.lru_id);
buffer.is_deleted = true;
}
void BufferCache::TouchBuffer(const Buffer& buffer) {
@@ -184,16 +193,21 @@ void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
PageManager& page_manager, TextureCache& texture_cache)
: m_graphics(graphics), m_scheduler(scheduler),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_texture_cache(texture_cache) {
: m_graphics(graphics), m_scheduler(scheduler),
m_fault_manager(graphics, scheduler, *this, CACHING_PAGEBITS, CACHING_NUMPAGES),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_bda_pagetable_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
BDA_PAGETABLE_SIZE),
m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_texture_cache(texture_cache) {
std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
m_gds_buffer.Flush(0, m_gds_buffer.Size());
SetVulkanObjectNameF(m_graphics.device, m_bda_pagetable_buffer.Handle(),
"BDA Page Table Buffer");
const auto null_id =
m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16);
EXIT_IF(null_id != NULL_BUFFER_ID);
@@ -316,8 +330,8 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
auto& command = m_scheduler.Current();
EXIT_IF(command.IsInvalid());
auto begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
auto end = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
auto begin = vaddr & ~(CACHING_PAGESIZE - 1);
auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
auto first = m_buffers.lower_bound(begin);
if (first != m_buffers.begin()) {
const auto previous = std::prev(first);
@@ -333,8 +347,9 @@ BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
end = std::max(end, buffer.CpuAddress() + buffer.Size());
}
const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags, end - begin);
const auto id = m_slot_buffers.insert(
m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin);
auto& buffer = m_slot_buffers[id];
SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
"Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
@@ -433,7 +448,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
if (!is_written && size <= CACHING_PAGE_SIZE &&
if (!is_written && size <= CACHING_PAGESIZE &&
!m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
const auto alignment = std::max<uint64_t>(
@@ -737,4 +752,24 @@ void BufferCache::RunGarbageCollector() {
}
}
void BufferCache::ProcessFaultBuffer() {
m_fault_manager.ProcessFaultBuffer();
}
void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {
const auto end = vaddr + size;
auto it = m_buffers.upper_bound(vaddr);
if (it != m_buffers.begin()) {
--it;
}
for (; it != m_buffers.end() && it->first < end; ++it) {
auto& buffer = m_slot_buffers[it->second];
const auto start = std::max(buffer.CpuAddress(), vaddr);
const auto finish = std::min(buffer.CpuAddress() + buffer.Size(), end);
if (start < finish) {
(void)SynchronizeBuffer(buffer, start, finish - start, false, false);
}
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+16 -3
View File
@@ -7,6 +7,7 @@
#include "common/slotVector.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/faultManager.h"
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
@@ -26,7 +27,11 @@ inline constexpr BufferId NULL_BUFFER_ID {0};
class BufferCache {
public:
static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
static constexpr uint32_t CACHING_PAGEBITS = 14;
static constexpr uint64_t CACHING_PAGESIZE = uint64_t {1} << CACHING_PAGEBITS;
static constexpr uint64_t CACHING_NUMPAGES = uint64_t {1} << (40 - CACHING_PAGEBITS);
static constexpr uint64_t BDA_PAGETABLE_SIZE =
CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
TextureCache& texture_cache);
@@ -51,6 +56,8 @@ public:
EXIT("BufferCache: invalid utility-buffer usage\n");
}
[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
[[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept { return &m_bda_pagetable_buffer; }
[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return m_fault_manager.GetFaultBuffer(); }
[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
@@ -60,14 +67,16 @@ public:
[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
void ProcessFaultBuffer();
void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
friend struct BufferCacheTestAccess;
struct DownloadCopy;
using PageTable = MultiLevelPageTable<BufferId, 14, 40, 16>;
static_assert(CACHING_PAGE_SIZE == (uint64_t {1} << PageTable::kPageBits));
using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
@@ -78,6 +87,8 @@ private:
[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
void Register(BufferId id);
void Unregister(BufferId id);
template <bool insert>
void ChangeRegister(BufferId id);
void DeleteBuffer(BufferId id);
[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
bool is_written, bool is_texel_buffer);
@@ -90,7 +101,9 @@ private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
FaultManager m_fault_manager;
Buffer m_gds_buffer;
Buffer m_bda_pagetable_buffer;
Common::SlotVector<Buffer> m_slot_buffers;
Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
std::map<uint64_t, BufferId> m_buffers;
src/graphics/host_gpu/renderer/cache/faultManager.cpp
Vendored
+167
View File
@@ -0,0 +1,167 @@
#include "graphics/host_gpu/renderer/cache/faultManager.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "gpu_tiler_shaders/fault_buffer_process_spv.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <bit>
#include <cinttypes>
#include <cstring>
#include <limits>
namespace Libs::Graphics {
namespace {
constexpr size_t MaxPageFaults = 1024;
constexpr size_t PageFaultAreaSize = MaxPageFaults * sizeof(uint64_t);
} // namespace
FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler,
BufferCache& buffer_cache, uint32_t caching_pagebits,
uint64_t caching_num_pages)
: m_graphics(graphics), m_scheduler(scheduler), m_buffer_cache(buffer_cache),
m_caching_pagesize(uint64_t {1} << caching_pagebits),
m_caching_num_pages(caching_num_pages), m_fault_buffer_size(caching_num_pages / 8),
m_fault_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
m_fault_buffer_size),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 0, AllFlags,
MaxPendingFaults * PageFaultAreaSize) {
SetVulkanObjectNameF(m_graphics.device, m_fault_buffer.Handle(), "Fault Buffer");
const vk::DescriptorSetLayoutBinding bindings[] {
{0, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
{1, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
};
vk::DescriptorSetLayoutCreateInfo layout_info {};
layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
layout_info.bindingCount = std::size(bindings);
layout_info.pBindings = bindings;
RequireVulkanSuccess(
m_graphics.device.createDescriptorSetLayout(&layout_info, nullptr,
&m_fault_process_desc_layout),
"create fault-buffer descriptor layout");
vk::ShaderModuleCreateInfo module_info {};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
module_info.pCode = FAULT_BUFFER_PROCESS_SPV;
vk::ShaderModule module = nullptr;
RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
"create fault-buffer shader module");
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &m_fault_process_desc_layout;
RequireVulkanSuccess(
m_graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
&m_fault_process_pipeline_layout),
"create fault-buffer pipeline layout");
vk::PipelineShaderStageCreateInfo stage {};
stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stage.stage = vk::ShaderStageFlagBits::eCompute;
stage.module = module;
stage.pName = "main";
vk::ComputePipelineCreateInfo pipeline_info {};
pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo;
pipeline_info.stage = stage;
pipeline_info.layout = m_fault_process_pipeline_layout;
const auto result = m_graphics.device.createComputePipelines(
nullptr, 1, &pipeline_info, nullptr, &m_fault_process_pipeline);
m_graphics.device.destroyShaderModule(module, nullptr);
RequireVulkanSuccess(result, "create fault-buffer pipeline");
SetVulkanObjectNameF(m_graphics.device, m_fault_process_pipeline, "Fault Buffer Parser");
}
FaultManager::~FaultManager() {
m_graphics.device.destroyPipeline(m_fault_process_pipeline, nullptr);
m_graphics.device.destroyPipelineLayout(m_fault_process_pipeline_layout, nullptr);
m_graphics.device.destroyDescriptorSetLayout(m_fault_process_desc_layout, nullptr);
}
void FaultManager::ProcessFaultBuffer() {
if (const auto wait_tick = m_fault_areas[m_current_area]; wait_tick != 0) {
m_scheduler.Wait(wait_tick);
m_scheduler.PopPendingOperations();
}
const auto offset = m_current_area * PageFaultAreaSize;
auto* mapped = m_download_buffer.Mapped().data() + offset;
std::memset(mapped, 0, PageFaultAreaSize);
m_download_buffer.Flush(offset, PageFaultAreaSize);
vk::BufferMemoryBarrier2 pre_barrier {};
pre_barrier.sType = vk::StructureType::eBufferMemoryBarrier2;
pre_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
pre_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
pre_barrier.dstStageMask = vk::PipelineStageFlagBits2::eComputeShader;
pre_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderRead;
pre_barrier.buffer = m_fault_buffer.Handle();
pre_barrier.offset = 0;
pre_barrier.size = m_fault_buffer_size;
auto post_barrier = pre_barrier;
post_barrier.srcStageMask = vk::PipelineStageFlagBits2::eComputeShader;
post_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
post_barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands;
post_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderWrite;
const vk::DescriptorBufferInfo infos[] {
{m_fault_buffer.Handle(), 0, m_fault_buffer_size},
{m_download_buffer.Handle(), offset, PageFaultAreaSize},
};
std::array<vk::WriteDescriptorSet, 2> writes {};
for (uint32_t index = 0; index < writes.size(); ++index) {
writes[index].sType = vk::StructureType::eWriteDescriptorSet;
writes[index].dstBinding = index;
writes[index].descriptorCount = 1;
writes[index].descriptorType = vk::DescriptorType::eStorageBuffer;
writes[index].pBufferInfo = &infos[index];
}
m_scheduler.EndRendering();
auto command = m_scheduler.Current().Handle();
vk::DependencyInfo dependency {};
dependency.sType = vk::StructureType::eDependencyInfo;
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
dependency.bufferMemoryBarrierCount = 1;
dependency.pBufferMemoryBarriers = &pre_barrier;
command.pipelineBarrier2(dependency);
command.bindPipeline(vk::PipelineBindPoint::eCompute, m_fault_process_pipeline);
command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute,
m_fault_process_pipeline_layout, 0, writes);
const auto num_threads = m_caching_num_pages / 32;
const auto num_workgroups = (num_threads + 63) / 64;
command.dispatch(static_cast<uint32_t>(num_workgroups), 1, 1);
dependency.pBufferMemoryBarriers = &post_barrier;
command.pipelineBarrier2(dependency);
const auto area = m_current_area;
m_scheduler.DeferOperation([this, mapped, offset, area] {
m_download_buffer.Invalidate(offset, PageFaultAreaSize);
m_fault_ranges.Clear();
const auto* faults = std::bit_cast<const uint64_t*>(mapped);
const auto count = static_cast<uint32_t>(faults[0]);
for (uint32_t index = 1; index <= count; ++index) {
m_fault_ranges.Add(faults[index], m_caching_pagesize);
LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
}
m_fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());
(void)m_buffer_cache.FindBuffer(start, end - start);
});
m_fault_areas[area] = 0;
});
m_fault_areas[m_current_area++] = m_scheduler.CurrentTick();
m_current_area %= MaxPendingFaults;
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/faultManager.h
Vendored
+46
View File
@@ -0,0 +1,46 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
#include "common/abi.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <array>
#include <cstdint>
namespace Libs::Graphics {
class BufferCache;
class FaultManager {
static constexpr size_t MaxPendingFaults = 8;
public:
FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, BufferCache& buffer_cache,
uint32_t caching_pagebits, uint64_t caching_num_pages);
~FaultManager();
KYTY_CLASS_NO_COPY(FaultManager);
[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return &m_fault_buffer; }
void ProcessFaultBuffer();
private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
BufferCache& m_buffer_cache;
RangeSet m_fault_ranges;
uint64_t m_caching_pagesize;
uint64_t m_caching_num_pages;
uint64_t m_fault_buffer_size;
Buffer m_fault_buffer;
Buffer m_download_buffer;
std::array<uint64_t, MaxPendingFaults> m_fault_areas {};
uint32_t m_current_area = 0;
vk::DescriptorSetLayout m_fault_process_desc_layout = nullptr;
vk::Pipeline m_fault_process_pipeline = nullptr;
vk::PipelineLayout m_fault_process_pipeline_layout = nullptr;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+12
View File
@@ -76,7 +76,19 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
m_gpu->SendCommandSync(unmap);
}
void GpuResourceManager::PrepareBda() {
std::shared_lock lock(m_mapped_ranges_mutex);
m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
});
m_fault_process_pending = true;
}
void GpuResourceManager::RunGarbageCollector() {
if (m_fault_process_pending) {
m_fault_process_pending = false;
m_buffer_cache.ProcessFaultBuffer();
}
m_texture_cache.ProcessDownloadImages();
m_texture_cache.RunGarbageCollector();
m_buffer_cache.RunGarbageCollector();
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+2
View File
@@ -30,6 +30,7 @@ public:
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
void PrepareBda();
void RunGarbageCollector();
private:
@@ -40,6 +41,7 @@ private:
mutable std::shared_mutex m_mapped_ranges_mutex;
RangeSet m_mapped_ranges;
GuestGpu* m_gpu = nullptr;
bool m_fault_process_pending = false;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/streamBuffer.cpp
Vendored
+26 -11
View File
@@ -73,8 +73,12 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
buffer_info.usage = flags;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
const VmaAllocationCreateFlags bda_flag =
with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
VmaAllocationCreateInfo allocation_info {};
allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
allocation_info.flags =
VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
allocation_info.usage = AllocationUsage(usage);
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
? VkMemoryPropertyFlags {}
@@ -99,6 +103,13 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
m_buffer->memory.type = allocation_result.memoryType;
m_buffer->memory.unique_id = VulkanNextMemoryUniqueId();
graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
vk::BufferDeviceAddressInfo address_info {};
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
address_info.buffer = m_buffer->buffer;
m_device_address = graphics.device.getBufferAddress(address_info);
EXIT_IF(m_device_address == 0);
}
VkMemoryPropertyFlags properties = 0;
vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
@@ -122,6 +133,11 @@ vk::Buffer Buffer::Handle() const noexcept {
return m_buffer->buffer;
}
vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
EXIT_IF(m_device_address == 0);
return m_device_address;
}
bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
}
@@ -141,6 +157,15 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
}
}
void Buffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
if (!m_is_coherent && size != 0) {
const auto result =
vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
}
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
vk::AccessFlags destination) const {
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
@@ -310,16 +335,6 @@ void StreamBuffer::Commit() {
watch.tick = tick;
}
void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset);
if (IsCoherent() || size == 0) {
return;
}
const auto result = vmaInvalidateAllocation(Graphics().allocator,
NativeBuffer().memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
EXIT_IF(source == nullptr);
const auto [data, offset] = Map(size, alignment);
src/graphics/host_gpu/renderer/cache/streamBuffer.h
Vendored
+3 -3
View File
@@ -48,12 +48,14 @@ public:
[[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; }
[[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; }
[[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; }
[[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept;
[[nodiscard]] uint64_t Offset(uint64_t address) const noexcept {
return address - m_cpu_address;
}
[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
void Write(uint64_t offset, const void* source, uint64_t size);
void Flush(uint64_t offset, uint64_t size);
void Invalidate(uint64_t offset, uint64_t size);
void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
uint64_t destination_offset, uint64_t size,
vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite,
@@ -84,6 +86,7 @@ private:
MemoryUsage m_usage = MemoryUsage::DeviceLocal;
uint64_t m_cpu_address = 0;
uint64_t m_size = 0;
vk::DeviceAddress m_device_address = 0;
std::unique_ptr<VulkanBuffer> m_buffer;
std::span<uint8_t> m_mapped;
bool m_is_coherent = false;
@@ -97,9 +100,6 @@ public:
[[nodiscard]] std::pair<uint8_t*, uint64_t> Map(uint64_t size, uint64_t alignment = 0,
bool allow_wait = true);
void Commit();
// Download mappings become visible to the CPU only after their GPU completion tick is free.
// Call this from the scheduler's deferred completion operation before reading Mapped().
void Invalidate(uint64_t offset, uint64_t size);
[[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0);
private:
src/graphics/host_gpu/renderer/commandScheduler.cpp
+1 -1
View File
@@ -196,8 +196,8 @@ void CommandScheduler::Finish() {
Submit();
}
m_master.Wait(CurrentTick() - 1);
PopPendingOperations();
BeginNext();
PopPendingOperations();
}
void CommandScheduler::Wait(uint64_t tick) {
src/graphics/host_gpu/renderer/pipeline/descriptors.cpp
+13 -78
View File
@@ -216,48 +216,6 @@ static BufferView NativeStorageBuffer(RenderContext&
return result;
}
static BufferView
NativeAddressBuffer(RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource,
const ShaderRecompiler::IR::ResourceSnapshot::Address& address,
ShaderType stage, uint32_t slot, uint32_t& address_offset, BufferId id) {
BufferView result;
address_offset = 0;
if (address.binding_base == 0) {
BindNullStorageBuffer(context, result);
return result;
}
if (resource.written) {
EXIT("writable address resources are unsupported\n");
}
const auto limit =
resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
? ShaderRecompiler::IR::FlatAddressWindowSize
: static_cast<uint64_t>(
context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
uint64_t size = 0;
const auto access = HostMemoryAccess::Mapped;
if (!HostMemoryQueryRange(address.binding_base, limit, access, size)) {
EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
address.binding_base);
}
const auto& graphics = context.GetGraphics();
const auto alignment = graphics.StorageMinAlignment();
auto [buffer, offset] =
context.GetBufferCache().ObtainBuffer(address.binding_base, size, false, false, id);
const auto aligned_offset = offset - offset % alignment;
const auto adjustment = offset - aligned_offset;
const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
const auto visible_size = std::min(size, static_cast<uint64_t>(max_range - adjustment));
address_offset = static_cast<uint32_t>(adjustment);
result.buffer = buffer->Handle();
result.offset = aligned_offset;
result.range = static_cast<vk::DeviceSize>(visible_size + adjustment);
SetVulkanObjectNameF(graphics.device, result.buffer,
"Kyty.{}.AddressBuffer[slot={} guest=0x{:016x} size=0x{:x}]",
ShaderStageResourceName(stage), slot, address.binding_base, visible_size);
return result;
}
static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) {
bool supported_dimension = false;
switch (resource.dimension) {
@@ -1000,25 +958,6 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
prepared.buffer_ids.push_back(cache.FindBuffer(address, size));
}
prepared.address_ids.clear();
prepared.address_ids.reserve(program.info.addresses.size());
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
const auto& resource = program.info.addresses[i];
const auto& address = snapshot.addresses[i];
if (address.binding_base == 0) {
prepared.address_ids.emplace_back();
continue;
}
EXIT_IF(resource.written);
const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
? ShaderRecompiler::IR::FlatAddressWindowSize
: static_cast<uint64_t>(m_context.GetGraphics()
.GetPhysicalDeviceProperties()
.limits.maxStorageBufferRange);
uint64_t size = 0;
EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size));
prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size));
}
}
void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
@@ -1028,8 +967,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
const auto& snapshot = *prepared.snapshot;
auto& resources = prepared.resources;
const auto& layout = program.bindings;
EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size() ||
prepared.address_ids.size() != program.info.addresses.size());
EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size());
resources.buffers.clear();
resources.buffers.reserve(program.info.buffers.size());
@@ -1049,15 +987,6 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
buffer_offset, prepared.buffer_ids[i]));
pack_memory_offset(i, buffer_offset);
}
resources.addresses.clear();
resources.addresses.reserve(program.info.addresses.size());
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
uint32_t address_offset = 0;
resources.addresses.push_back(NativeAddressBuffer(m_context, program.info.addresses[i],
snapshot.addresses[i], program.stage, i,
address_offset, prepared.address_ids[i]));
pack_memory_offset(static_cast<uint32_t>(program.info.buffers.size()) + i, address_offset);
}
if (!prepared.flattened_srt.empty()) {
resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt);
}
@@ -1129,6 +1058,10 @@ RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
if (bindings.pixel) {
FindBuffers(*bindings.pixel);
}
if (bindings.vertex.program->info.uses_dma ||
(bindings.pixel && bindings.pixel->program->info.uses_dma)) {
m_context.GetGpuResources().PrepareBda();
}
RebindBuffers(bindings.vertex);
if (bindings.pixel) {
RebindBuffers(*bindings.pixel);
@@ -1241,13 +1174,15 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
}
break;
case BindingKind::AddressMemory:
for (const auto resource: binding.resources) {
const auto& view = descriptors.addresses.at(resource);
EXIT_IF(view.buffer == nullptr);
m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
}
case BindingKind::BdaPagetable:
case BindingKind::FaultBuffer: {
auto& cache = m_context.GetBufferCache();
const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable
? cache.GetBdaPageTableBuffer()
: cache.GetFaultBuffer();
m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size());
break;
}
case BindingKind::FlattenedSrt:
case BindingKind::UserData:
case BindingKind::Gds: {
src/graphics/host_gpu/renderer/pipeline/descriptors.h
-2
View File
@@ -38,7 +38,6 @@ struct NativeDescriptors {
std::vector<BufferView> buffers;
std::vector<TextureBinding> images;
std::vector<vk::Sampler> samplers;
std::vector<BufferView> addresses;
BufferView gds;
BufferView flattened_srt;
BufferView user_data;
@@ -49,7 +48,6 @@ struct PreparedBindings {
const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
NativeDescriptors resources;
std::vector<BufferId> buffer_ids;
std::vector<BufferId> address_ids;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
bool committed = false;
src/graphics/host_gpu/renderer/renderCompute.cpp
+5 -2
View File
@@ -82,8 +82,8 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
const auto& resources = *input.stage.resources;
if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 ||
!program.info.images.empty() || !program.info.samplers.empty() ||
!program.info.addresses.empty() || !resources.images.empty() ||
!resources.samplers.empty() || !resources.addresses.empty()) {
program.info.uses_dma || !resources.images.empty() ||
!resources.samplers.empty()) {
return false;
}
const auto& resource = program.info.buffers.front();
@@ -337,6 +337,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader);
auto bindings = PrepareBindings(input_info.stage);
FindBuffers(bindings);
if (program.info.uses_dma) {
m_context.GetGpuResources().PrepareBda();
}
RebindBuffers(bindings);
RebindImages(bindings);
src/graphics/host_gpu/shaders/fault_buffer_process.comp
+35
View File
@@ -0,0 +1,35 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#version 450
#extension GL_ARB_gpu_shader_int64 : enable
layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
layout(std430, binding = 0) buffer input_buf {
uint fault_buffer[];
};
layout(std430, binding = 1) buffer output_buf {
uint64_t download_buffer[];
};
layout(std430, binding = 1) buffer output_buf32 {
uint download_buffer32[];
};
void main() {
const uint id = gl_GlobalInvocationID.x;
uint word = fault_buffer[id];
fault_buffer[id] = 0u;
const uint base_bit = id * 32u;
while (word != 0u) {
const uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u;
if (store_index >= MAX_PAGE_FAULTS) {
return;
}
const uint bit = findLSB(word);
word &= word - 1;
const uint page = base_bit + bit;
download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS;
}
}
src/graphics/host_gpu/vma.cpp
+4 -1
View File
@@ -72,7 +72,10 @@ bool GraphicContext::CreateAllocator() {
info.device = device;
info.pVulkanFunctions = &functions;
info.vulkanApiVersion = VULKAN_TARGET_API_VERSION;
info.flags = memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0;
info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
if (memory_budget_ext_enabled) {
info.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
}
const auto result = static_cast<vk::Result>(vmaCreateAllocator(&info, &allocator));
if (result != vk::Result::eSuccess) {
src/graphics/presentation/window/vulkanWindow.cpp
+6 -13
View File
@@ -594,20 +594,16 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.bufferDeviceAddress != VK_TRUE);
#if !defined(__APPLE__)
EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
#endif
features12.timelineSemaphore = VK_TRUE;
features12.shaderOutputLayer = VK_TRUE;
if (Config::GpuAssistedValidationEnabled()) {
if (supported_features12.bufferDeviceAddress != VK_TRUE) {
EXIT("--gpu-assisted-validation requires bufferDeviceAddress, unsupported by this "
"device\n");
}
features12.bufferDeviceAddress = VK_TRUE;
}
features12.bufferDeviceAddress = VK_TRUE;
vk::PhysicalDeviceFeatures device_features {};
device_features.fragmentStoresAndAtomics = VK_TRUE;
@@ -625,12 +621,9 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
device_features.shaderClipDistance = VK_TRUE;
device_features.shaderCullDistance = VK_TRUE;
device_features.largePoints = VK_TRUE;
device_features.vertexPipelineStoresAndAtomics = VK_TRUE;
graphics.sample_rate_shading_enabled = true;
device_features.vertexPipelineStoresAndAtomics =
supported_features2.features.vertexPipelineStoresAndAtomics;
if (Config::GpuAssistedValidationEnabled()) {
device_features.shaderInt64 = supported_features2.features.shaderInt64;
}
device_features.shaderInt64 = VK_TRUE;
vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
src/graphics/shader/recompiler/ShaderRecompiler.cpp
-1
View File
@@ -665,7 +665,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
runtime.read_memory = ReadZeroMemory;
}
runtime.userdata = options.read_memory_data;
runtime.flat_memory_base = options.flat_memory_base;
if (!IR::MaterializeResources(ir, runtime, resources, error)) {
return false;
}
src/graphics/shader/recompiler/ShaderRecompiler.h
-2
View File
@@ -6,7 +6,6 @@
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/shader.h"
#include <optional>
#include <span>
#include <vector>
@@ -20,7 +19,6 @@ struct CompileOptions {
uint32_t scratch_dwords = 0;
uint64_t shader_hash = 0;
uint64_t shader_base = 0;
std::optional<uint64_t> flat_memory_base;
uint32_t push_constant_offset = 0;
bool dump_ir = true;
bool early_dump = false;
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+10 -7
View File
@@ -70,8 +70,9 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
if (uses_gds) {
Expect(Kind::Gds);
}
if (!program.info.addresses.empty()) {
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
if (program.info.uses_dma) {
Expect(Kind::BdaPagetable);
Expect(Kind::FaultBuffer);
}
const bool uses_flattened_runtime =
!program.srt_reads.empty() ||
@@ -104,8 +105,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
program.bindings.push_constant_offset + program.bindings.push_constant_size >
IR::NativePushConstantSize ||
program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
program.bindings.memory_offset_count !=
program.info.buffers.size() + program.info.addresses.size() ||
program.bindings.memory_offset_count != program.info.buffers.size() ||
(!has_user_storage &&
program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
(has_user_storage && program.bindings.push_constant_size != 0) ||
@@ -146,8 +146,8 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
if (planning_only_handle(inst)) {
break;
}
if (dense >= program.info.addresses.size()) {
return Fail(error, "typed address handle has an invalid dense resource");
if (inst.NumArgs() != 2 || !program.info.uses_dma) {
return Fail(error, "typed address handle has invalid DMA metadata");
}
break;
case IR::ValueOpcode::GetScratchResource:
@@ -188,7 +188,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
if (address_access != IR::AddressAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
return Fail(error, "address operation has invalid memory metadata");
@@ -198,6 +199,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
return Fail(error, "scratch operation has no per-thread storage");
}
requirements.function_scratch = true;
} else if (address_access == IR::AddressAccess::Write) {
return Fail(error, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
}
}
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+22 -3
View File
@@ -36,8 +36,10 @@ enum : uint32_t {
ExecutionModeSignedZeroInfNanPreserve = 4461,
ExecutionModeDerivativeGroupQuadsKHR = 5289,
AddressingModelLogical = 0,
AddressingModelPhysicalStorageBuffer64 = 5348,
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
CapabilityInt64 = 11,
CapabilityImageGatherExtended = 25,
CapabilityClipDistance = 32,
CapabilityCullDistance = 33,
@@ -52,6 +54,7 @@ enum : uint32_t {
CapabilityShaderViewportIndexLayerEXT = 5254,
CapabilityFragmentBarycentricKHR = 5284,
CapabilityComputeDerivativeGroupQuadsKHR = 5288,
CapabilityPhysicalStorageBufferAddresses = 5347,
StorageClassUniformConstant = 0,
StorageClassInput = 1,
StorageClassOutput = 3,
@@ -60,6 +63,7 @@ enum : uint32_t {
StorageClassPushConstant = 9,
StorageClassImage = 11,
StorageClassStorageBuffer = 12,
StorageClassPhysicalStorageBuffer = 5349,
FunctionControlNone = 0,
SelectionControlNone = 0,
LoopControlNone = 0,
@@ -148,7 +152,9 @@ enum : uint32_t {
OpConstantComposite = 44,
OpUndef = 1,
OpFunction = 54,
OpFunctionParameter = 55,
OpFunctionEnd = 56,
OpFunctionCall = 57,
OpVariable = 59,
OpImageTexelPointer = 60,
OpLoad = 61,
@@ -178,6 +184,8 @@ enum : uint32_t {
OpConvertFToS = 110,
OpConvertSToF = 111,
OpConvertUToF = 112,
OpUConvert = 113,
OpConvertUToPtr = 120,
OpBitcast = 124,
OpSNegate = 126,
OpFNegate = 127,
@@ -256,11 +264,16 @@ enum : uint32_t {
OpSwitch = 251,
OpKill = 252,
OpReturn = 253,
OpReturnValue = 254,
OpGroupNonUniformBallot = 339,
OpGroupNonUniformBallotFindLSB = 343,
OpGroupNonUniformShuffle = 345,
};
enum : uint32_t {
MemoryAccessAlignedMask = 0x2,
};
enum : uint32_t {
GlslRoundEven = 2,
GlslTrunc = 3,
@@ -345,9 +358,10 @@ struct EmitterState {
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t storage_buffer_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers + IR::ShaderInfo::MaxAddresses>
memory_byte_offsets {};
uint32_t address_memory_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
uint32_t bda_pagetable_variable = 0;
uint32_t fault_buffer_variable = 0;
uint32_t bda_pointer_function = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
@@ -384,6 +398,7 @@ uint32_t TypeBool(EmitterState& state);
uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
uint32_t TypeU32(EmitterState& state);
uint32_t TypeU64(EmitterState& state);
uint32_t TypeDeviceAddress(EmitterState& state);
uint32_t TypeU32Pair(EmitterState& state);
uint32_t TypeI32(EmitterState& state);
uint32_t TypeI32Pair(EmitterState& state);
@@ -397,6 +412,8 @@ uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t point
uint32_t TypeFunction(EmitterState& state);
uint32_t TypeStorageBufferPointer(EmitterState& state);
uint32_t TypeStorageBufferElementPointer(EmitterState& state);
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
uint32_t TypePhysicalU32Pointer(EmitterState& state);
uint32_t TypePushConstantElementPointer(EmitterState& state);
uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
@@ -849,6 +866,8 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error);
void DefineGetBdaPointer(EmitterState& state);
// These templates accept local lambdas from several emitter translation units.
template <typename Fn>
void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+194 -51
View File
@@ -1,5 +1,7 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include <algorithm>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -95,60 +97,146 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad
return result;
}
uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t low, uint32_t high) {
uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
uint32_t high) {
auto& state = ctx.state;
auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u));
}
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
if (mem.kind == IR::ResourceKind::Scratch) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid = Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
}
const auto base = state.program.info.addresses[mem.resource].specialized_base;
if (mem.kind == IR::ResourceKind::Flat ||
(mem.kind == IR::ResourceKind::Global &&
state.program.info.addresses[mem.resource].unbased)) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto base_low = ConstantU32(state, static_cast<uint32_t>(base));
const auto base_high = ConstantU32(state, static_cast<uint32_t>(base >> 32u));
const auto relative = Binary(state, OpISub, TypeU32(state), low, base_low);
const auto borrow = Binary(state, OpULessThan, TypeBool(state), low, base_low);
const auto relative_high = Binary(
state, OpISub, TypeU32(state), Binary(state, OpISub, TypeU32(state), high, base_high),
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0)));
const auto valid =
Binary(state, OpIEqual, TypeBool(state), relative_high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, relative, ConstantU32(state, UINT32_MAX));
}
const auto initial = base + static_cast<uint64_t>(static_cast<int64_t>(immediate));
auto relative_high = ConstantU32(state, static_cast<uint32_t>(initial >> 32u));
const auto relative = AddU64Low(state, ConstantU32(state, static_cast<uint32_t>(initial)),
relative_high, low, ConstantU32(state, 0), relative_high);
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid =
Binary(state, OpIEqual, TypeBool(state), relative_high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, relative, ConstantU32(state, UINT32_MAX));
Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
}
uint32_t AddAddressMemoryOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
uint32_t address) {
auto& state = ctx.state;
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
const auto offset =
state.memory_byte_offsets[state.program.info.buffers.size() + binding.array_index];
const auto corrected = Binary(state, OpIAdd, TypeU32(state), address, offset);
const auto not_invalid =
Binary(state, OpINotEqual, TypeBool(state), address, ConstantU32(state, UINT32_MAX));
const auto no_overflow =
Binary(state, OpUGreaterThanEqual, TypeBool(state), corrected, address);
return Select(state, TypeU32(state), AndCondition(state, not_invalid, no_overflow), corrected,
ConstantU32(state, UINT32_MAX));
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
return state.builder.Constant(OpConstant, TypeDeviceAddress(state),
{static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u)});
}
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
const auto low64 = Unary(state, OpUConvert, TypeDeviceAddress(state), low);
const auto high64 = Binary(state, OpShiftLeftLogical, TypeDeviceAddress(state),
Unary(state, OpUConvert, TypeDeviceAddress(state), high),
ConstantDeviceAddress(state, 32));
return Binary(state, OpBitwiseOr, TypeDeviceAddress(state), low64, high64);
}
uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
auto& state = ctx.state;
auto low = ctx.Arg(inst, 1);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u));
}
uint32_t address = 0;
if (mem.address_is_full) {
address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2));
} else {
const auto* handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetAddressResource ||
handle->NumArgs() != 2) {
ctx.Fail(inst, "has no address base pair");
return ConstantDeviceAddress(state, 0);
}
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
address = Binary(state, OpIAdd, TypeDeviceAddress(state), base,
Unary(state, OpUConvert, TypeDeviceAddress(state), low));
}
auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
return immediate == 0
? address
: Binary(state, OpIAdd, TypeDeviceAddress(state), address,
ConstantDeviceAddress(state,
static_cast<uint64_t>(static_cast<int64_t>(immediate))));
}
uint32_t FaultElementPointer(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.fault_buffer_variable, ConstantU32(state, 0), index});
return pointer;
}
void RecordBdaFault(EmitterState& state, uint32_t page) {
const auto word = Binary(state, OpShiftRightLogical, TypeU32(state), page,
ConstantU32(state, 5));
const auto bit = Binary(
state, OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1),
Binary(state, OpBitwiseAnd, TypeU32(state), page, ConstantU32(state, 31)));
const auto pointer = FaultElementPointer(state, word);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
state.builder.AddFunction(
{OpStore, pointer, Binary(state, OpBitwiseOr, TypeU32(state), value, bit)});
}
uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state;
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpFunctionCall, TypeDeviceAddress(state), result, state.bda_pointer_function, address});
return result;
}
uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state;
const auto bda = GetBdaPointer(ctx, address);
const auto present = Binary(state, OpINotEqual, TypeBool(state), bda,
ConstantDeviceAddress(state, 0));
return EmitValueOrZeroIfCondition(state, present, [&]() {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, bda});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, TypeU32(state), value, pointer, MemoryAccessAlignedMask, sizeof(uint32_t)});
return value;
});
}
uint32_t LoadBda(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t bits) {
auto& state = ctx.state;
const auto address = GuestAddress(ctx, inst, mem);
const auto active = ctx.Arg(inst, inst.NumArgs() - 1);
return EmitValueOrZeroIfCondition(state, active, [&]() {
const auto aligned = Binary(state, OpBitwiseAnd, TypeDeviceAddress(state), address,
ConstantDeviceAddress(state, ~uint64_t {3}));
const auto first = LoadBdaDword(ctx, aligned);
const auto byte = Binary(state, OpBitwiseAnd, TypeU32(state),
Unary(state, OpUConvert, TypeU32(state), address),
ConstantU32(state, 3));
const auto crosses = bits == 8u
? ConstantBool(state, false)
: Binary(state, bits == 16u ? OpUGreaterThan : OpINotEqual,
TypeBool(state), byte,
ConstantU32(state, bits == 16u ? 2u : 0u));
const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() {
return LoadBdaDword(
ctx, Binary(state, OpIAdd, TypeDeviceAddress(state), aligned,
ConstantDeviceAddress(state, sizeof(uint32_t))));
});
const auto shift = Binary(state, OpShiftLeftLogical, TypeU32(state), byte,
ConstantU32(state, 3));
const auto upper_shift = Binary(
state, OpShiftLeftLogical, TypeU32(state),
Binary(state, OpBitwiseAnd, TypeU32(state),
Binary(state, OpISub, TypeU32(state), ConstantU32(state, 4), byte),
ConstantU32(state, 3)),
ConstantU32(state, 3));
const auto merged = Binary(
state, OpBitwiseOr, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), first, shift),
Binary(state, OpShiftLeftLogical, TypeU32(state), second, upper_shift));
return bits == 32u
? merged
: Binary(state, OpBitwiseAnd, TypeU32(state), merged,
ConstantU32(state, bits == 8u ? 0xffu : 0xffffu));
});
}
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
@@ -163,11 +251,10 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0),
ConstantU32(ctx.state, mem.offset));
}
const auto address = AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
if (mem.kind == IR::ResourceKind::Scratch) {
return address;
if (mem.kind != IR::ResourceKind::Scratch) {
EXIT("physical address memory must use the BDA emitter\n");
}
return AddAddressMemoryOffset(ctx, mem, address);
return ScratchByteAddress(ctx, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
}
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
@@ -923,6 +1010,58 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
} // namespace
void DefineGetBdaPointer(EmitterState& state) {
if (!state.program.info.uses_dma) {
return;
}
const auto type = TypeDeviceAddress(state);
const auto function_type = state.builder.Type(OpTypeFunction, {type, type});
state.bda_pointer_function = state.builder.AllocateId();
const auto address = state.builder.AllocateId();
const auto entry_label = state.builder.AllocateId();
state.builder.AddName(state.bda_pointer_function, "get_bda_pointer");
state.builder.AddFunction(
{OpFunction, type, state.bda_pointer_function, FunctionControlNone, function_type});
state.builder.AddFunction({OpFunctionParameter, type, address});
EmitLabel(state, entry_label);
const auto page64 = Binary(
state, OpShiftRightLogical, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
const auto page = Unary(state, OpUConvert, TypeU32(state), page64);
const auto entry_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeDeviceAddressStoragePointer(state), entry_pointer,
state.bda_pagetable_variable, ConstantU32(state, 0), page});
const auto base = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, base, entry_pointer});
const auto missing =
Binary(state, OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0));
const auto fault_label = state.builder.AllocateId();
const auto available_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction(
{OpBranchConditional, missing, fault_label, available_label});
EmitLabel(state, fault_label);
RecordBdaFault(state, page);
state.builder.AddFunction({OpBranch, merge_label});
EmitLabel(state, available_label);
const auto offset = Binary(
state, OpBitwiseAnd, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1));
const auto available = Binary(state, OpIAdd, type, base, offset);
state.builder.AddFunction({OpBranch, merge_label});
EmitLabel(state, merge_label);
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpPhi, type, result, ConstantDeviceAddress(state, 0), fault_label,
available, available_label});
state.builder.AddFunction({OpReturnValue, result});
state.builder.AddFunction({OpFunctionEnd});
}
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto op = inst.GetOpcode();
@@ -981,6 +1120,10 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
}
const auto address_info = IR::AddressOpcodeInfoOf(op);
const bool load_address = address_info.access == IR::AddressAccess::Read;
if (load_address && ctx.Memory(inst).kind != IR::ResourceKind::Scratch) {
ctx.Define(inst, LoadBda(ctx, inst, ctx.Memory(inst), address_info.data_bits));
return true;
}
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadBufferU32;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+2 -14
View File
@@ -108,20 +108,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
return access;
case IR::ResourceKind::ScalarAddress:
case IR::ResourceKind::Flat:
case IR::ResourceKind::Global: {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource,
"address memory binding was not emitted");
}
const auto binding = ResourceForDescriptor(
state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
access.object_pointer, state.address_memory_variable,
ConstantU32(state, binding.array_index)});
break;
}
case IR::ResourceKind::Global:
EXIT("physical address memory must use the BDA emitter\n");
case IR::ResourceKind::ScalarBuffer:
case IR::ResourceKind::Buffer: {
if (state.storage_buffer_variable == 0) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+45 -11
View File
@@ -28,6 +28,10 @@ uint32_t TypeU64(EmitterState& state) {
return TypeU32Vector(state, 2);
}
uint32_t TypeDeviceAddress(EmitterState& state) {
return state.builder.Type(OpTypeInt, {64, 0});
}
uint32_t TypeU32Pair(EmitterState& state) {
const auto element = TypeU32(state);
return state.builder.Type(OpTypeStruct, {element, element});
@@ -90,6 +94,14 @@ uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeU32(state));
}
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeDeviceAddress(state));
}
uint32_t TypePhysicalU32Pointer(EmitterState& state) {
return TypePointer(state, StorageClassPhysicalStorageBuffer, TypeU32(state));
}
uint32_t TypePushConstantElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassPushConstant, TypeU32(state));
}
@@ -130,6 +142,15 @@ uint32_t SampleMaskArrayType(EmitterState& state) {
return state.builder.Type(OpTypeArray, {TypeI32(state), ConstantU32(state, 1)});
}
uint32_t BdaPagetableType(EmitterState& state) {
const auto array = state.builder.DecoratedType(
OpTypeRuntimeArray, {TypeDeviceAddress(state)},
{{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}});
return state.builder.DecoratedType(
OpTypeStruct, {array},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}});
}
uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
return state.builder.Type(OpTypeArray, {TypeF32(state), ConstantU32(state, count)});
}
@@ -143,13 +164,14 @@ void DefineDescriptorVariables(EmitterState& state) {
state.storage_buffer_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer);
}
if (DescriptorBinding(state, IR::DescriptorBindingKind::AddressMemory) != nullptr) {
const auto count =
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::AddressMemory));
const auto array_type = state.builder.Type(OpTypeArray, {StorageBufferType(state), count});
const auto pointer_type = TypePointer(state, StorageClassStorageBuffer, array_type);
state.address_memory_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer);
if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) {
state.bda_pagetable_variable = state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassStorageBuffer, BdaPagetableType(state)),
StorageClassStorageBuffer);
}
if (DescriptorBinding(state, IR::DescriptorBindingKind::FaultBuffer) != nullptr) {
state.fault_buffer_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer);
}
if (state.program.bindings.push_constant_size != 0) {
const auto pointer_type =
@@ -564,9 +586,13 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) {
if (state.storage_buffer_variable != 0) {
Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
}
if (state.address_memory_variable != 0) {
Decorate(state.address_memory_variable, "address_memory",
IR::DescriptorBindingKind::AddressMemory);
if (state.bda_pagetable_variable != 0) {
Decorate(state.bda_pagetable_variable, "bda_pagetable",
IR::DescriptorBindingKind::BdaPagetable);
}
if (state.fault_buffer_variable != 0) {
Decorate(state.fault_buffer_variable, "fault_buffer",
IR::DescriptorBindingKind::FaultBuffer);
}
constexpr const char* SampledNames[] = {"sampled_1d",
"sampled_1d_array",
@@ -644,6 +670,11 @@ void DefineModule(EmitterState& state) {
state.builder.RequireCapability(CapabilityShader);
state.builder.RequireCapability(CapabilitySignedZeroInfNanPreserve);
if (state.program.info.uses_dma) {
state.builder.RequireCapability(CapabilityInt64);
state.builder.RequireCapability(CapabilityPhysicalStorageBufferAddresses);
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
}
if (state.clip_distance_variable != 0) {
state.builder.RequireCapability(CapabilityClipDistance);
}
@@ -682,7 +713,10 @@ void DefineModule(EmitterState& state) {
state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
}
state.builder.RequireExtension("SPV_KHR_float_controls");
state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
state.builder.AddMemoryModel(
{state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64
: AddressingModelLogical,
MemoryModelGLSL450});
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
state.interface_variables);
// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+1
View File
@@ -566,6 +566,7 @@ bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* e
SetError(error, ctx.error.c_str());
return false;
}
DefineGetBdaPointer(state);
for (const auto* block: program.blocks) {
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
src/graphics/shader/recompiler/frontend/translate/Memory.cpp
+4 -6
View File
@@ -136,12 +136,10 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
memory.offen = decoded.offen;
memory.resource = ResourceIndexFromOperand(decoded.src1);
memory.sampler = ResourceIndexFromOperand(decoded.src2);
if (memory.kind == ResourceKind::ScalarAddress) {
memory.resource = RawScalarLoadBase(decoded.src0);
} else if (memory.kind == ResourceKind::ScalarBuffer) {
if (memory.kind == ResourceKind::ScalarBuffer) {
memory.resource = ResourceIndexFromOperand(decoded.src0);
} else if (memory.kind == ResourceKind::Lds || memory.kind == ResourceKind::Gds ||
memory.kind == ResourceKind::Scratch) {
} else if (IsAddressResourceKind(memory.kind) || memory.kind == ResourceKind::Lds ||
memory.kind == ResourceKind::Gds) {
memory.resource = 0;
memory.sampler = 0;
}
@@ -421,7 +419,7 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) {
bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
const auto memory = MemoryInfoFromDecoded(inst);
const auto resource =
raw ? GetScalarAddressResource(memory.resource) : GetBufferResource(memory);
raw ? GetScalarAddressResource(RawScalarLoadBase(inst.src0)) : GetBufferResource(memory);
const auto offset = ReadU32(inst.src1);
std::array<IR::Value, 16> loaded {};
for (uint32_t component = 0; component < memory.data_dwords; component++) {
src/graphics/shader/recompiler/ir/ShaderIR.h
+3 -17
View File
@@ -176,20 +176,6 @@ struct SampledResourcePair {
bool operator==(const SampledResourcePair& other) const = default;
};
struct AddressResource {
uint32_t source = UINT32_MAX;
uint32_t first_use_pc = 0;
ResourceKind kind = ResourceKind::Flat;
int32_t min_offset = 0;
uint64_t specialized_base = 0;
bool unbased = true;
bool read = false;
bool written = false;
bool atomic = false;
bool operator==(const AddressResource& other) const = default;
};
enum class StageInputKind {
VertexIndex,
InstanceIndex,
@@ -321,7 +307,8 @@ enum class DescriptorBindingKind {
StorageAtomic3D,
Samplers,
Gds,
AddressMemory,
BdaPagetable,
FaultBuffer,
FlattenedSrt,
UserData,
Count,
@@ -431,13 +418,11 @@ struct BindingLayout {
struct ShaderInfo {
static constexpr uint32_t MaxBuffers = 32;
static constexpr uint32_t MaxAddresses = 32;
static constexpr uint32_t MaxImages = 32;
static constexpr uint32_t MaxSamplers = 32;
static constexpr uint32_t MaxSampledPairs = 64;
std::vector<BufferResource> buffers;
std::vector<AddressResource> addresses;
std::vector<ImageResource> images;
std::vector<SamplerResource> samplers;
std::vector<SampledResourcePair> sampled_pairs;
@@ -446,6 +431,7 @@ struct ShaderInfo {
std::array<uint8_t, 32> vertex_fetch_components {};
int32_t vertex_offset_sgpr = -1;
bool has_bitwise_xor = false;
bool uses_dma = false;
bool operator==(const ShaderInfo& other) const = default;
};
src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp
+4 -8
View File
@@ -93,8 +93,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
return false;
}
next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
next.memory_offset_count =
static_cast<uint32_t>(program.info.buffers.size() + program.info.addresses.size());
next.memory_offset_count = static_cast<uint32_t>(program.info.buffers.size());
if (!program.info.buffers.empty()) {
std::vector<uint32_t> resources(program.info.buffers.size());
@@ -157,12 +156,9 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
if (uses_gds) {
AddBinding(next, DescriptorBindingKind::Gds);
}
if (!program.info.addresses.empty()) {
std::vector<uint32_t> resources(program.info.addresses.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
if (program.info.uses_dma) {
AddBinding(next, DescriptorBindingKind::BdaPagetable);
AddBinding(next, DescriptorBindingKind::FaultBuffer);
}
const bool uses_flattened_runtime =
!program.srt_reads.empty() ||
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
+2 -68
View File
@@ -267,13 +267,6 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
return true;
}
uint64_t AddressSpecialization(const AddressResource& resource,
const ResourceSnapshot::Address& snapshot) {
return resource.kind == ResourceKind::Flat || resource.unbased
? snapshot.binding_base
: snapshot.guest_base - snapshot.binding_base;
}
size_t FlattenedRuntimeDwords(const Program& program) {
size_t size = program.srt_reads.size();
for (uint32_t resource = 0; resource < program.info.images.size(); resource++) {
@@ -298,8 +291,7 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
}
if (snapshot.buffers.size() != program.info.buffers.size() ||
snapshot.images.size() != program.info.images.size() ||
snapshot.samplers.size() != program.info.samplers.size() ||
snapshot.addresses.size() != program.info.addresses.size()) {
snapshot.samplers.size() != program.info.samplers.size()) {
if (error != nullptr) {
*error = "resource snapshot does not match dense shader topology";
}
@@ -343,14 +335,6 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
}
return true;
};
for (uint32_t i = 0; i < snapshot.addresses.size(); i++) {
if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) {
if (error != nullptr) {
*error = fmt::format("address resource {} binds above its guest base", i);
}
return false;
}
}
std::unordered_set<uint32_t> indirect_resources;
for (const auto& table: snapshot.indirect_images) {
const auto* source = table.resource < program.info.images.size()
@@ -526,15 +510,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
}
}
}
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
if (program.info.addresses[i].specialized_base !=
AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) {
if (error != nullptr) {
*error = fmt::format("address resource {} no longer matches specialization", i);
}
return false;
}
}
return true;
}
@@ -550,7 +525,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
std::vector<DescriptorSourceRequest> requests;
std::vector<uint8_t> clean_flat_slots(program.srt_reads.size());
requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u +
program.info.samplers.size() + program.info.addresses.size());
program.info.samplers.size());
for (const auto& buffer: program.info.buffers) {
requests.push_back({buffer.source, buffer.first_use_pc});
}
@@ -576,12 +551,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
for (const auto& sampler: program.info.samplers) {
requests.push_back({sampler.source, sampler.first_use_pc});
}
for (const auto& address: program.info.addresses) {
if (!address.unbased) {
requests.push_back({address.source, address.first_use_pc});
}
}
std::vector<DescriptorValue> values;
std::vector<uint32_t> flattened_srt;
if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, clean_flat_slots,
@@ -686,37 +655,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
return false;
}
next.samplers.assign(cursor, cursor + program.info.samplers.size());
cursor += program.info.samplers.size();
for (const auto& address: program.info.addresses) {
if (!address.unbased) {
const auto value = *cursor++;
auto base = (static_cast<uint64_t>(value.dwords[0]) |
static_cast<uint64_t>(value.dwords[1]) << 32u) &
AddressMask;
if (address.kind == ResourceKind::ScalarAddress) {
base &= ~uint64_t {3};
}
const auto before = static_cast<uint64_t>(-static_cast<int64_t>(address.min_offset));
uint64_t binding_base = 0;
if (address.kind == ResourceKind::Flat) {
binding_base = base & ~(FlatAddressWindowSize - 1u);
} else if (base >= before) {
binding_base = base - before;
}
next.addresses.push_back({base, binding_base});
} else {
if (!runtime.flat_memory_base.has_value()) {
if (error != nullptr) {
*error = fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
"guest-address translation",
address.kind == ResourceKind::Flat ? "FLAT" : "global",
address.first_use_pc);
}
return false;
}
next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base});
}
}
next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
if (!ValidateResourceSnapshot(program, next, error)) {
return false;
@@ -791,10 +729,6 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
next.buffers[i].descriptor_format = descriptor.Format();
next.buffers[i].descriptor_swizzle = descriptor.DstSelXYZW();
}
for (uint32_t i = 0; i < next.addresses.size(); i++) {
next.addresses[i].specialized_base =
AddressSpecialization(next.addresses[i], next_snapshot.addresses[i]);
}
for (uint32_t i = 0; i < next.images.size(); i++) {
const auto& descriptor = next_snapshot.images[i];
auto& image = next.images[i];
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h
-9
View File
@@ -5,15 +5,7 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
constexpr uint64_t FlatAddressWindowSize = 0x10000;
struct ResourceSnapshot {
struct Address {
uint64_t guest_base = 0;
uint64_t binding_base = 0;
bool operator==(const Address& other) const = default;
};
struct IndirectImage {
uint32_t resource = 0;
uint32_t capacity = 0;
@@ -25,7 +17,6 @@ struct ResourceSnapshot {
std::vector<DescriptorValue> buffers;
std::vector<DescriptorValue> images;
std::vector<DescriptorValue> samplers;
std::vector<Address> addresses;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
std::vector<IndirectImage> indirect_images;
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+6 -157
View File
@@ -7,7 +7,6 @@
#include <algorithm>
#include <fmt/format.h>
#include <span>
#include <unordered_set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -96,10 +95,10 @@ class Tracker {
public:
explicit Tracker(Program& program): m_program(program), m_info(program.info) {
m_info.buffers.clear();
m_info.addresses.clear();
m_info.images.clear();
m_info.samplers.clear();
m_info.sampled_pairs.clear();
m_info.uses_dma = false;
}
bool Run(std::string* error) {
@@ -180,90 +179,6 @@ private:
m_program.shader_hash, StageName(m_program.stage), pc, reason));
}
struct AddressPart {
Value value;
bool rooted = false;
};
AddressPart FindAddressPart(Value value, Value active,
std::unordered_set<const Inst*>& visiting) const {
value = value.Resolve();
if (value.IsImmediate()) {
return {value, false};
}
const auto* inst = value.TryInstruction();
if (inst == nullptr || !visiting.insert(inst).second) {
return {};
}
const auto finish = [&](AddressPart part) {
visiting.erase(inst);
return part;
};
std::string reason;
if (ValidateRuntimeValue(m_program, value, reason)) {
return finish({value, true});
}
const auto merge = [&](AddressPart left, AddressPart right, bool require_both) {
if (left.value.IsEmpty() || right.value.IsEmpty()) {
return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left);
}
return EquivalentValue(m_program, left.value, right.value)
? AddressPart {left.value, left.rooted || right.rooted}
: AddressPart {};
};
switch (inst->GetOpcode()) {
case ValueOpcode::SelectU32:
if (EquivalentValue(m_program, inst->Arg(0), active)) {
return finish(FindAddressPart(inst->Arg(1), active, visiting));
}
return finish(merge(FindAddressPart(inst->Arg(1), active, visiting),
FindAddressPart(inst->Arg(2), active, visiting), true));
case ValueOpcode::Phi: {
const auto invariant = ResolveInvariantPhi(m_program, value);
return finish(invariant.IsEmpty() ? AddressPart {}
: FindAddressPart(invariant, active, visiting));
}
case ValueOpcode::IAdd32:
case ValueOpcode::ISub32: {
const auto left = FindAddressPart(inst->Arg(0), active, visiting);
const auto right = FindAddressPart(inst->Arg(1), active, visiting);
if (left.rooted == right.rooted ||
(inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) {
return finish({});
}
return finish(left.rooted ? left : right);
}
case ValueOpcode::CompositeExtractU32x2: {
const auto* source = inst->Arg(0).ResolveInstruction();
if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) {
return finish({});
}
const auto left = FindAddressPart(source->Arg(0), active, visiting);
const auto right = FindAddressPart(source->Arg(1), active, visiting);
if (left.rooted == right.rooted) {
return finish({});
}
return finish(left.rooted ? left : right);
}
default: return finish({});
}
}
bool MakeFlatAddressSource(const Inst& handle, Value active,
DescriptorSource& descriptor) const {
std::unordered_set<const Inst*> visiting;
auto low = FindAddressPart(handle.Arg(0), active, visiting);
visiting.clear();
auto high = FindAddressPart(handle.Arg(1), active, visiting);
if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) {
return false;
}
descriptor.dword_count = 2;
descriptor.dwords[0] = low.value;
descriptor.dwords[1] = high.value;
return true;
}
bool MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust,
DescriptorSource& descriptor, uint32_t pc, std::string* error) const {
if (handle.NumArgs() != width) {
@@ -577,42 +492,14 @@ private:
return true;
}
bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle,
uint32_t& source, bool& unbased, std::string* error) {
handle = value.Resolve().TryInstruction();
bool ValidateAddressHandle(Value value, uint32_t pc, std::string* error) const {
const auto* handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) {
return Fail(pc, error, "address operation requires GetAddressResource");
}
if (handle->NumArgs() != 2) {
return Fail(pc, error, "GetAddressResource must have two address dwords");
}
DescriptorSource descriptor;
const auto& memory = m_program.memory_info[memory_inst.Flags<MemoryFlags>().index];
if (memory.address_is_full) {
const auto active = memory_inst.Arg(memory_inst.NumArgs() - 1u);
if (memory.kind != ResourceKind::Flat ||
!MakeFlatAddressSource(*handle, active, descriptor)) {
unbased = true;
source = UINT32_MAX;
return true;
}
} else if (!MakeSource(*handle, 2, false, false, descriptor, pc, error)) {
return false;
}
std::string reason;
uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) {
if (memory.address_is_full) {
unbased = true;
source = UINT32_MAX;
return true;
}
return Fail(
pc, error,
fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason));
}
unbased = false;
source = InternSource(descriptor);
return true;
}
@@ -706,39 +593,6 @@ private:
return static_cast<uint32_t>(m_info.samplers.size() - 1);
}
uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory,
AddressAccess access, uint32_t pc) {
auto immediate = static_cast<int32_t>(memory.offset);
if (memory.kind == ResourceKind::ScalarAddress) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto min_offset = unbased ? 0 : std::min(immediate, 0);
for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
auto& address = m_info.addresses[i];
if (address.source == source && address.unbased == unbased &&
address.kind == memory.kind) {
address.first_use_pc = std::min(address.first_use_pc, pc);
address.min_offset = std::min(address.min_offset, min_offset);
address.read = address.read || access == AddressAccess::Read;
address.written = address.written || access == AddressAccess::Write;
return i;
}
}
if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
return UINT32_MAX;
}
AddressResource address;
address.source = source;
address.first_use_pc = pc;
address.kind = memory.kind;
address.min_offset = min_offset;
address.unbased = unbased;
address.read = access == AddressAccess::Read;
address.written = access == AddressAccess::Write;
m_info.addresses.push_back(address);
return static_cast<uint32_t>(m_info.addresses.size() - 1);
}
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) {
@@ -839,16 +693,11 @@ private:
}
return true;
}
bool unbased = false;
if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) {
if (!ValidateAddressHandle(inst.Arg(0), flags.pc, error)) {
return false;
}
resource = AddAddress(source, unbased, memory, address_info.access, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "address resource limit exceeded");
}
return AddHandlePatch(handle, resource, flags.pc, error) &&
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
m_info.uses_dma = true;
return true;
}
if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) {
src/graphics/shader/recompiler/ir/passes/SrtWalker.h
-2
View File
@@ -3,7 +3,6 @@
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <optional>
#include <span>
namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -17,7 +16,6 @@ struct SrtRuntime {
uint64_t shader_base = 0;
SrtMemoryReader read_memory = nullptr;
void* userdata = nullptr;
std::optional<uint64_t> flat_memory_base;
SrtMemoryReader read_specialization_memory = nullptr;
};
src/graphics/shader/shader.cpp
+2 -10
View File
@@ -994,10 +994,10 @@ static void LogShaderProgramCacheHit(const char* stage, uint64_t shader_hash, ui
static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program) {
std::string ret = fmt::format(
"push={} groups={} user={} buffers={} images={} samplers={} addresses={} srt={}",
"push={} groups={} user={} buffers={} images={} samplers={} dma={} srt={}",
program.bindings.push_constant_size, program.bindings.descriptors.size(),
program.bindings.user_data_registers.size(), program.info.buffers.size(),
program.info.images.size(), program.info.samplers.size(), program.info.addresses.size(),
program.info.images.size(), program.info.samplers.size(), program.info.uses_dma,
program.srt_reads.size());
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto& buffer = program.info.buffers[i];
@@ -1014,9 +1014,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog
static_cast<uint32_t>(image.conversion_format), image.shader_swizzle,
image.indirect_root, image.indirect_mapping_offset, image.indirect_mapping_capacity);
}
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
ret += fmt::format(" a{}[base=0x{:x}]", i, program.info.addresses[i].specialized_base);
}
return ret;
}
@@ -1056,11 +1053,6 @@ static void ShaderAppendNativeSpecialization(std::vector<uint32_t>&
ids.push_back(static_cast<uint32_t>(image.indirect_resources.size()));
ids.insert(ids.end(), image.indirect_resources.begin(), image.indirect_resources.end());
}
ids.push_back(static_cast<uint32_t>(program.info.addresses.size()));
for (const auto& address: program.info.addresses) {
ids.push_back(static_cast<uint32_t>(address.specialized_base));
ids.push_back(static_cast<uint32_t>(address.specialized_base >> 32u));
}
}
static std::span<const uint32_t> AddShaderProgramPermutation(const char* stage,
tests/ResourceTrackingTests.cpp
+23 -36
View File
@@ -295,7 +295,7 @@ void TestInvariantIndirectImageMaterialization() {
const auto same_snapshot = [](const ResourceSnapshot &lhs,
const ResourceSnapshot &rhs) {
return lhs.buffers == rhs.buffers && lhs.images == rhs.images &&
lhs.samplers == rhs.samplers && lhs.addresses == rhs.addresses &&
lhs.samplers == rhs.samplers &&
lhs.flattened_srt == rhs.flattened_srt &&
lhs.user_data == rhs.user_data &&
lhs.indirect_images.empty() == rhs.indirect_images.empty();
@@ -848,7 +848,7 @@ void TestSrtFlatteningAndRuntimeMemoization() {
Check(fixture.program.srt_reads.size() == 1,
"shared typed scalar read did not receive one flat SRT slot");
Check(fixture.program.info.buffers.size() == 1 &&
fixture.program.info.addresses.empty(),
!fixture.program.info.uses_dma,
"planning-only scalar reads leaked into resource topology");
Check(fixture.program.memory_info[0].planning_only,
"canonical runtime scalar read was not marked planning-only");
@@ -910,7 +910,7 @@ void TestDynamicSrtReadRemainsExplicit() {
Check(fixture.program.srt_reads.empty() &&
fixture.program.dynamic_reads.size() == 1 &&
fixture.program.info.addresses.size() == 1,
fixture.program.info.uses_dma,
"dynamic scalar read was incorrectly flattened or lost");
std::array<uint32_t, 3> user_data{0x1000u, 0u, 4u};
TestMemory memory;
@@ -932,11 +932,13 @@ void TestDynamicSrtReadRemainsExplicit() {
FindBinding(fixture.program.bindings,
DescriptorBindingKind::FlattenedSrt) == nullptr &&
FindBinding(fixture.program.bindings,
DescriptorBindingKind::AddressMemory) != nullptr,
DescriptorBindingKind::BdaPagetable) != nullptr &&
FindBinding(fixture.program.bindings,
DescriptorBindingKind::FaultBuffer) != nullptr,
"dynamic scalar read received the wrong resource bindings");
Check(fixture.program.bindings.memory_offset_dword ==
fixture.program.bindings.user_data_registers.size() &&
fixture.program.bindings.memory_offset_count == 2u &&
fixture.program.bindings.memory_offset_count == 1u &&
fixture.program.bindings.ShaderDataDwords() ==
fixture.program.bindings.memory_offset_dword + 1u,
"unified memory-offset layout is inconsistent");
@@ -1029,7 +1031,7 @@ void TestInvariantLoopPhi() {
"loop-invariant descriptor phi was not evaluated through typed SSA");
}
void TestAddressMaterializationAndSpecialization() {
void TestDmaAddressMaterialization() {
Fixture fixture;
const auto based =
fixture.Address(fixture.UserData(0), fixture.UserData(1), 4);
@@ -1050,29 +1052,19 @@ void TestAddressMaterializationAndSpecialization() {
fixture.AddMemory(flat, 8));
fixture.PlanAndTrack();
Check(fixture.program.info.addresses.size() == 2 &&
!fixture.program.info.addresses[0].unbased &&
fixture.program.info.addresses[0].min_offset == -8 &&
fixture.program.info.addresses[1].unbased,
"typed based and unbased addresses were classified incorrectly");
Check(fixture.program.info.uses_dma,
"typed address operations did not enable DMA");
std::array<uint32_t, 2> user_data{0x2008u, 0u};
SrtRuntime runtime{.user_data = user_data, .flat_memory_base = 0x9000u};
SrtRuntime runtime{.user_data = user_data};
ResourceSnapshot snapshot;
std::string error;
Check(MaterializeResources(fixture.program, runtime, snapshot, &error),
"address resources did not materialize");
Check(snapshot.addresses.size() == 2 &&
snapshot.addresses[0].guest_base == 0x2008u &&
snapshot.addresses[0].binding_base == 0x2000u &&
snapshot.addresses[1].binding_base == 0x9000u,
"materialized address windows are incorrect");
Check(SpecializeResources(fixture.program, snapshot, &error) &&
fixture.program.info.addresses[0].specialized_base == 8u &&
fixture.program.info.addresses[1].specialized_base == 0x9000u,
"typed address specialization was not applied");
"DMA shader resources did not materialize");
Check(SpecializeResources(fixture.program, snapshot, &error),
"DMA shader resources were not specialized");
}
void TestExecMaskedFlatAddressProvenance() {
void TestDynamicFlatAddressesUseDma() {
Fixture fixture;
const auto low_root = fixture.UserData(0);
const auto high_root = fixture.UserData(1);
@@ -1092,18 +1084,14 @@ void TestExecMaskedFlatAddressProvenance() {
fixture.AddMemory(flat, 0xa4));
fixture.PlanAndTrack();
Check(fixture.program.info.addresses.size() == 1 &&
!fixture.program.info.addresses[0].unbased,
"exec-masked FLAT address lost its active user-data root");
Check(fixture.program.info.uses_dma,
"exec-masked FLAT address did not enable DMA");
std::array<uint32_t, 3> user_data{0x23456780u, 1u, 1u};
SrtRuntime runtime{.user_data = user_data};
ResourceSnapshot snapshot;
std::string error;
Check(MaterializeResources(fixture.program, runtime, snapshot, &error) &&
snapshot.addresses.size() == 1 &&
snapshot.addresses[0].guest_base == 0x0000000123456780ull &&
snapshot.addresses[0].binding_base == 0x0000000123450000ull,
"exec-masked FLAT address materialized the wrong user-data root");
Check(MaterializeResources(fixture.program, runtime, snapshot, &error),
"exec-masked FLAT shader resources did not materialize");
Fixture mismatch;
const auto mismatch_active = mismatch.Emit(ValueOpcode::INotEqual32,
@@ -1122,9 +1110,8 @@ void TestExecMaskedFlatAddressProvenance() {
{mismatch_address, mismatch_low, mismatch_high, other_active},
mismatch.AddMemory(flat, 0xa4));
mismatch.PlanAndTrack();
Check(mismatch.program.info.addresses.size() == 1 &&
mismatch.program.info.addresses[0].unbased,
"FLAT address used a select arm guarded by a different active mask");
Check(mismatch.program.info.uses_dma,
"dynamic FLAT address did not enable DMA");
}
void TestBufferSwizzleSpecialization() {
@@ -1393,8 +1380,8 @@ int main() {
Run("phi validation", TestPhiValidation);
Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
Run("invariant loop phi", TestInvariantLoopPhi);
Run("address materialization", TestAddressMaterializationAndSpecialization);
Run("exec-masked FLAT address", TestExecMaskedFlatAddressProvenance);
Run("DMA address materialization", TestDmaAddressMaterialization);
Run("dynamic FLAT address", TestDynamicFlatAddressesUseDma);
Run("buffer swizzle specialization", TestBufferSwizzleSpecialization);
Run("shader info and bindings", TestShaderInfoAndBindingLayout);
Run("graphics push constants", TestGraphicsPushConstantLayout);
tests/ShaderRecompilerComputeTests.cpp
+195 -54
View File
@@ -919,6 +919,11 @@ void CheckLeastRecentlyUsedCacheOrdering() {
std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache");
}
struct BdaMapping {
uint64_t guest_base = 0;
u32 backing_offset = 0;
};
struct TestCase {
const char *name = "";
std::vector<u32> code;
@@ -959,15 +964,15 @@ struct TestCase {
bool compile_only = false;
size_t storage_buffer_range_dwords = 0;
std::vector<u32> storage_buffer_offsets;
std::vector<u32> address_memory_offsets;
std::vector<BdaMapping> bda_mappings;
bool expand_shader_data_storage = false;
bool expected_force_point_sampler = false;
std::optional<uint64_t> flat_memory_base;
std::vector<u32> gds_initial;
std::vector<u32> expected_gds;
std::vector<std::pair<std::string, size_t>> decoded_counts;
std::vector<std::pair<std::string, size_t>> ir_counts;
u32 expected_storage_mip_descriptors = 0;
std::string expected_compile_error;
};
struct GraphicsCase {
@@ -1168,7 +1173,6 @@ CompiledShader CompileCase(const TestCase &test) {
options.user_data = user_data.data();
options.read_memory = ReadTestMemory;
options.read_memory_data = const_cast<std::vector<u32> *>(&test.initial);
options.flat_memory_base = test.flat_memory_base;
options.scratch_dwords = test.compute_info.scratch_size_dwords;
if (test.has_compute_info) {
options.wave_size = test.compute_info.wave_size;
@@ -1177,8 +1181,14 @@ CompiledShader CompileCase(const TestCase &test) {
ShaderRecompiler::CompileResult result;
std::string error;
if (!ShaderRecompiler::TryRecompile(test.code, options, result, &error)) {
if (!test.expected_compile_error.empty() &&
error.find(test.expected_compile_error) != std::string::npos) {
return {};
}
Fail(test.name, "decode/IR", error.c_str());
}
Require(test.name, "decode/IR", test.expected_compile_error.empty(),
"shader unexpectedly compiled despite the required rejection");
for (const auto &[text, expected] : test.decoded_counts) {
const auto actual = CountText(result.decoded_dump, text);
Require(test.name, "decoded RDNA2", actual == expected,
@@ -1267,17 +1277,6 @@ CompiledShader CompileCase(const TestCase &test) {
packed_user_data[result.program.bindings.memory_offset_dword + i / 4u] |=
offset << ((i % 4u) * 8u);
}
for (u32 i = 0; i < result.program.info.addresses.size(); i++) {
u32 offset = 0;
if (i < test.address_memory_offsets.size()) {
offset = test.address_memory_offsets[i];
}
Require(test.name, "shader data", offset < 256,
"address-memory offset is not representable");
const auto index = buffer_count + i;
packed_user_data[result.program.bindings.memory_offset_dword +
index / 4u] |= offset << ((index % 4u) * 8u);
}
return {std::move(result.spirv), std::move(result.program),
std::move(result.resources.flattened_srt),
std::move(packed_user_data)};
@@ -1469,6 +1468,7 @@ public:
struct Buffer {
vk::Buffer buffer = nullptr;
vk::DeviceMemory memory = nullptr;
vk::DeviceAddress device_address = 0;
vk::DeviceSize size = 0;
bool coherent = false;
};
@@ -3157,7 +3157,7 @@ public:
"chunked host copy lost bytes across its scratch boundary");
constexpr uint64_t index_offset = 0x180000;
constexpr uint64_t index_page = BufferCache::CACHING_PAGE_SIZE;
constexpr uint64_t index_page = BufferCache::CACHING_PAGESIZE;
constexpr uint64_t index_span = 3 * index_page;
const uint64_t index_begin = base + index_offset;
Require(name, "registered-range empty lookup",
@@ -3386,7 +3386,7 @@ public:
"Buffer ownership");
constexpr uint64_t starvation_offset = 0x100000;
constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGE_SIZE;
constexpr uint64_t starvation_stride = 2 * BufferCache::CACHING_PAGESIZE;
constexpr uint32_t starvation_count = 33;
constexpr uint32_t starvation_value = 0x31415926u;
BufferCacheTestAccess::SetGarbageCollectionThresholds(
@@ -3443,7 +3443,7 @@ public:
constexpr uint64_t lookup_only_offset = 0x218000;
constexpr uint64_t obtained_offset = 0x220000;
constexpr uint64_t residency_size =
2 * BufferCache::CACHING_PAGE_SIZE;
2 * BufferCache::CACHING_PAGESIZE;
BufferCacheTestAccess::SetGarbageCollectionThresholds(
cache, std::numeric_limits<uint64_t>::max(),
std::numeric_limits<uint64_t>::max());
@@ -4199,7 +4199,7 @@ public:
const uint64_t mip_guest_size = mip_total.size + 256;
Require(name, "formatted mip fixture",
mip_sizes[0].offset == 0 &&
mip_prefix_size > BufferCache::CACHING_PAGE_SIZE &&
mip_prefix_size > BufferCache::CACHING_PAGESIZE &&
mip_prefix_size < mip_guest_size &&
mip_guest_size % sizeof(uint32_t) == 0,
"single-mip fixture did not expose a cache-sized fitting backing "
@@ -8946,7 +8946,7 @@ public:
Buffer CreateStorageBuffer(const char *shader_name,
const std::vector<u32> &initial,
size_t dword_count) {
size_t dword_count, bool device_address = false) {
Buffer ret;
ret.size = static_cast<vk::DeviceSize>(std::max<size_t>(dword_count, 1u) *
sizeof(u32));
@@ -8955,6 +8955,9 @@ public:
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = ret.size;
buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer;
if (device_address) {
buffer_info.usage |= vk::BufferUsageFlagBits::eShaderDeviceAddress;
}
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
@@ -8979,12 +8982,26 @@ public:
alloc.sType = vk::StructureType::eMemoryAllocateInfo;
alloc.allocationSize = req.size;
alloc.memoryTypeIndex = memory_type;
vk::MemoryAllocateFlagsInfo flags{};
if (device_address) {
flags.sType = vk::StructureType::eMemoryAllocateFlagsInfo;
flags.flags = vk::MemoryAllocateFlagBits::eDeviceAddress;
alloc.pNext = &flags;
}
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&alloc, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
if (device_address) {
vk::BufferDeviceAddressInfo address_info{};
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
address_info.buffer = ret.buffer;
ret.device_address = m_device.getBufferAddress(address_info);
Require(shader_name, "dispatch", ret.device_address != 0,
"storage buffer has no device address");
}
std::vector<u32> contents(dword_count, 0);
for (size_t i = 0; i < initial.size() && i < contents.size(); i++) {
@@ -9366,7 +9383,9 @@ public:
pool_sizes.push_back({type, count});
};
add_pool_size(vk::DescriptorType::eStorageBuffer,
Count(Kind::Buffers) + Count(Kind::AddressMemory) +
Count(Kind::Buffers) +
(Binding(Kind::BdaPagetable) != nullptr ? 1u : 0u) +
(Binding(Kind::FaultBuffer) != nullptr ? 1u : 0u) +
Count(Kind::Gds) +
(Binding(Kind::FlattenedSrt) != nullptr ? 1u : 0u) +
(Binding(Kind::UserData) != nullptr ? 1u : 0u));
@@ -9411,7 +9430,6 @@ public:
std::vector<vk::WriteDescriptorSet> writes;
std::vector<vk::DescriptorBufferInfo> buffer_infos;
std::vector<vk::DescriptorBufferInfo> address_memory_infos;
std::vector<vk::DescriptorImageInfo> sampled_infos;
std::vector<vk::DescriptorImageInfo> storage_infos;
std::vector<vk::DescriptorImageInfo> storage_uint_infos;
@@ -9422,6 +9440,33 @@ public:
vk::DescriptorBufferInfo flattened_info{};
vk::DescriptorBufferInfo user_data_info{};
vk::DescriptorBufferInfo gds_info{};
vk::DescriptorBufferInfo bda_pagetable_info{};
vk::DescriptorBufferInfo fault_buffer_info{};
const bool uses_bda = Binding(Kind::BdaPagetable) != nullptr;
Require(test.name, "dispatch",
uses_bda == (Binding(Kind::FaultBuffer) != nullptr),
"BDA page table and fault buffer must be bound together");
if (uses_bda) {
EnsureBdaBuffers(test.name);
bda_pagetable_info = {m_bda_pagetable_buffer.buffer, 0,
m_bda_pagetable_buffer.size};
fault_buffer_info = {m_fault_buffer.buffer, 0, m_fault_buffer.size};
const std::array special_buffers{
std::pair{Kind::BdaPagetable, &bda_pagetable_info},
std::pair{Kind::FaultBuffer, &fault_buffer_info},
};
for (const auto &[kind, info] : special_buffers) {
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(kind);
write.descriptorCount = 1;
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = info;
writes.push_back(write);
}
}
const auto *buffers = Binding(Kind::Buffers);
if (buffers != nullptr) {
@@ -9450,21 +9495,6 @@ public:
write.pBufferInfo = buffer_infos.data();
writes.push_back(write);
}
if (const auto *address = Binding(Kind::AddressMemory);
address != nullptr) {
address_memory_infos.resize(address->resources.size());
for (auto &info : address_memory_infos) {
info = {buffer.buffer, 0, buffer.size};
}
vk::WriteDescriptorSet write{};
write.sType = vk::StructureType::eWriteDescriptorSet;
write.dstSet = descriptor_set;
write.dstBinding = Native(Kind::AddressMemory);
write.descriptorCount = static_cast<u32>(address_memory_infos.size());
write.descriptorType = vk::DescriptorType::eStorageBuffer;
write.pBufferInfo = address_memory_infos.data();
writes.push_back(write);
}
if (const auto *flattened = Binding(Kind::FlattenedSrt);
flattened != nullptr) {
flattened_buffer = CreateStorageBuffer(test.name, compiled.flattened_srt,
@@ -9590,6 +9620,52 @@ public:
}
vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch");
if (uses_bda) {
Require(test.name, "dispatch", buffer.device_address != 0,
"BDA test backing has no device address");
Require(test.name, "dispatch", !test.bda_mappings.empty(),
"BDA test has no explicit guest mapping");
cmd.fillBuffer(m_bda_pagetable_buffer.buffer, 0,
m_bda_pagetable_buffer.size, 0);
cmd.fillBuffer(m_fault_buffer.buffer, 0, m_fault_buffer.size, 0);
for (const auto &[guest_base, backing] : test.bda_mappings) {
const auto page_offset = guest_base &
(BufferCache::CACHING_PAGESIZE - 1);
Require(test.name, "dispatch",
backing >= page_offset && backing < buffer.size,
"BDA test mapping begins outside its backing buffer");
const auto bytes = page_offset + buffer.size - backing;
const auto pages =
(bytes + BufferCache::CACHING_PAGESIZE - 1) /
BufferCache::CACHING_PAGESIZE;
auto address = buffer.device_address + backing - page_offset;
auto page = guest_base >> BufferCache::CACHING_PAGEBITS;
for (uint64_t mapped = 0; mapped < pages; mapped++) {
cmd.updateBuffer(m_bda_pagetable_buffer.buffer,
(page + mapped) * sizeof(vk::DeviceAddress),
sizeof(address), &address);
address += BufferCache::CACHING_PAGESIZE;
}
}
std::array<vk::BufferMemoryBarrier, 2> barriers{};
barriers[0].sType = vk::StructureType::eBufferMemoryBarrier;
barriers[0].srcAccessMask = vk::AccessFlagBits::eTransferWrite;
barriers[0].dstAccessMask = vk::AccessFlagBits::eShaderRead;
barriers[0].srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barriers[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barriers[0].buffer = m_bda_pagetable_buffer.buffer;
barriers[0].offset = 0;
barriers[0].size = m_bda_pagetable_buffer.size;
barriers[1] = barriers[0];
barriers[1].dstAccessMask = vk::AccessFlagBits::eShaderRead |
vk::AccessFlagBits::eShaderWrite;
barriers[1].buffer = m_fault_buffer.buffer;
barriers[1].size = m_fault_buffer.size;
cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eComputeShader, {}, 0,
nullptr, static_cast<u32>(barriers.size()),
barriers.data(), 0, nullptr);
}
cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline);
cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0,
1, &descriptor_set, 0, nullptr);
@@ -10951,6 +11027,7 @@ private:
allocator_info.device = m_device;
allocator_info.pVulkanFunctions = &functions;
allocator_info.vulkanApiVersion = VK_API_VERSION_1_3;
allocator_info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
RequireVk("VulkanHarness", "runtime context",
static_cast<vk::Result>(vmaCreateAllocator(
&allocator_info, &m_runtime_context.allocator)),
@@ -11001,13 +11078,18 @@ private:
if ((queues[i].queueFlags &
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) ==
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) {
vk::PhysicalDeviceVulkan12Features features12{};
features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
barycentric.pNext = &features12;
vk::PhysicalDeviceFeatures2 features{};
features.sType = vk::StructureType::ePhysicalDeviceFeatures2;
features.pNext = &barycentric;
physical.getFeatures2(&features);
if (barycentric.fragmentShaderBarycentric != true) {
if (barycentric.fragmentShaderBarycentric != true ||
features.features.shaderInt64 != true ||
features12.bufferDeviceAddress != true) {
continue;
}
m_physical_device = physical;
@@ -11051,6 +11133,11 @@ private:
Require("VulkanHarness", "dispatch",
available_features.sampleRateShading == true,
"sample-rate shading is not supported");
Require("VulkanHarness", "dispatch", available_features.shaderInt64 == true,
"shaderInt64 is not supported");
Require("VulkanHarness", "dispatch",
available_features12.bufferDeviceAddress == true,
"bufferDeviceAddress is not supported");
float priority = 1.0f;
vk::DeviceQueueCreateInfo queue_info{};
@@ -11067,6 +11154,7 @@ private:
device_features12.sType =
vk::StructureType::ePhysicalDeviceVulkan12Features;
device_features12.timelineSemaphore = true;
device_features12.bufferDeviceAddress = true;
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
barycentric.pNext = &device_features12;
@@ -11081,6 +11169,7 @@ private:
vk::PhysicalDeviceFeatures device_features{};
device_features.shaderStorageImageWriteWithoutFormat = true;
device_features.sampleRateShading = true;
device_features.shaderInt64 = true;
device_info.pEnabledFeatures = &device_features;
constexpr const char *device_extensions[] = {
VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME,
@@ -11105,6 +11194,8 @@ private:
void Destroy() {
if (m_device != nullptr) {
RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle");
DestroyBuffer(&m_fault_buffer);
DestroyBuffer(&m_bda_pagetable_buffer);
if (m_runtime_context.allocator != nullptr) {
m_renderer.reset();
vmaDestroyAllocator(m_runtime_context.allocator);
@@ -11276,6 +11367,53 @@ private:
image->layout = final_layout;
}
Buffer CreateDeviceBuffer(const char *shader_name, vk::DeviceSize size,
vk::BufferUsageFlags usage) {
Buffer ret;
ret.size = size;
vk::BufferCreateInfo buffer_info{};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = size;
buffer_info.usage = usage;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
RequireVk(shader_name, "dispatch",
m_device.createBuffer(&buffer_info, nullptr, &ret.buffer),
"vkCreateBuffer");
vk::MemoryRequirements requirements{};
m_device.getBufferMemoryRequirements(ret.buffer, &requirements);
u32 memory_type = 0;
Require(shader_name, "dispatch",
FindMemoryType(requirements.memoryTypeBits,
vk::MemoryPropertyFlagBits::eDeviceLocal,
&memory_type) ||
FindMemoryType(requirements.memoryTypeBits, {}, &memory_type),
"no memory type for BDA support buffer");
vk::MemoryAllocateInfo allocation{};
allocation.sType = vk::StructureType::eMemoryAllocateInfo;
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = memory_type;
RequireVk(shader_name, "dispatch",
m_device.allocateMemory(&allocation, nullptr, &ret.memory),
"vkAllocateMemory");
RequireVk(shader_name, "dispatch",
m_device.bindBufferMemory(ret.buffer, ret.memory, 0),
"vkBindBufferMemory");
return ret;
}
void EnsureBdaBuffers(const char *shader_name) {
if (m_bda_pagetable_buffer.buffer != nullptr) {
return;
}
const auto usage = vk::BufferUsageFlagBits::eStorageBuffer |
vk::BufferUsageFlagBits::eTransferDst;
m_bda_pagetable_buffer =
CreateDeviceBuffer(shader_name, BufferCache::BDA_PAGETABLE_SIZE, usage);
m_fault_buffer = CreateDeviceBuffer(
shader_name, BufferCache::CACHING_NUMPAGES / 8, usage);
}
Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size,
vk::BufferUsageFlags usage,
const std::vector<u32> &contents) {
@@ -11349,6 +11487,8 @@ private:
vk::CommandPool m_command_pool = nullptr;
u32 m_queue_family = 0;
vk::PhysicalDeviceMemoryProperties m_memory_properties{};
Buffer m_bda_pagetable_buffer;
Buffer m_fault_buffer;
GraphicContext m_runtime_context{};
std::unique_ptr<RenderContext> m_renderer;
};
@@ -11392,6 +11532,10 @@ void CompareGraphicsWords(const GraphicsCase &test,
void RunCase(VulkanHarness *vulkan, const TestCase &test) {
auto compiled = CompileCase(test);
if (!test.expected_compile_error.empty()) {
std::printf("[compute] %-32s ok\n", test.name);
return;
}
if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) {
Require(test.name, "resource specialization",
!compiled.program.info.images.empty() &&
@@ -11410,15 +11554,15 @@ void RunCase(VulkanHarness *vulkan, const TestCase &test) {
std::printf("[compute] %-32s ok\n", test.name);
return;
}
const auto dwords = std::max<size_t>(
{test.initial.size(), test.expected.size(), static_cast<size_t>(1)});
auto buffer = vulkan->CreateStorageBuffer(test.name, test.initial, dwords);
using Kind = ShaderRecompiler::IR::DescriptorBindingKind;
auto Has = [&](Kind kind) {
return ShaderRecompiler::IR::FindBinding(compiled.program.bindings, kind) !=
nullptr;
};
const auto dwords = std::max<size_t>(
{test.initial.size(), test.expected.size(), static_cast<size_t>(1)});
auto buffer = vulkan->CreateStorageBuffer(
test.name, test.initial, dwords, Has(Kind::BdaPagetable));
VulkanHarness::Image sampled_image;
VulkanHarness::Image storage_image;
VulkanHarness::Image storage_image_uint;
@@ -17341,16 +17485,16 @@ TestCase FlatLoadVariants() {
O::FLAT_LOAD_USHORT, O::FLAT_LOAD_SSHORT, O::FLAT_LOAD_DWORD,
O::FLAT_LOAD_DWORDX2, O::FLAT_LOAD_DWORDX3,
O::FLAT_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
test.flat_memory_base = 0;
test.address_memory_offsets = {static_cast<u32>(byte_offset)};
test.bda_mappings = {{0, static_cast<u32>(byte_offset)}};
return test;
}
TestCase FlatSubdwordLoadsApplyByteOffset() {
using O = ShaderOpcode;
constexpr size_t byte_offset = 3;
std::vector<u32> code;
AppendVMovU32(&code, 20, 0);
AppendVMovU32(&code, 20, static_cast<u32>(byte_offset));
AppendVMovU32(&code, 21, 0);
code.push_back(EncodeFlat0(0x08, 0, 0));
code.push_back(EncodeFlat1(0, 0x7d, 0, 20));
@@ -17360,7 +17504,6 @@ TestCase FlatSubdwordLoadsApplyByteOffset() {
AppendStoreVgpr(&code, 1, 1);
AppendEnd(&code);
constexpr size_t byte_offset = 3;
constexpr u32 guest_word = 0x80ff7f01u;
std::vector<u32> host_memory(
(byte_offset + sizeof(guest_word) + sizeof(u32) - 1u) / sizeof(u32));
@@ -17372,8 +17515,7 @@ TestCase FlatSubdwordLoadsApplyByteOffset() {
test.code = std::move(code);
test.initial = std::move(host_memory);
test.expected = {0x01u, 0x0000ff7fu};
test.flat_memory_base = 0;
test.address_memory_offsets = {static_cast<u32>(byte_offset)};
test.bda_mappings = {{0, 0}};
test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_UBYTE, O::FLAT_LOAD_USHORT,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
@@ -17457,8 +17599,7 @@ TestCase FlatVirtualAddressRebasesGuestAllocation() {
test.code = std::move(code);
test.initial = {0xfeedfaceu, 0xcafebabeu, 0, 0x12345678u};
test.expected = {0x12345678u, 0x12345678u, 0};
test.flat_memory_base = GuestBase;
test.address_memory_offsets = {8, 8, 8};
test.bda_mappings = {{GuestBase, 8}};
test.opcodes = {O::V_MOV_B32, O::FLAT_LOAD_DWORD, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
return test;
@@ -17480,10 +17621,9 @@ TestCase GlobalSignedImmediateRebasesBeforeSaddr() {
TestCase test;
test.name = "GlobalSignedImmediateRebasesBeforeSaddr";
test.code = std::move(code);
test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x22222222u,
0x12345678u};
test.initial = {0xfeedfaceu, 0xcafebabeu, 0x11111111u, 0x12345678u};
test.expected = {0x12345678u};
test.address_memory_offsets = {8};
test.bda_mappings = {{GuestBase, 8}};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
@@ -17517,7 +17657,7 @@ TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() {
test.expected = {0x11111111u, 0x11111111u};
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::FLAT_LOAD_DWORD,
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
test.flat_memory_base = 0;
test.bda_mappings = {{0, 0}};
return test;
}
@@ -17596,7 +17736,8 @@ TestCase FlatStoreVariants() {
{O::V_MOV_B32, O::FLAT_STORE_BYTE, O::FLAT_STORE_SHORT,
O::FLAT_STORE_DWORD, O::FLAT_STORE_DWORDX2,
O::FLAT_STORE_DWORDX3, O::FLAT_STORE_DWORDX4, O::S_ENDPGM}};
test.flat_memory_base = 0;
test.expected_compile_error =
"writable FLAT/GLOBAL addresses require GPU ownership tracking";
return test;
}
tests/ShaderStageRuntimeTests.cpp
+6 -14
View File
@@ -73,12 +73,7 @@ UnbasedFlatProgram() {
program.srt_plan_complete = true;
program.resource_tracking_complete = true;
AddValueBlock(program);
AddressResource address;
address.source = UINT32_MAX;
address.first_use_pc = 0x80;
address.kind = ResourceKind::Flat;
address.unbased = true;
program.info.addresses.push_back(address);
program.info.uses_dma = true;
return std::make_shared<const Program>(std::move(program));
}
@@ -102,7 +97,7 @@ void TestMappedSrtUsesDirectReaderByDefault() {
"cache rematerialization did not use the direct reader by default");
}
void TestUnbasedFlatCacheHitFailsClosed() {
void TestUnbasedFlatCacheHitMaterializes() {
using namespace Libs::Graphics;
auto cached_program = UnbasedFlatProgram();
auto prior_program = std::make_shared<const ShaderRecompiler::IR::Program>();
@@ -110,12 +105,9 @@ void TestUnbasedFlatCacheHitFailsClosed() {
std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>();
ShaderStageRuntime stage{prior_program, prior_resources};
std::string error;
Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error) &&
error.find("requires runtime guest-address translation") !=
std::string::npos,
"unbased FLAT cache hit did not fail without a runtime translator");
Check(stage.program == prior_program && stage.resources == prior_resources,
"unbased FLAT cache rejection replaced the prior stage snapshot");
Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error), error.c_str());
Check(stage.program == cached_program && stage.resources != prior_resources,
"unbased FLAT cache hit did not publish its BDA stage snapshot");
}
} // namespace
@@ -130,7 +122,7 @@ void DbgExit(int) { std::abort(); }
int main() {
TestMappedSrtUsesDirectReaderByDefault();
TestUnbasedFlatCacheHitFailsClosed();
TestUnbasedFlatCacheHitMaterializes();
std::puts("ShaderStageRuntimeTests: all cases passed");
return 0;
}
tests/shaderCfgTests.cpp
+48 -61
View File
@@ -4319,14 +4319,14 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() {
error.c_str());
const auto *address_binding = ShaderRecompiler::IR::FindBinding(
raw.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::AddressMemory);
Check(raw.program.info.addresses.size() == 1u &&
raw.program.info.addresses[0].kind ==
ShaderRecompiler::IR::ResourceKind::ScalarAddress &&
raw.program.info.addresses[0].read &&
!raw.program.info.addresses[0].written &&
raw.program.info.buffers.empty() && address_binding != nullptr &&
address_binding->resources == std::vector<uint32_t>{0u} &&
ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable);
const auto *fault_binding = ShaderRecompiler::IR::FindBinding(
raw.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer);
Check(raw.program.info.uses_dma && raw.program.info.buffers.empty() &&
address_binding != nullptr && fault_binding != nullptr &&
address_binding->resources.empty() &&
fault_binding->resources.empty() &&
ShaderRecompiler::IR::FindBinding(
raw.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::Buffers) ==
@@ -4335,17 +4335,12 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() {
raw.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) ==
nullptr &&
raw.program.bindings.memory_offset_count == 1u,
"raw scalar load did not use only the address-memory domain");
raw.program.bindings.memory_offset_count == 0u,
"raw scalar load did not use only the DMA domain");
Check(count_live_memory_ops(
raw.program, ShaderRecompiler::IR::ValueOpcode::LoadAddressU32,
ShaderRecompiler::IR::ResourceKind::ScalarAddress) == 1u,
"raw scalar load did not remain a live typed address operation");
Check(raw.resources.addresses.size() == 1u &&
raw.resources.addresses[0].guest_base == 0x1000u &&
raw.resources.addresses[0].binding_base == 0x1000u &&
raw.program.info.addresses[0].specialized_base == 0u,
"raw scalar address specialization was incorrect");
Check(SpirvContainsOpcode(raw.spirv, 199),
"raw scalar SOFFSET alignment was not emitted");
CheckSpirvBinaryValidates(raw.spirv);
@@ -4377,12 +4372,12 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() {
ShaderRecompiler::IR::DescriptorBindingKind::Buffers);
Check(buffer.program.info.buffers.size() == 1u &&
buffer.program.info.buffers[0].scalar &&
buffer.program.info.addresses.empty() &&
!buffer.program.info.uses_dma &&
buffer_binding != nullptr &&
buffer_binding->resources == std::vector<uint32_t>{0u} &&
ShaderRecompiler::IR::FindBinding(
buffer.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::AddressMemory) ==
ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable) ==
nullptr &&
ShaderRecompiler::IR::FindBinding(
buffer.program.bindings,
@@ -6201,7 +6196,6 @@ void TestNewShaderRecompilerFlatOldBackedTranslation() {
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
options.flat_memory_base = 0;
ShaderRecompiler::CompileResult result;
std::string error;
@@ -6224,23 +6218,32 @@ void TestNewShaderRecompilerFlatOldBackedTranslation() {
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerUnbasedFlatRequiresTranslator() {
void TestNewShaderRecompilerUnbasedFlatUsesBda() {
const uint32_t shader[] = {
EncodeFlat0(0x0c, 0, 0),
EncodeFlat1(0, 0x7d, 0, 1),
EncodeFlat0(0x1c, 0, 0),
EncodeFlat1(0, 0x7d, 0, 2),
0xbf810000u,
EncodeExp0(0x00, 0x1),
EncodeExp1(0, 0, 0, 0),
EncodeSopp(0x01),
};
auto options = MakeCompileOptions(ShaderType::Compute);
auto options = MakeCompileOptions(ShaderType::Pixel);
ShaderRecompiler::CompileResult result;
std::string error;
Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) &&
Common::ContainsStr(error,
"requires runtime guest-address translation"),
"unbased FLAT compiled without an explicit translator");
Check(ShaderRecompiler::TryRecompile(shader, options, result, &error),
error.c_str());
Check(result.program.info.uses_dma &&
ShaderRecompiler::IR::FindBinding(
result.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable) !=
nullptr &&
ShaderRecompiler::IR::FindBinding(
result.program.bindings,
ShaderRecompiler::IR::DescriptorBindingKind::FaultBuffer) !=
nullptr,
"unbased FLAT did not compile through BDA");
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerFlatSignedLoadTranslation() {
@@ -6256,7 +6259,6 @@ void TestNewShaderRecompilerFlatSignedLoadTranslation() {
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
options.flat_memory_base = 0;
ShaderRecompiler::CompileResult result;
std::string error;
@@ -6311,7 +6313,6 @@ void TestNewShaderRecompilerFlatStoreTranslation() {
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
options.flat_memory_base = 0;
ShaderRecompiler::CompileResult result;
std::string error;
@@ -11966,49 +11967,40 @@ void TestNewShaderRecompilerUnsupportedMemoryDecode() {
"MTBUF", "opcode=0x08");
}
void TestNewShaderRecompilerFlatUserPointerProvenance() {
void TestNewShaderRecompilerFlatUserPointerUsesDma() {
const uint32_t shader[] = {
EncodeVop2(0x25, 0, 0, 2), // v_add_nc_u32 v0, s0, v2
EncodeVop1(0x01, 1, 1), // v_mov_b32 v1, s1
EncodeFlat0(0x1c, 0, 0),
EncodeFlat1(0, 0x7d, 3, 0), // flat_store_dword v3, v[0:1]
0xbf810000u,
EncodeFlat0(0x0c, 0, 0),
EncodeFlat1(3, 0x7d, 0, 0), // flat_load_dword v3, v[0:1]
EncodeExp0(0x00, 0x1),
EncodeExp1(3, 0, 0, 0),
EncodeSopp(0x01),
};
const uint32_t user_data[] = {0x34567000u, 0x00000012u};
auto options = MakeCompileOptions(ShaderType::Compute);
auto options = MakeCompileOptions(ShaderType::Pixel);
options.user_data = user_data;
options.user_data_count = static_cast<uint32_t>(std::size(user_data));
options.flat_memory_base = 0;
ShaderRecompiler::CompileResult result;
std::string error;
const bool compiled =
ShaderRecompiler::TryRecompile(shader, options, result, &error);
Check(compiled, error.c_str());
Check(result.program.info.addresses.size() == 1 &&
!result.program.info.addresses[0].unbased &&
result.program.info.addresses[0].source <
result.program.descriptor_sources.size(),
"FLAT user pointer provenance was not attached");
Check(result.resources.addresses.size() == 1 &&
result.resources.addresses[0].guest_base == 0x0000001234567000ull &&
result.resources.addresses[0].binding_base ==
0x0000001234560000ull &&
result.program.info.addresses[0].specialized_base ==
0x0000001234560000ull,
"FLAT user pointer did not materialize its guest base");
Check(result.program.info.uses_dma,
"FLAT user pointer did not enable DMA");
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() {
void TestNewShaderRecompilerFlatAddressDomainsUseDma() {
const uint32_t segmented_shader[] = {
EncodeVop2(0x25, 0, 0, 2),
EncodeVop1(0x01, 1, 1),
EncodeFlat0(0x1c, 2, 0),
EncodeFlat1(0, 0x7d, 3, 0), // global_store_dword v3, v[0:1]
EncodeFlat0(0x0c, 2, 0),
EncodeFlat1(3, 0x7d, 0, 0), // global_load_dword v3, v[0:1]
EncodeFlat0(0x1c, 1, 0),
EncodeFlat1(0, 0x7d, 4, 0), // scratch_store_dword v4, v[0:1]
EncodeFlat1(0, 0x7d, 3, 0), // scratch_store_dword v3, v[0:1]
0xbf810000u,
};
const uint32_t user_data[] = {0x34567000u, 0u, 0x1000u};
@@ -12016,19 +12008,14 @@ void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() {
auto options = MakeCompileOptions(ShaderType::Compute);
options.user_data = user_data;
options.user_data_count = static_cast<uint32_t>(std::size(user_data));
options.flat_memory_base = 0;
options.scratch_dwords = 1;
ShaderRecompiler::CompileResult result;
std::string error;
const bool compiled =
ShaderRecompiler::TryRecompile(segmented_shader, options, result, &error);
Check(compiled, error.c_str());
Check(result.program.info.addresses.size() == 1 &&
result.program.info.addresses[0].kind ==
ShaderRecompiler::IR::ResourceKind::Global &&
result.program.info.addresses[0].unbased &&
result.program.info.addresses[0].source == UINT32_MAX,
"GLOBAL null-SADDR did not remain an unbased guest address");
Check(result.program.info.uses_dma,
"GLOBAL null-SADDR did not enable DMA");
Check(Common::ContainsStr(result.ir_dump, "GetScratchResource") &&
result.program.scratch_dwords == 1,
"SCRATCH incorrectly entered guest address tracking");
@@ -12324,9 +12311,9 @@ int main() {
TestNewShaderRecompilerRejectsDppOn64BitCompares();
TestNewShaderRecompilerIrLookupMissFailsExplicitly();
TestPsInputCountRegisterDecode();
TestNewShaderRecompilerUnbasedFlatRequiresTranslator();
TestNewShaderRecompilerFlatUserPointerProvenance();
TestNewShaderRecompilerFlatAddressProvenanceBoundaries();
TestNewShaderRecompilerUnbasedFlatUsesBda();
TestNewShaderRecompilerFlatUserPointerUsesDma();
TestNewShaderRecompilerFlatAddressDomainsUseDma();
TestNewShaderRecompilerCfgStraightLine();
TestNewShaderRecompilerCfgIfElse();
TestNewShaderRecompilerCfgConsecutiveNativePhis();