raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-21 03:16:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

graphics: add RDNA2 FLAT BDA buffer cache and faults

Browse Source
This commit is contained in:
nmzik
2026-08-27 03:45:41 +02:00
parent 8072718462
commit e738535294
37 changed files with 1002 additions and 684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/rangeSet.h
+10 -1
View File
@@ -53,6 +53,15 @@ public:
}
}
void Clear() { m_ranges.clear(); }
template <typename Func>
void ForEach(Func&& func) const {
for (const auto& [begin, end]: m_ranges) {
func(begin, end);
}
}
[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
std::vector<Range> result;
ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
@@ -98,7 +107,7 @@ public:
private:
static uint64_t End(uint64_t address, uint64_t size) {
if (address == 0 || size == 0 || size > UINT64_MAX - address) {
if (size == 0 || size > UINT64_MAX - address) {
EXIT("invalid range-set address or size\n");
}
return address + size;
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+73 -38
View File
@@ -51,41 +51,50 @@ struct BufferCache::DownloadCopy {
};
void BufferCache::Register(BufferId id) {
auto& buffer = m_slot_buffers[id];
PageTable::PageRange pages {};
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
m_page_table[page] = id;
}
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
(void)it;
EXIT_IF(!inserted);
buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
m_total_used_memory += buffer.Size();
ChangeRegister<true>(id);
}
void BufferCache::Unregister(BufferId id) {
ChangeRegister<false>(id);
}
template <bool insert>
void BufferCache::ChangeRegister(BufferId id) {
auto& buffer = m_slot_buffers[id];
if (buffer.is_deleted) {
return;
}
PageTable::PageRange pages {};
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
auto* owner = m_page_table.Find(page);
if (owner != nullptr && *owner == id) {
*owner = {};
if constexpr (insert) {
m_page_table[page] = id;
} else {
m_page_table[page] = {};
}
}
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
if (buffer.Size() > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
const auto size_pages = pages.last_exclusive - pages.first;
if constexpr (insert) {
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
(void)it;
EXIT_IF(!inserted);
m_total_used_memory += buffer.Size();
buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
std::vector<vk::DeviceAddress> addresses;
addresses.reserve(size_pages);
for (uint64_t i = 0; i < size_pages; ++i) {
addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS));
}
WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress),
addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
} else {
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
EXIT_IF(buffer.Size() > m_total_used_memory);
m_total_used_memory -= buffer.Size();
m_lru_cache.Free(buffer.lru_id);
m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress),
size_pages * sizeof(vk::DeviceAddress), 0);
buffer.is_deleted = true;
}
m_total_used_memory -= buffer.Size();
m_lru_cache.Free(buffer.lru_id);
buffer.is_deleted = true;
}
void BufferCache::TouchBuffer(const Buffer& buffer) {
@@ -184,16 +193,21 @@ void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
PageManager& page_manager, TextureCache& texture_cache)
: m_graphics(graphics), m_scheduler(scheduler),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_texture_cache(texture_cache) {
: m_graphics(graphics), m_scheduler(scheduler),
m_fault_manager(graphics, scheduler, *this, CACHING_PAGEBITS, CACHING_NUMPAGES),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_bda_pagetable_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
BDA_PAGETABLE_SIZE),
m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_texture_cache(texture_cache) {
std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
m_gds_buffer.Flush(0, m_gds_buffer.Size());
SetVulkanObjectNameF(m_graphics.device, m_bda_pagetable_buffer.Handle(),
"BDA Page Table Buffer");
const auto null_id =
m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16);
EXIT_IF(null_id != NULL_BUFFER_ID);
@@ -316,8 +330,8 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
auto& command = m_scheduler.Current();
EXIT_IF(command.IsInvalid());
auto begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
auto end = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
auto begin = vaddr & ~(CACHING_PAGESIZE - 1);
auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
auto first = m_buffers.lower_bound(begin);
if (first != m_buffers.begin()) {
const auto previous = std::prev(first);
@@ -333,8 +347,9 @@ BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
end = std::max(end, buffer.CpuAddress() + buffer.Size());
}
const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags, end - begin);
const auto id = m_slot_buffers.insert(
m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin);
auto& buffer = m_slot_buffers[id];
SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
"Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
@@ -433,7 +448,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
if (!is_written && size <= CACHING_PAGE_SIZE &&
if (!is_written && size <= CACHING_PAGESIZE &&
!m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
const auto alignment = std::max<uint64_t>(
@@ -737,4 +752,24 @@ void BufferCache::RunGarbageCollector() {
}
}
void BufferCache::ProcessFaultBuffer() {
m_fault_manager.ProcessFaultBuffer();
}
void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {
const auto end = vaddr + size;
auto it = m_buffers.upper_bound(vaddr);
if (it != m_buffers.begin()) {
--it;
}
for (; it != m_buffers.end() && it->first < end; ++it) {
auto& buffer = m_slot_buffers[it->second];
const auto start = std::max(buffer.CpuAddress(), vaddr);
const auto finish = std::min(buffer.CpuAddress() + buffer.Size(), end);
if (start < finish) {
(void)SynchronizeBuffer(buffer, start, finish - start, false, false);
}
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+16 -3
View File
@@ -7,6 +7,7 @@
#include "common/slotVector.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/faultManager.h"
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
@@ -26,7 +27,11 @@ inline constexpr BufferId NULL_BUFFER_ID {0};
class BufferCache {
public:
static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
static constexpr uint32_t CACHING_PAGEBITS = 14;
static constexpr uint64_t CACHING_PAGESIZE = uint64_t {1} << CACHING_PAGEBITS;
static constexpr uint64_t CACHING_NUMPAGES = uint64_t {1} << (40 - CACHING_PAGEBITS);
static constexpr uint64_t BDA_PAGETABLE_SIZE =
CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
TextureCache& texture_cache);
@@ -51,6 +56,8 @@ public:
EXIT("BufferCache: invalid utility-buffer usage\n");
}
[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
[[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept { return &m_bda_pagetable_buffer; }
[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return m_fault_manager.GetFaultBuffer(); }
[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
@@ -60,14 +67,16 @@ public:
[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
void ProcessFaultBuffer();
void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
friend struct BufferCacheTestAccess;
struct DownloadCopy;
using PageTable = MultiLevelPageTable<BufferId, 14, 40, 16>;
static_assert(CACHING_PAGE_SIZE == (uint64_t {1} << PageTable::kPageBits));
using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
@@ -78,6 +87,8 @@ private:
[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
void Register(BufferId id);
void Unregister(BufferId id);
template <bool insert>
void ChangeRegister(BufferId id);
void DeleteBuffer(BufferId id);
[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
bool is_written, bool is_texel_buffer);
@@ -90,7 +101,9 @@ private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
FaultManager m_fault_manager;
Buffer m_gds_buffer;
Buffer m_bda_pagetable_buffer;
Common::SlotVector<Buffer> m_slot_buffers;
Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
std::map<uint64_t, BufferId> m_buffers;
src/graphics/host_gpu/renderer/cache/faultManager.cpp
Vendored
+167
View File
@@ -0,0 +1,167 @@
#include "graphics/host_gpu/renderer/cache/faultManager.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "gpu_tiler_shaders/fault_buffer_process_spv.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <bit>
#include <cinttypes>
#include <cstring>
#include <limits>
namespace Libs::Graphics {
namespace {
constexpr size_t MaxPageFaults = 1024;
constexpr size_t PageFaultAreaSize = MaxPageFaults * sizeof(uint64_t);
} // namespace
FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler,
BufferCache& buffer_cache, uint32_t caching_pagebits,
uint64_t caching_num_pages)
: m_graphics(graphics), m_scheduler(scheduler), m_buffer_cache(buffer_cache),
m_caching_pagesize(uint64_t {1} << caching_pagebits),
m_caching_num_pages(caching_num_pages), m_fault_buffer_size(caching_num_pages / 8),
m_fault_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
m_fault_buffer_size),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 0, AllFlags,
MaxPendingFaults * PageFaultAreaSize) {
SetVulkanObjectNameF(m_graphics.device, m_fault_buffer.Handle(), "Fault Buffer");
const vk::DescriptorSetLayoutBinding bindings[] {
{0, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
{1, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
};
vk::DescriptorSetLayoutCreateInfo layout_info {};
layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
layout_info.bindingCount = std::size(bindings);
layout_info.pBindings = bindings;
RequireVulkanSuccess(
m_graphics.device.createDescriptorSetLayout(&layout_info, nullptr,
&m_fault_process_desc_layout),
"create fault-buffer descriptor layout");
vk::ShaderModuleCreateInfo module_info {};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
module_info.pCode = FAULT_BUFFER_PROCESS_SPV;
vk::ShaderModule module = nullptr;
RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
"create fault-buffer shader module");
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &m_fault_process_desc_layout;
RequireVulkanSuccess(
m_graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
&m_fault_process_pipeline_layout),
"create fault-buffer pipeline layout");
vk::PipelineShaderStageCreateInfo stage {};
stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stage.stage = vk::ShaderStageFlagBits::eCompute;
stage.module = module;
stage.pName = "main";
vk::ComputePipelineCreateInfo pipeline_info {};
pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo;
pipeline_info.stage = stage;
pipeline_info.layout = m_fault_process_pipeline_layout;
const auto result = m_graphics.device.createComputePipelines(
nullptr, 1, &pipeline_info, nullptr, &m_fault_process_pipeline);
m_graphics.device.destroyShaderModule(module, nullptr);
RequireVulkanSuccess(result, "create fault-buffer pipeline");
SetVulkanObjectNameF(m_graphics.device, m_fault_process_pipeline, "Fault Buffer Parser");
}
FaultManager::~FaultManager() {
m_graphics.device.destroyPipeline(m_fault_process_pipeline, nullptr);
m_graphics.device.destroyPipelineLayout(m_fault_process_pipeline_layout, nullptr);
m_graphics.device.destroyDescriptorSetLayout(m_fault_process_desc_layout, nullptr);
}
void FaultManager::ProcessFaultBuffer() {
if (const auto wait_tick = m_fault_areas[m_current_area]; wait_tick != 0) {
m_scheduler.Wait(wait_tick);
m_scheduler.PopPendingOperations();
}
const auto offset = m_current_area * PageFaultAreaSize;
auto* mapped = m_download_buffer.Mapped().data() + offset;
std::memset(mapped, 0, PageFaultAreaSize);
m_download_buffer.Flush(offset, PageFaultAreaSize);
vk::BufferMemoryBarrier2 pre_barrier {};
pre_barrier.sType = vk::StructureType::eBufferMemoryBarrier2;
pre_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
pre_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
pre_barrier.dstStageMask = vk::PipelineStageFlagBits2::eComputeShader;
pre_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderRead;
pre_barrier.buffer = m_fault_buffer.Handle();
pre_barrier.offset = 0;
pre_barrier.size = m_fault_buffer_size;
auto post_barrier = pre_barrier;
post_barrier.srcStageMask = vk::PipelineStageFlagBits2::eComputeShader;
post_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
post_barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands;
post_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderWrite;
const vk::DescriptorBufferInfo infos[] {
{m_fault_buffer.Handle(), 0, m_fault_buffer_size},
{m_download_buffer.Handle(), offset, PageFaultAreaSize},
};
std::array<vk::WriteDescriptorSet, 2> writes {};
for (uint32_t index = 0; index < writes.size(); ++index) {
writes[index].sType = vk::StructureType::eWriteDescriptorSet;
writes[index].dstBinding = index;
writes[index].descriptorCount = 1;
writes[index].descriptorType = vk::DescriptorType::eStorageBuffer;
writes[index].pBufferInfo = &infos[index];
}
m_scheduler.EndRendering();
auto command = m_scheduler.Current().Handle();
vk::DependencyInfo dependency {};
dependency.sType = vk::StructureType::eDependencyInfo;
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
dependency.bufferMemoryBarrierCount = 1;
dependency.pBufferMemoryBarriers = &pre_barrier;
command.pipelineBarrier2(dependency);
command.bindPipeline(vk::PipelineBindPoint::eCompute, m_fault_process_pipeline);
command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute,
m_fault_process_pipeline_layout, 0, writes);
const auto num_threads = m_caching_num_pages / 32;
const auto num_workgroups = (num_threads + 63) / 64;
command.dispatch(static_cast<uint32_t>(num_workgroups), 1, 1);
dependency.pBufferMemoryBarriers = &post_barrier;
command.pipelineBarrier2(dependency);
const auto area = m_current_area;
m_scheduler.DeferOperation([this, mapped, offset, area] {
m_download_buffer.Invalidate(offset, PageFaultAreaSize);
m_fault_ranges.Clear();
const auto* faults = std::bit_cast<const uint64_t*>(mapped);
const auto count = static_cast<uint32_t>(faults[0]);
for (uint32_t index = 1; index <= count; ++index) {
m_fault_ranges.Add(faults[index], m_caching_pagesize);
LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
}
m_fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());
(void)m_buffer_cache.FindBuffer(start, end - start);
});
m_fault_areas[area] = 0;
});
m_fault_areas[m_current_area++] = m_scheduler.CurrentTick();
m_current_area %= MaxPendingFaults;
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/faultManager.h
Vendored
+46
View File
@@ -0,0 +1,46 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
#include "common/abi.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <array>
#include <cstdint>
namespace Libs::Graphics {
class BufferCache;
class FaultManager {
static constexpr size_t MaxPendingFaults = 8;
public:
FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, BufferCache& buffer_cache,
uint32_t caching_pagebits, uint64_t caching_num_pages);
~FaultManager();
KYTY_CLASS_NO_COPY(FaultManager);
[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return &m_fault_buffer; }
void ProcessFaultBuffer();
private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
BufferCache& m_buffer_cache;
RangeSet m_fault_ranges;
uint64_t m_caching_pagesize;
uint64_t m_caching_num_pages;
uint64_t m_fault_buffer_size;
Buffer m_fault_buffer;
Buffer m_download_buffer;
std::array<uint64_t, MaxPendingFaults> m_fault_areas {};
uint32_t m_current_area = 0;
vk::DescriptorSetLayout m_fault_process_desc_layout = nullptr;
vk::Pipeline m_fault_process_pipeline = nullptr;
vk::PipelineLayout m_fault_process_pipeline_layout = nullptr;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+12
View File
@@ -76,7 +76,19 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
m_gpu->SendCommandSync(unmap);
}
void GpuResourceManager::PrepareBda() {
std::shared_lock lock(m_mapped_ranges_mutex);
m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
});
m_fault_process_pending = true;
}
void GpuResourceManager::RunGarbageCollector() {
if (m_fault_process_pending) {
m_fault_process_pending = false;
m_buffer_cache.ProcessFaultBuffer();
}
m_texture_cache.ProcessDownloadImages();
m_texture_cache.RunGarbageCollector();
m_buffer_cache.RunGarbageCollector();
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+2
View File
@@ -30,6 +30,7 @@ public:
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
void PrepareBda();
void RunGarbageCollector();
private:
@@ -40,6 +41,7 @@ private:
mutable std::shared_mutex m_mapped_ranges_mutex;
RangeSet m_mapped_ranges;
GuestGpu* m_gpu = nullptr;
bool m_fault_process_pending = false;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/streamBuffer.cpp
Vendored
+26 -11
View File
@@ -73,8 +73,12 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
buffer_info.usage = flags;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
const VmaAllocationCreateFlags bda_flag =
with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
VmaAllocationCreateInfo allocation_info {};
allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
allocation_info.flags =
VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
allocation_info.usage = AllocationUsage(usage);
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
? VkMemoryPropertyFlags {}
@@ -99,6 +103,13 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
m_buffer->memory.type = allocation_result.memoryType;
m_buffer->memory.unique_id = VulkanNextMemoryUniqueId();
graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
vk::BufferDeviceAddressInfo address_info {};
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
address_info.buffer = m_buffer->buffer;
m_device_address = graphics.device.getBufferAddress(address_info);
EXIT_IF(m_device_address == 0);
}
VkMemoryPropertyFlags properties = 0;
vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
@@ -122,6 +133,11 @@ vk::Buffer Buffer::Handle() const noexcept {
return m_buffer->buffer;
}
vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
EXIT_IF(m_device_address == 0);
return m_device_address;
}
bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
}
@@ -141,6 +157,15 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
}
}
void Buffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
if (!m_is_coherent && size != 0) {
const auto result =
vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
}
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
vk::AccessFlags destination) const {
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
@@ -310,16 +335,6 @@ void StreamBuffer::Commit() {
watch.tick = tick;
}
void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset);
if (IsCoherent() || size == 0) {
return;
}
const auto result = vmaInvalidateAllocation(Graphics().allocator,
NativeBuffer().memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
EXIT_IF(source == nullptr);
const auto [data, offset] = Map(size, alignment);
src/graphics/host_gpu/renderer/cache/streamBuffer.h
Vendored
+3 -3
View File
@@ -48,12 +48,14 @@ public:
[[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; }
[[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; }
[[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; }
[[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept;
[[nodiscard]] uint64_t Offset(uint64_t address) const noexcept {
return address - m_cpu_address;
}
[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
void Write(uint64_t offset, const void* source, uint64_t size);
void Flush(uint64_t offset, uint64_t size);
void Invalidate(uint64_t offset, uint64_t size);
void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
uint64_t destination_offset, uint64_t size,
vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite,
@@ -84,6 +86,7 @@ private:
MemoryUsage m_usage = MemoryUsage::DeviceLocal;
uint64_t m_cpu_address = 0;
uint64_t m_size = 0;
vk::DeviceAddress m_device_address = 0;
std::unique_ptr<VulkanBuffer> m_buffer;
std::span<uint8_t> m_mapped;
bool m_is_coherent = false;
@@ -97,9 +100,6 @@ public:
[[nodiscard]] std::pair<uint8_t*, uint64_t> Map(uint64_t size, uint64_t alignment = 0,
bool allow_wait = true);
void Commit();
// Download mappings become visible to the CPU only after their GPU completion tick is free.
// Call this from the scheduler's deferred completion operation before reading Mapped().
void Invalidate(uint64_t offset, uint64_t size);
[[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0);
private:
src/graphics/host_gpu/renderer/commandScheduler.cpp
+1 -1
View File
@@ -196,8 +196,8 @@ void CommandScheduler::Finish() {
Submit();
}
m_master.Wait(CurrentTick() - 1);
PopPendingOperations();
BeginNext();
PopPendingOperations();
}
void CommandScheduler::Wait(uint64_t tick) {
src/graphics/host_gpu/renderer/pipeline/descriptors.cpp
+13 -78
View File
@@ -216,48 +216,6 @@ static BufferView NativeStorageBuffer(RenderContext&
return result;
}
static BufferView
NativeAddressBuffer(RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource,
const ShaderRecompiler::IR::ResourceSnapshot::Address& address,
ShaderType stage, uint32_t slot, uint32_t& address_offset, BufferId id) {
BufferView result;
address_offset = 0;
if (address.binding_base == 0) {
BindNullStorageBuffer(context, result);
return result;
}
if (resource.written) {
EXIT("writable address resources are unsupported\n");
}
const auto limit =
resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
? ShaderRecompiler::IR::FlatAddressWindowSize
: static_cast<uint64_t>(
context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
uint64_t size = 0;
const auto access = HostMemoryAccess::Mapped;
if (!HostMemoryQueryRange(address.binding_base, limit, access, size)) {
EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
address.binding_base);
}
const auto& graphics = context.GetGraphics();
const auto alignment = graphics.StorageMinAlignment();
auto [buffer, offset] =
context.GetBufferCache().ObtainBuffer(address.binding_base, size, false, false, id);
const auto aligned_offset = offset - offset % alignment;
const auto adjustment = offset - aligned_offset;
const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
const auto visible_size = std::min(size, static_cast<uint64_t>(max_range - adjustment));
address_offset = static_cast<uint32_t>(adjustment);
result.buffer = buffer->Handle();
result.offset = aligned_offset;
result.range = static_cast<vk::DeviceSize>(visible_size + adjustment);
SetVulkanObjectNameF(graphics.device, result.buffer,
"Kyty.{}.AddressBuffer[slot={} guest=0x{:016x} size=0x{:x}]",
ShaderStageResourceName(stage), slot, address.binding_base, visible_size);
return result;
}
static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) {
bool supported_dimension = false;
switch (resource.dimension) {
@@ -1000,25 +958,6 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
prepared.buffer_ids.push_back(cache.FindBuffer(address, size));
}
prepared.address_ids.clear();
prepared.address_ids.reserve(program.info.addresses.size());
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
const auto& resource = program.info.addresses[i];
const auto& address = snapshot.addresses[i];
if (address.binding_base == 0) {
prepared.address_ids.emplace_back();
continue;
}
EXIT_IF(resource.written);
const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
? ShaderRecompiler::IR::FlatAddressWindowSize
: static_cast<uint64_t>(m_context.GetGraphics()
.GetPhysicalDeviceProperties()
.limits.maxStorageBufferRange);
uint64_t size = 0;
EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size));
prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size));
}
}
void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
@@ -1028,8 +967,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
const auto& snapshot = *prepared.snapshot;
auto& resources = prepared.resources;
const auto& layout = program.bindings;
EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size() ||
prepared.address_ids.size() != program.info.addresses.size());
EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size());
resources.buffers.clear();
resources.buffers.reserve(program.info.buffers.size());
@@ -1049,15 +987,6 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
buffer_offset, prepared.buffer_ids[i]));
pack_memory_offset(i, buffer_offset);
}
resources.addresses.clear();
resources.addresses.reserve(program.info.addresses.size());
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
uint32_t address_offset = 0;
resources.addresses.push_back(NativeAddressBuffer(m_context, program.info.addresses[i],
snapshot.addresses[i], program.stage, i,
address_offset, prepared.address_ids[i]));
pack_memory_offset(static_cast<uint32_t>(program.info.buffers.size()) + i, address_offset);
}
if (!prepared.flattened_srt.empty()) {
resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt);
}
@@ -1129,6 +1058,10 @@ RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
if (bindings.pixel) {
FindBuffers(*bindings.pixel);
}
if (bindings.vertex.program->info.uses_dma ||
(bindings.pixel && bindings.pixel->program->info.uses_dma)) {
m_context.GetGpuResources().PrepareBda();
}
RebindBuffers(bindings.vertex);
if (bindings.pixel) {
RebindBuffers(*bindings.pixel);
@@ -1241,13 +1174,15 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
}
break;
case BindingKind::AddressMemory:
for (const auto resource: binding.resources) {
const auto& view = descriptors.addresses.at(resource);
EXIT_IF(view.buffer == nullptr);
m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
}
case BindingKind::BdaPagetable:
case BindingKind::FaultBuffer: {
auto& cache = m_context.GetBufferCache();
const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable
? cache.GetBdaPageTableBuffer()
: cache.GetFaultBuffer();
m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size());
break;
}
case BindingKind::FlattenedSrt:
case BindingKind::UserData:
case BindingKind::Gds: {
src/graphics/host_gpu/renderer/pipeline/descriptors.h
-2
View File
@@ -38,7 +38,6 @@ struct NativeDescriptors {
std::vector<BufferView> buffers;
std::vector<TextureBinding> images;
std::vector<vk::Sampler> samplers;
std::vector<BufferView> addresses;
BufferView gds;
BufferView flattened_srt;
BufferView user_data;
@@ -49,7 +48,6 @@ struct PreparedBindings {
const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
NativeDescriptors resources;
std::vector<BufferId> buffer_ids;
std::vector<BufferId> address_ids;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
bool committed = false;
src/graphics/host_gpu/renderer/renderCompute.cpp
+5 -2
View File
@@ -82,8 +82,8 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
const auto& resources = *input.stage.resources;
if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 ||
!program.info.images.empty() || !program.info.samplers.empty() ||
!program.info.addresses.empty() || !resources.images.empty() ||
!resources.samplers.empty() || !resources.addresses.empty()) {
program.info.uses_dma || !resources.images.empty() ||
!resources.samplers.empty()) {
return false;
}
const auto& resource = program.info.buffers.front();
@@ -337,6 +337,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader);
auto bindings = PrepareBindings(input_info.stage);
FindBuffers(bindings);
if (program.info.uses_dma) {
m_context.GetGpuResources().PrepareBda();
}
RebindBuffers(bindings);
RebindImages(bindings);
src/graphics/host_gpu/shaders/fault_buffer_process.comp
+35
View File
@@ -0,0 +1,35 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#version 450
#extension GL_ARB_gpu_shader_int64 : enable
layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
layout(std430, binding = 0) buffer input_buf {
uint fault_buffer[];
};
layout(std430, binding = 1) buffer output_buf {
uint64_t download_buffer[];
};
layout(std430, binding = 1) buffer output_buf32 {
uint download_buffer32[];
};
void main() {
const uint id = gl_GlobalInvocationID.x;
uint word = fault_buffer[id];
fault_buffer[id] = 0u;
const uint base_bit = id * 32u;
while (word != 0u) {
const uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u;
if (store_index >= MAX_PAGE_FAULTS) {
return;
}
const uint bit = findLSB(word);
word &= word - 1;
const uint page = base_bit + bit;
download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS;
}
}
src/graphics/host_gpu/vma.cpp
+4 -1
View File
@@ -72,7 +72,10 @@ bool GraphicContext::CreateAllocator() {
info.device = device;
info.pVulkanFunctions = &functions;
info.vulkanApiVersion = VULKAN_TARGET_API_VERSION;
info.flags = memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0;
info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
if (memory_budget_ext_enabled) {
info.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
}
const auto result = static_cast<vk::Result>(vmaCreateAllocator(&info, &allocator));
if (result != vk::Result::eSuccess) {
src/graphics/presentation/window/vulkanWindow.cpp
+6 -13
View File
@@ -594,20 +594,16 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE);
EXIT_NOT_IMPLEMENTED(supported_features12.bufferDeviceAddress != VK_TRUE);
#if !defined(__APPLE__)
EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
#endif
features12.timelineSemaphore = VK_TRUE;
features12.shaderOutputLayer = VK_TRUE;
if (Config::GpuAssistedValidationEnabled()) {
if (supported_features12.bufferDeviceAddress != VK_TRUE) {
EXIT("--gpu-assisted-validation requires bufferDeviceAddress, unsupported by this "
"device\n");
}
features12.bufferDeviceAddress = VK_TRUE;
}
features12.bufferDeviceAddress = VK_TRUE;
vk::PhysicalDeviceFeatures device_features {};
device_features.fragmentStoresAndAtomics = VK_TRUE;
@@ -625,12 +621,9 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
device_features.shaderClipDistance = VK_TRUE;
device_features.shaderCullDistance = VK_TRUE;
device_features.largePoints = VK_TRUE;
device_features.vertexPipelineStoresAndAtomics = VK_TRUE;
graphics.sample_rate_shading_enabled = true;
device_features.vertexPipelineStoresAndAtomics =
supported_features2.features.vertexPipelineStoresAndAtomics;
if (Config::GpuAssistedValidationEnabled()) {
device_features.shaderInt64 = supported_features2.features.shaderInt64;
}
device_features.shaderInt64 = VK_TRUE;
vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
src/graphics/shader/recompiler/ShaderRecompiler.cpp
-1
View File
@@ -665,7 +665,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
runtime.read_memory = ReadZeroMemory;
}
runtime.userdata = options.read_memory_data;
runtime.flat_memory_base = options.flat_memory_base;
if (!IR::MaterializeResources(ir, runtime, resources, error)) {
return false;
}
src/graphics/shader/recompiler/ShaderRecompiler.h
-2
View File
@@ -6,7 +6,6 @@
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/shader.h"
#include <optional>
#include <span>
#include <vector>
@@ -20,7 +19,6 @@ struct CompileOptions {
uint32_t scratch_dwords = 0;
uint64_t shader_hash = 0;
uint64_t shader_base = 0;
std::optional<uint64_t> flat_memory_base;
uint32_t push_constant_offset = 0;
bool dump_ir = true;
bool early_dump = false;
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+10 -7
View File
@@ -70,8 +70,9 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
if (uses_gds) {
Expect(Kind::Gds);
}
if (!program.info.addresses.empty()) {
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
if (program.info.uses_dma) {
Expect(Kind::BdaPagetable);
Expect(Kind::FaultBuffer);
}
const bool uses_flattened_runtime =
!program.srt_reads.empty() ||
@@ -104,8 +105,7 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
program.bindings.push_constant_offset + program.bindings.push_constant_size >
IR::NativePushConstantSize ||
program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
program.bindings.memory_offset_count !=
program.info.buffers.size() + program.info.addresses.size() ||
program.bindings.memory_offset_count != program.info.buffers.size() ||
(!has_user_storage &&
program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
(has_user_storage && program.bindings.push_constant_size != 0) ||
@@ -146,8 +146,8 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
if (planning_only_handle(inst)) {
break;
}
if (dense >= program.info.addresses.size()) {
return Fail(error, "typed address handle has an invalid dense resource");
if (inst.NumArgs() != 2 || !program.info.uses_dma) {
return Fail(error, "typed address handle has invalid DMA metadata");
}
break;
case IR::ValueOpcode::GetScratchResource:
@@ -188,7 +188,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
if (address_access != IR::AddressAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
return Fail(error, "address operation has invalid memory metadata");
@@ -198,6 +199,8 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
return Fail(error, "scratch operation has no per-thread storage");
}
requirements.function_scratch = true;
} else if (address_access == IR::AddressAccess::Write) {
return Fail(error, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
}
}
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+22 -3
View File
@@ -36,8 +36,10 @@ enum : uint32_t {
ExecutionModeSignedZeroInfNanPreserve = 4461,
ExecutionModeDerivativeGroupQuadsKHR = 5289,
AddressingModelLogical = 0,
AddressingModelPhysicalStorageBuffer64 = 5348,
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
CapabilityInt64 = 11,
CapabilityImageGatherExtended = 25,
CapabilityClipDistance = 32,
CapabilityCullDistance = 33,
@@ -52,6 +54,7 @@ enum : uint32_t {
CapabilityShaderViewportIndexLayerEXT = 5254,
CapabilityFragmentBarycentricKHR = 5284,
CapabilityComputeDerivativeGroupQuadsKHR = 5288,
CapabilityPhysicalStorageBufferAddresses = 5347,
StorageClassUniformConstant = 0,
StorageClassInput = 1,
StorageClassOutput = 3,
@@ -60,6 +63,7 @@ enum : uint32_t {
StorageClassPushConstant = 9,
StorageClassImage = 11,
StorageClassStorageBuffer = 12,
StorageClassPhysicalStorageBuffer = 5349,
FunctionControlNone = 0,
SelectionControlNone = 0,
LoopControlNone = 0,
@@ -148,7 +152,9 @@ enum : uint32_t {
OpConstantComposite = 44,
OpUndef = 1,
OpFunction = 54,
OpFunctionParameter = 55,
OpFunctionEnd = 56,
OpFunctionCall = 57,
OpVariable = 59,
OpImageTexelPointer = 60,
OpLoad = 61,
@@ -178,6 +184,8 @@ enum : uint32_t {
OpConvertFToS = 110,
OpConvertSToF = 111,
OpConvertUToF = 112,
OpUConvert = 113,
OpConvertUToPtr = 120,
OpBitcast = 124,
OpSNegate = 126,
OpFNegate = 127,
@@ -256,11 +264,16 @@ enum : uint32_t {
OpSwitch = 251,
OpKill = 252,
OpReturn = 253,
OpReturnValue = 254,
OpGroupNonUniformBallot = 339,
OpGroupNonUniformBallotFindLSB = 343,
OpGroupNonUniformShuffle = 345,
};
enum : uint32_t {
MemoryAccessAlignedMask = 0x2,
};
enum : uint32_t {
GlslRoundEven = 2,
GlslTrunc = 3,
@@ -345,9 +358,10 @@ struct EmitterState {
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t storage_buffer_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers + IR::ShaderInfo::MaxAddresses>
memory_byte_offsets {};
uint32_t address_memory_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
uint32_t bda_pagetable_variable = 0;
uint32_t fault_buffer_variable = 0;
uint32_t bda_pointer_function = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
@@ -384,6 +398,7 @@ uint32_t TypeBool(EmitterState& state);
uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
uint32_t TypeU32(EmitterState& state);
uint32_t TypeU64(EmitterState& state);
uint32_t TypeDeviceAddress(EmitterState& state);
uint32_t TypeU32Pair(EmitterState& state);
uint32_t TypeI32(EmitterState& state);
uint32_t TypeI32Pair(EmitterState& state);
@@ -397,6 +412,8 @@ uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t point
uint32_t TypeFunction(EmitterState& state);
uint32_t TypeStorageBufferPointer(EmitterState& state);
uint32_t TypeStorageBufferElementPointer(EmitterState& state);
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
uint32_t TypePhysicalU32Pointer(EmitterState& state);
uint32_t TypePushConstantElementPointer(EmitterState& state);
uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
@@ -849,6 +866,8 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error);
void DefineGetBdaPointer(EmitterState& state);
// These templates accept local lambdas from several emitter translation units.
template <typename Fn>
void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+194 -51
View File
@@ -1,5 +1,7 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include <algorithm>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
@@ -95,60 +97,146 @@ uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t ad
return result;
}
uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t low, uint32_t high) {
uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t low,
uint32_t high) {
auto& state = ctx.state;
auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u));
}
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
if (mem.kind == IR::ResourceKind::Scratch) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid = Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
}
const auto base = state.program.info.addresses[mem.resource].specialized_base;
if (mem.kind == IR::ResourceKind::Flat ||
(mem.kind == IR::ResourceKind::Global &&
state.program.info.addresses[mem.resource].unbased)) {
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto base_low = ConstantU32(state, static_cast<uint32_t>(base));
const auto base_high = ConstantU32(state, static_cast<uint32_t>(base >> 32u));
const auto relative = Binary(state, OpISub, TypeU32(state), low, base_low);
const auto borrow = Binary(state, OpULessThan, TypeBool(state), low, base_low);
const auto relative_high = Binary(
state, OpISub, TypeU32(state), Binary(state, OpISub, TypeU32(state), high, base_high),
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0)));
const auto valid =
Binary(state, OpIEqual, TypeBool(state), relative_high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, relative, ConstantU32(state, UINT32_MAX));
}
const auto initial = base + static_cast<uint64_t>(static_cast<int64_t>(immediate));
auto relative_high = ConstantU32(state, static_cast<uint32_t>(initial >> 32u));
const auto relative = AddU64Low(state, ConstantU32(state, static_cast<uint32_t>(initial)),
relative_high, low, ConstantU32(state, 0), relative_high);
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid =
Binary(state, OpIEqual, TypeBool(state), relative_high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, relative, ConstantU32(state, UINT32_MAX));
Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
}
uint32_t AddAddressMemoryOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
uint32_t address) {
auto& state = ctx.state;
const auto binding =
ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
const auto offset =
state.memory_byte_offsets[state.program.info.buffers.size() + binding.array_index];
const auto corrected = Binary(state, OpIAdd, TypeU32(state), address, offset);
const auto not_invalid =
Binary(state, OpINotEqual, TypeBool(state), address, ConstantU32(state, UINT32_MAX));
const auto no_overflow =
Binary(state, OpUGreaterThanEqual, TypeBool(state), corrected, address);
return Select(state, TypeU32(state), AndCondition(state, not_invalid, no_overflow), corrected,
ConstantU32(state, UINT32_MAX));
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
return state.builder.Constant(OpConstant, TypeDeviceAddress(state),
{static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u)});
}
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
const auto low64 = Unary(state, OpUConvert, TypeDeviceAddress(state), low);
const auto high64 = Binary(state, OpShiftLeftLogical, TypeDeviceAddress(state),
Unary(state, OpUConvert, TypeDeviceAddress(state), high),
ConstantDeviceAddress(state, 32));
return Binary(state, OpBitwiseOr, TypeDeviceAddress(state), low64, high64);
}
uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
auto& state = ctx.state;
auto low = ctx.Arg(inst, 1);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u));
}
uint32_t address = 0;
if (mem.address_is_full) {
address = DeviceAddressFromWords(state, low, ctx.Arg(inst, 2));
} else {
const auto* handle = inst.Arg(0).Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetAddressResource ||
handle->NumArgs() != 2) {
ctx.Fail(inst, "has no address base pair");
return ConstantDeviceAddress(state, 0);
}
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
address = Binary(state, OpIAdd, TypeDeviceAddress(state), base,
Unary(state, OpUConvert, TypeDeviceAddress(state), low));
}
auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
return immediate == 0
? address
: Binary(state, OpIAdd, TypeDeviceAddress(state), address,
ConstantDeviceAddress(state,
static_cast<uint64_t>(static_cast<int64_t>(immediate))));
}
uint32_t FaultElementPointer(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.fault_buffer_variable, ConstantU32(state, 0), index});
return pointer;
}
void RecordBdaFault(EmitterState& state, uint32_t page) {
const auto word = Binary(state, OpShiftRightLogical, TypeU32(state), page,
ConstantU32(state, 5));
const auto bit = Binary(
state, OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1),
Binary(state, OpBitwiseAnd, TypeU32(state), page, ConstantU32(state, 31)));
const auto pointer = FaultElementPointer(state, word);
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
state.builder.AddFunction(
{OpStore, pointer, Binary(state, OpBitwiseOr, TypeU32(state), value, bit)});
}
uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state;
const auto result = state.builder.AllocateId();
state.builder.AddFunction(
{OpFunctionCall, TypeDeviceAddress(state), result, state.bda_pointer_function, address});
return result;
}
uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state;
const auto bda = GetBdaPointer(ctx, address);
const auto present = Binary(state, OpINotEqual, TypeBool(state), bda,
ConstantDeviceAddress(state, 0));
return EmitValueOrZeroIfCondition(state, present, [&]() {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, bda});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, TypeU32(state), value, pointer, MemoryAccessAlignedMask, sizeof(uint32_t)});
return value;
});
}
uint32_t LoadBda(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
uint32_t bits) {
auto& state = ctx.state;
const auto address = GuestAddress(ctx, inst, mem);
const auto active = ctx.Arg(inst, inst.NumArgs() - 1);
return EmitValueOrZeroIfCondition(state, active, [&]() {
const auto aligned = Binary(state, OpBitwiseAnd, TypeDeviceAddress(state), address,
ConstantDeviceAddress(state, ~uint64_t {3}));
const auto first = LoadBdaDword(ctx, aligned);
const auto byte = Binary(state, OpBitwiseAnd, TypeU32(state),
Unary(state, OpUConvert, TypeU32(state), address),
ConstantU32(state, 3));
const auto crosses = bits == 8u
? ConstantBool(state, false)
: Binary(state, bits == 16u ? OpUGreaterThan : OpINotEqual,
TypeBool(state), byte,
ConstantU32(state, bits == 16u ? 2u : 0u));
const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() {
return LoadBdaDword(
ctx, Binary(state, OpIAdd, TypeDeviceAddress(state), aligned,
ConstantDeviceAddress(state, sizeof(uint32_t))));
});
const auto shift = Binary(state, OpShiftLeftLogical, TypeU32(state), byte,
ConstantU32(state, 3));
const auto upper_shift = Binary(
state, OpShiftLeftLogical, TypeU32(state),
Binary(state, OpBitwiseAnd, TypeU32(state),
Binary(state, OpISub, TypeU32(state), ConstantU32(state, 4), byte),
ConstantU32(state, 3)),
ConstantU32(state, 3));
const auto merged = Binary(
state, OpBitwiseOr, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), first, shift),
Binary(state, OpShiftLeftLogical, TypeU32(state), second, upper_shift));
return bits == 32u
? merged
: Binary(state, OpBitwiseAnd, TypeU32(state), merged,
ConstantU32(state, bits == 8u ? 0xffu : 0xffffu));
});
}
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
@@ -163,11 +251,10 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0),
ConstantU32(ctx.state, mem.offset));
}
const auto address = AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
if (mem.kind == IR::ResourceKind::Scratch) {
return address;
if (mem.kind != IR::ResourceKind::Scratch) {
EXIT("physical address memory must use the BDA emitter\n");
}
return AddAddressMemoryOffset(ctx, mem, address);
return ScratchByteAddress(ctx, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
}
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
@@ -923,6 +1010,58 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
} // namespace
void DefineGetBdaPointer(EmitterState& state) {
if (!state.program.info.uses_dma) {
return;
}
const auto type = TypeDeviceAddress(state);
const auto function_type = state.builder.Type(OpTypeFunction, {type, type});
state.bda_pointer_function = state.builder.AllocateId();
const auto address = state.builder.AllocateId();
const auto entry_label = state.builder.AllocateId();
state.builder.AddName(state.bda_pointer_function, "get_bda_pointer");
state.builder.AddFunction(
{OpFunction, type, state.bda_pointer_function, FunctionControlNone, function_type});
state.builder.AddFunction({OpFunctionParameter, type, address});
EmitLabel(state, entry_label);
const auto page64 = Binary(
state, OpShiftRightLogical, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
const auto page = Unary(state, OpUConvert, TypeU32(state), page64);
const auto entry_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeDeviceAddressStoragePointer(state), entry_pointer,
state.bda_pagetable_variable, ConstantU32(state, 0), page});
const auto base = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, base, entry_pointer});
const auto missing =
Binary(state, OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0));
const auto fault_label = state.builder.AllocateId();
const auto available_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction(
{OpBranchConditional, missing, fault_label, available_label});
EmitLabel(state, fault_label);
RecordBdaFault(state, page);
state.builder.AddFunction({OpBranch, merge_label});
EmitLabel(state, available_label);
const auto offset = Binary(
state, OpBitwiseAnd, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1));
const auto available = Binary(state, OpIAdd, type, base, offset);
state.builder.AddFunction({OpBranch, merge_label});
EmitLabel(state, merge_label);
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpPhi, type, result, ConstantDeviceAddress(state, 0), fault_label,
available, available_label});
state.builder.AddFunction({OpReturnValue, result});
state.builder.AddFunction({OpFunctionEnd});
}
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto op = inst.GetOpcode();
@@ -981,6 +1120,10 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
}
const auto address_info = IR::AddressOpcodeInfoOf(op);
const bool load_address = address_info.access == IR::AddressAccess::Read;
if (load_address && ctx.Memory(inst).kind != IR::ResourceKind::Scratch) {
ctx.Define(inst, LoadBda(ctx, inst, ctx.Memory(inst), address_info.data_bits));
return true;
}
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 ||
op == IR::ValueOpcode::LoadBufferU16 ||
op == IR::ValueOpcode::LoadBufferU32;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+2 -14
View File
@@ -108,20 +108,8 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
return access;
case IR::ResourceKind::ScalarAddress:
case IR::ResourceKind::Flat:
case IR::ResourceKind::Global: {
if (state.address_memory_variable == 0) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
mem.resource,
"address memory binding was not emitted");
}
const auto binding = ResourceForDescriptor(
state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
access.object_pointer, state.address_memory_variable,
ConstantU32(state, binding.array_index)});
break;
}
case IR::ResourceKind::Global:
EXIT("physical address memory must use the BDA emitter\n");
case IR::ResourceKind::ScalarBuffer:
case IR::ResourceKind::Buffer: {
if (state.storage_buffer_variable == 0) {
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+45 -11
View File
@@ -28,6 +28,10 @@ uint32_t TypeU64(EmitterState& state) {
return TypeU32Vector(state, 2);
}
uint32_t TypeDeviceAddress(EmitterState& state) {
return state.builder.Type(OpTypeInt, {64, 0});
}
uint32_t TypeU32Pair(EmitterState& state) {
const auto element = TypeU32(state);
return state.builder.Type(OpTypeStruct, {element, element});
@@ -90,6 +94,14 @@ uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeU32(state));
}
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeDeviceAddress(state));
}
uint32_t TypePhysicalU32Pointer(EmitterState& state) {
return TypePointer(state, StorageClassPhysicalStorageBuffer, TypeU32(state));
}
uint32_t TypePushConstantElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassPushConstant, TypeU32(state));
}
@@ -130,6 +142,15 @@ uint32_t SampleMaskArrayType(EmitterState& state) {
return state.builder.Type(OpTypeArray, {TypeI32(state), ConstantU32(state, 1)});
}
uint32_t BdaPagetableType(EmitterState& state) {
const auto array = state.builder.DecoratedType(
OpTypeRuntimeArray, {TypeDeviceAddress(state)},
{{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}});
return state.builder.DecoratedType(
OpTypeStruct, {array},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}});
}
uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
return state.builder.Type(OpTypeArray, {TypeF32(state), ConstantU32(state, count)});
}
@@ -143,13 +164,14 @@ void DefineDescriptorVariables(EmitterState& state) {
state.storage_buffer_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer);
}
if (DescriptorBinding(state, IR::DescriptorBindingKind::AddressMemory) != nullptr) {
const auto count =
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::AddressMemory));
const auto array_type = state.builder.Type(OpTypeArray, {StorageBufferType(state), count});
const auto pointer_type = TypePointer(state, StorageClassStorageBuffer, array_type);
state.address_memory_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer);
if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) {
state.bda_pagetable_variable = state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassStorageBuffer, BdaPagetableType(state)),
StorageClassStorageBuffer);
}
if (DescriptorBinding(state, IR::DescriptorBindingKind::FaultBuffer) != nullptr) {
state.fault_buffer_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer);
}
if (state.program.bindings.push_constant_size != 0) {
const auto pointer_type =
@@ -564,9 +586,13 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) {
if (state.storage_buffer_variable != 0) {
Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
}
if (state.address_memory_variable != 0) {
Decorate(state.address_memory_variable, "address_memory",
IR::DescriptorBindingKind::AddressMemory);
if (state.bda_pagetable_variable != 0) {
Decorate(state.bda_pagetable_variable, "bda_pagetable",
IR::DescriptorBindingKind::BdaPagetable);
}
if (state.fault_buffer_variable != 0) {
Decorate(state.fault_buffer_variable, "fault_buffer",
IR::DescriptorBindingKind::FaultBuffer);
}
constexpr const char* SampledNames[] = {"sampled_1d",
"sampled_1d_array",
@@ -644,6 +670,11 @@ void DefineModule(EmitterState& state) {
state.builder.RequireCapability(CapabilityShader);
state.builder.RequireCapability(CapabilitySignedZeroInfNanPreserve);
if (state.program.info.uses_dma) {
state.builder.RequireCapability(CapabilityInt64);
state.builder.RequireCapability(CapabilityPhysicalStorageBufferAddresses);
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
}
if (state.clip_distance_variable != 0) {
state.builder.RequireCapability(CapabilityClipDistance);
}
@@ -682,7 +713,10 @@ void DefineModule(EmitterState& state) {
state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
}
state.builder.RequireExtension("SPV_KHR_float_controls");
state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
state.builder.AddMemoryModel(
{state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64
: AddressingModelLogical,
MemoryModelGLSL450});
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
state.interface_variables);
// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+1
View File
@@ -566,6 +566,7 @@ bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* e
SetError(error, ctx.error.c_str());
return false;
}
DefineGetBdaPointer(state);
for (const auto* block: program.blocks) {
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
src/graphics/shader/recompiler/frontend/translate/Memory.cpp
+4 -6
View File
@@ -136,12 +136,10 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
memory.offen = decoded.offen;
memory.resource = ResourceIndexFromOperand(decoded.src1);
memory.sampler = ResourceIndexFromOperand(decoded.src2);
if (memory.kind == ResourceKind::ScalarAddress) {
memory.resource = RawScalarLoadBase(decoded.src0);
} else if (memory.kind == ResourceKind::ScalarBuffer) {
if (memory.kind == ResourceKind::ScalarBuffer) {
memory.resource = ResourceIndexFromOperand(decoded.src0);
} else if (memory.kind == ResourceKind::Lds || memory.kind == ResourceKind::Gds ||
memory.kind == ResourceKind::Scratch) {
} else if (IsAddressResourceKind(memory.kind) || memory.kind == ResourceKind::Lds ||
memory.kind == ResourceKind::Gds) {
memory.resource = 0;
memory.sampler = 0;
}
@@ -421,7 +419,7 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) {
bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
const auto memory = MemoryInfoFromDecoded(inst);
const auto resource =
raw ? GetScalarAddressResource(memory.resource) : GetBufferResource(memory);
raw ? GetScalarAddressResource(RawScalarLoadBase(inst.src0)) : GetBufferResource(memory);
const auto offset = ReadU32(inst.src1);
std::array<IR::Value, 16> loaded {};
for (uint32_t component = 0; component < memory.data_dwords; component++) {
src/graphics/shader/recompiler/ir/ShaderIR.h
+3 -17
View File
@@ -176,20 +176,6 @@ struct SampledResourcePair {
bool operator==(const SampledResourcePair& other) const = default;
};
struct AddressResource {
uint32_t source = UINT32_MAX;
uint32_t first_use_pc = 0;
ResourceKind kind = ResourceKind::Flat;
int32_t min_offset = 0;
uint64_t specialized_base = 0;
bool unbased = true;
bool read = false;
bool written = false;
bool atomic = false;
bool operator==(const AddressResource& other) const = default;
};
enum class StageInputKind {
VertexIndex,
InstanceIndex,
@@ -321,7 +307,8 @@ enum class DescriptorBindingKind {
StorageAtomic3D,
Samplers,
Gds,
AddressMemory,
BdaPagetable,
FaultBuffer,
FlattenedSrt,
UserData,
Count,
@@ -431,13 +418,11 @@ struct BindingLayout {
struct ShaderInfo {
static constexpr uint32_t MaxBuffers = 32;
static constexpr uint32_t MaxAddresses = 32;
static constexpr uint32_t MaxImages = 32;
static constexpr uint32_t MaxSamplers = 32;
static constexpr uint32_t MaxSampledPairs = 64;
std::vector<BufferResource> buffers;
std::vector<AddressResource> addresses;
std::vector<ImageResource> images;
std::vector<SamplerResource> samplers;
std::vector<SampledResourcePair> sampled_pairs;
@@ -446,6 +431,7 @@ struct ShaderInfo {
std::array<uint8_t, 32> vertex_fetch_components {};
int32_t vertex_offset_sgpr = -1;
bool has_bitwise_xor = false;
bool uses_dma = false;
bool operator==(const ShaderInfo& other) const = default;
};
src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp
+4 -8
View File
@@ -93,8 +93,7 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
return false;
}
next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
next.memory_offset_count =
static_cast<uint32_t>(program.info.buffers.size() + program.info.addresses.size());
next.memory_offset_count = static_cast<uint32_t>(program.info.buffers.size());
if (!program.info.buffers.empty()) {
std::vector<uint32_t> resources(program.info.buffers.size());
@@ -157,12 +156,9 @@ bool AllocateBindings(Program& program, uint32_t push_constant_offset, std::stri
if (uses_gds) {
AddBinding(next, DescriptorBindingKind::Gds);
}
if (!program.info.addresses.empty()) {
std::vector<uint32_t> resources(program.info.addresses.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
if (program.info.uses_dma) {
AddBinding(next, DescriptorBindingKind::BdaPagetable);
AddBinding(next, DescriptorBindingKind::FaultBuffer);
}
const bool uses_flattened_runtime =
!program.srt_reads.empty() ||
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp
+2 -68
View File
@@ -267,13 +267,6 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect,
return true;
}
uint64_t AddressSpecialization(const AddressResource& resource,
const ResourceSnapshot::Address& snapshot) {
return resource.kind == ResourceKind::Flat || resource.unbased
? snapshot.binding_base
: snapshot.guest_base - snapshot.binding_base;
}
size_t FlattenedRuntimeDwords(const Program& program) {
size_t size = program.srt_reads.size();
for (uint32_t resource = 0; resource < program.info.images.size(); resource++) {
@@ -298,8 +291,7 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
}
if (snapshot.buffers.size() != program.info.buffers.size() ||
snapshot.images.size() != program.info.images.size() ||
snapshot.samplers.size() != program.info.samplers.size() ||
snapshot.addresses.size() != program.info.addresses.size()) {
snapshot.samplers.size() != program.info.samplers.size()) {
if (error != nullptr) {
*error = "resource snapshot does not match dense shader topology";
}
@@ -343,14 +335,6 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn
}
return true;
};
for (uint32_t i = 0; i < snapshot.addresses.size(); i++) {
if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) {
if (error != nullptr) {
*error = fmt::format("address resource {} binds above its guest base", i);
}
return false;
}
}
std::unordered_set<uint32_t> indirect_resources;
for (const auto& table: snapshot.indirect_images) {
const auto* source = table.resource < program.info.images.size()
@@ -526,15 +510,6 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
}
}
}
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
if (program.info.addresses[i].specialized_base !=
AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) {
if (error != nullptr) {
*error = fmt::format("address resource {} no longer matches specialization", i);
}
return false;
}
}
return true;
}
@@ -550,7 +525,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
std::vector<DescriptorSourceRequest> requests;
std::vector<uint8_t> clean_flat_slots(program.srt_reads.size());
requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u +
program.info.samplers.size() + program.info.addresses.size());
program.info.samplers.size());
for (const auto& buffer: program.info.buffers) {
requests.push_back({buffer.source, buffer.first_use_pc});
}
@@ -576,12 +551,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
for (const auto& sampler: program.info.samplers) {
requests.push_back({sampler.source, sampler.first_use_pc});
}
for (const auto& address: program.info.addresses) {
if (!address.unbased) {
requests.push_back({address.source, address.first_use_pc});
}
}
std::vector<DescriptorValue> values;
std::vector<uint32_t> flattened_srt;
if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, clean_flat_slots,
@@ -686,37 +655,6 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
return false;
}
next.samplers.assign(cursor, cursor + program.info.samplers.size());
cursor += program.info.samplers.size();
for (const auto& address: program.info.addresses) {
if (!address.unbased) {
const auto value = *cursor++;
auto base = (static_cast<uint64_t>(value.dwords[0]) |
static_cast<uint64_t>(value.dwords[1]) << 32u) &
AddressMask;
if (address.kind == ResourceKind::ScalarAddress) {
base &= ~uint64_t {3};
}
const auto before = static_cast<uint64_t>(-static_cast<int64_t>(address.min_offset));
uint64_t binding_base = 0;
if (address.kind == ResourceKind::Flat) {
binding_base = base & ~(FlatAddressWindowSize - 1u);
} else if (base >= before) {
binding_base = base - before;
}
next.addresses.push_back({base, binding_base});
} else {
if (!runtime.flat_memory_base.has_value()) {
if (error != nullptr) {
*error = fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
"guest-address translation",
address.kind == ResourceKind::Flat ? "FLAT" : "global",
address.first_use_pc);
}
return false;
}
next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base});
}
}
next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
if (!ValidateResourceSnapshot(program, next, error)) {
return false;
@@ -791,10 +729,6 @@ bool SpecializeResources(Program& program, ResourceSnapshot& snapshot, std::stri
next.buffers[i].descriptor_format = descriptor.Format();
next.buffers[i].descriptor_swizzle = descriptor.DstSelXYZW();
}
for (uint32_t i = 0; i < next.addresses.size(); i++) {
next.addresses[i].specialized_base =
AddressSpecialization(next.addresses[i], next_snapshot.addresses[i]);
}
for (uint32_t i = 0; i < next.images.size(); i++) {
const auto& descriptor = next_snapshot.images[i];
auto& image = next.images[i];
src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h
-9
View File
@@ -5,15 +5,7 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
constexpr uint64_t FlatAddressWindowSize = 0x10000;
struct ResourceSnapshot {
struct Address {
uint64_t guest_base = 0;
uint64_t binding_base = 0;
bool operator==(const Address& other) const = default;
};
struct IndirectImage {
uint32_t resource = 0;
uint32_t capacity = 0;
@@ -25,7 +17,6 @@ struct ResourceSnapshot {
std::vector<DescriptorValue> buffers;
std::vector<DescriptorValue> images;
std::vector<DescriptorValue> samplers;
std::vector<Address> addresses;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
std::vector<IndirectImage> indirect_images;
src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp
+6 -157
View File
@@ -7,7 +7,6 @@
#include <algorithm>
#include <fmt/format.h>
#include <span>
#include <unordered_set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -96,10 +95,10 @@ class Tracker {
public:
explicit Tracker(Program& program): m_program(program), m_info(program.info) {
m_info.buffers.clear();
m_info.addresses.clear();
m_info.images.clear();
m_info.samplers.clear();
m_info.sampled_pairs.clear();
m_info.uses_dma = false;
}
bool Run(std::string* error) {
@@ -180,90 +179,6 @@ private:
m_program.shader_hash, StageName(m_program.stage), pc, reason));
}
struct AddressPart {
Value value;
bool rooted = false;
};
AddressPart FindAddressPart(Value value, Value active,
std::unordered_set<const Inst*>& visiting) const {
value = value.Resolve();
if (value.IsImmediate()) {
return {value, false};
}
const auto* inst = value.TryInstruction();
if (inst == nullptr || !visiting.insert(inst).second) {
return {};
}
const auto finish = [&](AddressPart part) {
visiting.erase(inst);
return part;
};
std::string reason;
if (ValidateRuntimeValue(m_program, value, reason)) {
return finish({value, true});
}
const auto merge = [&](AddressPart left, AddressPart right, bool require_both) {
if (left.value.IsEmpty() || right.value.IsEmpty()) {
return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left);
}
return EquivalentValue(m_program, left.value, right.value)
? AddressPart {left.value, left.rooted || right.rooted}
: AddressPart {};
};
switch (inst->GetOpcode()) {
case ValueOpcode::SelectU32:
if (EquivalentValue(m_program, inst->Arg(0), active)) {
return finish(FindAddressPart(inst->Arg(1), active, visiting));
}
return finish(merge(FindAddressPart(inst->Arg(1), active, visiting),
FindAddressPart(inst->Arg(2), active, visiting), true));
case ValueOpcode::Phi: {
const auto invariant = ResolveInvariantPhi(m_program, value);
return finish(invariant.IsEmpty() ? AddressPart {}
: FindAddressPart(invariant, active, visiting));
}
case ValueOpcode::IAdd32:
case ValueOpcode::ISub32: {
const auto left = FindAddressPart(inst->Arg(0), active, visiting);
const auto right = FindAddressPart(inst->Arg(1), active, visiting);
if (left.rooted == right.rooted ||
(inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) {
return finish({});
}
return finish(left.rooted ? left : right);
}
case ValueOpcode::CompositeExtractU32x2: {
const auto* source = inst->Arg(0).ResolveInstruction();
if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) {
return finish({});
}
const auto left = FindAddressPart(source->Arg(0), active, visiting);
const auto right = FindAddressPart(source->Arg(1), active, visiting);
if (left.rooted == right.rooted) {
return finish({});
}
return finish(left.rooted ? left : right);
}
default: return finish({});
}
}
bool MakeFlatAddressSource(const Inst& handle, Value active,
DescriptorSource& descriptor) const {
std::unordered_set<const Inst*> visiting;
auto low = FindAddressPart(handle.Arg(0), active, visiting);
visiting.clear();
auto high = FindAddressPart(handle.Arg(1), active, visiting);
if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) {
return false;
}
descriptor.dword_count = 2;
descriptor.dwords[0] = low.value;
descriptor.dwords[1] = high.value;
return true;
}
bool MakeSource(const Inst& handle, uint32_t width, bool sampler, bool sample_adjust,
DescriptorSource& descriptor, uint32_t pc, std::string* error) const {
if (handle.NumArgs() != width) {
@@ -577,42 +492,14 @@ private:
return true;
}
bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle,
uint32_t& source, bool& unbased, std::string* error) {
handle = value.Resolve().TryInstruction();
bool ValidateAddressHandle(Value value, uint32_t pc, std::string* error) const {
const auto* handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) {
return Fail(pc, error, "address operation requires GetAddressResource");
}
if (handle->NumArgs() != 2) {
return Fail(pc, error, "GetAddressResource must have two address dwords");
}
DescriptorSource descriptor;
const auto& memory = m_program.memory_info[memory_inst.Flags<MemoryFlags>().index];
if (memory.address_is_full) {
const auto active = memory_inst.Arg(memory_inst.NumArgs() - 1u);
if (memory.kind != ResourceKind::Flat ||
!MakeFlatAddressSource(*handle, active, descriptor)) {
unbased = true;
source = UINT32_MAX;
return true;
}
} else if (!MakeSource(*handle, 2, false, false, descriptor, pc, error)) {
return false;
}
std::string reason;
uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) {
if (memory.address_is_full) {
unbased = true;
source = UINT32_MAX;
return true;
}
return Fail(
pc, error,
fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason));
}
unbased = false;
source = InternSource(descriptor);
return true;
}
@@ -706,39 +593,6 @@ private:
return static_cast<uint32_t>(m_info.samplers.size() - 1);
}
uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory,
AddressAccess access, uint32_t pc) {
auto immediate = static_cast<int32_t>(memory.offset);
if (memory.kind == ResourceKind::ScalarAddress) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto min_offset = unbased ? 0 : std::min(immediate, 0);
for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
auto& address = m_info.addresses[i];
if (address.source == source && address.unbased == unbased &&
address.kind == memory.kind) {
address.first_use_pc = std::min(address.first_use_pc, pc);
address.min_offset = std::min(address.min_offset, min_offset);
address.read = address.read || access == AddressAccess::Read;
address.written = address.written || access == AddressAccess::Write;
return i;
}
}
if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
return UINT32_MAX;
}
AddressResource address;
address.source = source;
address.first_use_pc = pc;
address.kind = memory.kind;
address.min_offset = min_offset;
address.unbased = unbased;
address.read = access == AddressAccess::Read;
address.written = access == AddressAccess::Write;
m_info.addresses.push_back(address);
return static_cast<uint32_t>(m_info.addresses.size() - 1);
}
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) {
@@ -839,16 +693,11 @@ private:
}
return true;
}
bool unbased = false;
if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) {
if (!ValidateAddressHandle(inst.Arg(0), flags.pc, error)) {
return false;
}
resource = AddAddress(source, unbased, memory, address_info.access, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "address resource limit exceeded");
}
return AddHandlePatch(handle, resource, flags.pc, error) &&
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
m_info.uses_dma = true;
return true;
}
if (!ImageResourceKindMatches(memory.kind, image_info.resource_class)) {
src/graphics/shader/recompiler/ir/passes/SrtWalker.h
-2
View File
@@ -3,7 +3,6 @@
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <optional>
#include <span>
namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -17,7 +16,6 @@ struct SrtRuntime {
uint64_t shader_base = 0;
SrtMemoryReader read_memory = nullptr;
void* userdata = nullptr;
std::optional<uint64_t> flat_memory_base;
SrtMemoryReader read_specialization_memory = nullptr;
};
src/graphics/shader/shader.cpp
+2 -10
View File
@@ -994,10 +994,10 @@ static void LogShaderProgramCacheHit(const char* stage, uint64_t shader_hash, ui
static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Program& program) {
std::string ret = fmt::format(
"push={} groups={} user={} buffers={} images={} samplers={} addresses={} srt={}",
"push={} groups={} user={} buffers={} images={} samplers={} dma={} srt={}",
program.bindings.push_constant_size, program.bindings.descriptors.size(),
program.bindings.user_data_registers.size(), program.info.buffers.size(),
program.info.images.size(), program.info.samplers.size(), program.info.addresses.size(),
program.info.images.size(), program.info.samplers.size(), program.info.uses_dma,
program.srt_reads.size());
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto& buffer = program.info.buffers[i];
@@ -1014,9 +1014,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog
static_cast<uint32_t>(image.conversion_format), image.shader_swizzle,
image.indirect_root, image.indirect_mapping_offset, image.indirect_mapping_capacity);
}
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
ret += fmt::format(" a{}[base=0x{:x}]", i, program.info.addresses[i].specialized_base);
}
return ret;
}
@@ -1056,11 +1053,6 @@ static void ShaderAppendNativeSpecialization(std::vector<uint32_t>&
ids.push_back(static_cast<uint32_t>(image.indirect_resources.size()));
ids.insert(ids.end(), image.indirect_resources.begin(), image.indirect_resources.end());
}
ids.push_back(static_cast<uint32_t>(program.info.addresses.size()));
for (const auto& address: program.info.addresses) {
ids.push_back(static_cast<uint32_t>(address.specialized_base));
ids.push_back(static_cast<uint32_t>(address.specialized_base >> 32u));
}
}
static std::span<const uint32_t> AddShaderProgramPermutation(const char* stage,