raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 02:47:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

graphics: add RDNA2 FLAT BDA buffer cache and faults

Browse Source
This commit is contained in:
nmzik
2026-08-27 03:45:41 +02:00
parent 8072718462
commit e738535294
37 changed files with 1002 additions and 684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/rangeSet.h
+10 -1
View File
@@ -53,6 +53,15 @@ public:
}
}
void Clear() { m_ranges.clear(); }
template <typename Func>
void ForEach(Func&& func) const {
for (const auto& [begin, end]: m_ranges) {
func(begin, end);
}
}
[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
std::vector<Range> result;
ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
@@ -98,7 +107,7 @@ public:
private:
static uint64_t End(uint64_t address, uint64_t size) {
if (address == 0 || size == 0 || size > UINT64_MAX - address) {
if (size == 0 || size > UINT64_MAX - address) {
EXIT("invalid range-set address or size\n");
}
return address + size;
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+73 -38
View File
@@ -51,41 +51,50 @@ struct BufferCache::DownloadCopy {
};
void BufferCache::Register(BufferId id) {
auto& buffer = m_slot_buffers[id];
PageTable::PageRange pages {};
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
m_page_table[page] = id;
}
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
(void)it;
EXIT_IF(!inserted);
buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
m_total_used_memory += buffer.Size();
ChangeRegister<true>(id);
}
void BufferCache::Unregister(BufferId id) {
ChangeRegister<false>(id);
}
template <bool insert>
void BufferCache::ChangeRegister(BufferId id) {
auto& buffer = m_slot_buffers[id];
if (buffer.is_deleted) {
return;
}
PageTable::PageRange pages {};
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
auto* owner = m_page_table.Find(page);
if (owner != nullptr && *owner == id) {
*owner = {};
if constexpr (insert) {
m_page_table[page] = id;
} else {
m_page_table[page] = {};
}
}
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
if (buffer.Size() > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
const auto size_pages = pages.last_exclusive - pages.first;
if constexpr (insert) {
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
(void)it;
EXIT_IF(!inserted);
m_total_used_memory += buffer.Size();
buffer.lru_id = m_lru_cache.Insert(id, m_gc_tick);
std::vector<vk::DeviceAddress> addresses;
addresses.reserve(size_pages);
for (uint64_t i = 0; i < size_pages; ++i) {
addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS));
}
WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress),
addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
} else {
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
EXIT_IF(buffer.Size() > m_total_used_memory);
m_total_used_memory -= buffer.Size();
m_lru_cache.Free(buffer.lru_id);
m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress),
size_pages * sizeof(vk::DeviceAddress), 0);
buffer.is_deleted = true;
}
m_total_used_memory -= buffer.Size();
m_lru_cache.Free(buffer.lru_id);
buffer.is_deleted = true;
}
void BufferCache::TouchBuffer(const Buffer& buffer) {
@@ -184,16 +193,21 @@ void BufferCache::DownloadBufferMemory(std::span<const DownloadCopy> copies) {
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
PageManager& page_manager, TextureCache& texture_cache)
: m_graphics(graphics), m_scheduler(scheduler),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_texture_cache(texture_cache) {
: m_graphics(graphics), m_scheduler(scheduler),
m_fault_manager(graphics, scheduler, *this, CACHING_PAGEBITS, CACHING_NUMPAGES),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_bda_pagetable_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
BDA_PAGETABLE_SIZE),
m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_texture_cache(texture_cache) {
std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
m_gds_buffer.Flush(0, m_gds_buffer.Size());
SetVulkanObjectNameF(m_graphics.device, m_bda_pagetable_buffer.Handle(),
"BDA Page Table Buffer");
const auto null_id =
m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16);
EXIT_IF(null_id != NULL_BUFFER_ID);
@@ -316,8 +330,8 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
auto& command = m_scheduler.Current();
EXIT_IF(command.IsInvalid());
auto begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
auto end = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
auto begin = vaddr & ~(CACHING_PAGESIZE - 1);
auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
auto first = m_buffers.lower_bound(begin);
if (first != m_buffers.begin()) {
const auto previous = std::prev(first);
@@ -333,8 +347,9 @@ BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
end = std::max(end, buffer.CpuAddress() + buffer.Size());
}
const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags, end - begin);
const auto id = m_slot_buffers.insert(
m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin);
auto& buffer = m_slot_buffers[id];
SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
"Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
@@ -433,7 +448,7 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
if (!is_written && size <= CACHING_PAGE_SIZE &&
if (!is_written && size <= CACHING_PAGESIZE &&
!m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
const auto alignment = std::max<uint64_t>(
@@ -737,4 +752,24 @@ void BufferCache::RunGarbageCollector() {
}
}
void BufferCache::ProcessFaultBuffer() {
m_fault_manager.ProcessFaultBuffer();
}
void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {
const auto end = vaddr + size;
auto it = m_buffers.upper_bound(vaddr);
if (it != m_buffers.begin()) {
--it;
}
for (; it != m_buffers.end() && it->first < end; ++it) {
auto& buffer = m_slot_buffers[it->second];
const auto start = std::max(buffer.CpuAddress(), vaddr);
const auto finish = std::min(buffer.CpuAddress() + buffer.Size(), end);
if (start < finish) {
(void)SynchronizeBuffer(buffer, start, finish - start, false, false);
}
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+16 -3
View File
@@ -7,6 +7,7 @@
#include "common/slotVector.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/faultManager.h"
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
@@ -26,7 +27,11 @@ inline constexpr BufferId NULL_BUFFER_ID {0};
class BufferCache {
public:
static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
static constexpr uint32_t CACHING_PAGEBITS = 14;
static constexpr uint64_t CACHING_PAGESIZE = uint64_t {1} << CACHING_PAGEBITS;
static constexpr uint64_t CACHING_NUMPAGES = uint64_t {1} << (40 - CACHING_PAGEBITS);
static constexpr uint64_t BDA_PAGETABLE_SIZE =
CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
TextureCache& texture_cache);
@@ -51,6 +56,8 @@ public:
EXIT("BufferCache: invalid utility-buffer usage\n");
}
[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
[[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept { return &m_bda_pagetable_buffer; }
[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return m_fault_manager.GetFaultBuffer(); }
[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
@@ -60,14 +67,16 @@ public:
[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
void ProcessFaultBuffer();
void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
friend struct BufferCacheTestAccess;
struct DownloadCopy;
using PageTable = MultiLevelPageTable<BufferId, 14, 40, 16>;
static_assert(CACHING_PAGE_SIZE == (uint64_t {1} << PageTable::kPageBits));
using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
@@ -78,6 +87,8 @@ private:
[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
void Register(BufferId id);
void Unregister(BufferId id);
template <bool insert>
void ChangeRegister(BufferId id);
void DeleteBuffer(BufferId id);
[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
bool is_written, bool is_texel_buffer);
@@ -90,7 +101,9 @@ private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
FaultManager m_fault_manager;
Buffer m_gds_buffer;
Buffer m_bda_pagetable_buffer;
Common::SlotVector<Buffer> m_slot_buffers;
Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
std::map<uint64_t, BufferId> m_buffers;
src/graphics/host_gpu/renderer/cache/faultManager.cpp
Vendored
+167
View File
@@ -0,0 +1,167 @@
#include "graphics/host_gpu/renderer/cache/faultManager.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "gpu_tiler_shaders/fault_buffer_process_spv.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <bit>
#include <cinttypes>
#include <cstring>
#include <limits>
namespace Libs::Graphics {
namespace {
constexpr size_t MaxPageFaults = 1024;
constexpr size_t PageFaultAreaSize = MaxPageFaults * sizeof(uint64_t);
} // namespace
FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler,
BufferCache& buffer_cache, uint32_t caching_pagebits,
uint64_t caching_num_pages)
: m_graphics(graphics), m_scheduler(scheduler), m_buffer_cache(buffer_cache),
m_caching_pagesize(uint64_t {1} << caching_pagebits),
m_caching_num_pages(caching_num_pages), m_fault_buffer_size(caching_num_pages / 8),
m_fault_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
m_fault_buffer_size),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 0, AllFlags,
MaxPendingFaults * PageFaultAreaSize) {
SetVulkanObjectNameF(m_graphics.device, m_fault_buffer.Handle(), "Fault Buffer");
const vk::DescriptorSetLayoutBinding bindings[] {
{0, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
{1, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
};
vk::DescriptorSetLayoutCreateInfo layout_info {};
layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo;
layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
layout_info.bindingCount = std::size(bindings);
layout_info.pBindings = bindings;
RequireVulkanSuccess(
m_graphics.device.createDescriptorSetLayout(&layout_info, nullptr,
&m_fault_process_desc_layout),
"create fault-buffer descriptor layout");
vk::ShaderModuleCreateInfo module_info {};
module_info.sType = vk::StructureType::eShaderModuleCreateInfo;
module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
module_info.pCode = FAULT_BUFFER_PROCESS_SPV;
vk::ShaderModule module = nullptr;
RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
"create fault-buffer shader module");
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo;
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &m_fault_process_desc_layout;
RequireVulkanSuccess(
m_graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
&m_fault_process_pipeline_layout),
"create fault-buffer pipeline layout");
vk::PipelineShaderStageCreateInfo stage {};
stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
stage.stage = vk::ShaderStageFlagBits::eCompute;
stage.module = module;
stage.pName = "main";
vk::ComputePipelineCreateInfo pipeline_info {};
pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo;
pipeline_info.stage = stage;
pipeline_info.layout = m_fault_process_pipeline_layout;
const auto result = m_graphics.device.createComputePipelines(
nullptr, 1, &pipeline_info, nullptr, &m_fault_process_pipeline);
m_graphics.device.destroyShaderModule(module, nullptr);
RequireVulkanSuccess(result, "create fault-buffer pipeline");
SetVulkanObjectNameF(m_graphics.device, m_fault_process_pipeline, "Fault Buffer Parser");
}
FaultManager::~FaultManager() {
m_graphics.device.destroyPipeline(m_fault_process_pipeline, nullptr);
m_graphics.device.destroyPipelineLayout(m_fault_process_pipeline_layout, nullptr);
m_graphics.device.destroyDescriptorSetLayout(m_fault_process_desc_layout, nullptr);
}
void FaultManager::ProcessFaultBuffer() {
if (const auto wait_tick = m_fault_areas[m_current_area]; wait_tick != 0) {
m_scheduler.Wait(wait_tick);
m_scheduler.PopPendingOperations();
}
const auto offset = m_current_area * PageFaultAreaSize;
auto* mapped = m_download_buffer.Mapped().data() + offset;
std::memset(mapped, 0, PageFaultAreaSize);
m_download_buffer.Flush(offset, PageFaultAreaSize);
vk::BufferMemoryBarrier2 pre_barrier {};
pre_barrier.sType = vk::StructureType::eBufferMemoryBarrier2;
pre_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
pre_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
pre_barrier.dstStageMask = vk::PipelineStageFlagBits2::eComputeShader;
pre_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderRead;
pre_barrier.buffer = m_fault_buffer.Handle();
pre_barrier.offset = 0;
pre_barrier.size = m_fault_buffer_size;
auto post_barrier = pre_barrier;
post_barrier.srcStageMask = vk::PipelineStageFlagBits2::eComputeShader;
post_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
post_barrier.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands;
post_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderWrite;
const vk::DescriptorBufferInfo infos[] {
{m_fault_buffer.Handle(), 0, m_fault_buffer_size},
{m_download_buffer.Handle(), offset, PageFaultAreaSize},
};
std::array<vk::WriteDescriptorSet, 2> writes {};
for (uint32_t index = 0; index < writes.size(); ++index) {
writes[index].sType = vk::StructureType::eWriteDescriptorSet;
writes[index].dstBinding = index;
writes[index].descriptorCount = 1;
writes[index].descriptorType = vk::DescriptorType::eStorageBuffer;
writes[index].pBufferInfo = &infos[index];
}
m_scheduler.EndRendering();
auto command = m_scheduler.Current().Handle();
vk::DependencyInfo dependency {};
dependency.sType = vk::StructureType::eDependencyInfo;
dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion;
dependency.bufferMemoryBarrierCount = 1;
dependency.pBufferMemoryBarriers = &pre_barrier;
command.pipelineBarrier2(dependency);
command.bindPipeline(vk::PipelineBindPoint::eCompute, m_fault_process_pipeline);
command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute,
m_fault_process_pipeline_layout, 0, writes);
const auto num_threads = m_caching_num_pages / 32;
const auto num_workgroups = (num_threads + 63) / 64;
command.dispatch(static_cast<uint32_t>(num_workgroups), 1, 1);
dependency.pBufferMemoryBarriers = &post_barrier;
command.pipelineBarrier2(dependency);
const auto area = m_current_area;
m_scheduler.DeferOperation([this, mapped, offset, area] {
m_download_buffer.Invalidate(offset, PageFaultAreaSize);
m_fault_ranges.Clear();
const auto* faults = std::bit_cast<const uint64_t*>(mapped);
const auto count = static_cast<uint32_t>(faults[0]);
for (uint32_t index = 1; index <= count; ++index) {
m_fault_ranges.Add(faults[index], m_caching_pagesize);
LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
}
m_fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());
(void)m_buffer_cache.FindBuffer(start, end - start);
});
m_fault_areas[area] = 0;
});
m_fault_areas[m_current_area++] = m_scheduler.CurrentTick();
m_current_area %= MaxPendingFaults;
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/faultManager.h
Vendored
+46
View File
@@ -0,0 +1,46 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
#include "common/abi.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <array>
#include <cstdint>
namespace Libs::Graphics {
class BufferCache;
class FaultManager {
static constexpr size_t MaxPendingFaults = 8;
public:
FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, BufferCache& buffer_cache,
uint32_t caching_pagebits, uint64_t caching_num_pages);
~FaultManager();
KYTY_CLASS_NO_COPY(FaultManager);
[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return &m_fault_buffer; }
void ProcessFaultBuffer();
private:
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
BufferCache& m_buffer_cache;
RangeSet m_fault_ranges;
uint64_t m_caching_pagesize;
uint64_t m_caching_num_pages;
uint64_t m_fault_buffer_size;
Buffer m_fault_buffer;
Buffer m_download_buffer;
std::array<uint64_t, MaxPendingFaults> m_fault_areas {};
uint32_t m_current_area = 0;
vk::DescriptorSetLayout m_fault_process_desc_layout = nullptr;
vk::Pipeline m_fault_process_pipeline = nullptr;
vk::PipelineLayout m_fault_process_pipeline_layout = nullptr;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+12
View File
@@ -76,7 +76,19 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
m_gpu->SendCommandSync(unmap);
}
void GpuResourceManager::PrepareBda() {
std::shared_lock lock(m_mapped_ranges_mutex);
m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
});
m_fault_process_pending = true;
}
void GpuResourceManager::RunGarbageCollector() {
if (m_fault_process_pending) {
m_fault_process_pending = false;
m_buffer_cache.ProcessFaultBuffer();
}
m_texture_cache.ProcessDownloadImages();
m_texture_cache.RunGarbageCollector();
m_buffer_cache.RunGarbageCollector();
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+2
View File
@@ -30,6 +30,7 @@ public:
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
void PrepareBda();
void RunGarbageCollector();
private:
@@ -40,6 +41,7 @@ private:
mutable std::shared_mutex m_mapped_ranges_mutex;
RangeSet m_mapped_ranges;
GuestGpu* m_gpu = nullptr;
bool m_fault_process_pending = false;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/streamBuffer.cpp
Vendored
+26 -11
View File
@@ -73,8 +73,12 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
buffer_info.usage = flags;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
const VmaAllocationCreateFlags bda_flag =
with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
VmaAllocationCreateInfo allocation_info {};
allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
allocation_info.flags =
VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
allocation_info.usage = AllocationUsage(usage);
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
? VkMemoryPropertyFlags {}
@@ -99,6 +103,13 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
m_buffer->memory.type = allocation_result.memoryType;
m_buffer->memory.unique_id = VulkanNextMemoryUniqueId();
graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
vk::BufferDeviceAddressInfo address_info {};
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
address_info.buffer = m_buffer->buffer;
m_device_address = graphics.device.getBufferAddress(address_info);
EXIT_IF(m_device_address == 0);
}
VkMemoryPropertyFlags properties = 0;
vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
@@ -122,6 +133,11 @@ vk::Buffer Buffer::Handle() const noexcept {
return m_buffer->buffer;
}
vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
EXIT_IF(m_device_address == 0);
return m_device_address;
}
bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
}
@@ -141,6 +157,15 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
}
}
void Buffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
if (!m_is_coherent && size != 0) {
const auto result =
vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
}
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
vk::AccessFlags destination) const {
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
@@ -310,16 +335,6 @@ void StreamBuffer::Commit() {
watch.tick = tick;
}
void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset);
if (IsCoherent() || size == 0) {
return;
}
const auto result = vmaInvalidateAllocation(Graphics().allocator,
NativeBuffer().memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
EXIT_IF(source == nullptr);
const auto [data, offset] = Map(size, alignment);
src/graphics/host_gpu/renderer/cache/streamBuffer.h
Vendored
+3 -3
View File
@@ -48,12 +48,14 @@ public:
[[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; }
[[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; }
[[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; }
[[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept;
[[nodiscard]] uint64_t Offset(uint64_t address) const noexcept {
return address - m_cpu_address;
}
[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
void Write(uint64_t offset, const void* source, uint64_t size);
void Flush(uint64_t offset, uint64_t size);
void Invalidate(uint64_t offset, uint64_t size);
void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
uint64_t destination_offset, uint64_t size,
vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite,
@@ -84,6 +86,7 @@ private:
MemoryUsage m_usage = MemoryUsage::DeviceLocal;
uint64_t m_cpu_address = 0;
uint64_t m_size = 0;
vk::DeviceAddress m_device_address = 0;
std::unique_ptr<VulkanBuffer> m_buffer;
std::span<uint8_t> m_mapped;
bool m_is_coherent = false;
@@ -97,9 +100,6 @@ public:
[[nodiscard]] std::pair<uint8_t*, uint64_t> Map(uint64_t size, uint64_t alignment = 0,
bool allow_wait = true);
void Commit();
// Download mappings become visible to the CPU only after their GPU completion tick is free.
// Call this from the scheduler's deferred completion operation before reading Mapped().
void Invalidate(uint64_t offset, uint64_t size);
[[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0);
private:
src/graphics/host_gpu/renderer/commandScheduler.cpp
+1 -1
View File
@@ -196,8 +196,8 @@ void CommandScheduler::Finish() {
Submit();
}
m_master.Wait(CurrentTick() - 1);
PopPendingOperations();
BeginNext();
PopPendingOperations();
}
void CommandScheduler::Wait(uint64_t tick) {
src/graphics/host_gpu/renderer/pipeline/descriptors.cpp
+13 -78
View File
@@ -216,48 +216,6 @@ static BufferView NativeStorageBuffer(RenderContext&
return result;
}
static BufferView
NativeAddressBuffer(RenderContext& context, const ShaderRecompiler::IR::AddressResource& resource,
const ShaderRecompiler::IR::ResourceSnapshot::Address& address,
ShaderType stage, uint32_t slot, uint32_t& address_offset, BufferId id) {
BufferView result;
address_offset = 0;
if (address.binding_base == 0) {
BindNullStorageBuffer(context, result);
return result;
}
if (resource.written) {
EXIT("writable address resources are unsupported\n");
}
const auto limit =
resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
? ShaderRecompiler::IR::FlatAddressWindowSize
: static_cast<uint64_t>(
context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
uint64_t size = 0;
const auto access = HostMemoryAccess::Mapped;
if (!HostMemoryQueryRange(address.binding_base, limit, access, size)) {
EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
address.binding_base);
}
const auto& graphics = context.GetGraphics();
const auto alignment = graphics.StorageMinAlignment();
auto [buffer, offset] =
context.GetBufferCache().ObtainBuffer(address.binding_base, size, false, false, id);
const auto aligned_offset = offset - offset % alignment;
const auto adjustment = offset - aligned_offset;
const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
const auto visible_size = std::min(size, static_cast<uint64_t>(max_range - adjustment));
address_offset = static_cast<uint32_t>(adjustment);
result.buffer = buffer->Handle();
result.offset = aligned_offset;
result.range = static_cast<vk::DeviceSize>(visible_size + adjustment);
SetVulkanObjectNameF(graphics.device, result.buffer,
"Kyty.{}.AddressBuffer[slot={} guest=0x{:016x} size=0x{:x}]",
ShaderStageResourceName(stage), slot, address.binding_base, visible_size);
return result;
}
static bool IsSupportedSampledColorResource(const ShaderRecompiler::IR::ImageResource& resource) {
bool supported_dimension = false;
switch (resource.dimension) {
@@ -1000,25 +958,6 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
prepared.buffer_ids.push_back(cache.FindBuffer(address, size));
}
prepared.address_ids.clear();
prepared.address_ids.reserve(program.info.addresses.size());
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
const auto& resource = program.info.addresses[i];
const auto& address = snapshot.addresses[i];
if (address.binding_base == 0) {
prepared.address_ids.emplace_back();
continue;
}
EXIT_IF(resource.written);
const auto limit = resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
? ShaderRecompiler::IR::FlatAddressWindowSize
: static_cast<uint64_t>(m_context.GetGraphics()
.GetPhysicalDeviceProperties()
.limits.maxStorageBufferRange);
uint64_t size = 0;
EXIT_IF(!HostMemoryQueryRange(address.binding_base, limit, HostMemoryAccess::Mapped, size));
prepared.address_ids.push_back(cache.FindBuffer(address.binding_base, size));
}
}
void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
@@ -1028,8 +967,7 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
const auto& snapshot = *prepared.snapshot;
auto& resources = prepared.resources;
const auto& layout = program.bindings;
EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size() ||
prepared.address_ids.size() != program.info.addresses.size());
EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size());
resources.buffers.clear();
resources.buffers.reserve(program.info.buffers.size());
@@ -1049,15 +987,6 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
buffer_offset, prepared.buffer_ids[i]));
pack_memory_offset(i, buffer_offset);
}
resources.addresses.clear();
resources.addresses.reserve(program.info.addresses.size());
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
uint32_t address_offset = 0;
resources.addresses.push_back(NativeAddressBuffer(m_context, program.info.addresses[i],
snapshot.addresses[i], program.stage, i,
address_offset, prepared.address_ids[i]));
pack_memory_offset(static_cast<uint32_t>(program.info.buffers.size()) + i, address_offset);
}
if (!prepared.flattened_srt.empty()) {
resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt);
}
@@ -1129,6 +1058,10 @@ RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
if (bindings.pixel) {
FindBuffers(*bindings.pixel);
}
if (bindings.vertex.program->info.uses_dma ||
(bindings.pixel && bindings.pixel->program->info.uses_dma)) {
m_context.GetGpuResources().PrepareBda();
}
RebindBuffers(bindings.vertex);
if (bindings.pixel) {
RebindBuffers(*bindings.pixel);
@@ -1241,13 +1174,15 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
}
break;
case BindingKind::AddressMemory:
for (const auto resource: binding.resources) {
const auto& view = descriptors.addresses.at(resource);
EXIT_IF(view.buffer == nullptr);
m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
}
case BindingKind::BdaPagetable:
case BindingKind::FaultBuffer: {
auto& cache = m_context.GetBufferCache();
const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable
? cache.GetBdaPageTableBuffer()
: cache.GetFaultBuffer();
m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size());
break;
}
case BindingKind::FlattenedSrt:
case BindingKind::UserData:
case BindingKind::Gds: {
src/graphics/host_gpu/renderer/pipeline/descriptors.h
-2
View File
@@ -38,7 +38,6 @@ struct NativeDescriptors {
std::vector<BufferView> buffers;
std::vector<TextureBinding> images;
std::vector<vk::Sampler> samplers;
std::vector<BufferView> addresses;
BufferView gds;
BufferView flattened_srt;
BufferView user_data;
@@ -49,7 +48,6 @@ struct PreparedBindings {
const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
NativeDescriptors resources;
std::vector<BufferId> buffer_ids;
std::vector<BufferId> address_ids;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
bool committed = false;
src/graphics/host_gpu/renderer/renderCompute.cpp
+5 -2
View File
@@ -82,8 +82,8 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
const auto& resources = *input.stage.resources;
if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 ||
!program.info.images.empty() || !program.info.samplers.empty() ||
!program.info.addresses.empty() || !resources.images.empty() ||
!resources.samplers.empty() || !resources.addresses.empty()) {
program.info.uses_dma || !resources.images.empty() ||
!resources.samplers.empty()) {
return false;
}
const auto& resource = program.info.buffers.front();
@@ -337,6 +337,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader);
auto bindings = PrepareBindings(input_info.stage);
FindBuffers(bindings);
if (program.info.uses_dma) {
m_context.GetGpuResources().PrepareBda();
}
RebindBuffers(bindings);
RebindImages(bindings);
src/graphics/host_gpu/shaders/fault_buffer_process.comp
+35
View File
@@ -0,0 +1,35 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#version 450
#extension GL_ARB_gpu_shader_int64 : enable
layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
layout(std430, binding = 0) buffer input_buf {
uint fault_buffer[];
};
layout(std430, binding = 1) buffer output_buf {
uint64_t download_buffer[];
};
layout(std430, binding = 1) buffer output_buf32 {
uint download_buffer32[];
};
void main() {
const uint id = gl_GlobalInvocationID.x;
uint word = fault_buffer[id];
fault_buffer[id] = 0u;
const uint base_bit = id * 32u;
while (word != 0u) {
const uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u;
if (store_index >= MAX_PAGE_FAULTS) {
return;
}
const uint bit = findLSB(word);
word &= word - 1;
const uint page = base_bit + bit;
download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS;
}
}
src/graphics/host_gpu/vma.cpp
+4 -1
View File
@@ -72,7 +72,10 @@ bool GraphicContext::CreateAllocator() {
info.device = device;
info.pVulkanFunctions = &functions;
info.vulkanApiVersion = VULKAN_TARGET_API_VERSION;
info.flags = memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0;
info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
if (memory_budget_ext_enabled) {
info.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
}
const auto result = static_cast<vk::Result>(vmaCreateAllocator(&info, &allocator));
if (result != vk::Result::eSuccess) {