raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 19:07:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(renderer): simplify buffer and texture cache ownership

Browse Source
This commit is contained in:
nmzik
2026-08-21 20:11:38 +02:00
parent 49ea1680f8
commit 26c4168160
20 changed files with 1116 additions and 1411 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+242 -309
View File
@@ -28,60 +28,25 @@ constexpr uint64_t GdsBufferSize = 64 * 1024;
} // namespace
uint64_t BufferCache::AlignDown(uint64_t value) noexcept {
return value & ~(CACHING_PAGE_SIZE - 1);
}
uint64_t BufferCache::AlignUp(uint64_t value) {
if (value > UINT64_MAX - (CACHING_PAGE_SIZE - 1)) {
EXIT("BufferCache: address alignment overflow\n");
}
return (value + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
}
void BufferCache::Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
const void* source, uint64_t size) {
void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source,
uint64_t size) {
auto* bytes = static_cast<const uint8_t*>(source);
while (size != 0) {
const auto chunk = std::min(size, m_staging_buffer.Size());
const auto stage_offset = m_staging_buffer.Copy(bytes, chunk, 4);
destination.CopyFrom(command, m_staging_buffer, stage_offset, destination_offset, chunk,
vk::AccessFlagBits::eHostWrite);
const auto chunk = std::min(size, m_staging_buffer.Size());
const auto offset = m_staging_buffer.Copy(bytes, chunk, 4);
buffer.CopyFrom(m_scheduler.Current(), m_staging_buffer, offset, buffer.Offset(address),
chunk, vk::AccessFlagBits::eHostWrite);
bytes += chunk;
destination_offset += chunk;
address += chunk;
size -= chunk;
}
}
bool BufferCache::ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept {
if (merged.address == 0 || merged.size == 0 || candidate.address == 0 || candidate.size == 0 ||
merged.size > UINT64_MAX - merged.address ||
candidate.size > UINT64_MAX - candidate.address) {
EXIT("BufferCache: invalid overlap-merge range\n");
}
const auto merged_end = merged.address + merged.size;
const auto candidate_end = candidate.address + candidate.size;
if (merged.address >= candidate_end || candidate.address >= merged_end) {
return false;
}
const auto address = std::min(merged.address, candidate.address);
const auto end = std::max(merged_end, candidate_end);
merged = {.address = address, .size = end - address};
return true;
}
struct BufferCache::CachedBuffer {
uint64_t vaddr = 0;
uint64_t size = 0;
std::shared_ptr<Buffer> buffer;
uint64_t tick_accessed_last = 0;
};
struct BufferCache::DownloadCopy {
std::shared_ptr<Buffer> owner;
uint64_t source_offset = 0;
uint64_t address = 0;
uint64_t size = 0;
Buffer* buffer = nullptr;
uint64_t source_offset = 0;
uint64_t address = 0;
uint64_t size = 0;
};
struct BufferCache::DownloadRange {
@@ -90,15 +55,37 @@ struct BufferCache::DownloadRange {
uint64_t offset = 0;
};
struct BufferCache::RetiredBuffer {
uint64_t address = 0;
uint64_t size = 0;
std::shared_ptr<Buffer> owner;
};
void BufferCache::Unregister(BufferId id) {
auto& buffer = m_slot_buffers[id];
if (buffer.is_deleted) {
return;
}
const auto found = m_buffers.find(buffer.CpuAddress());
EXIT_IF(found == m_buffers.end() || found->second != id);
m_buffers.erase(found);
if (buffer.Size() > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
}
m_total_used_memory -= buffer.Size();
buffer.is_deleted = true;
}
void BufferCache::DeleteBuffer(BufferId id) {
auto* buffer = m_slot_buffers.try_get(id);
if (buffer == nullptr || buffer->is_deleted) {
return;
}
Unregister(id);
if (m_scheduler.Active()) {
m_scheduler.DeferOperation([this, id] { m_slot_buffers.erase(id); });
} else {
m_slot_buffers.erase(id);
}
}
std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() ||
copy.size > copy.owner->Size() - copy.source_offset) {
if (copy.buffer == nullptr || copy.size == 0 || copy.source_offset > copy.buffer->Size() ||
copy.size > copy.buffer->Size() - copy.source_offset) {
EXIT("BufferCache: invalid download copy\n");
}
const auto begin = copy.source_offset & ~uint64_t {3};
@@ -107,7 +94,7 @@ std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy&
EXIT("BufferCache: download copy alignment overflow\n");
}
const auto end = (copy.source_offset + copy.size + 3) & ~uint64_t {3};
if (end > copy.owner->Size()) {
if (end > copy.buffer->Size()) {
EXIT("BufferCache: aligned download copy exceeds its owner\n");
}
return {begin, end - begin};
@@ -144,7 +131,7 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
for (const auto& copy: copies) {
const auto [source_begin, envelope_size] = DownloadEnvelope(copy);
const auto prefix = copy.source_offset - source_begin;
download.CopyFrom(m_scheduler.Current(), *copy.owner, source_begin, base_offset + cursor,
download.CopyFrom(m_scheduler.Current(), *copy.buffer, source_begin, base_offset + cursor,
envelope_size, vk::AccessFlagBits::eMemoryWrite, vk::AccessFlags {},
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlagBits::eHostRead);
@@ -163,27 +150,29 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
}
}
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
if (copies.empty()) {
return;
}
void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, BufferId id,
uint64_t address, uint64_t size) {
EXIT_IF(copies.empty());
auto downloads = RecordDownloads(copies);
m_scheduler.DeferOperation(
[this, downloads = std::move(downloads), retire = std::move(retire)]() mutable {
PublishDownloads(downloads);
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
// ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
// interval on those pages was downloaded and removed. Clearing the original
// query therefore cannot orphan a dirty sibling on an edge page.
m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size);
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
}
m_memory_tracker.UntrackMemory(retire.address, retire.size);
});
auto retired = downloads;
m_scheduler.DeferPriorityOperation(
[this, downloads = std::move(downloads)]() mutable { PublishDownloads(downloads); });
m_scheduler.DeferOperation([this, retired = std::move(retired), id, address, size]() mutable {
for (const auto& range: retired) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
// ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
// interval on those pages was downloaded and removed. Clearing the original
// query therefore cannot orphan a dirty sibling on an edge page.
m_memory_tracker.UnmarkRegionAsGpuModified(address, size);
if (m_memory_tracker.IsRegionGpuModified(address, size) ||
!m_gpu_modified_ranges.Intersections(address, size).empty()) {
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
}
m_memory_tracker.UntrackMemory(address, size);
EXIT_IF(!m_slot_buffers[id].is_deleted);
m_slot_buffers.erase(id);
});
}
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -198,6 +187,10 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
m_texture_cache(texture_cache) {
std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
m_gds_buffer.Flush(0, m_gds_buffer.Size());
const auto null_id =
m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16);
EXIT_IF(null_id != NULL_BUFFER_ID);
SetVulkanObjectNameF(m_graphics.device, GetBuffer(null_id).Handle(), "Kyty.NullBuffer");
if (!m_graphics.CanReportMemoryUsage()) {
return;
}
@@ -216,38 +209,16 @@ BufferCache::~BufferCache() {
if (!m_gpu_modified_ranges.Empty()) {
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
}
for (const auto& [vaddr, cached]: m_buffers) {
for (const auto& [vaddr, id]: m_buffers) {
(void)vaddr;
if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) {
const auto& buffer = m_slot_buffers[id];
if (m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size())) {
EXIT("BufferCache: destroyed with GPU-modified buffer\n");
}
}
m_buffers.clear();
}
StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept {
switch (usage) {
case MemoryUsage::Upload: return m_staging_buffer;
case MemoryUsage::Stream: return m_stream_buffer;
case MemoryUsage::Download: return m_download_buffer;
case MemoryUsage::DeviceLocal: return m_device_buffer;
}
EXIT("BufferCache: invalid utility-buffer usage\n");
}
BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint64_t alignment) {
EXIT_IF(data == nullptr || size == 0);
if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false); mapped != nullptr) {
std::memcpy(mapped, data, static_cast<size_t>(size));
m_stream_buffer.Commit();
return {{}, m_stream_buffer.Handle(), offset};
}
auto owner =
std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0, AllFlags, size);
owner->Write(0, data, size);
return {owner, owner->Handle(), 0};
}
void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
@@ -288,14 +259,14 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
if (owner == m_buffers.begin()) {
EXIT("BufferCache: invalidation readback has no buffer owner\n");
}
auto& cached = *std::prev(owner)->second;
if (!cached.buffer->IsInBounds(copy_address, 1)) {
auto& buffer = m_slot_buffers[std::prev(owner)->second];
if (!buffer.IsInBounds(copy_address, 1)) {
EXIT("BufferCache: invalidation readback is outside its buffer owner\n");
}
const auto copy_size =
std::min(range.size - copied, cached.vaddr + cached.size - copy_address);
copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
copy_address, copy_size});
const auto copy_size = std::min(
range.size - copied, buffer.CpuAddress() + buffer.Size() - copy_address);
copies.push_back(
{&buffer, buffer.Offset(copy_address), copy_address, copy_size});
copied += copy_size;
}
}
@@ -330,38 +301,39 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
std::vector<DownloadCopy> copies;
std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
for (const auto& [begin, cached]: m_buffers) {
if (vaddr < begin + cached->size && begin < vaddr + size) {
retired_buffers.emplace_back(begin, cached->size);
for (const auto& [begin, id]: m_buffers) {
const auto& buffer = m_slot_buffers[id];
if (vaddr < begin + buffer.Size() && begin < vaddr + size) {
retired_buffers.emplace_back(begin, buffer.Size());
}
}
for (const auto& [begin, cached]: m_buffers) {
if (vaddr >= begin + cached->size || begin >= vaddr + size ||
!m_memory_tracker.IsRegionGpuModified(begin, cached->size)) {
for (const auto& [begin, id]: m_buffers) {
const auto& buffer = m_slot_buffers[id];
if (vaddr >= begin + buffer.Size() || begin >= vaddr + size ||
!m_memory_tracker.IsRegionGpuModified(begin, buffer.Size())) {
continue;
}
const auto dirty = m_gpu_modified_ranges.Intersections(begin, cached->size);
const auto dirty = m_gpu_modified_ranges.Intersections(begin, buffer.Size());
if (dirty.empty()) {
EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n");
}
modified_buffers.emplace_back(begin, cached->size);
modified_buffers.emplace_back(begin, buffer.Size());
}
for (const auto& [begin, bytes]: modified_buffers) {
auto owner = m_buffers.find(begin);
if (owner == m_buffers.end() || owner->second->size != bytes) {
if (owner == m_buffers.end() || m_slot_buffers[owner->second].Size() != bytes) {
EXIT("BufferCache: unmap owner changed during collection\n");
}
auto& cached = *owner->second;
auto& buffer = m_slot_buffers[owner->second];
m_memory_tracker.ForEachDownloadRange<false>(
begin, cached.size,
begin, buffer.Size(),
[&](uint64_t address, uint64_t bytes) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
"unmap");
},
[&](uint64_t address, uint64_t bytes) noexcept {
for (const auto& range: m_gpu_modified_ranges.Intersections(address, bytes)) {
copies.push_back(
{cached.buffer, range.address - begin, range.address, range.size});
copies.push_back({&buffer, range.address - begin, range.address, range.size});
}
});
}
@@ -385,86 +357,92 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
EXIT("BufferCache: unmap retained dirty byte ranges\n");
}
m_memory_tracker.UntrackMemory(vaddr, size);
for (auto it = m_buffers.begin(); it != m_buffers.end();) {
if (vaddr < it->first + it->second->size && it->first < vaddr + size) {
if (it->second->size > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
}
m_total_used_memory -= it->second->size;
it = m_buffers.erase(it);
} else {
++it;
for (const auto& [begin, bytes]: retired_buffers) {
(void)bytes;
const auto found = m_buffers.find(begin);
if (found != m_buffers.end()) {
DeleteBuffer(found->second);
}
}
}
BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
uint64_t size) {
const auto begin = AlignDown(vaddr);
const auto end = AlignUp(vaddr + size);
BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
if (vaddr == 0) {
return NULL_BUFFER_ID;
}
if (size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid buffer discovery request\n");
}
auto& command = m_scheduler.Current();
EXIT_IF(command.IsInvalid() || command.IsExecute());
auto begin = vaddr & ~(CACHING_PAGE_SIZE - 1);
auto end = (vaddr + size + CACHING_PAGE_SIZE - 1) & ~(CACHING_PAGE_SIZE - 1);
const auto next = m_buffers.upper_bound(vaddr);
if (next != m_buffers.begin()) {
auto owner = std::prev(next);
if (owner->second->buffer->IsInBounds(vaddr, size)) {
owner->second->tick_accessed_last = m_gc_tick;
return *owner->second;
const auto owner = std::prev(next);
auto& buffer = m_slot_buffers[owner->second];
if (buffer.IsInBounds(vaddr, size)) {
buffer.tick_accessed_last = m_gc_tick;
return owner->second;
}
}
CacheRange merged {.address = begin, .size = end - begin};
using Iterator = decltype(m_buffers.begin());
std::vector<Iterator> overlaps;
auto merge = [&](const Buffer& buffer) {
const auto buffer_begin = buffer.CpuAddress();
const auto buffer_end = buffer_begin + buffer.Size();
if (begin >= buffer_end || buffer_begin >= end) {
return false;
}
begin = std::min(begin, buffer_begin);
end = std::max(end, buffer_end);
return true;
};
std::vector<BufferId> overlaps;
auto first = m_buffers.lower_bound(begin);
if (first != m_buffers.begin()) {
auto previous = std::prev(first);
if (ResolveOverlap(merged, {previous->second->vaddr, previous->second->size})) {
const auto previous = std::prev(first);
if (merge(m_slot_buffers[previous->second])) {
first = previous;
}
}
for (auto candidate = first; candidate != m_buffers.end(); ++candidate) {
if (candidate->first >= merged.address + merged.size) {
if (candidate->first >= end) {
break;
}
if (ResolveOverlap(merged, {candidate->second->vaddr, candidate->second->size})) {
overlaps.push_back(candidate);
if (merge(m_slot_buffers[candidate->second])) {
overlaps.push_back(candidate->second);
}
}
auto cached = std::make_unique<CachedBuffer>();
cached->vaddr = merged.address;
cached->size = merged.size;
cached->tick_accessed_last = m_gc_tick;
cached->buffer = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal,
merged.address, AllFlags, merged.size);
SetVulkanObjectNameF(m_graphics.device, cached->buffer->Handle(),
"Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", cached->vaddr,
cached->size);
const auto id = m_slot_buffers.insert(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
AllFlags, end - begin);
auto& buffer = m_slot_buffers[id];
buffer.tick_accessed_last = m_gc_tick;
SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
"Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
for (const auto overlap: overlaps) {
const auto& old = *overlap->second;
cached->buffer->CopyFrom(command, *old.buffer, 0, old.vaddr - cached->vaddr, old.size);
command.RetainResourceUntilFence(old.buffer);
const auto& old = m_slot_buffers[overlap];
buffer.CopyFrom(command, old, 0, old.CpuAddress() - begin, old.Size());
}
for (const auto overlap: overlaps) {
if (overlap->second->size > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
}
m_total_used_memory -= overlap->second->size;
m_buffers.erase(overlap);
DeleteBuffer(overlap);
}
m_total_used_memory += cached->size;
return *m_buffers.emplace(cached->vaddr, std::move(cached)).first->second;
m_total_used_memory += buffer.Size();
const auto [it, inserted] = m_buffers.emplace(begin, id);
(void)it;
EXIT_IF(!inserted);
return id;
}
bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
uint64_t size, bool is_written, bool is_texel_buffer) {
bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, bool is_written,
bool is_texel_buffer) {
std::vector<std::pair<uint64_t, uint64_t>> uploads;
m_memory_tracker.ForEachUploadRange(
vaddr, size, is_written,
[&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); },
[&]() noexcept {
for (const auto& [address, bytes]: uploads) {
Upload(command, buffer, buffer.Offset(address),
reinterpret_cast<const void*>(address), bytes);
WriteDataBuffer(buffer, address, reinterpret_cast<const void*>(address), bytes);
}
});
if (is_texel_buffer && !is_written) {
@@ -473,71 +451,51 @@ bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint
return false;
}
BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written, bool is_read, bool is_formatted) {
if (command.IsInvalid() || command.IsExecute()) {
std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t size,
bool is_written, bool is_texel_buffer,
BufferId id) {
auto& command = m_scheduler.Current();
if (command.IsInvalid() || command.IsExecute() || vaddr == 0 || size == 0 ||
vaddr >= TRACKER_ADDRESS_SIZE || size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
if (is_read && !is_written && size <= CACHING_PAGE_SIZE &&
if (!is_written && size <= CACHING_PAGE_SIZE &&
!m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
const auto alignment = std::max<uint64_t>(
m_graphics.physical_device_properties.limits.minUniformBufferOffsetAlignment, 1);
if (auto [mapped, offset] = m_stream_buffer.Map(size, alignment, false);
mapped != nullptr) {
if (Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size)) {
m_stream_buffer.Commit();
return {{}, m_stream_buffer.Handle(), offset};
}
} else {
auto owner = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::Upload, 0,
AllFlags, size);
if (Libs::LibKernel::Memory::TryReadBacking(vaddr, owner->Mapped().data(), size)) {
owner->Flush(0, size);
return {owner, owner->Handle(), 0};
}
}
auto [mapped, offset] = m_stream_buffer.Map(size, alignment);
EXIT_IF(mapped == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, mapped, size));
m_stream_buffer.Commit();
return {&m_stream_buffer, offset};
}
if (is_formatted && is_written) {
(void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true);
auto* buffer = m_slot_buffers.try_get(id);
if (buffer == nullptr || buffer->is_deleted || !buffer->IsInBounds(vaddr, size)) {
id = FindBuffer(vaddr, size);
buffer = &m_slot_buffers[id];
}
auto& cached = GetOrCreateBuffer(command, vaddr, size);
(void)SynchronizeBuffer(command, *cached.buffer, vaddr, size, is_written,
is_formatted && is_read);
(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
if (is_written) {
m_gpu_modified_ranges.Add(vaddr, size);
}
auto owner = cached.buffer;
uint64_t offset = cached.buffer->Offset(vaddr);
return {owner, owner->Handle(), offset};
return {buffer, buffer->Offset(vaddr)};
}
std::shared_ptr<Buffer> BufferCache::ObtainNullBuffer() {
if (m_null_buffer != nullptr) {
return m_null_buffer;
}
m_null_buffer = std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal, 0,
AllFlags, 16);
const std::array<uint8_t, 16> zeros {};
Upload(m_scheduler.Current(), *m_null_buffer, 0, zeros.data(), zeros.size());
return m_null_buffer;
}
ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid image source\n");
}
auto find_owner = [&]() -> CachedBuffer* {
auto find_owner = [&]() -> Buffer* {
auto owner = m_buffers.upper_bound(vaddr);
if (owner == m_buffers.begin()) {
return nullptr;
}
--owner;
return owner->second->buffer->IsInBounds(vaddr, size) ? owner->second.get() : nullptr;
auto* buffer = m_slot_buffers.try_get(owner->second);
return buffer != nullptr && buffer->IsInBounds(vaddr, size) ? buffer : nullptr;
};
{
@@ -553,17 +511,18 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
if (!IsRegionRegistered(vaddr, size)) {
EXIT("BufferCache: GPU-dirty image source has no native buffer\n");
}
auto& cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size);
owner = &cached;
const auto resolved = m_buffers.find(cached.vaddr);
if (resolved == m_buffers.end() || resolved->second.get() != &cached ||
!cached.buffer->IsInBounds(vaddr, size)) {
const auto id = FindBuffer(vaddr, size);
auto& buffer = m_slot_buffers[id];
owner = &buffer;
const auto resolved = m_buffers.find(buffer.CpuAddress());
if (resolved == m_buffers.end() || resolved->second != id ||
!buffer.IsInBounds(vaddr, size)) {
EXIT("BufferCache: merged image source does not contain the requested range\n");
}
}
if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) {
owner->tick_accessed_last = m_gc_tick;
return {owner->buffer.get(), owner->buffer->Offset(vaddr)};
return {owner, owner->Offset(vaddr)};
}
if (has_dirty_buffer_source && owner == nullptr) {
EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
@@ -588,8 +547,7 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
return {&m_staging_buffer, stage_offset};
}
auto& cached = *owner;
cached.tick_accessed_last = m_gc_tick;
owner->tick_accessed_last = m_gc_tick;
std::vector<std::pair<uint64_t, uint64_t>> uploads;
m_memory_tracker.ForEachUploadRange(
vaddr, size, false,
@@ -598,12 +556,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
},
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
owner->CopyFrom(m_scheduler.Current(), m_staging_buffer,
stage_offset + address - vaddr, owner->Offset(address), upload_size,
vk::AccessFlagBits::eHostWrite);
}
});
return {cached.buffer.get(), cached.buffer->Offset(vaddr)};
return {owner, owner->Offset(vaddr)};
}
void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data) {
@@ -613,16 +571,16 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data)
Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size());
const auto end = vaddr + data.size();
for (auto& [address, cached]: m_buffers) {
const auto cached_end = address + cached->size;
for (const auto& [address, id]: m_buffers) {
auto& buffer = m_slot_buffers[id];
const auto buffer_end = address + buffer.Size();
const auto begin = std::max(vaddr, address);
const auto range_end = std::min(end, cached_end);
const auto range_end = std::min(end, buffer_end);
if (begin >= range_end) {
continue;
}
Upload(m_scheduler.Current(), *cached->buffer, cached->buffer->Offset(begin),
data.data() + begin - vaddr, range_end - begin);
cached->tick_accessed_last = m_gc_tick;
WriteDataBuffer(buffer, begin, data.data() + begin - vaddr, range_end - begin);
buffer.tick_accessed_last = m_gc_tick;
}
}
@@ -660,12 +618,11 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
}
}
auto& command = m_scheduler.Current();
auto dst = ObtainBuffer(command, vaddr, size, true, false, true);
EXIT_IF(dst.buffer == nullptr || dst.owner == nullptr);
command.RetainResourceUntilFence(dst.owner);
auto owner = std::static_pointer_cast<Buffer>(dst.owner);
owner->Fill(dst.offset, size, value);
m_texture_cache.InvalidateMemoryFromGPU(vaddr, size);
const auto id = FindBuffer(vaddr, size);
auto [dst, dst_offset] = ObtainBuffer(vaddr, size, true, true, id);
EXIT_IF(dst == nullptr);
dst->Fill(dst_offset, size, value);
}
void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
@@ -707,30 +664,20 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
}
auto& command = m_scheduler.Current();
auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true)
: BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr};
auto dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true)
: BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr};
EXIT_IF(src.buffer == nullptr || dst.buffer == nullptr || (dst_memory && dst.owner == nullptr));
if (src.owner != nullptr) {
command.RetainResourceUntilFence(src.owner);
if (dst_memory) {
m_texture_cache.InvalidateMemoryFromGPU(dst_vaddr, size);
}
if (dst.owner != nullptr) {
command.RetainResourceUntilFence(dst.owner);
}
if (src.buffer == dst.buffer && src.offset < dst.offset + size &&
dst.offset < src.offset + size) {
const auto src_id = src_memory ? FindBuffer(src_vaddr, size) : BufferId {};
const auto dst_id = dst_memory ? FindBuffer(dst_vaddr, size) : BufferId {};
auto [src, src_offset] = src_memory ? ObtainBuffer(src_vaddr, size, false, true, src_id)
: std::pair {&m_gds_buffer, src_vaddr};
auto [dst, dst_offset] = dst_memory ? ObtainBuffer(dst_vaddr, size, true, true, dst_id)
: std::pair {&m_gds_buffer, dst_vaddr};
EXIT_IF(src == nullptr || dst == nullptr);
if (src == dst && src_offset < dst_offset + size && dst_offset < src_offset + size) {
EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
}
auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
: src_gds ? m_gds_buffer
: m_stream_buffer;
auto& destination =
dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
}
destination.CopyFrom(command, source, src.offset, dst.offset, size);
dst->CopyFrom(command, *src, src_offset, dst_offset, size);
}
bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) {
@@ -744,8 +691,8 @@ bool BufferCache::IsRegionRegistered(uint64_t vaddr, uint64_t size) {
if (candidate == m_buffers.begin()) {
return false;
}
const auto& [address, cached] = *std::prev(candidate);
return address + cached->size > vaddr;
const auto& [address, id] = *std::prev(candidate);
return address + m_slot_buffers[id].Size() > vaddr;
}
bool BufferCache::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
@@ -773,70 +720,56 @@ void BufferCache::RunGarbageCollector() {
const uint64_t age = std::min<uint64_t>(aggressive ? 80 : 160, tick);
const size_t limit = aggressive ? 64 : 32;
std::vector<RetiredBuffer> retires;
std::vector<std::pair<RetiredBuffer, std::vector<DownloadCopy>>> dirty_retires;
{
std::vector<uint64_t> candidates;
for (const auto& [address, owner]: m_buffers) {
const auto& cached = *owner;
if (tick - std::min(tick, cached.tick_accessed_last) < age) {
continue;
}
std::vector<uint64_t> candidates;
for (const auto& [address, id]: m_buffers) {
const auto& buffer = m_slot_buffers[id];
if (tick - std::min(tick, buffer.tick_accessed_last) >= age) {
candidates.push_back(address);
}
std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) {
return m_buffers.at(left)->tick_accessed_last < m_buffers.at(right)->tick_accessed_last;
});
if (candidates.size() > limit) {
candidates.resize(limit);
}
for (const auto address: candidates) {
auto& cached = *m_buffers.at(address);
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
cached.size, "garbage collection");
retires.push_back({address, cached.size, cached.buffer});
// GC runs immediately before submission. Preserve every source referenced by commands
// already recorded in the active batch.
m_scheduler.Current().RetainResourceUntilFence(cached.buffer);
}
for (const auto& retire: retires) {
if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
continue;
}
auto& copies = dirty_retires.emplace_back(retire, std::vector<DownloadCopy> {}).second;
}
std::ranges::sort(candidates, [&](uint64_t left, uint64_t right) {
return m_slot_buffers[m_buffers.at(left)].tick_accessed_last <
m_slot_buffers[m_buffers.at(right)].tick_accessed_last;
});
if (candidates.size() > limit) {
candidates.resize(limit);
}
for (const auto address: candidates) {
auto found = m_buffers.find(address);
EXIT_IF(found == m_buffers.end());
const auto id = found->second;
auto& buffer = m_slot_buffers[id];
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, buffer.CpuAddress(),
buffer.Size(), "garbage collection");
const bool dirty = m_memory_tracker.IsRegionGpuModified(buffer.CpuAddress(), buffer.Size());
std::vector<DownloadCopy> copies;
if (dirty) {
m_memory_tracker.ForEachDownloadRange<false>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
"garbage collection");
buffer.CpuAddress(), buffer.Size(),
[&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address,
dirty_size, "garbage collection");
},
[&](uint64_t address, uint64_t size) noexcept {
for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
copies.push_back({retire.owner, range.address - retire.address,
[&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
for (const auto range:
m_gpu_modified_ranges.Intersections(dirty_address, dirty_size)) {
copies.push_back({&buffer, range.address - buffer.CpuAddress(),
range.address, range.size});
}
});
}
}
for (auto& [retire, copies]: dirty_retires) {
QueueGarbageDownload(copies, std::move(retire));
}
for (const auto& retire: retires) {
auto found = m_buffers.find(retire.address);
if (found == m_buffers.end() || found->second->size != retire.size ||
found->second->buffer != retire.owner) {
EXIT("BufferCache: garbage-collection owner changed during download\n");
const auto retire_address = buffer.CpuAddress();
const auto retire_size = buffer.Size();
if (dirty) {
EXIT_IF(copies.empty());
Unregister(id);
QueueGarbageDownload(copies, id, retire_address, retire_size);
} else {
m_memory_tracker.UntrackMemory(retire_address, retire_size);
DeleteBuffer(id);
}
if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
m_memory_tracker.UntrackMemory(retire.address, retire.size);
}
if (retire.size > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
}
m_total_used_memory -= retire.size;
m_buffers.erase(found);
}
}
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+49 -63
View File
@@ -3,12 +3,12 @@
#include "common/abi.h"
#include "common/common.h"
#include "common/slotVector.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <map>
#include <memory>
#include <span>
#include <utility>
#include <vector>
@@ -16,49 +16,44 @@
namespace Libs::Graphics {
struct GraphicContext;
class CommandBuffer;
class CommandScheduler;
class TextureCache;
struct BufferBinding {
std::shared_ptr<void> owner;
vk::Buffer buffer = nullptr;
uint64_t offset = 0;
};
struct ImageBufferSource {
Buffer* buffer = nullptr;
uint64_t offset = 0;
};
using BufferId = Common::SlotId;
inline constexpr BufferId NULL_BUFFER_ID {0};
class BufferCache {
public:
static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
static constexpr uint64_t GetBufferOffset(uint64_t vaddr) {
return vaddr & (CACHING_PAGE_SIZE - 1);
}
BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
TextureCache& texture_cache);
~BufferCache();
KYTY_CLASS_NO_COPY(BufferCache);
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written = false, bool is_read = true,
bool is_formatted = false);
[[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept;
[[nodiscard]] Buffer& GetGdsBuffer() noexcept { return m_gds_buffer; }
[[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; }
[[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size,
uint64_t alignment);
[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
[[nodiscard]] ImageBufferSource ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
bool src_gds = false);
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] Buffer& GetBuffer(BufferId id) { return m_slot_buffers[id]; }
[[nodiscard]] BufferId FindBuffer(uint64_t vaddr, uint64_t size);
[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBuffer(uint64_t vaddr, uint64_t size,
bool is_written,
bool is_texel_buffer = false,
BufferId id = {});
[[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept {
switch (usage) {
case MemoryUsage::Upload: return m_staging_buffer;
case MemoryUsage::Stream: return m_stream_buffer;
case MemoryUsage::Download: return m_download_buffer;
case MemoryUsage::DeviceLocal: return m_device_buffer;
}
EXIT("BufferCache: invalid utility-buffer usage\n");
}
[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
bool src_gds);
// Cache-index and exact dirty-range queries require GPU-thread serialization.
[[nodiscard]] bool IsRegionRegistered(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
@@ -69,51 +64,42 @@ public:
private:
friend struct BufferCacheTestAccess;
struct CacheRange {
uint64_t address = 0;
uint64_t size = 0;
};
struct CachedBuffer;
struct DownloadCopy;
struct DownloadRange;
struct RetiredBuffer;
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
}
[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
const void* source, uint64_t size);
[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
uint64_t size);
[[nodiscard]] bool SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
uint64_t size, bool is_written, bool is_texel_buffer);
void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
void Unregister(BufferId id);
void DeleteBuffer(BufferId id);
[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
bool is_written, bool is_texel_buffer);
[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, BufferId id, uint64_t address,
uint64_t size);
void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
Buffer m_gds_buffer;
std::shared_ptr<Buffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
RangeSet m_gpu_modified_ranges;
MemoryTracker m_memory_tracker;
StreamBuffer m_staging_buffer;
StreamBuffer m_stream_buffer;
StreamBuffer m_download_buffer;
StreamBuffer m_device_buffer;
TextureCache& m_texture_cache;
uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 1ull * 1024 * 1024 * 1024;
uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0;
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
Buffer m_gds_buffer;
Common::SlotVector<Buffer> m_slot_buffers;
std::map<uint64_t, BufferId> m_buffers;
RangeSet m_gpu_modified_ranges;
MemoryTracker m_memory_tracker;
StreamBuffer m_staging_buffer;
StreamBuffer m_stream_buffer;
StreamBuffer m_download_buffer;
StreamBuffer m_device_buffer;
TextureCache& m_texture_cache;
uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 1ull * 1024 * 1024 * 1024;
uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/streamBuffer.h
Vendored
+4
View File
@@ -65,6 +65,10 @@ public:
vk::AccessFlagBits::eMemoryWrite);
void Fill(uint64_t offset, uint64_t size, uint32_t value);
// BufferCache state lives directly on the resource.
bool is_deleted = false;
uint64_t tick_accessed_last = 0;
protected:
[[nodiscard]] GraphicContext& Graphics() const noexcept { return *m_graphics; }
[[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; }
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
+188 -355
View File
File diff suppressed because it is too large Load Diff
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
+33 -50
View File
@@ -4,17 +4,17 @@
#include "common/abi.h"
#include "common/common.h"
#include "common/lruCache.h"
#include "common/slotVector.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/regionManager.h"
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/image/tiler.h"
#include <compare>
#include <map>
#include <memory>
#include <set>
#include <utility>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace Libs::Graphics {
@@ -25,8 +25,6 @@ class BufferCache;
class CommandBuffer;
class CommandScheduler;
class RenderExecutor;
class StreamBuffer;
class TileManager;
struct TextureCacheTestAccess;
class TextureCache {
@@ -57,15 +55,17 @@ public:
[[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc);
[[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc);
[[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc);
[[nodiscard]] Image& GetImage(ImageId id);
[[nodiscard]] const Image& GetImage(ImageId id) const;
void MarkGpuWritten(ImageId id);
[[nodiscard]] Image& GetImage(ImageId id) {
auto& image = m_slot_images[id];
TouchImage(image);
return image;
}
void MarkGpuWritten(ImageId id);
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
uint32_t packed_clear);
void InvalidateMemory(uint64_t address, uint64_t size);
[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
bool formatted_buffer_write = false);
void InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
[[nodiscard]] bool IsMeta(uint64_t address);
@@ -86,16 +86,11 @@ private:
struct ColorTransferPlan;
struct DownloadPlan;
struct Slot {
std::shared_ptr<Image> image;
uint32_t generation = 1;
};
struct MetaDataInfo {
// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
// PendingDcc retains that exact fill until FindRenderTarget classifies the address,
// without exposing an unconfirmed buffer address to the normal metadata heuristics.
// Keep all surface metadata in this one shadPS4-style entry so CMask/FMask can be
// Keep all surface metadata in one entry so CMask/FMask can be
// registered beside HTile and DCC without introducing parallel tracking paths.
enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc };
@@ -114,26 +109,21 @@ private:
using ImageIds = InlinePageOwnerList<ImageId, 16>;
using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
[[nodiscard]] Image& ResolveImage(ImageId id);
[[nodiscard]] const Image& ResolveImage(ImageId id) const;
[[nodiscard]] std::shared_ptr<Image> ResolveOwner(ImageId id) const;
[[nodiscard]] ImageId InsertImage(const ImageInfo& info);
[[nodiscard]] ImageId GetNullImage(const ImageDesc& desc);
void RegisterImage(ImageId id);
void UnregisterImage(ImageId id);
void DeleteImage(ImageId id);
void FreeImage(ImageId id);
void RetainImage(CommandBuffer& command, ImageId id);
void TouchImage(Image& image);
void TrackImage(ImageId id);
void TrackImageHead(ImageId id);
void TrackImageTail(ImageId id);
void UntrackImage(ImageId id);
void UntrackImageHead(ImageId id);
void UntrackImageTail(ImageId id);
void MarkAsMaybeDirty(ImageId id, Image& image);
void TrackImageDownload(ImageId id);
void TrackImageDownloadLocked(ImageId id, Image& image);
[[nodiscard]] ImageId InsertImage(const ImageInfo& info);
[[nodiscard]] ImageId GetNullImage(const ImageDesc& desc);
void RegisterImage(ImageId id);
void UnregisterImage(ImageId id);
void DeleteImage(ImageId id);
void FreeImage(ImageId id);
void TouchImage(Image& image);
void TrackImage(ImageId id);
void TrackImageHead(ImageId id);
void TrackImageTail(ImageId id);
void UntrackImage(ImageId id);
void UntrackImageHead(ImageId id);
void UntrackImageTail(ImageId id);
void MarkAsMaybeDirty(ImageId id, Image& image);
void TrackImageDownload(ImageId id, Image& image);
[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
bool exact_format);
[[nodiscard]] static BindingType UploadBinding(const Image& image);
@@ -162,31 +152,24 @@ private:
[[nodiscard]] bool CopyD16(Image& destination, Image& source);
void CopyImage(ImageId destination, ImageId source);
void AssociateStencil(ImageId depth, GuestRange stencil);
void AssociateStencilLocked(ImageId depth, GuestRange stencil);
void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer);
void ValidateImageDesc(const ImageDesc& desc) const;
void InvalidateCpuAliases(uint64_t address, uint64_t size);
void ClearGpuModified(ImageId id);
void DownloadImage(ImageId id);
[[nodiscard]] bool TryDownloadImage(ImageId id);
[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset);
void InvalidateCpuAliases(uint64_t address, uint64_t size);
[[nodiscard]] bool TryDownloadImage(ImageId id);
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
TrackingSpinLock m_lock;
PageManager& m_page_manager;
BlitHelper m_blit_helper;
std::unique_ptr<TileManager> m_tiler;
TileManager m_tiler;
BufferCache& m_buffer_cache;
std::vector<Slot> m_slots;
std::vector<uint32_t> m_free_slots;
Common::SlotVector<Image> m_slot_images;
ImagePageTable m_image_page_table;
std::map<vk::Format, ImageId> m_null_images;
std::unordered_map<vk::Format, ImageId> m_null_images;
Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
std::set<ImageId> m_download_images;
std::unordered_set<ImageId> m_download_images;
std::map<uint64_t, MetaDataInfo> m_surface_metas;
uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 0;