mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-21 19:36:55 +02:00
graphics: account for device memory the way VMA sees it
The caches decided when to collect from the driver's process-wide usage estimate, which varied by about 4x between reads of the same moment on this setup and is not what the allocator is holding. Read VMA's own accounting instead, and let the collector stand down on the same figure it stood up on rather than a stale tally. Adds a small deferral guard so a completion callback cannot re-enter the resource caches while the caller still holds their state.
This commit is contained in:
Mac
@@ -0,0 +1,24 @@
|
||||
#include "graphics/host_gpu/deferralBlock.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
namespace {
|
||||
thread_local uint32_t g_deferral_block_depth = 0;
|
||||
} // namespace
|
||||
|
||||
DeferralBlock::DeferralBlock() noexcept {
|
||||
g_deferral_block_depth++;
|
||||
}
|
||||
|
||||
DeferralBlock::~DeferralBlock() noexcept {
|
||||
EXIT_IF(g_deferral_block_depth == 0);
|
||||
g_deferral_block_depth--;
|
||||
}
|
||||
|
||||
bool DeferralBlocked() noexcept {
|
||||
return g_deferral_block_depth != 0;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
@@ -0,0 +1,20 @@
|
||||
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_DEFERRALBLOCK_H_
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_DEFERRALBLOCK_H_
|
||||
|
||||
#include "common/assert.h"
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
class DeferralBlock {
|
||||
public:
|
||||
DeferralBlock() noexcept;
|
||||
~DeferralBlock() noexcept;
|
||||
DeferralBlock(const DeferralBlock&) = delete;
|
||||
DeferralBlock& operator=(const DeferralBlock&) = delete;
|
||||
};
|
||||
|
||||
[[nodiscard]] bool DeferralBlocked() noexcept;
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
#endif
|
||||
@@ -29,6 +29,8 @@ struct GraphicContext {
|
||||
vk::Device device = nullptr;
|
||||
VmaAllocator allocator = nullptr;
|
||||
bool memory_budget_ext_enabled = false;
|
||||
bool diagnostic_checkpoints_enabled = false;
|
||||
bool device_fault_enabled = false;
|
||||
bool rt_extensions_enabled = false;
|
||||
bool compute_subgroup_size_control_enabled = false;
|
||||
bool compute_wave64_supported = false;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_MEMORYTRACKER_H_
|
||||
|
||||
#include "graphics/host_gpu/deferralBlock.h"
|
||||
#include "common/assert.h"
|
||||
#include "graphics/host_gpu/pageManager.h"
|
||||
#include "graphics/host_gpu/rangeSet.h"
|
||||
@@ -34,6 +35,7 @@ public:
|
||||
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
|
||||
static_assert(std::is_invocable_v<Flush&>);
|
||||
CheckNotInUploadCallback();
|
||||
const DeferralBlock deferral_block;
|
||||
ValidateRange(vaddr, size);
|
||||
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
@@ -68,6 +70,7 @@ public:
|
||||
static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
|
||||
static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
|
||||
CheckNotInUploadCallback();
|
||||
const DeferralBlock deferral_block;
|
||||
std::vector<RegionManager*> managers;
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
|
||||
managers.push_back(manager);
|
||||
@@ -108,6 +111,7 @@ public:
|
||||
static_assert(std::is_nothrow_invocable_v<RangeFunc&, uint64_t, uint64_t>);
|
||||
static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
|
||||
CheckNotInUploadCallback();
|
||||
const DeferralBlock deferral_block;
|
||||
Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
|
||||
const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
|
||||
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
|
||||
+175
-14
@@ -1,6 +1,10 @@
|
||||
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include <iterator>
|
||||
#include <optional>
|
||||
#include <cstdlib>
|
||||
#include <atomic>
|
||||
#include "common/logging/log.h"
|
||||
#include "common/profiler.h"
|
||||
#include "graphics/guest_gpu/graphicsRun.h"
|
||||
@@ -32,6 +36,12 @@ constexpr uint64_t GdsBufferSize = 64 * 1024;
|
||||
void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source,
|
||||
uint64_t size) {
|
||||
auto* bytes = static_cast<const uint8_t*>(source);
|
||||
if (const auto dest_offset = buffer.Offset(address); dest_offset >= buffer.Size()) {
|
||||
return;
|
||||
} else if (const auto room = buffer.Size() - dest_offset; size > room) {
|
||||
static std::atomic<uint32_t> clamp_log {0};
|
||||
size = room;
|
||||
}
|
||||
while (size != 0) {
|
||||
const auto chunk = std::min(size, m_staging_buffer.Size());
|
||||
const auto offset = m_staging_buffer.Copy(bytes, chunk, 4);
|
||||
@@ -108,9 +118,33 @@ void BufferCache::DeleteBuffer(BufferId id) {
|
||||
if (buffer == nullptr || buffer->is_deleted) {
|
||||
return;
|
||||
}
|
||||
const auto vaddr = buffer->CpuAddress();
|
||||
const auto size = buffer->Size();
|
||||
const auto handle = static_cast<const void*>(buffer->Handle());
|
||||
if (m_memory_tracker.IsRegionGpuModified(vaddr, size)) {
|
||||
std::vector<DownloadCopy> copies;
|
||||
m_memory_tracker.ForEachDownloadRange<false>(
|
||||
vaddr, size,
|
||||
[&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
|
||||
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, dirty_address,
|
||||
dirty_size, "buffer eviction");
|
||||
},
|
||||
[&](uint64_t dirty_address, uint64_t dirty_size) noexcept {
|
||||
for (const auto range: m_gpu_modified_ranges.Intersections(dirty_address, dirty_size)) {
|
||||
copies.push_back(
|
||||
{buffer, range.address - vaddr, range.address, range.size});
|
||||
}
|
||||
});
|
||||
EXIT_IF(copies.empty());
|
||||
DownloadBufferMemory(copies);
|
||||
m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
|
||||
}
|
||||
Unregister(id);
|
||||
if (m_scheduler.Active()) {
|
||||
m_scheduler.DeferOperation([this, id] { m_slot_buffers.erase(id); });
|
||||
const auto defer_tick = m_scheduler.CurrentTick();
|
||||
m_scheduler.DeferOperation([this, id] {
|
||||
m_slot_buffers.erase(id);
|
||||
});
|
||||
} else {
|
||||
m_slot_buffers.erase(id);
|
||||
}
|
||||
@@ -199,10 +233,10 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
|
||||
m_bda_pagetable_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
|
||||
BDA_PAGETABLE_SIZE),
|
||||
m_memory_tracker(page_manager),
|
||||
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
|
||||
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
|
||||
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
|
||||
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
|
||||
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, UPLOAD_BUFFER_SIZE),
|
||||
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, STREAM_BUFFER_SIZE),
|
||||
m_download_buffer(graphics, scheduler, MemoryUsage::Download, DOWNLOAD_BUFFER_SIZE),
|
||||
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, DEVICE_LOCAL_BUFFER_SIZE),
|
||||
m_texture_cache(texture_cache) {
|
||||
std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
|
||||
m_gds_buffer.Flush(0, m_gds_buffer.Size());
|
||||
@@ -291,6 +325,10 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
|
||||
}
|
||||
}
|
||||
});
|
||||
static std::atomic<uint64_t> resolve_last_vaddr {0};
|
||||
static std::atomic<uint64_t> resolve_repeat {0};
|
||||
bool do_log = false;
|
||||
uint64_t logged_repeat = 0;
|
||||
if (copies.empty()) {
|
||||
if (!is_write) {
|
||||
return;
|
||||
@@ -307,8 +345,18 @@ void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write)
|
||||
if (is_write) {
|
||||
m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
|
||||
}
|
||||
if (do_log) {
|
||||
uint64_t total_downloaded = 0;
|
||||
for (const auto& copy: copies) {
|
||||
total_downloaded += copy.size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
inline uint64_t ClampUnboundedSize(uint64_t vaddr, uint64_t size);
|
||||
} // namespace
|
||||
|
||||
BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
|
||||
if (vaddr == 0) {
|
||||
return NULL_BUFFER_ID;
|
||||
@@ -327,6 +375,7 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
|
||||
}
|
||||
|
||||
BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
|
||||
size = ClampUnboundedSize(vaddr, size);
|
||||
auto& command = m_scheduler.Current();
|
||||
EXIT_IF(command.IsInvalid());
|
||||
auto begin = vaddr & ~(CACHING_PAGESIZE - 1);
|
||||
@@ -346,6 +395,11 @@ BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
|
||||
end = std::max(end, buffer.CpuAddress() + buffer.Size());
|
||||
}
|
||||
|
||||
const auto overlap_count = static_cast<size_t>(std::distance(first, last));
|
||||
|
||||
constexpr uint64_t outsized = 256ull << 20u;
|
||||
constexpr uint32_t outsized_limit = 16u;
|
||||
|
||||
const auto id = m_slot_buffers.insert(
|
||||
m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
|
||||
AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin);
|
||||
@@ -373,6 +427,7 @@ bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t siz
|
||||
[&](uint64_t address, uint64_t bytes) noexcept {
|
||||
copies.emplace_back(total_size, buffer.Offset(address), bytes);
|
||||
total_size += bytes;
|
||||
total_size += bytes;
|
||||
},
|
||||
[&]() noexcept { source = UploadCopies(buffer, copies, total_size); });
|
||||
if (source) {
|
||||
@@ -408,6 +463,33 @@ bool BufferCache::SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t siz
|
||||
return false;
|
||||
}
|
||||
|
||||
namespace {
|
||||
constexpr uint64_t UnboundedBufferSize = 0xffffffffull;
|
||||
inline uint64_t UnboundedThresholdBytes() {
|
||||
constexpr uint64_t bytes = UnboundedBufferSize;
|
||||
return bytes;
|
||||
}
|
||||
inline uint64_t UnboundedClampBytes() {
|
||||
constexpr uint64_t bytes = 64ull * 1024 * 1024;
|
||||
return bytes;
|
||||
}
|
||||
inline uint64_t ClampUnboundedSize(uint64_t vaddr, uint64_t size) {
|
||||
if (size >= UnboundedThresholdBytes()) {
|
||||
static std::atomic<uint32_t> clamp_log {0};
|
||||
bool mapped_known = false;
|
||||
const auto mapped = Libs::LibKernel::Memory::MappedExtentFrom(vaddr, &mapped_known);
|
||||
if (mapped != 0) {
|
||||
size = std::min(mapped, UnboundedClampBytes());
|
||||
} else if (!mapped_known) {
|
||||
size = UnboundedClampBytes();
|
||||
} else {
|
||||
size = 0x1000;
|
||||
}
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
vk::Buffer BufferCache::UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
|
||||
uint64_t total_size) {
|
||||
if (copies.empty()) {
|
||||
@@ -442,10 +524,21 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
|
||||
bool is_written, bool is_texel_buffer,
|
||||
BufferId id) {
|
||||
auto& command = m_scheduler.Current();
|
||||
size = ClampUnboundedSize(vaddr, size);
|
||||
if (command.IsInvalid() || !GuestRange {vaddr, size}.Valid()) {
|
||||
EXIT("BufferCache: buffer request requires a recording command buffer\n");
|
||||
}
|
||||
|
||||
{
|
||||
static const auto watch = [] {
|
||||
std::pair<uint64_t, uint64_t> range {0, 0};
|
||||
return range;
|
||||
}();
|
||||
if (watch.second != 0 && vaddr < watch.first + watch.second && watch.first < vaddr + size) {
|
||||
static std::atomic<uint32_t> log_count {0};
|
||||
}
|
||||
}
|
||||
|
||||
if (!is_written && size <= CACHING_PAGESIZE &&
|
||||
!m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
|
||||
m_memory_tracker.IsRegionCpuModified(vaddr, size)) {
|
||||
@@ -463,18 +556,52 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBuffer(uint64_t vaddr, uint64_t
|
||||
id = FindBuffer(vaddr, size);
|
||||
buffer = &m_slot_buffers[id];
|
||||
}
|
||||
{
|
||||
constexpr uint64_t bvh_check = 0ull;
|
||||
const bool is_bvh = [&] {
|
||||
if (bvh_check == 0 || bvh_check + 64 > size || size < 8) {
|
||||
return false;
|
||||
}
|
||||
if (Libs::LibKernel::Memory::MappedExtentFrom(vaddr) < 8) {
|
||||
return false;
|
||||
}
|
||||
const auto* magic = reinterpret_cast<const uint32_t*>(vaddr);
|
||||
return magic[0] == 0x5f525350u && magic[1] == 0x4c485642u; // "PSR_" "BVHL"
|
||||
}();
|
||||
if (is_bvh) {
|
||||
static std::atomic<uint32_t> check_log {0};
|
||||
if (check_log.fetch_add(1, std::memory_order_relaxed) < 8) {
|
||||
const auto probe = vaddr + bvh_check;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TouchBuffer(*buffer);
|
||||
(void)SynchronizeBuffer(*buffer, vaddr, size, is_written, is_texel_buffer);
|
||||
if (is_written) {
|
||||
m_gpu_modified_ranges.Add(vaddr, size);
|
||||
}
|
||||
constexpr uint64_t traced_page = 0ull;
|
||||
return {buffer, buffer->Offset(vaddr)};
|
||||
}
|
||||
|
||||
static void ImageWatchCheck(const char* who, uint64_t vaddr, uint64_t size) {
|
||||
static const auto watch = [] {
|
||||
std::pair<uint64_t, uint64_t> range {0, 0};
|
||||
return range;
|
||||
}();
|
||||
if (watch.second == 0 || vaddr >= watch.first + watch.second || watch.first >= vaddr + size) {
|
||||
return;
|
||||
}
|
||||
static std::atomic<uint32_t> log_count {0};
|
||||
}
|
||||
|
||||
std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t size) {
|
||||
if (!GuestRange {vaddr, size}.Valid()) {
|
||||
EXIT("BufferCache: invalid image source\n");
|
||||
}
|
||||
|
||||
ImageWatchCheck("ObtainBufferForImage", vaddr, size);
|
||||
auto find_owner = [&]() -> Buffer* {
|
||||
const auto* owner = m_page_table.Find(vaddr >> PageTable::kPageBits);
|
||||
if (owner == nullptr || !*owner) {
|
||||
@@ -491,33 +618,47 @@ std::pair<Buffer*, uint64_t> BufferCache::ObtainBufferForImage(uint64_t vaddr, u
|
||||
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
|
||||
"image source");
|
||||
|
||||
auto* owner = find_owner();
|
||||
auto* owner = find_owner();
|
||||
bool unowned_gpu = false;
|
||||
if (has_dirty_buffer_source && owner == nullptr) {
|
||||
if (!IsRegionRegistered(vaddr, size)) {
|
||||
EXIT("BufferCache: GPU-dirty image source has no native buffer\n");
|
||||
unowned_gpu = true;
|
||||
} else {
|
||||
const auto id = FindBuffer(vaddr, size);
|
||||
auto& buffer = m_slot_buffers[id];
|
||||
owner = &buffer;
|
||||
const auto resolved = m_buffers.find(buffer.CpuAddress());
|
||||
if (resolved == m_buffers.end() || resolved->second != id ||
|
||||
!buffer.IsInBounds(vaddr, size)) {
|
||||
EXIT("BufferCache: merged image source does not contain the requested range\n");
|
||||
}
|
||||
}
|
||||
owner = &m_slot_buffers[FindBuffer(vaddr, size)];
|
||||
}
|
||||
if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) {
|
||||
TouchBuffer(*owner);
|
||||
return {owner, owner->Offset(vaddr)};
|
||||
}
|
||||
if (has_dirty_buffer_source && owner == nullptr) {
|
||||
if (has_dirty_buffer_source && owner == nullptr && !unowned_gpu) {
|
||||
EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
|
||||
}
|
||||
}
|
||||
|
||||
auto [staging, stage_offset] = m_staging_buffer.Map(size, 16);
|
||||
if (staging == nullptr || (!Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size) &&
|
||||
!Libs::LibKernel::Memory::TryReadPrtBacking(vaddr, staging, size))) {
|
||||
EXIT("BufferCache: failed to read mapped guest image backing\n");
|
||||
if (staging == nullptr) {
|
||||
EXIT("BufferCache: staging reservation failed for guest image, size=0x%016" PRIx64
|
||||
" staging=0x%016" PRIx64 " vaddr=0x%016" PRIx64 "\n",
|
||||
size, m_staging_buffer.Size(), vaddr);
|
||||
}
|
||||
if (!Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size) &&
|
||||
!Libs::LibKernel::Memory::TryReadPrtBacking(vaddr, staging, size)) {
|
||||
std::memset(staging, 0, static_cast<size_t>(size));
|
||||
}
|
||||
m_staging_buffer.Commit();
|
||||
|
||||
const bool has_dirty_buffer_source = m_gpu_modified_ranges.Intersects(vaddr, size);
|
||||
auto* owner = find_owner();
|
||||
if (has_dirty_buffer_source && owner == nullptr) {
|
||||
EXIT("BufferCache: GPU-dirty image source lost its native owner\n");
|
||||
return {&m_staging_buffer, stage_offset};
|
||||
}
|
||||
if (owner == nullptr ||
|
||||
(m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) {
|
||||
@@ -692,7 +833,27 @@ void BufferCache::RunGarbageCollector() {
|
||||
return;
|
||||
}
|
||||
|
||||
const bool aggressive = m_total_used_memory >= m_critical_gc_memory;
|
||||
{
|
||||
static std::atomic<uint64_t> buf_log {0};
|
||||
if ((buf_log.fetch_add(1, std::memory_order_relaxed) % 256) == 0) {
|
||||
uint64_t bytes = 0;
|
||||
for (const auto& [address, owner]: m_buffers) {
|
||||
bytes += m_slot_buffers[owner].Size();
|
||||
}
|
||||
{
|
||||
std::vector<std::pair<uint64_t, uint64_t>> sizes;
|
||||
sizes.reserve(m_buffers.size());
|
||||
for (const auto& [address, owner]: m_buffers) {
|
||||
sizes.emplace_back(m_slot_buffers[owner].Size(), address);
|
||||
}
|
||||
std::ranges::sort(sizes, std::greater<>());
|
||||
for (size_t i = 0; i < sizes.size() && i < 4; i++) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const bool aggressive = m_total_used_memory >= m_critical_gc_memory;
|
||||
const uint64_t age = std::min<uint64_t>(aggressive ? 80 : 160, tick);
|
||||
const size_t limit = aggressive ? 64 : 32;
|
||||
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
#include "graphics/guest_gpu/graphicsRun.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <cinttypes>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
|
||||
@@ -14,6 +20,23 @@ GpuResourceManager::~GpuResourceManager() = default;
|
||||
bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
|
||||
constexpr uint64_t fault_size = 8;
|
||||
if (!IsMapped(fault_vaddr, fault_size)) {
|
||||
static std::atomic<uint32_t> log_count {0};
|
||||
if (log_count.fetch_add(1, std::memory_order_relaxed) < 32) {
|
||||
uint64_t region_start = 0;
|
||||
uint64_t region_size = 0;
|
||||
uint32_t protect = 0;
|
||||
uint32_t state = 0;
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
MEMORY_BASIC_INFORMATION mbi {};
|
||||
if (VirtualQuery(reinterpret_cast<const void*>(fault_vaddr), &mbi, sizeof(mbi)) != 0) {
|
||||
region_start = reinterpret_cast<uint64_t>(mbi.BaseAddress);
|
||||
region_size = static_cast<uint64_t>(mbi.RegionSize);
|
||||
protect = static_cast<uint32_t>(mbi.Protect);
|
||||
state = static_cast<uint32_t>(mbi.State);
|
||||
}
|
||||
#endif
|
||||
std::shared_lock lock(m_mapped_ranges_mutex);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (access == PageFaultAccess::Write) {
|
||||
|
||||
@@ -33,6 +33,8 @@ public:
|
||||
void PrepareBda();
|
||||
void RunGarbageCollector();
|
||||
|
||||
[[nodiscard]] uint64_t GetPageSize() const { return m_page_manager.GetPageSize(); }
|
||||
|
||||
private:
|
||||
PageManager m_page_manager;
|
||||
CommandScheduler& m_scheduler;
|
||||
|
||||
+14
-7
@@ -1,11 +1,14 @@
|
||||
#include <cinttypes>
|
||||
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
#include "common/profiler.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/vma.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
@@ -73,12 +76,8 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
|
||||
buffer_info.usage = flags;
|
||||
buffer_info.sharingMode = vk::SharingMode::eExclusive;
|
||||
|
||||
const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
|
||||
const VmaAllocationCreateFlags bda_flag =
|
||||
with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
|
||||
VmaAllocationCreateInfo allocation_info {};
|
||||
allocation_info.flags =
|
||||
VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
|
||||
allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
|
||||
allocation_info.usage = AllocationUsage(usage);
|
||||
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
|
||||
? VkMemoryPropertyFlags {}
|
||||
@@ -188,10 +187,13 @@ vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::Acce
|
||||
void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
|
||||
uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before,
|
||||
vk::AccessFlags destination_before, vk::AccessFlags source_after,
|
||||
vk::AccessFlags destination_after) {
|
||||
vk::AccessFlags destination_after, std::source_location where) {
|
||||
if (size == 0 || source_offset > source.m_size || size > source.m_size - source_offset ||
|
||||
destination_offset > m_size || size > m_size - destination_offset) {
|
||||
EXIT("Buffer: invalid copy range\n");
|
||||
EXIT("Buffer: invalid copy range from %s:%u: src_size=0x%" PRIx64 " src_off=0x%" PRIx64
|
||||
" dst_size=0x%" PRIx64 " dst_off=0x%" PRIx64 " size=0x%" PRIx64 "\n",
|
||||
where.file_name(), where.line(), source.m_size, source_offset, m_size,
|
||||
destination_offset, size);
|
||||
}
|
||||
if (source.Handle() == Handle() && source_offset < destination_offset + size &&
|
||||
destination_offset < source_offset + size) {
|
||||
@@ -228,6 +230,11 @@ void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) {
|
||||
if (((offset | size) & 3u) != 0) {
|
||||
EXIT("Buffer: fill range must be dword aligned\n");
|
||||
}
|
||||
if (offset > m_size || size > m_size - offset) {
|
||||
EXIT("Buffer: fill range outside the buffer, offset=0x%" PRIx64 " size=0x%" PRIx64
|
||||
" capacity=0x%" PRIx64 "\n",
|
||||
offset, size, m_size);
|
||||
}
|
||||
auto& command = Scheduler().Current();
|
||||
command.EndRendering();
|
||||
const auto before =
|
||||
|
||||
+3
-1
@@ -1,6 +1,7 @@
|
||||
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_
|
||||
|
||||
#include <source_location>
|
||||
#include "common/abi.h"
|
||||
#include "common/common.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
@@ -64,7 +65,8 @@ public:
|
||||
vk::AccessFlags source_after = vk::AccessFlagBits::eMemoryRead |
|
||||
vk::AccessFlagBits::eMemoryWrite,
|
||||
vk::AccessFlags destination_after = vk::AccessFlagBits::eMemoryRead |
|
||||
vk::AccessFlagBits::eMemoryWrite);
|
||||
vk::AccessFlagBits::eMemoryWrite,
|
||||
std::source_location where = std::source_location::current());
|
||||
void Fill(uint64_t offset, uint64_t size, uint32_t value);
|
||||
|
||||
// BufferCache state lives directly on the resource.
|
||||
|
||||
+86
-23
@@ -17,8 +17,10 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <bit>
|
||||
#include <cinttypes>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <mutex>
|
||||
@@ -76,10 +78,15 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
|
||||
const auto budget =
|
||||
static_cast<int64_t>(std::min<uint64_t>(m_graphics.GetTotalMemoryBudget(), INT64_MAX));
|
||||
const auto threshold = std::min<int64_t>(budget, 8 * GiB);
|
||||
const bool retain = false;
|
||||
m_pressure_gc_memory = static_cast<uint64_t>(
|
||||
std::max<int64_t>(std::min(budget - 6 * threshold / 10, budget - GiB), GiB + GiB / 2));
|
||||
retain ? std::max<int64_t>(budget - GiB, GiB + GiB / 2)
|
||||
: std::max<int64_t>(std::min(budget - 6 * threshold / 10, budget - GiB),
|
||||
GiB + GiB / 2));
|
||||
m_critical_gc_memory = static_cast<uint64_t>(
|
||||
std::max<int64_t>(std::min(budget - 2 * threshold / 10, budget - GiB / 2), 3 * GiB));
|
||||
retain ? std::max<int64_t>(budget - GiB / 2, 3 * GiB)
|
||||
: std::max<int64_t>(std::min(budget - 2 * threshold / 10, budget - GiB / 2),
|
||||
3 * GiB));
|
||||
m_trigger_gc_memory = static_cast<uint64_t>(std::max<int64_t>((budget - threshold) / 2, 0));
|
||||
}
|
||||
}
|
||||
@@ -149,6 +156,10 @@ bool TextureCache::SafeToDownload(const Image& image) {
|
||||
return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size);
|
||||
}
|
||||
|
||||
bool TextureCache::IsLiveId(ImageId id) const {
|
||||
return m_slot_images.is_allocated(id);
|
||||
}
|
||||
|
||||
ImageId TextureCache::InsertImage(const ImageInfo& info) {
|
||||
const auto id = m_slot_images.insert(m_graphics, m_scheduler, info);
|
||||
if (!info.data.Empty()) {
|
||||
@@ -942,14 +953,7 @@ void TextureCache::UploadImage(Image& image, const ImageDesc& desc, Buffer& sour
|
||||
if (desc.type != BindingType::DepthTarget) {
|
||||
auto plan = BuildColorTransfer(image, desc.type, TransferDirection::Upload);
|
||||
if (!plan.valid) {
|
||||
EXIT("TextureCache: invalid color upload: binding=%u addr=0x%016" PRIx64
|
||||
" size=0x%016" PRIx64 " format=%u tile=%u family=%u extent=%ux%ux%u "
|
||||
"pitch=%u levels=%u layers=%u samples=%u\n",
|
||||
static_cast<uint32_t>(desc.type), info.data.address, info.data.size,
|
||||
static_cast<uint32_t>(info.guest_format), static_cast<uint32_t>(info.tile_mode),
|
||||
static_cast<uint32_t>(plan.layout.surface.texture.block.family), info.extent.width,
|
||||
info.extent.height, info.extent.depth, info.pitch, info.resources.levels,
|
||||
info.resources.layers, info.samples);
|
||||
return;
|
||||
}
|
||||
TileManager::Result linear {source.Handle(), source_offset, info.data.size};
|
||||
if (plan.tiled) {
|
||||
@@ -1075,6 +1079,7 @@ void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
constexpr uint64_t watch = 0ull;
|
||||
if (!cpu_dirty) {
|
||||
return;
|
||||
}
|
||||
@@ -1670,12 +1675,16 @@ bool TextureCache::TryDownloadImage(ImageId id) {
|
||||
if (!plan.valid || !SafeToDownload(image)) {
|
||||
return false;
|
||||
}
|
||||
const auto range = image.info.data;
|
||||
auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download);
|
||||
auto [mapped, offset] =
|
||||
download.Map(range.size, std::max<uint64_t>(image.info.bytes_per_block, 4));
|
||||
const auto range = image.info.data;
|
||||
auto& download = m_buffer_cache.GetUtilityBuffer(MemoryUsage::Download);
|
||||
const auto alignment = std::max<uint64_t>(image.info.bytes_per_block, 4);
|
||||
auto [mapped, offset] = download.Map(range.size, alignment);
|
||||
if (mapped == nullptr) {
|
||||
EXIT("TextureCache: failed to map reusable download buffer\n");
|
||||
EXIT("TextureCache: failed to map reusable download buffer, size=0x%" PRIx64
|
||||
" alignment=0x%" PRIx64 " download_buffer=0x%" PRIx64 " (%s)\n",
|
||||
range.size, alignment, download.Size(),
|
||||
range.size > download.Size() ? "request exceeds the buffer"
|
||||
: "buffer could not be drained");
|
||||
}
|
||||
download.Commit();
|
||||
if (!LibKernel::Memory::TryReadBacking(range.address, mapped, range.size)) {
|
||||
@@ -1743,7 +1752,16 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si
|
||||
void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
|
||||
const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
|
||||
const auto page_end = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
|
||||
for (const auto id: FindImagesInRegion(address, size, true)) {
|
||||
const auto found_images = FindImagesInRegion(address, size, true);
|
||||
static const bool log_fault_addr = false;
|
||||
static std::atomic<uint64_t> last_vaddr {0};
|
||||
static std::atomic<uint64_t> repeat_count {0};
|
||||
if (log_fault_addr) {
|
||||
const auto previous = last_vaddr.exchange(address);
|
||||
const auto repeat = (previous == address) ? repeat_count.fetch_add(1) + 1
|
||||
: (repeat_count.store(0), uint64_t {0});
|
||||
}
|
||||
for (const auto id: found_images) {
|
||||
auto owner = m_slot_images.try_get(id);
|
||||
if (owner == nullptr || owner->depth_id) {
|
||||
continue;
|
||||
@@ -1883,6 +1901,13 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
|
||||
}
|
||||
}
|
||||
|
||||
static std::atomic<uint64_t> g_gc_skip_unresolved {0};
|
||||
static std::atomic<uint64_t> g_gc_skip_tiled {0};
|
||||
static std::atomic<uint64_t> g_gc_skip_unpressured {0};
|
||||
static std::atomic<uint64_t> g_gc_skip_download {0};
|
||||
static std::atomic<uint64_t> g_gc_deleted {0};
|
||||
static std::atomic<uint64_t> g_gc_candidates {0};
|
||||
|
||||
void TextureCache::RunGarbageCollector() {
|
||||
std::scoped_lock lock {m_lock};
|
||||
const uint64_t tick = m_gc_tick++;
|
||||
@@ -1893,15 +1918,22 @@ void TextureCache::RunGarbageCollector() {
|
||||
return;
|
||||
}
|
||||
const auto collect = [&](bool allow_aggressive) {
|
||||
bool pressured = m_total_used_memory >= m_pressure_gc_memory;
|
||||
bool aggressive = allow_aggressive && m_total_used_memory >= m_critical_gc_memory;
|
||||
const uint64_t age = std::min<uint64_t>(aggressive ? 160 : pressured ? 80 : 16, tick);
|
||||
const uint64_t device_used =
|
||||
m_graphics.CanReportMemoryUsage() ? m_graphics.GetDeviceMemoryUsage() : 0;
|
||||
const uint64_t used_memory = std::max(m_total_used_memory, device_used);
|
||||
bool pressured = used_memory >= m_pressure_gc_memory;
|
||||
static const bool force_aggressive =
|
||||
false;
|
||||
bool aggressive =
|
||||
allow_aggressive && (force_aggressive || used_memory >= m_critical_gc_memory);
|
||||
const uint64_t age = std::min<uint64_t>(aggressive ? 4 : pressured ? 8 : 16, tick);
|
||||
size_t deletions = aggressive ? 40 : pressured ? 20 : 10;
|
||||
std::vector<ImageId> candidates;
|
||||
candidates.reserve(deletions);
|
||||
// Deleting depth recursively deletes its stencil association, so finish LRU traversal
|
||||
// first.
|
||||
m_lru_cache.ForEachItemBelow(tick - age, [&](ImageId id) {
|
||||
g_gc_candidates.fetch_add(1, std::memory_order_relaxed);
|
||||
candidates.push_back(id);
|
||||
return candidates.size() == deletions;
|
||||
});
|
||||
@@ -1912,36 +1944,67 @@ void TextureCache::RunGarbageCollector() {
|
||||
--deletions;
|
||||
auto owner = m_slot_images.try_get(id);
|
||||
if (owner == nullptr || !owner->registered || owner->depth_id) {
|
||||
g_gc_skip_unresolved.fetch_add(1, std::memory_order_relaxed);
|
||||
continue;
|
||||
}
|
||||
if (owner->IsGpuModified()) {
|
||||
const bool safe = SafeToDownload(*owner);
|
||||
if (safe && owner->info.IsTiled()) {
|
||||
if (safe && owner->info.IsTiled() && !aggressive) {
|
||||
g_gc_skip_tiled.fetch_add(1, std::memory_order_relaxed);
|
||||
continue;
|
||||
}
|
||||
if (safe && !pressured) {
|
||||
g_gc_skip_unpressured.fetch_add(1, std::memory_order_relaxed);
|
||||
continue;
|
||||
}
|
||||
if (safe && !TryDownloadImage(id)) {
|
||||
g_gc_skip_download.fetch_add(1, std::memory_order_relaxed);
|
||||
continue;
|
||||
}
|
||||
owner->ClearGpuModified();
|
||||
}
|
||||
g_gc_deleted.fetch_add(1, std::memory_order_relaxed);
|
||||
DeleteImage(id);
|
||||
if (m_total_used_memory < m_critical_gc_memory && aggressive) {
|
||||
if (used_memory < m_critical_gc_memory && aggressive) {
|
||||
deletions >>= 2;
|
||||
aggressive = false;
|
||||
}
|
||||
if (m_total_used_memory < m_pressure_gc_memory && pressured) {
|
||||
if (used_memory < m_pressure_gc_memory && pressured) {
|
||||
deletions >>= 1;
|
||||
pressured = false;
|
||||
}
|
||||
}
|
||||
};
|
||||
const auto before = m_total_used_memory;
|
||||
collect(false);
|
||||
if (m_total_used_memory >= m_critical_gc_memory) {
|
||||
static const bool force_aggressive_pass =
|
||||
false;
|
||||
if (force_aggressive_pass || m_total_used_memory >= m_critical_gc_memory) {
|
||||
collect(true);
|
||||
}
|
||||
if (m_graphics.CanReportMemoryUsage()) {
|
||||
m_total_used_memory = m_graphics.GetDeviceMemoryUsage();
|
||||
}
|
||||
{
|
||||
static std::atomic<uint64_t> gc_log_count {0};
|
||||
if ((gc_log_count.fetch_add(1, std::memory_order_relaxed) % 256) == 0) {
|
||||
{
|
||||
uint64_t live = 0;
|
||||
uint64_t live_bytes = 0;
|
||||
uint64_t tiled = 0;
|
||||
m_slot_images.ForEach([&](ImageId, Image& image) {
|
||||
if (!image.registered) {
|
||||
return;
|
||||
}
|
||||
live++;
|
||||
live_bytes += image.info.data.size + image.info.stencil.size;
|
||||
if (image.info.IsTiled()) {
|
||||
tiled++;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::ProcessDownloadImages() {
|
||||
|
||||
@@ -60,6 +60,7 @@ public:
|
||||
TouchImage(image);
|
||||
return image;
|
||||
}
|
||||
[[nodiscard]] bool IsLiveId(ImageId id) const;
|
||||
void MarkGpuWritten(ImageId id);
|
||||
|
||||
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#include "graphics/host_gpu/deferralBlock.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
|
||||
#include "common/assert.h"
|
||||
@@ -98,6 +99,11 @@ bool CommandScheduler::InDeferredOperation() noexcept {
|
||||
return g_deferred_callback_scheduler != nullptr;
|
||||
}
|
||||
|
||||
size_t CommandScheduler::PendingOperationCount() const {
|
||||
std::lock_guard lock(m_operation_mutex);
|
||||
return m_pending_operations.size();
|
||||
}
|
||||
|
||||
CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics)
|
||||
: m_master(graphics), m_context(context), m_graphics(graphics),
|
||||
m_command_pool(graphics, m_master), m_command(*this),
|
||||
@@ -221,6 +227,9 @@ void CommandScheduler::PopPendingOperations() {
|
||||
}
|
||||
|
||||
void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) {
|
||||
if (DeferralBlocked()) {
|
||||
return;
|
||||
}
|
||||
if (refresh_gpu_tick) {
|
||||
m_master.Refresh();
|
||||
}
|
||||
@@ -402,6 +411,9 @@ uint64_t CommandScheduler::Submit(SubmitInfo submit) {
|
||||
}
|
||||
|
||||
if (result != vk::Result::eSuccess) {
|
||||
if (result == vk::Result::eErrorDeviceLost) {
|
||||
GpuCrashDiagnosticsDump(graphics, "vkQueueSubmit");
|
||||
}
|
||||
ReportVulkanFatal("vkQueueSubmit", result, tick, m_command.m_debug_op,
|
||||
m_command.m_debug_submit_id, m_command.m_debug_arg0,
|
||||
m_command.m_debug_arg1, m_command.m_debug_arg2, m_command.m_debug_arg3,
|
||||
|
||||
@@ -38,6 +38,7 @@ public:
|
||||
void PopPendingOperations();
|
||||
void DrainPriorityOperations();
|
||||
void WaitPriorityOperations(uint64_t tick);
|
||||
[[nodiscard]] size_t PendingOperationCount() const;
|
||||
void DeferOperation(Common::UniqueFunction<void>&& operation);
|
||||
void DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
|
||||
[[nodiscard]] static bool InDeferredOperation() noexcept;
|
||||
@@ -93,7 +94,7 @@ private:
|
||||
CommandBuffer m_command;
|
||||
std::queue<PendingOperation> m_pending_operations;
|
||||
std::queue<PendingOperation> m_priority_operations;
|
||||
std::mutex m_operation_mutex;
|
||||
mutable std::mutex m_operation_mutex;
|
||||
std::condition_variable m_operation_available;
|
||||
std::jthread m_priority_thread;
|
||||
bool m_priority_active = false;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#include <cstdlib>
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
|
||||
#if defined(__clang__)
|
||||
@@ -38,6 +39,8 @@ MemoryStats g_memory_stats;
|
||||
void TrackAllocationImpl(const VulkanMemory& memory) {
|
||||
g_memory_stats.allocated[memory.type] += memory.requirements.size;
|
||||
g_memory_stats.count[memory.type]++;
|
||||
|
||||
constexpr uint64_t big = 0ull;
|
||||
}
|
||||
|
||||
void UntrackAllocationImpl(const VulkanMemory& memory) {
|
||||
@@ -113,6 +116,18 @@ void GraphicContext::LogMemoryBudget() const {
|
||||
static_cast<uint64_t>(budgets[i].statistics.allocationBytes),
|
||||
static_cast<uint64_t>(budgets[i].statistics.blockBytes));
|
||||
}
|
||||
|
||||
VmaTotalStatistics stats {};
|
||||
vmaCalculateStatistics(allocator, &stats);
|
||||
for (uint32_t i = 0; i < properties.memoryTypeCount; i++) {
|
||||
const auto vma_bytes = static_cast<uint64_t>(stats.memoryType[i].statistics.allocationBytes);
|
||||
const auto vma_count = static_cast<uint64_t>(stats.memoryType[i].statistics.allocationCount);
|
||||
if (vma_bytes == 0 && vma_count == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto tracked = g_memory_stats.allocated[i].load(std::memory_order_relaxed);
|
||||
const auto tracked_count = g_memory_stats.count[i].load(std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t GraphicContext::GetDeviceMemoryUsage() const {
|
||||
@@ -129,7 +144,7 @@ uint64_t GraphicContext::GetDeviceMemoryUsage() const {
|
||||
static_cast<bool>(physical_device_memory_properties.memoryHeaps[heap].flags &
|
||||
vk::MemoryHeapFlagBits::eDeviceLocal);
|
||||
if (!discrete || device_local) {
|
||||
usage += budgets[heap].usage;
|
||||
usage += budgets[heap].statistics.allocationBytes;
|
||||
}
|
||||
}
|
||||
return usage;
|
||||
|
||||
Reference in New Issue
Block a user