raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 19:07:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: sync only GPU-dirty readbacks

Browse Source
This commit is contained in:
nmzik
2026-08-03 22:13:41 +02:00
parent e8752aca7e
commit 51d0e02543
13 changed files with 373 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/guest_gpu/command_processor/commandProcessor.h
+7 -21
View File
@@ -50,23 +50,10 @@ public:
void Reset();
void BufferInit();
void BufferFlush();
void BufferFlushAndWait();
void BufferWait();
void BeginReadbackTransaction() {
if (m_readback_active) {
EXIT("nested command-processor readback transaction\n");
}
m_readback_active = true;
}
void EndReadbackTransaction() {
if (!m_readback_active) {
EXIT("command-processor readback transaction is not active\n");
}
m_readback_active = false;
}
void BufferInit();
void BufferFlush();
void BufferFlushAndWait();
void BufferWait();
HW::Context& GetCtx() { return m_ctx; }
HW::UserConfig& GetUcfg() { return m_ucfg; }
HW::Shader& GetShCtx() { return m_sh_ctx; }
@@ -173,10 +160,9 @@ private:
// Persistent draw state: indirect draws update it for subsequent draws.
uint32_t m_num_instances = 1;
uint32_t m_de_count = 0;
uint32_t m_ce_count = 0;
bool m_ce_complete = false;
bool m_readback_active = false;
uint32_t m_de_count = 0;
uint32_t m_ce_count = 0;
bool m_ce_complete = false;
uint32_t m_const_ram[0x3000] = {0};
src/graphics/guest_gpu/graphicsRun.cpp
+33 -26
View File
@@ -36,6 +36,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr;
static thread_local Pm4Execution* g_current_execution = nullptr;
static thread_local bool g_gpu_mutex_owned = false;
static thread_local bool g_gpu_thread = false;
static thread_local GpuState* g_gpu_state = nullptr;
class GpuMutexLock final {
public:
@@ -98,11 +99,10 @@ public:
void SubmitFlipPreparation(uint64_t request_id);
void Done();
void Shutdown();
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
void SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command);
int GetFrameNum();
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
int GetFrameNum();
[[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; }
private:
@@ -127,6 +127,7 @@ private:
void WaitLocked();
void Enqueue(Submission submission);
void WaitForIdle();
void ProcessCommands();
bool Process(Submission& submission);
static void ThreadRun(void* data);
CommandProcessor& GetProcessor(uint32_t queue_id);
@@ -139,6 +140,7 @@ private:
Common::CondVar m_idle;
std::array<std::deque<Submission>, QueueCount> m_queues;
std::deque<Common::UniqueFunction<void>> m_commands;
std::atomic_uint32_t m_pending_commands {0};
uint32_t m_next_queue = 0;
uint32_t m_submission_count = 0;
bool m_processing = false;
@@ -153,6 +155,8 @@ private:
uint64_t m_submit_id = 0;
std::atomic_int m_done_num = 0;
std::jthread m_thread;
friend class CommandProcessor;
};
static bool GraphicsRunDebugDumpEnabled() {
@@ -195,9 +199,25 @@ void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
Common::LockGuard lock(m_queue_mutex);
EXIT_IF(!m_accepting);
m_commands.push_back(std::move(command));
m_pending_commands.fetch_add(1, std::memory_order_release);
m_work_available.Signal();
}
void GpuState::ProcessCommands() {
EXIT_IF(!IsGpuThread());
while (m_pending_commands.load(std::memory_order_acquire) != 0) {
Common::UniqueFunction<void> command;
{
Common::LockGuard lock(m_queue_mutex);
EXIT_IF(m_commands.empty());
command = std::move(m_commands.front());
m_commands.pop_front();
EXIT_IF(m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
}
command();
}
}
void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
EXIT_IF(!command);
if (IsGpuThread()) {
@@ -212,17 +232,6 @@ void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
done.acquire();
}
void GpuState::SendCommandSyncWithProcessor(
Common::UniqueFunction<void, CommandProcessor&>&& command) {
EXIT_IF(!command);
SendCommandSync([this, operation = std::move(command)]() mutable {
EXIT_IF(g_current_processor != nullptr);
g_current_processor = m_gfx_cp.get();
operation(*m_gfx_cp);
g_current_processor = nullptr;
});
}
void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
GpuMutexLock lock(m_submission_mutex);
@@ -509,6 +518,7 @@ void GpuState::ThreadRun(void* data) {
EXIT_IF(gpu == nullptr);
KYTY_PROFILER_THREAD("Thread_Gpu");
g_gpu_thread = true;
g_gpu_state = gpu;
for (;;) {
Submission submission;
@@ -529,6 +539,7 @@ void GpuState::ThreadRun(void* data) {
} else if (!gpu->m_commands.empty()) {
command = std::move(gpu->m_commands.front());
gpu->m_commands.pop_front();
EXIT_IF(gpu->m_pending_commands.fetch_sub(1, std::memory_order_acq_rel) == 0);
gpu->m_processing = true;
} else {
int selected_queue = -1;
@@ -560,6 +571,7 @@ void GpuState::ThreadRun(void* data) {
}
if (should_stop) {
gpu->m_gfx_cp->BufferWait();
g_gpu_state = nullptr;
g_gpu_thread = false;
return;
}
@@ -741,6 +753,9 @@ void CommandProcessor::SuspendPm4() {
void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
while (execution.m_buffer_stack.size() > stop_depth) {
if (g_gpu_state != nullptr) {
g_gpu_state->ProcessCommands();
}
const auto buffer_index = execution.m_buffer_stack.size() - 1;
auto& cursor = execution.m_buffer_stack[buffer_index];
if (cursor.deferred_advance_dw != 0) {
@@ -1650,10 +1665,6 @@ void Gpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
m_state->SendCommandSync(std::move(command));
}
void Gpu::SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command) {
m_state->SendCommandSyncWithProcessor(std::move(command));
}
void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) {
EXIT_IF(draw_commands == nullptr || draw_size_dw == 0);
@@ -1679,12 +1690,8 @@ int Gpu::GetFrameNum() const {
return m_state->GetFrameNum();
}
bool Gpu::IsCommandProcessorThread() noexcept {
return g_current_processor != nullptr;
}
CommandProcessor* Gpu::CurrentCommandProcessor() noexcept {
return g_current_processor;
bool Gpu::IsGpuThread() noexcept {
return GpuState::IsGpuThread();
}
} // namespace Libs::Graphics
src/graphics/guest_gpu/graphicsRun.h
+1 -4
View File
@@ -9,7 +9,6 @@
namespace Libs::Graphics {
class CommandProcessor;
class GpuState;
class RenderContext;
@@ -23,7 +22,6 @@ public:
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
void SendCommandSyncWithProcessor(Common::UniqueFunction<void, CommandProcessor&>&& command);
void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false);
@@ -33,8 +31,7 @@ public:
void Done();
[[nodiscard]] int GetFrameNum() const;
[[nodiscard]] static bool IsCommandProcessorThread() noexcept;
[[nodiscard]] static CommandProcessor* CurrentCommandProcessor() noexcept;
[[nodiscard]] static bool IsGpuThread() noexcept;
private:
std::unique_ptr<GpuState> m_state;
src/graphics/host_gpu/memoryTracker.h
+17 -33
View File
@@ -29,45 +29,29 @@ public:
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UntrackMemory(uint64_t vaddr, uint64_t size);
// Removes protection from a range and flushes GPU-owned data when required.
template <typename Flush>
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
static_assert(std::is_invocable_v<Flush&>);
CheckNotInUploadCallback();
ValidateRange(vaddr, size);
const auto update_cpu_state = [this, vaddr, size] {
std::lock_guard access(m_access_mutex);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
});
std::vector<std::unique_lock<TrackingSpinLock>> locks;
locks.reserve(managers.size());
for (auto* manager: managers) {
locks.emplace_back(manager->lock);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const bool should_flush = [&] {
// Perform both the GPU modification check and CPU state change with the lock in
// case the GPU thread is racing to mark the page modified. If a flush is needed,
// on_flush performs the CPU state change.
std::scoped_lock lock(manager->lock);
if (manager->IsModified<DirtySource::Gpu>(offset, bytes)) {
return true;
}
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
return false;
}();
if (should_flush) {
on_flush();
}
const bool gpu_modified = Iterate<false>(
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
});
if (gpu_modified) {
return true;
}
Iterate<false>(vaddr, size,
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
manager->ChangeState<DirtySource::Cpu, true>(
manager->GetCpuAddr() + offset, bytes);
});
return false;
};
if (!update_cpu_state()) {
return;
}
std::forward<Flush>(on_flush)();
if (update_cpu_state()) {
EXIT("memory invalidation retained GPU-owned pages\n");
}
});
}
#if KYTY_BUILD == KYTY_BUILD_DEBUG
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+37 -5
View File
@@ -3,15 +3,18 @@
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/profiler.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "kernel/memory.h"
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cstring>
#include <utility>
#include <vector>
@@ -288,10 +291,29 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
return;
}
m_memory_tracker.InvalidateRegion(vaddr, size,
[this, vaddr, size] { ReadMemory(vaddr, size); });
[this, vaddr, size] { ReadMemory(vaddr, size, true); });
}
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
if (Gpu::IsGpuThread()) {
ReadMemoryOnGpu(vaddr, size, is_write);
return;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported buffer readback from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported buffer readback from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
m_scheduler.Context().GetGpu().SendCommandSync(
[this, vaddr, size, is_write] { ReadMemoryOnGpu(vaddr, size, is_write); });
}
void BufferCache::ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write) {
std::vector<DownloadCopy> copies;
{
FaultSafeCacheLock lock(this, m_mutex);
@@ -322,9 +344,16 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
}
}
});
}
if (copies.empty()) {
return;
if (copies.empty()) {
if (!is_write) {
return;
}
// A preceding read fault can consume the last GPU-owned copy after this write
// invalidation has already chosen to flush. Complete the CPU ownership handoff even
// though this callback no longer has bytes to download.
m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
return;
}
}
auto downloads = RecordDownloads(copies);
m_scheduler.FinishCurrent();
@@ -336,6 +365,9 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
}
// The enumeration above covered whole dirty pages and every exact interval on them.
m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
if (is_write) {
m_memory_tracker.MarkRegionAsCpuModified(vaddr, size);
}
}
}
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+2 -1
View File
@@ -46,7 +46,7 @@ public:
KYTY_CLASS_NO_COPY(BufferCache);
void InvalidateMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written = false, bool is_read = true,
@@ -97,6 +97,7 @@ private:
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+8 -57
View File
@@ -1,7 +1,6 @@
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
#include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
namespace Libs::Graphics {
@@ -18,69 +17,21 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
if (!IsMapped(fault_vaddr, fault_size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported guest-memory fault from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
if (access == PageFaultAccess::Write) {
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
} else {
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
}
bool handled = false;
const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
if (access == PageFaultAccess::Write) {
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
} else {
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
}
handled = true;
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return handled;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
return handled;
return true;
}
bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
if (!IsMapped(vaddr, size)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return true;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
return true;
}
src/graphics/host_gpu/renderer/cache/resourceMutex.cpp
Vendored
+2 -48
View File
@@ -4,18 +4,10 @@
namespace Libs::Graphics {
ResourceMutex::FaultScope::FaultScope(ResourceMutex& mutex): m_mutex(mutex) {
m_resource_preowned = m_mutex.BeginFault();
}
ResourceMutex::FaultScope::~FaultScope() {
m_mutex.EndFault(m_resource_preowned);
}
ResourceMutex::~ResourceMutex() {
std::lock_guard state(m_state);
if (m_resource_owner != std::thread::id {} || m_fault_owner != std::thread::id {}) {
EXIT("ResourceMutex destroyed with an active owner or fault transaction\n");
if (m_resource_owner != std::thread::id {}) {
EXIT("ResourceMutex destroyed with an active owner\n");
}
}
@@ -26,9 +18,6 @@ void ResourceMutex::lock() {
if (m_resource_owner == current) {
EXIT("recursive resource transaction\n");
}
if (m_fault_owner == current) {
EXIT("resource transaction re-entered from a page-fault callback\n");
}
}
m_resource.lock();
std::lock_guard state(m_state);
@@ -55,39 +44,4 @@ bool ResourceMutex::IsOwnedByCurrentThread() {
return m_resource_owner == std::this_thread::get_id();
}
bool ResourceMutex::BeginFault() {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_fault_owner == current) {
EXIT("nested resource page-fault transaction\n");
}
if (m_resource_owner == current) {
m_fault_owner = current;
return true;
}
}
lock();
std::lock_guard state(m_state);
if (m_resource_owner != current || m_fault_owner != std::thread::id {}) {
EXIT("resource fault transaction acquired inconsistent ownership\n");
}
m_fault_owner = current;
return false;
}
void ResourceMutex::EndFault(bool resource_preowned) {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_fault_owner != current || m_resource_owner != current) {
EXIT("resource page-fault transaction ended with inconsistent ownership\n");
}
m_fault_owner = {};
}
if (!resource_preowned) {
unlock();
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/resourceMutex.h
Vendored
+1 -19
View File
@@ -8,21 +8,9 @@
namespace Libs::Graphics {
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
// command-processor faults drain pending guest processors before entering this transaction.
// Owner-tracked shared buffer/image transaction.
class ResourceMutex final {
public:
class FaultScope final {
public:
explicit FaultScope(ResourceMutex& mutex);
~FaultScope();
KYTY_CLASS_NO_COPY(FaultScope);
private:
ResourceMutex& m_mutex;
bool m_resource_preowned = false;
};
ResourceMutex() = default;
~ResourceMutex();
KYTY_CLASS_NO_COPY(ResourceMutex);
@@ -32,15 +20,9 @@ public:
[[nodiscard]] bool IsOwnedByCurrentThread();
private:
friend class FaultScope;
[[nodiscard]] bool BeginFault();
void EndFault(bool resource_preowned);
std::mutex m_resource;
std::mutex m_state;
std::thread::id m_resource_owner;
std::thread::id m_fault_owner;
};
} // namespace Libs::Graphics