raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 10:56:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

remove readback hacks, allows to read/write guest memory directly

Browse Source
This commit is contained in:
nmzik
2026-07-17 05:16:00 +02:00
parent 013e9a4485
commit e903c93848
27 changed files with 522 additions and 1228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/bufferCache.cpp
+13 -297
View File
@@ -16,7 +16,6 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <memory>
@@ -37,27 +36,6 @@ constexpr uint64_t AlignReadbackCopySize(uint64_t size) noexcept {
return (size + READBACK_COPY_ALIGNMENT - 1) & ~(READBACK_COPY_ALIGNMENT - 1);
}
constexpr bool ShouldLogCommandProcessorReadback(uint64_t serial) noexcept {
return serial <= 32 || (serial % 256) == 0;
}
struct CommandProcessorReadbackState {
BufferCache* cache = nullptr;
PageFaultAccess access = PageFaultAccess::Unknown;
uint64_t vaddr = 0;
uint64_t size = 0;
uint64_t page = 0;
uint64_t serial = 0;
uint64_t dirty_bytes = 0;
uint64_t packed_bytes = 0;
uint32_t dirty_count = 0;
std::chrono::steady_clock::time_point started {};
bool installed = false;
};
thread_local CommandProcessorReadbackState g_command_processor_readback;
std::atomic<uint64_t> g_command_processor_readback_serial {0};
struct SharedVulkanBufferOwner {
explicit SharedVulkanBufferOwner(GraphicContext* context): ctx(context) {
EXIT_IF(ctx == nullptr);
@@ -192,10 +170,6 @@ bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept
return queue < 64 && (queue_mask & ~(uint64_t {1} << queue)) == 0;
}
bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept {
return queue < 64 && queue_mask == (uint64_t {1} << queue);
}
struct BufferCache::CachedBuffer {
uint64_t vaddr = 0;
uint64_t size = 0;
@@ -204,14 +178,6 @@ struct BufferCache::CachedBuffer {
std::shared_ptr<VulkanBuffer> buffer;
};
struct BufferCache::CommandProcessorReadbackResources {
GraphicContext* ctx = nullptr;
int queue = -1;
VulkanBuffer buffer {};
void* mapped = nullptr;
std::unique_ptr<CommandBuffer> command;
};
struct BufferCache::ReadbackWorker {
static constexpr uint32_t MAX_RANGES = READBACK_MAX_RANGES;
enum class State : uint32_t {
@@ -305,20 +271,20 @@ struct BufferCache::ReadbackWorker {
}
void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept {
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld();
const bool command_thread = GraphicsRunIsCommandProcessorThread();
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread;
const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now;
if (GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() ||
g_cache_lock_owner != nullptr || ctx == nullptr || command == nullptr ||
mapped == nullptr || readback.buffer == nullptr) {
if (unsafe_gpu_lock || LabelInCallback() || g_cache_lock_owner != nullptr ||
ctx == nullptr || command == nullptr || mapped == nullptr ||
readback.buffer == nullptr) {
EXIT("BufferCache: unsafe readback request context, command_thread=%d "
"submission_lock=%d "
"gpu_lock=%d label_callback=%d cache_lock=%p ctx=%p command=%p mapped=%p "
"buffer=%p\n",
GraphicsRunIsCommandProcessorThread(), GraphicsRunSubmissionLockHeld(),
GraphicsRunGpuLockHeld(), LabelInCallback(),
static_cast<const void*>(g_cache_lock_owner), static_cast<const void*>(ctx),
static_cast<const void*>(command.get()), static_cast<const void*>(mapped),
static_cast<const void*>(readback.buffer));
command_thread, GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(),
LabelInCallback(), static_cast<const void*>(g_cache_lock_owner),
static_cast<const void*>(ctx), static_cast<const void*>(command.get()),
static_cast<const void*>(mapped), static_cast<const void*>(readback.buffer));
}
State expected = State::Idle;
while (!state.compare_exchange_weak(expected, State::Claimed, std::memory_order_acq_rel)) {
@@ -584,18 +550,6 @@ BufferCache::BufferCache(PageManager& page_manager, ResourceMutex& resource_mute
BufferCache::~BufferCache() {
m_readback.reset();
if (m_cp_readback_resources != nullptr) {
auto& resources = *m_cp_readback_resources;
resources.command.reset();
if (resources.mapped != nullptr) {
VulkanUnmapMemory(resources.ctx, &resources.buffer.memory);
resources.mapped = nullptr;
}
if (resources.buffer.buffer != nullptr) {
VulkanDeleteBuffer(resources.ctx, &resources.buffer);
}
m_cp_readback_resources.reset();
}
if (!m_gpu_modified_ranges.Empty()) {
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
}
@@ -623,14 +577,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6
switch (phase) {
case PageFaultPhase::Invalidate: break;
case PageFaultPhase::Complete:
return CompleteCommandProcessorReadback(access, vaddr, size) ||
m_readback->Complete(access, vaddr, size) ||
return m_readback->Complete(access, vaddr, size) ||
m_memory_tracker.CompleteCpuFault(vaddr, size, access, false);
case PageFaultPhase::Release:
if (!ReleaseCommandProcessorReadback(access, vaddr, size)) {
m_readback->Release(access, vaddr, size);
}
return true;
case PageFaultPhase::Release: m_readback->Release(access, vaddr, size); return true;
default:
EXIT("BufferCache: unsupported page-fault phase %u\n", static_cast<uint32_t>(phase));
}
@@ -639,242 +588,9 @@ bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint6
return action == CpuFaultAction::Continue;
}
if (GraphicsRunIsCommandProcessorThread()) {
if (!GraphicsRunIsCommandProcessorGuestAccess()) {
EXIT("BufferCache: command-processor fault outside a coherent guest-memory access, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
RequestCommandProcessorReadback(access, vaddr, size);
} else {
m_readback->Request(access, vaddr, size);
}
return true;
}
void BufferCache::RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) {
if (!GraphicsRunIsCommandProcessorThread() || g_command_processor_readback.cache != nullptr) {
EXIT("BufferCache: invalid nested command-processor readback\n");
}
const auto serial =
g_command_processor_readback_serial.fetch_add(1, std::memory_order_relaxed) + 1;
if (serial == 0) {
EXIT("BufferCache: command-processor readback serial overflow\n");
}
const bool log_detail = ShouldLogCommandProcessorReadback(serial);
if (log_detail) {
LOGF("BufferCache: command-processor readback request serial=%" PRIu64 " addr=0x%016" PRIx64
" size=0x%016" PRIx64 " access=%u poll=%d\n",
serial, vaddr, size, static_cast<uint32_t>(access), GraphicsRunIsGuestMemoryPoll());
Log::Flush();
}
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto queue = static_cast<uint32_t>(GraphicsRunBeginCommandProcessorReadback());
if (log_detail) {
LOGF("BufferCache: command-processor queue synchronized serial=%" PRIu64
" queue=%u page=0x%016" PRIx64 "\n",
serial, queue, page);
Log::Flush();
}
if (g_cache_lock_owner != nullptr) {
EXIT("BufferCache: command-processor readback entered with a cache lock\n");
}
g_cache_lock_owner = this;
m_mutex.Lock();
if (log_detail) {
LOGF("BufferCache: command-processor cache locked serial=%" PRIu64 " page=0x%016" PRIx64
"\n",
serial, page);
Log::Flush();
}
auto it = m_buffers.upper_bound(vaddr);
if (it == m_buffers.begin()) {
EXIT("BufferCache: command-processor readback address has no cached buffer, "
"addr=0x%016" PRIx64 "\n",
vaddr);
}
--it;
auto& cached = *it->second;
if (vaddr < cached.vaddr || vaddr >= cached.vaddr + cached.size || page < cached.vaddr ||
TRACKER_PAGE_SIZE > cached.size - (page - cached.vaddr) || cached.ctx == nullptr ||
cached.buffer == nullptr || cached.buffer->buffer == nullptr) {
EXIT("BufferCache: invalid command-processor readback owner, fault=0x%016" PRIx64
" page=0x%016" PRIx64 " buffer=0x%016" PRIx64 "+0x%016" PRIx64 "\n",
vaddr, page, cached.vaddr, cached.size);
}
if (!CanReadbackBufferCacheQueueMask(cached.queue_mask, queue)) {
EXIT("BufferCache: command-processor readback cannot synchronize cross-queue ownership, "
"addr=0x%016" PRIx64 " used_queues=0x%016" PRIx64 " current_queue=%u\n",
vaddr, cached.queue_mask, queue);
}
const auto dirty = m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE);
if (dirty.empty()) {
EXIT("BufferCache: command-processor GPU-dirty page has no dirty byte ranges, "
"page=0x%016" PRIx64 "\n",
page);
}
ValidateDirtyPages(m_gpu_modified_ranges, page, TRACKER_PAGE_SIZE, "command-processor");
if (m_cp_readback_resources == nullptr) {
auto resources = std::make_unique<CommandProcessorReadbackResources>();
resources->ctx = cached.ctx;
resources->buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
resources->buffer.memory.property = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
VulkanCreateBuffer(cached.ctx, READBACK_CAPACITY, &resources->buffer);
VulkanMapMemory(cached.ctx, &resources->buffer.memory, &resources->mapped);
m_cp_readback_resources = std::move(resources);
}
auto& resources = *m_cp_readback_resources;
if (resources.ctx != cached.ctx || resources.mapped == nullptr ||
resources.buffer.buffer == nullptr) {
EXIT("BufferCache: invalid command-processor readback resources\n");
}
if (queue >= GraphicContext::QUEUES_NUM || cached.ctx->queues[queue].family == UINT32_MAX ||
cached.ctx->queues[queue].vk_queue == nullptr) {
EXIT("BufferCache: command-processor readback has invalid Vulkan queue %u\n", queue);
}
if (resources.command == nullptr || resources.queue != static_cast<int>(queue)) {
resources.command.reset();
resources.command = std::make_unique<CommandBuffer>(static_cast<int>(queue));
resources.queue = static_cast<int>(queue);
}
if (resources.command->IsInvalid() ||
resources.command->GetQueue() != static_cast<int>(queue)) {
EXIT("BufferCache: failed to initialize owning-queue readback command buffer, queue=%u\n",
queue);
}
if (dirty.size() > ReadbackWorker::MAX_RANGES) {
EXIT("BufferCache: too many command-processor dirty ranges, page=0x%016" PRIx64
" count=%" PRIu64 " limit=%u\n",
page, static_cast<uint64_t>(dirty.size()), ReadbackWorker::MAX_RANGES);
}
std::array<VkBufferCopy, ReadbackWorker::MAX_RANGES> copies {};
std::array<VkBufferMemoryBarrier, ReadbackWorker::MAX_RANGES> barriers {};
uint64_t dirty_bytes = 0;
uint64_t packed_bytes = 0;
uint32_t dirty_count = 0;
for (const auto& range: dirty) {
if ((range.address - cached.vaddr) % 4 != 0 || range.size % 4 != 0) {
EXIT("BufferCache: invalid command-processor dirty range, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " page=0x%016" PRIx64 " packed=0x%016" PRIx64 "\n",
range.address, range.size, page, dirty_bytes);
}
copies[dirty_count] = {.srcOffset = range.address - cached.vaddr,
.dstOffset = packed_bytes,
.size = range.size};
auto& barrier = barriers[dirty_count];
barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER;
barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = cached.buffer->buffer;
barrier.offset = copies[dirty_count].srcOffset;
barrier.size = range.size;
const auto slot_size = AlignReadbackCopySize(range.size);
if (packed_bytes > READBACK_CAPACITY || slot_size > READBACK_CAPACITY - packed_bytes) {
EXIT("BufferCache: aligned command-processor readback exceeds persistent capacity, "
"packed=%" PRIu64 " size=%" PRIu64 " capacity=%" PRIu64 "\n",
packed_bytes, slot_size, READBACK_CAPACITY);
}
packed_bytes += slot_size;
dirty_bytes += range.size;
dirty_count++;
}
if (dirty_count == 0 || dirty_bytes == 0) {
EXIT("BufferCache: command-processor readback produced no packed ranges\n");
}
const auto started = std::chrono::steady_clock::now();
if (log_detail) {
LOGF("BufferCache: command-processor readback begin serial=%" PRIu64
" queue=%u page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 " packed=%" PRIu64 "\n",
serial, queue, page, dirty_count, dirty_bytes, packed_bytes);
Log::Flush();
}
auto* vk_command = resources.command->GetPool()->buffers[resources.command->GetIndex()];
resources.command->Begin();
vkCmdPipelineBarrier(vk_command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, dirty_count,
barriers.data(), 0, nullptr);
vkCmdCopyBuffer(vk_command, cached.buffer->buffer, resources.buffer.buffer, dirty_count,
copies.data());
resources.command->End();
resources.command->Execute();
resources.command->WaitForFenceAndReset();
if (log_detail) {
LOGF("BufferCache: command-processor GPU copy complete serial=%" PRIu64 "\n", serial);
Log::Flush();
}
for (uint32_t i = 0; i < dirty_count; i++) {
Libs::LibKernel::Memory::WriteBacking(
dirty[i].address, static_cast<const uint8_t*>(resources.mapped) + copies[i].dstOffset,
dirty[i].size);
}
g_command_processor_readback = {.cache = this,
.access = access,
.vaddr = vaddr,
.size = size,
.page = page,
.serial = serial,
.dirty_bytes = dirty_bytes,
.packed_bytes = packed_bytes,
.dirty_count = dirty_count,
.started = started};
}
bool BufferCache::CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept {
auto& pending = g_command_processor_readback;
if (pending.cache == nullptr) {
return false;
}
if (pending.cache != this || pending.access != access || pending.vaddr != vaddr ||
pending.size != size || pending.installed) {
EXIT("BufferCache: mismatched active command-processor readback completion\n");
}
if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) {
EXIT("BufferCache: failed to complete command-processor readback, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " access=%u\n",
vaddr, size, static_cast<uint32_t>(access));
}
m_gpu_modified_ranges.Subtract(pending.page, TRACKER_PAGE_SIZE);
pending.installed = true;
if (ShouldLogCommandProcessorReadback(pending.serial)) {
LOGF("BufferCache: command-processor readback complete serial=%" PRIu64
" page=0x%016" PRIx64 " ranges=%u bytes=%" PRIu64 "\n",
pending.serial, pending.page, pending.dirty_count, pending.dirty_bytes);
}
return true;
}
bool BufferCache::ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept {
auto& pending = g_command_processor_readback;
if (pending.cache == nullptr) {
return false;
}
if (pending.cache != this || pending.access != access || pending.vaddr != vaddr ||
pending.size != size || !pending.installed) {
EXIT("BufferCache: mismatched active command-processor readback release\n");
}
const auto serial = pending.serial;
const auto page = pending.page;
const auto dirty_count = pending.dirty_count;
const auto dirty_bytes = pending.dirty_bytes;
const auto elapsed_micro = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - pending.started)
.count();
pending = {};
m_mutex.Unlock();
g_cache_lock_owner = nullptr;
GraphicsRunEndCommandProcessorReadback();
if (ShouldLogCommandProcessorReadback(serial)) {
LOGF("BufferCache: command-processor readback release serial=%" PRIu64 " page=0x%016" PRIx64
" ranges=%u bytes=%" PRIu64 " elapsed_us=%" PRIi64 "\n",
serial, page, dirty_count, dirty_bytes, static_cast<int64_t>(elapsed_micro));
GraphicsRunFinishCommandProcessors();
}
m_readback->Request(access, vaddr, size);
return true;
}
src/graphics/host_gpu/renderer/bufferCache.h
+3 -13
View File
@@ -39,7 +39,6 @@ struct BufferCacheRange {
[[nodiscard]] bool MergeOverlappingBufferCacheRange(BufferCacheRange* merged,
BufferCacheRange candidate) noexcept;
[[nodiscard]] bool CanMergeBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept;
[[nodiscard]] bool CanReadbackBufferCacheQueueMask(uint64_t queue_mask, uint32_t queue) noexcept;
class BufferCache {
public:
@@ -61,11 +60,9 @@ public:
// Emulator-owned, CPU-current scratch only. Guest ranges must use ObtainBuffer so page
// ownership is resolved before any CPU access.
[[nodiscard]] bool UploadHostData(CommandBuffer* command, GraphicContext* ctx, const void* src,
uint64_t size, uint64_t alignment,
VulkanBuffer** out_buffer, uint64_t* out_offset,
uint64_t* out_range);
[[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command,
GraphicContext* ctx);
uint64_t size, uint64_t alignment, VulkanBuffer** out_buffer,
uint64_t* out_offset, uint64_t* out_range);
[[nodiscard]] VulkanBuffer* ObtainNullBuffer(CommandBuffer* command, GraphicContext* ctx);
[[nodiscard]] BufferImageCopySource ObtainBufferForImage(uint64_t vaddr, uint64_t size);
void FillBuffer(CommandBuffer* command, GraphicContext* ctx, uint64_t vaddr, uint64_t size,
uint32_t value);
@@ -84,19 +81,12 @@ public:
private:
struct CachedBuffer;
struct ReadbackWorker;
struct CommandProcessorReadbackResources;
void RequestCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr, uint64_t size);
[[nodiscard]] bool CompleteCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept;
[[nodiscard]] bool ReleaseCommandProcessorReadback(PageFaultAccess access, uint64_t vaddr,
uint64_t size) noexcept;
Common::Mutex m_mutex;
std::vector<VulkanBuffer*> m_delete_later;
std::shared_ptr<VulkanBuffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
std::unique_ptr<ReadbackWorker> m_readback;
std::unique_ptr<CommandProcessorReadbackResources> m_cp_readback_resources;
RangeSet m_gpu_modified_ranges;
MemoryTracker m_memory_tracker;
PageManager& m_page_manager;
src/graphics/host_gpu/renderer/context.cpp
+4 -3
View File
@@ -332,7 +332,7 @@ void CommandBuffer::Execute() {
m_execute = true;
m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@@ -396,7 +396,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore signal_semaphore) {
m_execute = true;
m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@@ -464,7 +464,7 @@ void CommandBuffer::ExecuteWithSemaphore(VkSemaphore wait_semaphore,
m_execute = true;
m_fence_waited = false;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
m_submit_seq = g_command_buffer_submit_seq.fetch_add(1, std::memory_order_relaxed) + 1;
if (result != VK_SUCCESS) {
LOGF("vkQueueSubmit failed: %s (%d), queue=%d index=%u submit_seq=%" PRIu64
@@ -518,6 +518,7 @@ void CommandBuffer::WaitForFenceAndReset() {
m_fence_waited = false;
vkResetCommandBuffer(m_pool->buffers[m_index],
VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT);
m_recording_generation++;
}
m_host_stream.Reset();
if (was_executed) {
src/graphics/host_gpu/renderer/descriptors.cpp
+1 -1
View File
@@ -205,7 +205,7 @@ NativeAddressBuffer(uint64_t submit_id, CommandBuffer* command_buffer,
->GetPhysicalDeviceProperties()
.limits.maxStorageBufferRange);
uint64_t size = 0;
const auto access = HostMemoryAccess::Read;
const auto access = HostMemoryAccess::Mapped;
if (!HostMemoryQueryRange(address.binding_base, limit, access, &size)) {
EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
address.binding_base);
src/graphics/host_gpu/renderer/gpuResourceManager.cpp
+16 -37
View File
@@ -1,6 +1,7 @@
#include "graphics/host_gpu/renderer/gpuResourceManager.h"
#include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/renderer/render.h"
@@ -37,17 +38,24 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
"addr=0x%016" PRIx64 " access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
if (!GraphicsRunIsCommandProcessorGuestAccess()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
if (auto* cp = GraphicsRunCurrentCommandProcessor(); cp != nullptr) {
cp->BeginReadbackTransaction();
bool handled = false;
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handled = m_page_manager.HandleFault(access, fault_vaddr);
}
ResourceMutex::FaultScope fault(m_resource_mutex);
return m_page_manager.HandleFault(access, fault_vaddr);
cp->EndReadbackTransaction();
return handled;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
// Stop command-processor jobs before taking the shared cache transaction. External readback
// workers inherit this paused state and therefore never form resource -> submission lock inversion.
// workers inherit this paused state and therefore never form resource -> submission lock
// inversion.
GraphicsRunSubmissionLock submissions;
ResourceMutex::FaultScope fault(m_resource_mutex);
return m_page_manager.HandleFault(access, fault_vaddr);
@@ -57,35 +65,6 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
return m_page_manager.IsMapped(vaddr, size);
}
std::unique_lock<ResourceMutex> GpuResourceManager::LockCommandProcessorGuestAccess(
uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("invalid command-processor guest-memory access, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
return std::unique_lock(m_resource_mutex);
}
std::unique_lock<ResourceMutex> GpuResourceManager::LockAsynchronousGuestWrite(uint64_t vaddr,
uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("invalid asynchronous guest-memory write, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
std::unique_lock transaction(m_resource_mutex);
if (m_buffer_cache.HasPageOverlap(vaddr, size) || m_texture_cache.HasPageOverlap(vaddr, size) ||
m_texture_cache.HasMetaOverlap(vaddr, size)) {
EXIT("asynchronous guest-memory write to a cached buffer/image range is unsupported, "
"addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
return transaction;
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
m_page_manager.OnGpuMap(vaddr, size, access);
}
src/graphics/host_gpu/renderer/gpuResourceManager.h
+1 -10
View File
@@ -9,7 +9,6 @@
#include "graphics/host_gpu/renderer/textureCache.h"
#include <cstdint>
#include <mutex>
namespace Libs::Graphics {
@@ -26,14 +25,6 @@ public:
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
// Serializes synchronous command-processor guest-memory access against buffer/image ownership
// publication. Address metadata keeps this seam ready for a future page-range lock.
[[nodiscard]] std::unique_lock<ResourceMutex>
LockCommandProcessorGuestAccess(uint64_t vaddr, uint64_t size);
// Keeps an asynchronous host callback's validation and guest write in one cache transaction.
// Cached buffer/image destinations are unsupported and fail before the callback can fault.
[[nodiscard]] std::unique_lock<ResourceMutex> LockAsynchronousGuestWrite(uint64_t vaddr,
uint64_t size);
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void FillBuffer(CommandBuffer* command, uint64_t vaddr, uint64_t size, uint32_t value);
@@ -48,7 +39,7 @@ private:
PageManager m_page_manager;
ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
TextureCache m_texture_cache;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/render.h
+31 -29
View File
@@ -87,25 +87,27 @@ public:
[[nodiscard]] int GetQueue() const { return m_queue; }
VulkanCommandPool* GetPool() { return m_pool; }
[[nodiscard]] bool IsExecute() const { return m_execute; }
[[nodiscard]] uint64_t GetRecordingGeneration() const { return m_recording_generation; }
private:
friend class BufferCache;
void RecycleDescriptorsAfterFence();
VulkanCommandPool* m_pool = nullptr;
uint32_t m_index = static_cast<uint32_t>(-1);
int m_queue = -1;
bool m_execute = false;
bool m_fence_waited = false;
uint64_t m_submit_seq = 0;
uint32_t m_debug_op = 0;
uint64_t m_debug_submit_id = 0;
uint32_t m_debug_arg0 = 0;
uint32_t m_debug_arg1 = 0;
uint32_t m_debug_arg2 = 0;
uint32_t m_debug_arg3 = 0;
uint64_t m_debug_arg4 = 0;
VulkanCommandPool* m_pool = nullptr;
uint32_t m_index = static_cast<uint32_t>(-1);
int m_queue = -1;
bool m_execute = false;
bool m_fence_waited = false;
uint64_t m_submit_seq = 0;
uint64_t m_recording_generation = 0;
uint32_t m_debug_op = 0;
uint64_t m_debug_submit_id = 0;
uint32_t m_debug_arg0 = 0;
uint32_t m_debug_arg1 = 0;
uint32_t m_debug_arg2 = 0;
uint32_t m_debug_arg3 = 0;
uint64_t m_debug_arg4 = 0;
std::vector<VulkanBuffer*> m_delete_after_fence;
FenceResourceRetainer m_fence_resources;
std::vector<VulkanDescriptorSet*> m_descriptor_sets_after_fence;
@@ -129,8 +131,8 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context
void GraphicsRenderInit();
void GraphicsRenderCreateContext();
[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency,
uint64_t* value);
[[nodiscard]] bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency,
uint64_t* value);
[[nodiscard]] uint64_t GraphicsRenderReadReferenceClock();
[[nodiscard]] bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t group_x,
@@ -140,21 +142,22 @@ void GraphicsRenderCreateContext();
[[nodiscard]] bool ResolveHtileClearTarget(const HW::DepthRenderTarget& target,
uint64_t descriptor_size, HtileClearTarget* resolved);
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr);
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer,
uint64_t* dst_gpu_addr);
void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t value);
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t dw_offset,
uint32_t dw_num);
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr, uint64_t value);
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer,
uint64_t* dst_gpu_addr,
uint64_t value);
void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t value);
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
uint32_t* dst_gpu_addr, uint32_t dw_offset,
uint32_t dw_num);
uint64_t GraphicsRenderPrepareDisplayBufferFlip(CommandBuffer* buffer, int handle, int index,
int flip_mode, int64_t flip_arg);
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
uint64_t submit_id, CommandBuffer* buffer, uint32_t* dst_gpu_addr, uint32_t value, int handle,
int index, int flip_mode, int64_t flip_arg, uint64_t request_id);
void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer* buffer,
@@ -193,7 +196,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id)
void GraphicsRenderTriggerAgcUserInterrupt();
void GraphicsRenderTriggerEopEvent(uint32_t context_id);
void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value);
void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/renderCompute.cpp
+30 -24
View File
@@ -1,7 +1,3 @@
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/emulatorConfig.h"
@@ -19,7 +15,9 @@
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/framebufferCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "graphics/host_gpu/renderer/renderState.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
@@ -547,30 +545,38 @@ void RenderDispatchDirect(uint64_t submit_id, CommandBuffer* buffer, HW::Context
return;
}
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline(
&input_info, &sh_ctx->GetCs(), cs_shader);
for (;;) {
const auto recording_generation = buffer->GetRecordingGeneration();
auto* vk_buffer = buffer->GetPool()->buffers[buffer->GetIndex()];
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateComputePipeline(
&input_info, &sh_ctx->GetCs(), cs_shader);
vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
vkCmdBindPipeline(vk_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
const auto address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout,
input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute);
const auto address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline_layout,
input_info.stage, VK_SHADER_STAGE_COMPUTE_BIT, DescriptorCache::Stage::Compute);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z);
vkCmdDispatch(vk_buffer, thread_group_x, thread_group_y, thread_group_z);
bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes;
has_storage_writes =
std::any_of(program.info.images.begin(), program.info.images.end(),
[](const auto& image) {
return image.written &&
(image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage ||
image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint);
}) ||
has_storage_writes;
if (has_storage_writes) {
ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
has_storage_writes = MarkShaderAddressWrites(address_writes) || has_storage_writes;
has_storage_writes =
std::any_of(
program.info.images.begin(), program.info.images.end(),
[](const auto& image) {
return image.written &&
(image.kind == ShaderRecompiler::IR::ResourceKind::StorageImage ||
image.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint);
}) ||
has_storage_writes;
if (has_storage_writes) {
ShaderWriteBarrier(vk_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
}
break;
}
}
src/graphics/host_gpu/renderer/renderDraw.cpp
+138 -118
View File
@@ -1,7 +1,3 @@
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/emulatorConfig.h"
@@ -19,7 +15,9 @@
#include "graphics/host_gpu/renderer/descriptorCache.h"
#include "graphics/host_gpu/renderer/framebufferCache.h"
#include "graphics/host_gpu/renderer/pipelineCache.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderInternal.h"
#include "graphics/host_gpu/renderer/renderState.h"
#include "graphics/host_gpu/renderer/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/shaderSubgroup.h"
@@ -603,11 +601,12 @@ struct DrawEmitInfo {
uint32_t first_vertex = 0;
};
struct DrawIndexBufferBind {
bool enabled = false;
VulkanBuffer* buffer = nullptr;
VkDeviceSize offset = 0;
VkIndexType type = VK_INDEX_TYPE_UINT16;
struct DrawIndexBufferSource {
bool enabled = false;
uint64_t address = 0;
const void* host_data = nullptr;
uint64_t size = 0;
VkIndexType type = VK_INDEX_TYPE_UINT16;
};
static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) {
@@ -802,6 +801,32 @@ static void BindDrawVertexBuffers(uint64_t submit_id, CommandBuffer* buffer,
}
}
static void BindDrawIndexBuffer(CommandBuffer* buffer, VkCommandBuffer vk_buffer,
const DrawIndexBufferSource& source) {
if (!source.enabled) {
return;
}
EXIT_IF(source.size == 0);
VulkanBuffer* index_buffer = nullptr;
VkDeviceSize index_offset = 0;
if (source.host_data != nullptr) {
VkDeviceSize range = 0;
if (!g_render_ctx->GetBufferCache()->UploadHostData(
buffer, g_render_ctx->GetGraphicCtx(), source.host_data, source.size, 16,
&index_buffer, &index_offset, &range)) {
EXIT("failed to upload host index buffer\n");
}
} else {
auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(
buffer, g_render_ctx->GetGraphicCtx(), source.address, source.size);
index_buffer = binding.first;
index_offset = binding.second;
}
EXIT_IF(index_buffer == nullptr);
vkCmdBindIndexBuffer(vk_buffer, index_buffer->buffer, index_offset, source.type);
}
static void LogDrawStateIfNeeded(HW::Context* ctx, HW::UserConfig* ucfg, const DrawCallInfo& draw,
const DrawRenderState& state, bool always_log,
bool force_legacy_rect_log, uint32_t index_type_and_size,
@@ -898,7 +923,7 @@ static void EmitDrawPrimitives(const HW::UserConfig* ucfg, VkCommandBuffer vk_bu
static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx,
HW::UserConfig* ucfg, HW::Shader* sh_ctx, const DrawCallInfo& draw,
DrawRenderState* state, VkPrimitiveTopology topology,
const DrawEmitInfo& emit, const DrawIndexBufferBind& index_bind,
const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
bool log_pipeline_phase, bool set_bind_debug, bool set_auto_debug) {
EXIT_IF(buffer == nullptr);
EXIT_IF(ctx == nullptr);
@@ -907,86 +932,93 @@ static void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer* buffer, HW::C
EXIT_IF(draw.name == nullptr);
EXIT_IF(state == nullptr);
if (log_pipeline_phase) {
LogDrawPhase(draw.name, "CreatePipeline");
}
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline(
state->framebuffer, state->color_info, state->color_count, &state->depth_info,
&state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active,
state->vs_shader, state->ps_shader);
for (;;) {
const auto recording_generation = buffer->GetRecordingGeneration();
if (log_pipeline_phase) {
LogDrawPhase(draw.name, "CreatePipeline");
}
auto* pipeline = g_render_ctx->GetPipelineCache()->CreateGraphicsPipeline(
state->framebuffer, state->color_info, state->color_count, &state->depth_info,
&state->vs_input_info, ctx, sh_ctx, &state->ps_input_info, topology, state->ps_active,
state->vs_shader, state->ps_shader);
if (set_bind_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
}
vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
const auto dynamic_params =
BuildGraphicsDynamicParams(*ctx, state->color_info, state->color_count, state->depth_info);
SetDynamicParams(state->vk_buffer, dynamic_params);
if (set_bind_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x100u);
}
vkCmdBindPipeline(state->vk_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
const auto dynamic_params = BuildGraphicsDynamicParams(
*ctx, state->color_info, state->color_count, state->depth_info);
SetDynamicParams(state->vk_buffer, dynamic_params);
// EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1);
BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info);
// EXIT_NOT_IMPLEMENTED(vs_input_info.buffers_num > 1);
BindDrawVertexBuffers(submit_id, buffer, draw, state->vk_buffer, state->vs_input_info);
LogDrawPhase(draw.name, "BindDescriptorsVS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
}
const auto vs_address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout,
state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex);
std::vector<ShaderAddressWriteRange> ps_address_writes;
if (state->ps_active) {
LogDrawPhase(draw.name, "BindDescriptorsPS");
LogDrawPhase(draw.name, "BindDescriptorsVS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
}
ps_address_writes =
BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
pipeline->pipeline_layout, state->ps_input_info.stage,
VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel);
}
const auto vs_address_writes = BindDescriptors(
submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline_layout,
state->vs_input_info.stage, VK_SHADER_STAGE_VERTEX_BIT, DescriptorCache::Stage::Vertex);
if (index_bind.enabled) {
EXIT_NOT_IMPLEMENTED(index_bind.buffer == nullptr);
vkCmdBindIndexBuffer(state->vk_buffer, index_bind.buffer->buffer, index_bind.offset,
index_bind.type);
}
LogDrawPhase(draw.name, "BeginRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
}
buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count,
&state->depth_info);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
}
EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
}
buffer->EndRenderPass();
VkPipelineStageFlags shader_write_stages = 0;
const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage);
const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes);
if (vs_wrote_buffers || vs_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
}
if (state->ps_active) {
const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage);
const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes);
if (ps_wrote_buffers || ps_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
std::vector<ShaderAddressWriteRange> ps_address_writes;
if (state->ps_active) {
LogDrawPhase(draw.name, "BindDescriptorsPS");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
}
ps_address_writes =
BindDescriptors(submit_id, buffer, VK_PIPELINE_BIND_POINT_GRAPHICS,
pipeline->pipeline_layout, state->ps_input_info.stage,
VK_SHADER_STAGE_FRAGMENT_BIT, DescriptorCache::Stage::Pixel);
}
}
if (shader_write_stages != 0) {
ShaderWriteBarrier(state->vk_buffer, shader_write_stages);
}
LogDrawPhase(draw.name, "EndRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
// Index data may use this command buffer's host stream. Resolve it only after the other
// fault-capable bindings, and rebuild it whenever a fault reset changes the generation.
BindDrawIndexBuffer(buffer, state->vk_buffer, index_source);
if (buffer->GetRecordingGeneration() != recording_generation) {
continue;
}
LogDrawPhase(draw.name, "BeginRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x400u);
}
buffer->BeginRenderPass(state->framebuffer, state->color_info, state->color_count,
&state->depth_info);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x500u);
}
EmitDrawPrimitives(ucfg, state->vk_buffer, state->vs_input_info, draw, emit);
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x600u);
}
buffer->EndRenderPass();
VkPipelineStageFlags shader_write_stages = 0;
const bool vs_wrote_buffers = HasShaderBufferWrites(state->vs_input_info.stage);
const bool vs_wrote_addresses = MarkShaderAddressWrites(vs_address_writes);
if (vs_wrote_buffers || vs_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
}
if (state->ps_active) {
const bool ps_wrote_buffers = HasShaderBufferWrites(state->ps_input_info.stage);
const bool ps_wrote_addresses = MarkShaderAddressWrites(ps_address_writes);
if (ps_wrote_buffers || ps_wrote_addresses) {
shader_write_stages |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
}
}
if (shader_write_stages != 0) {
ShaderWriteBarrier(state->vk_buffer, shader_write_stages);
}
LogDrawPhase(draw.name, "EndRenderPass");
if (set_auto_debug) {
SetDrawDebugPhase(buffer, submit_id, draw, 0x700u);
}
break;
}
}
@@ -1068,7 +1100,7 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
index_type = VK_INDEX_TYPE_UINT32;
index_size = 4 * static_cast<uint64_t>(index_count);
break;
// Some games use it
// Some games use it - need vulkan extension
case Prospero::IndexType::kIndex8:
index_type = VK_INDEX_TYPE_UINT16;
index_size = static_cast<uint64_t>(index_count);
@@ -1086,6 +1118,25 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
const DrawCallInfo draw {"DrawIndex", CommandBufferDebugOp::DrawIndex,
index_count, flags,
instance_count, first_instance};
std::vector<uint16_t> expanded_indices;
if (expand_index8_to_u16) {
EXIT_NOT_IMPLEMENTED(index_addr == nullptr);
const auto* src = static_cast<const uint8_t*>(index_addr);
expanded_indices.resize(index_count);
for (uint32_t i = 0; i < index_count; i++) {
expanded_indices[i] = src[i];
}
}
DrawIndexBufferSource index_source {};
index_source.enabled = true;
index_source.address = reinterpret_cast<uint64_t>(index_addr);
index_source.host_data =
expanded_indices.empty() ? nullptr : static_cast<const void*>(expanded_indices.data());
index_source.size = expanded_indices.empty()
? index_size
: expanded_indices.size() * sizeof(uint16_t);
index_source.type = index_type;
DrawRenderState state {};
if (!PrepareDrawRenderState(submit_id, buffer, ctx, ucfg, sh_ctx, draw,
@@ -1100,43 +1151,12 @@ void RenderDrawIndex(uint64_t submit_id, CommandBuffer* buffer, HW::Context* ctx
const auto vertex_offset =
ResolveVertexOffset(ucfg->GetIndexOffset(), state.vs_input_info) + vertex_offset_add;
VkDeviceSize index_offset = 0;
VulkanBuffer* indices = nullptr;
if (expand_index8_to_u16) {
EXIT_NOT_IMPLEMENTED(index_addr == nullptr);
const auto* src = static_cast<const uint8_t*>(index_addr);
std::vector<uint16_t> expanded_indices(index_count);
for (uint32_t i = 0; i < index_count; i++) {
expanded_indices[i] = src[i];
}
VkDeviceSize range = 0;
if (!g_render_ctx->GetBufferCache()->UploadHostData(
buffer, g_render_ctx->GetGraphicCtx(), expanded_indices.data(),
expanded_indices.size() * sizeof(uint16_t), 16, &indices, &index_offset, &range)) {
EXIT("failed to upload expanded 8-bit index buffer\n");
}
} else {
auto binding = g_render_ctx->GetBufferCache()->ObtainBuffer(
buffer, g_render_ctx->GetGraphicCtx(), reinterpret_cast<uint64_t>(index_addr),
index_size);
indices = binding.first;
index_offset = binding.second;
}
EXIT_NOT_IMPLEMENTED(indices == nullptr);
DrawIndexBufferBind index_bind {};
index_bind.enabled = true;
index_bind.buffer = indices;
index_bind.offset = index_offset;
index_bind.type = index_type;
DrawEmitInfo emit {};
emit.indexed = true;
emit.vertex_offset = vertex_offset;
ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit,
index_bind, true, true, false);
index_source, true, true, false);
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
@@ -1243,9 +1263,9 @@ void RenderDrawIndexAuto(uint64_t submit_id, CommandBuffer* buffer, HW::Context*
emit.draw_vertex_count = draw_vertex_count;
emit.first_vertex = static_cast<uint32_t>(vertex_offset);
DrawIndexBufferBind index_bind {};
DrawIndexBufferSource index_source {};
ExecutePreparedDraw(submit_id, buffer, ctx, ucfg, sh_ctx, draw, &state, topology, emit,
index_bind, false, false, true);
index_source, false, false, true);
}
bool IsSameColorResolveSubresource(const RenderColorInfo& src, const RenderColorInfo& dst) {
src/graphics/host_gpu/renderer/resourceMutex.h
+4 -4
View File
@@ -8,8 +8,8 @@
namespace Libs::Graphics {
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions before
// taking this mutex; synchronous command-processor accesses reuse their already-owned transaction.
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
// command-processor faults drain their queue before entering this transaction.
class ResourceMutex final {
public:
class FaultScope final {
@@ -27,8 +27,8 @@ public:
~ResourceMutex();
KYTY_CLASS_NO_COPY(ResourceMutex);
void lock();
void unlock();
void lock();
void unlock();
[[nodiscard]] bool IsOwnedByCurrentThread();
private:
src/graphics/host_gpu/renderer/sync.cpp
+30 -105
View File
@@ -13,31 +13,29 @@
#include "kernel/pthread.h"
#include "libs/errno.h"
#include <cstring>
#include <limits>
namespace Libs::Graphics {
constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
constexpr int GRAPHICS_EVENT_EOP = 0x40;
constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
constexpr int GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
constexpr int GRAPHICS_EVENT_EOP = 0x40;
constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
bool GraphicsScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t* value) {
if (host_frequency == 0 || value == nullptr) {
return false;
}
const auto whole_seconds = host_ticks / host_frequency;
const auto remainder = host_ticks % host_frequency;
constexpr auto MAX_VALUE = std::numeric_limits<uint64_t>::max();
const auto whole_seconds = host_ticks / host_frequency;
const auto remainder = host_ticks % host_frequency;
constexpr auto MAX_VALUE = std::numeric_limits<uint64_t>::max();
if (whole_seconds > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY ||
remainder > MAX_VALUE / GRAPHICS_REFERENCE_CLOCK_FREQUENCY) {
return false;
}
const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY;
const auto fractional_value =
(remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency;
const auto whole_value = whole_seconds * GRAPHICS_REFERENCE_CLOCK_FREQUENCY;
const auto fractional_value = (remainder * GRAPHICS_REFERENCE_CLOCK_FREQUENCY) / host_frequency;
if (whole_value > MAX_VALUE - fractional_value) {
return false;
}
@@ -50,40 +48,19 @@ uint64_t GraphicsRenderReadReferenceClock() {
const auto host_ticks = LibKernel::KernelReadTsc();
uint64_t value = 0;
if (!GraphicsScaleReferenceClock(host_ticks, host_frequency, &value)) {
EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n",
host_ticks, host_frequency);
EXIT("cannot scale host clock, ticks=0x%016" PRIx64 " frequency=%" PRIu64 "\n", host_ticks,
host_frequency);
}
return value;
}
static void SubmitLabel32(CommandBuffer* buffer, uint32_t* dst, uint32_t value,
LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr,
const uint64_t* args = nullptr) {
auto* label =
LabelCreate32(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args);
static void SubmitLabel(CommandBuffer* buffer, LabelCallback callback_1 = nullptr,
LabelCallback callback_2 = nullptr, const uint64_t* args = nullptr) {
auto* label = LabelCreate(g_render_ctx->GetGraphicCtx(), callback_1, callback_2, args);
LabelSet(buffer, label);
LabelDelete(label);
}
static void SubmitLabel64(CommandBuffer* buffer, uint64_t* dst, uint64_t value,
LabelCallback callback_1 = nullptr, LabelCallback callback_2 = nullptr,
const uint64_t* args = nullptr) {
auto* label =
LabelCreate64(g_render_ctx->GetGraphicCtx(), dst, value, callback_1, callback_2, args);
LabelSet(buffer, label);
LabelDelete(label);
}
template <typename T>
static void PublishImmediateFence(T* dst, T value) {
static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
EXIT_IF(dst == nullptr);
// EOP/EOS fence payloads are constant and published while parsing the packet. Kyty performs the
// direct mapped write under CommandProcessorGuestAccessScope: a protected access is first resolved by
// BufferCache inline, then this instruction retries so the EOP payload wins over prior GPU bytes.
std::memcpy(dst, &value, sizeof(value));
}
static bool CompleteDisplayBufferFlip(const uint64_t* args) {
if (g_render_ctx == nullptr || args == nullptr) {
EXIT("GPU flip completion has invalid state, render_ctx=%p args=%p\n",
@@ -114,8 +91,6 @@ void GraphicsRenderWriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer* buffer,
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 4, value,
0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buffer,
@@ -128,28 +103,6 @@ void GraphicsRenderWriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer* buff
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id,
dw_offset, dw_num, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(dw_offset),
static_cast<uint64_t>(dw_num),
reinterpret_cast<uint64_t>(dst_gpu_addr), 0};
SubmitLabel32(
buffer, dst_gpu_addr, 0,
[](const uint64_t* args) {
auto dw_offset = static_cast<uint32_t>(args[0]);
auto dw_num = static_cast<uint32_t>(args[1]);
auto* dst_gpu_addr = reinterpret_cast<uint32_t*>(args[2]);
if (dst_gpu_addr == nullptr || dw_num == 0) {
EXIT("invalid asynchronous EOP GDS write\n");
}
const auto size = static_cast<uint64_t>(dw_num) * sizeof(uint32_t);
auto transaction = g_render_ctx->GetGpuResources()->LockAsynchronousGuestWrite(
reinterpret_cast<uint64_t>(dst_gpu_addr), size);
g_render_ctx->GetGdsBuffer()->Read(g_render_ctx->GetGraphicCtx(), dst_gpu_addr,
dw_offset, dw_num);
return false;
},
nullptr, args);
}
void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
@@ -162,12 +115,10 @@ void GraphicsRenderWriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer* buffer,
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8,
static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0,
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer* buffer,
uint64_t* dst_gpu_addr) {
uint64_t* dst_gpu_addr, uint64_t value) {
EXIT_IF(g_render_ctx == nullptr);
EXIT_IF(dst_gpu_addr == nullptr);
EXIT_IF(buffer == nullptr);
@@ -176,8 +127,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWrite), submit_id, 8, 0, 0,
0, reinterpret_cast<uint64_t>(dst_gpu_addr));
const auto value = GraphicsRenderReadReferenceClock();
PublishImmediateFence(dst_gpu_addr, value);
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
@@ -185,7 +134,8 @@ void GraphicsRenderWriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffe
void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id,
CommandBuffer* buffer,
uint64_t* dst_gpu_addr) {
uint64_t* dst_gpu_addr,
uint64_t value) {
EXIT_IF(g_render_ctx == nullptr);
EXIT_IF(dst_gpu_addr == nullptr);
EXIT_IF(buffer == nullptr);
@@ -194,8 +144,6 @@ void GraphicsRenderWriteAtEndOfPipeClockCounterWithWriteBack(uint64_t subm
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8, 0,
0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
const auto value = GraphicsRenderReadReferenceClock();
PublishImmediateFence(dst_gpu_addr, value);
LOGF_COLOR(Log::Color::BrightGreen,
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
@@ -211,8 +159,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 8,
static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u), 0,
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer* buffer,
@@ -224,8 +170,6 @@ void GraphicsRenderWriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4,
value, 0, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
}
void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id,
@@ -241,12 +185,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submi
context_id, static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u),
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel64(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -266,12 +208,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submi
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBack), submit_id, 4,
context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel32(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -292,12 +232,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBu
context_id, static_cast<uint32_t>(value),
static_cast<uint32_t>(value >> 32u),
reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel64(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -316,12 +254,10 @@ void GraphicsRenderWriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBu
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopInterrupt), submit_id, 4,
context_id, value, 0, reinterpret_cast<uint64_t>(dst_gpu_addr));
PublishImmediateFence(dst_gpu_addr, value);
uint64_t args[LABEL_ARGS_MAX] = {context_id != 0 ? context_id : value};
SubmitLabel32(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -360,11 +296,9 @@ void GraphicsRenderWriteAtEndOfPipeWithInterruptWriteBackFlip32(
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopWriteBackFlip), submit_id,
static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg));
PublishImmediateFence(dst_gpu_addr, value);
const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(
buffer, nullptr, 0, CompleteDisplayBufferFlip,
SubmitLabel(
buffer, CompleteDisplayBufferFlip,
[](const uint64_t* /*args*/) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(0);
@@ -385,10 +319,8 @@ void GraphicsRenderWriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer*
buffer->SetDebugInfo(static_cast<uint32_t>(CommandBufferDebugOp::EopFlip), submit_id,
static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
static_cast<uint32_t>(flip_mode), value, static_cast<uint64_t>(flip_arg));
PublishImmediateFence(dst_gpu_addr, value);
const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args);
SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args);
}
void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* buffer, int handle,
@@ -403,7 +335,7 @@ void GraphicsRenderWriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer* b
static_cast<uint32_t>(flip_mode), 0, static_cast<uint64_t>(flip_arg));
const uint64_t args[LABEL_ARGS_MAX] = {request_id};
SubmitLabel32(buffer, nullptr, 0, CompleteDisplayBufferFlip, nullptr, args);
SubmitLabel(buffer, CompleteDisplayBufferFlip, nullptr, args);
}
void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t context_id) {
@@ -413,8 +345,8 @@ void GraphicsRenderTriggerEopEventAtEndOfPipe(CommandBuffer* buffer, uint32_t co
uint64_t args[LABEL_ARGS_MAX] = {static_cast<uint64_t>(context_id)};
SubmitLabel32(
buffer, nullptr, 0, nullptr,
SubmitLabel(
buffer, nullptr,
[](const uint64_t* args) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->TriggerEopEvent(static_cast<uint32_t>(args[0]));
@@ -489,13 +421,6 @@ int GraphicsRenderDeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id)
return result;
}
void GraphicsRenderClearGds(uint64_t dw_offset, uint32_t dw_num, uint32_t clear_value) {
EXIT_IF(g_render_ctx == nullptr);
g_render_ctx->GetGdsBuffer()->Clear(g_render_ctx->GetGraphicCtx(), dw_offset, dw_num,
clear_value);
}
void GraphicsRenderReadGds(uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
EXIT_IF(g_render_ctx == nullptr);
src/graphics/host_gpu/renderer/textureCache.cpp
+25 -41
View File
@@ -8,7 +8,6 @@
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/guest_gpu/tile.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/host_gpu/objects/label.h"
#include "graphics/host_gpu/objects/textureCommon.h"
#include "graphics/host_gpu/renderer/bufferCache.h"
@@ -232,14 +231,15 @@ struct TextureCache::ReadbackWorker {
}
void Request(PageFaultAccess fault_access, uint64_t fault_vaddr, uint64_t fault_size) noexcept {
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld();
const bool command_thread = GraphicsRunIsCommandProcessorThread();
const bool submissions_prepaused_now = GraphicsRunSubmissionLockHeld() || command_thread;
const bool unsafe_gpu_lock = GraphicsRunGpuLockHeld() && !submissions_prepaused_now;
if ((fault_access != PageFaultAccess::Read && fault_access != PageFaultAccess::Write) ||
GraphicsRunIsCommandProcessorThread() || unsafe_gpu_lock || LabelInCallback() ||
g_texture_cache_lock_owner != nullptr || g_texture_fault_owner != &cache) {
unsafe_gpu_lock || LabelInCallback() || g_texture_cache_lock_owner != nullptr ||
g_texture_fault_owner != &cache) {
EXIT("TextureCache: unsafe image readback request, access=%u command_thread=%d "
"submission_lock=%d gpu_lock=%d label_callback=%d cache_lock=%p fault_owner=%p\n",
static_cast<uint32_t>(fault_access), GraphicsRunIsCommandProcessorThread(),
static_cast<uint32_t>(fault_access), command_thread,
GraphicsRunSubmissionLockHeld(), GraphicsRunGpuLockHeld(), LabelInCallback(),
g_texture_cache_lock_owner, g_texture_fault_owner);
}
@@ -1557,13 +1557,12 @@ VkImageView TextureCache::GetRenderTargetStorageView(GraphicContext* c
VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext* ctx,
StorageTextureVulkanImage* image,
const ImageInfo& info) {
const auto shape = SelectStorageSampledViewShape(
info.type, info.depth, image != nullptr ? image->layers : 0);
const auto shape =
SelectStorageSampledViewShape(info.type, info.depth, image != nullptr ? image->layers : 0);
if (ctx == nullptr || image == nullptr || image->image == nullptr ||
shape == StorageSampledViewShape::Unsupported ||
info.base_array != 0 || info.levels != image->mip_levels ||
info.base_level >= info.levels || info.view_levels == 0 ||
info.base_level + info.view_levels > info.levels) {
shape == StorageSampledViewShape::Unsupported || info.base_array != 0 ||
info.levels != image->mip_levels || info.base_level >= info.levels ||
info.view_levels == 0 || info.base_level + info.view_levels > info.levels) {
EXIT("TextureCache: invalid sampled view of storage texture, image=%p type=%u depth=%u"
" base=%u levels=%u view_levels=%u image_levels=%u base_array=%u\n",
static_cast<const void*>(image), info.type, info.depth, info.base_level, info.levels,
@@ -1589,23 +1588,21 @@ VkImageView TextureCache::GetStorageTextureSampledView(GraphicContext*
usage.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO;
usage.usage = VK_IMAGE_USAGE_SAMPLED_BIT;
VkImageViewCreateInfo create {};
create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
create.pNext = &usage;
create.image = image->image;
create.viewType = shape == StorageSampledViewShape::Image3D
? VK_IMAGE_VIEW_TYPE_3D
: shape == StorageSampledViewShape::Image2DArray
? VK_IMAGE_VIEW_TYPE_2D_ARRAY
: VK_IMAGE_VIEW_TYPE_2D;
create.format = view_format;
create.components = TextureGetComponentMapping(info.swizzle);
create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
create.subresourceRange.baseMipLevel = info.base_level;
create.subresourceRange.levelCount = info.view_levels;
create.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
create.pNext = &usage;
create.image = image->image;
create.viewType = shape == StorageSampledViewShape::Image3D ? VK_IMAGE_VIEW_TYPE_3D
: shape == StorageSampledViewShape::Image2DArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY
: VK_IMAGE_VIEW_TYPE_2D;
create.format = view_format;
create.components = TextureGetComponentMapping(info.swizzle);
create.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
create.subresourceRange.baseMipLevel = info.base_level;
create.subresourceRange.levelCount = info.view_levels;
create.subresourceRange.baseArrayLayer = 0;
create.subresourceRange.layerCount =
shape == StorageSampledViewShape::Image2DArray ? info.depth : 1;
VkImageView view = nullptr;
VkImageView view = nullptr;
const auto result = vkCreateImageView(ctx->device, &create, nullptr, &view);
if (result != VK_SUCCESS || view == nullptr) {
EXIT("TextureCache: failed to create sampled view of storage texture, result=%d"
@@ -1926,15 +1923,6 @@ VulkanImage* TextureCache::FindTexture(CommandBuffer* command, GraphicContext* c
cached->gpu_modified, cached->ctx == ctx);
}
}
uint64_t readable = 0;
if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) {
EXIT("TextureCache: new sampled image has unreadable guest backing, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " readable=0x%016" PRIx64
" format=%u extent=%ux%ux%u pitch=%u levels=%u tile=%u type=%u\n",
info.address, info.size, readable, info.format, info.width, info.height, info.depth,
info.pitch, info.levels, info.tile, info.type);
}
m_images.reserve(m_images.size() + 1);
auto cached = std::make_shared<CachedImage>();
cached->kind = CachedImage::Kind::Texture;
@@ -2057,13 +2045,6 @@ StorageTextureVulkanImage* TextureCache::FindStorageTexture(CommandBuffer* com
}
RetireStoragePageNeighbors(ctx, info);
uint64_t readable = 0;
if (!HostMemoryQueryReadable(info.address, info.size, &readable) || readable < info.size) {
EXIT("TextureCache: new storage image has unreadable guest backing, addr=0x%016" PRIx64
" size=0x%016" PRIx64 " readable=0x%016" PRIx64 "\n",
info.address, info.size, readable);
}
m_images.reserve(m_images.size() + 1);
auto cached = std::make_shared<CachedImage>();
cached->kind = CachedImage::Kind::StorageTexture;
@@ -3639,6 +3620,9 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint
}
if (needs_readback) {
if (GraphicsRunIsCommandProcessorThread()) {
GraphicsRunFinishCommandProcessors();
}
m_readback->Request(access, vaddr, size);
action = m_memory_tracker.BeginCpuFault(vaddr, size, access);
if (action != CpuFaultAction::Download) {