raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 19:07:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(graphics): split renderer and recompiler into focused modules

Browse Source
This commit is contained in:
nmzik
2026-07-27 22:08:04 +02:00
parent 2b9cba457e
commit a6b61d7aa0
108 changed files with 245 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+1243
View File
File diff suppressed because it is too large Load Diff
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+145
View File
@@ -0,0 +1,145 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BUFFERCACHE_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BUFFERCACHE_H_
#include "common/abi.h"
#include "common/common.h"
#include "common/threads.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include <map>
#include <memory>
#include <mutex>
#include <span>
#include <utility>
#include <vector>
namespace Libs::Graphics {
struct GraphicContext;
class CommandBuffer;
class CommandScheduler;
class TextureCache;
class ResourceMutex;
struct BufferBinding {
std::shared_ptr<void> owner;
vk::Buffer buffer = nullptr;
uint64_t offset = 0;
};
struct ImageBufferSource {
Buffer* buffer = nullptr;
uint64_t offset = 0;
bool gpu_owned = false;
};
class BufferCache {
public:
static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
static constexpr uint64_t GetBufferOffset(uint64_t vaddr) {
return vaddr & (CACHING_PAGE_SIZE - 1);
}
BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
TextureCache& texture_cache, ResourceMutex& resource_mutex);
~BufferCache();
KYTY_CLASS_NO_COPY(BufferCache);
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
bool is_written = false, bool is_read = true,
bool is_formatted = false);
[[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept;
[[nodiscard]] Buffer& GetGdsBuffer() noexcept { return m_gds_buffer; }
[[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; }
[[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size,
uint64_t alignment);
[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
[[nodiscard]] ImageBufferSource ObtainBufferForImage(uint64_t vaddr, uint64_t size);
[[nodiscard]] std::pair<std::shared_ptr<Buffer>, uint64_t>
ObtainBufferForImageWrite(uint64_t vaddr, uint64_t size);
void DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
bool src_gds = false);
[[nodiscard]] bool HasPageOverlap(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
void InvalidateImageAliases(uint64_t vaddr, uint64_t size);
void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
void RunGarbageCollector();
private:
friend struct BufferCacheTestAccess;
struct CacheRange {
uint64_t address = 0;
uint64_t size = 0;
};
struct CachedBuffer;
struct DownloadCopy;
struct DownloadRange;
struct RetiredBuffer;
struct FaultReadback;
struct PendingBackingPublication;
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
}
[[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right,
uint64_t right_size) noexcept;
[[nodiscard]] static std::pair<uint64_t, uint64_t>
DownloadEnvelope(const DownloadCopy& copy);
[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
const void* source, uint64_t size);
[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
uint64_t size);
[[nodiscard]] std::vector<DownloadRange>
RecordDownloads(std::span<const DownloadCopy> copies);
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
uint64_t size, bool upload);
void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation);
void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
Buffer m_gds_buffer;
Common::Mutex m_mutex;
std::shared_ptr<Buffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
std::unique_ptr<FaultReadback> m_fault_readback;
RangeSet m_gpu_modified_ranges;
RangeSet m_image_invalidated_ranges;
std::mutex m_publication_mutex;
std::vector<PendingBackingPublication> m_pending_backing_publications;
MemoryTracker m_memory_tracker;
StreamBuffer m_staging_buffer;
StreamBuffer m_stream_buffer;
StreamBuffer m_download_buffer;
StreamBuffer m_device_buffer;
PageManager& m_page_manager;
TextureCache& m_texture_cache;
ResourceMutex& m_resource_mutex;
uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 1ull * 1024 * 1024 * 1024;
uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_BUFFERCACHE_H_
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+141
View File
@@ -0,0 +1,141 @@
#include "graphics/host_gpu/renderer/cache/gpuResourceManager.h"
#include "common/assert.h"
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
namespace Libs::Graphics {
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
: m_page_manager(FaultThunk, this),
m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
GpuResourceManager::~GpuResourceManager() = default;
bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept {
return static_cast<GpuResourceManager*>(context)->InvalidateMemory(access, vaddr, size, phase);
}
bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
// Let the authoritative image materialize first. A clean overlapping buffer marks a write
// fault CPU-dirty when it begins ownership transfer; doing that before image preflight would
// make the image appear to race a real CPU write. Completion and release retain buffer-first
// ordering so its pending fault is gone before TextureCache publishes the downloaded backing.
if (phase == PageFaultPhase::Invalidate) {
const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
return buffer_handled || image_handled;
}
const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
return buffer_handled || image_handled;
}
bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
if (!m_page_manager.IsMapped(fault_vaddr, 1)) {
return false;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported guest-memory fault from an asynchronous GPU completion, "
"addr=0x%016" PRIx64 " access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
bool handled = false;
const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
(void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1);
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handled = m_page_manager.HandleFault(access, fault_vaddr);
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return handled;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported page fault from a pre-owned resource transaction, addr=0x%016" PRIx64
" access=%u\n",
fault_vaddr, static_cast<uint32_t>(access));
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
return handled;
}
void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) {
if (!m_page_manager.HasAnyMapping(vaddr, size)) {
return;
}
if (CommandScheduler::InDeferredOperation()) {
EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
const auto handle_range = [this, vaddr, size] {
if (!m_page_manager.HandleWriteRange(vaddr, size)) {
EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
vaddr, size);
}
};
const auto resolve = [this, &handle_range](CommandProcessor& cp) {
cp.BeginReadbackTransaction();
{
ResourceMutex::FaultScope fault(m_resource_mutex);
handle_range();
}
cp.EndReadbackTransaction();
};
if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
resolve(*cp);
return;
}
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64
" size=0x%016" PRIx64 "\n",
vaddr, size);
}
EXIT_IF(m_gpu == nullptr);
m_gpu->SendCommandSyncWithProcessor(resolve);
}
bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
return m_page_manager.IsMapped(vaddr, size);
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
m_page_manager.OnGpuMap(vaddr, size, access);
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (!IsMapped(vaddr, size)) {
EXIT("cannot unmap an unmapped GPU resource range\n");
}
const auto unmap = [this, vaddr, size, access] {
m_texture_cache.UnmapMemory(vaddr, size);
m_buffer_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size, access);
};
if (m_gpu == nullptr) {
if (m_resource_mutex.IsOwnedByCurrentThread()) {
EXIT("cannot synchronously unmap from a resource transaction\n");
}
unmap();
return;
}
Gpu::SubmissionLock submissions(*m_gpu);
m_gpu->SendCommandSync(unmap);
}
void GpuResourceManager::RunGarbageCollector() {
m_texture_cache.ProcessDownloadImages();
m_texture_cache.RunGarbageCollector();
m_buffer_cache.RunGarbageCollector();
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+50
View File
@@ -0,0 +1,50 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
#include "common/abi.h"
#include "common/common.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/renderer/cache/bufferCache.h"
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include <cstdint>
namespace Libs::Graphics {
class CommandScheduler;
class Gpu;
class GpuResourceManager {
public:
GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler);
~GpuResourceManager();
KYTY_CLASS_NO_COPY(GpuResourceManager);
[[nodiscard]] BufferCache& GetBufferCache() { return m_buffer_cache; }
[[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; }
void SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
void PrepareHostWrite(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void RunGarbageCollector();
private:
static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
PageManager m_page_manager;
ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
Gpu* m_gpu = nullptr;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_GPURESOURCEMANAGER_H_
src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h
Vendored
+360
View File
@@ -0,0 +1,360 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
#include "common/assert.h"
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <list>
#include <memory>
#include <utility>
#include <vector>
namespace Libs::Graphics {
// Sparse two-level lookup for texture-cache page ownership.
// Entries are selected by guest page number; queries never allocate L1 buckets.
template <typename EntryT, size_t PageBits = 20, size_t AddressSpaceBits = 40,
size_t FirstLevelBits = 10>
class MultiLevelPageTable final {
public:
using Entry = EntryT;
static_assert(AddressSpaceBits < 64);
static_assert(PageBits < AddressSpaceBits);
static_assert(FirstLevelBits > 0 && FirstLevelBits <= AddressSpaceBits - PageBits);
static constexpr size_t kPageBits = PageBits;
static constexpr size_t kAddressSpaceBits = AddressSpaceBits;
static constexpr size_t kFirstLevelBits = FirstLevelBits;
static constexpr size_t kSecondLevelBits = AddressSpaceBits - FirstLevelBits - PageBits;
static constexpr size_t kFirstLevelEntries = size_t {1} << FirstLevelBits;
static constexpr size_t kBucketEntries = size_t {1} << kSecondLevelBits;
static constexpr size_t kPageCount = size_t {1} << (AddressSpaceBits - PageBits);
static constexpr uint64_t kAddressSpaceSize = uint64_t {1} << AddressSpaceBits;
struct PageRange final {
size_t first = 0;
size_t last_exclusive = 0;
};
MultiLevelPageTable(): m_first_level(kFirstLevelEntries) {}
[[nodiscard]] Entry* Find(size_t page) {
if (!IsValidPage(page)) {
return nullptr;
}
auto& bucket = m_first_level[FirstLevelIndex(page)];
return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
}
[[nodiscard]] const Entry* Find(size_t page) const {
if (!IsValidPage(page)) {
return nullptr;
}
const auto& bucket = m_first_level[FirstLevelIndex(page)];
return bucket == nullptr ? nullptr : &(*bucket)[SecondLevelIndex(page)];
}
[[nodiscard]] Entry& GetOrCreate(size_t page) {
if (!IsValidPage(page)) {
EXIT("MultiLevelPageTable page is outside the guest address space");
}
auto& bucket = m_first_level[FirstLevelIndex(page)];
if (bucket == nullptr) {
bucket = std::make_unique<Bucket>();
++m_allocated_buckets;
}
return (*bucket)[SecondLevelIndex(page)];
}
[[nodiscard]] Entry& operator[](size_t page) { return GetOrCreate(page); }
[[nodiscard]] static constexpr bool IsValidPage(size_t page) { return page < kPageCount; }
// Returns the half-open page interval touched by a non-empty byte range.
// An end exactly at 2^AddressSpaceBits is valid; wrapping or crossing it is not.
[[nodiscard]] static constexpr bool TryGetPageRange(uint64_t address, uint64_t size,
PageRange& range) {
if (size == 0 || address >= kAddressSpaceSize || size > kAddressSpaceSize - address) {
return false;
}
const uint64_t end = address + size;
range.first = static_cast<size_t>(address >> PageBits);
range.last_exclusive = static_cast<size_t>(((end - 1) >> PageBits) + 1);
return true;
}
[[nodiscard]] size_t AllocatedBucketCount() const { return m_allocated_buckets; }
private:
using Bucket = std::array<Entry, kBucketEntries>;
[[nodiscard]] static constexpr size_t FirstLevelIndex(size_t page) {
return page >> kSecondLevelBits;
}
[[nodiscard]] static constexpr size_t SecondLevelIndex(size_t page) {
return page & (kBucketEntries - 1);
}
std::vector<std::unique_ptr<Bucket>> m_first_level;
size_t m_allocated_buckets = 0;
};
// Removes only the requested owner. Other owners in the same page entry remain intact.
template <typename Container, typename Value>
[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
const auto it = std::find(owners.begin(), owners.end(), owner);
if (it == owners.end()) {
return false;
}
owners.erase(it);
return true;
}
// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB
// lifetime accounting. OwnerT only needs equality.
template <typename OwnerT>
class MultiRangePageOwnerIndex final {
private:
struct Registration;
using MembershipList = std::vector<const Registration*>;
public:
struct ByteRange final {
uint64_t address = 0;
uint64_t size = 0;
};
using CoarseTable = MultiLevelPageTable<MembershipList, 20, 40, 10>;
using TrackingTable = MultiLevelPageTable<MembershipList, 12, 40, 18>;
[[nodiscard]] bool Register(const OwnerT& owner, const std::vector<ByteRange>& ranges) {
if (FindRegistration(owner) != m_registrations.end()) {
return false;
}
auto normalized = Normalize(ranges);
if (normalized.empty()) {
return false;
}
m_registrations.push_back({owner, std::move(normalized)});
const Registration* registration = &m_registrations.back();
for (const size_t page: CollectPages<20>(registration->ranges)) {
m_coarse_pages[page].push_back(registration);
}
for (const size_t page: CollectPages<12>(registration->ranges)) {
m_tracking_pages[page].push_back(registration);
}
return true;
}
// No state changes if an expected membership is absent. Final releases are
// sorted and coalesced 4 KiB spans whose final owner disappeared.
[[nodiscard]] bool Unregister(const OwnerT& owner, std::vector<ByteRange>& final_releases) {
final_releases.clear();
auto registration = FindRegistration(owner);
if (registration == m_registrations.end()) {
return false;
}
const auto coarse_pages = CollectPages<20>(registration->ranges);
const auto tracking_pages = CollectPages<12>(registration->ranges);
const Registration* registration_ptr = &*registration;
if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) ||
!HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) {
return false;
}
std::vector<size_t> final_pages;
for (const size_t page: coarse_pages) {
(void)EraseExact(*m_coarse_pages.Find(page), registration_ptr);
}
for (const size_t page: tracking_pages) {
auto& owners = *m_tracking_pages.Find(page);
if (owners.size() == 1) {
final_pages.push_back(page);
}
(void)EraseExact(owners, registration_ptr);
}
m_registrations.erase(registration);
final_releases = CoalesceTrackingPages(final_pages);
return true;
}
[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size) const {
return Query(address, size, [](const OwnerT&) { return true; });
}
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size,
Predicate&& predicate) const {
return QueryImpl(address, size, true, std::forward<Predicate>(predicate));
}
// Fault paths use page candidates: exact byte-disjoint owners sharing a
// touched 4 KiB page are intentionally retained.
[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size) const {
return QueryCandidates(address, size, [](const OwnerT&) { return true; });
}
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size,
Predicate&& predicate) const {
return QueryImpl(address, size, false, std::forward<Predicate>(predicate));
}
[[nodiscard]] size_t CoarseMembershipCount(size_t page) const {
const auto* owners = m_coarse_pages.Find(page);
return owners == nullptr ? 0 : owners->size();
}
[[nodiscard]] size_t TrackingMembershipCount(size_t page) const {
const auto* owners = m_tracking_pages.Find(page);
return owners == nullptr ? 0 : owners->size();
}
private:
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> QueryImpl(uint64_t address, uint64_t size, bool strict_bytes,
Predicate&& predicate) const {
typename CoarseTable::PageRange coarse_range {};
typename TrackingTable::PageRange tracking_range {};
if (!CoarseTable::TryGetPageRange(address, size, coarse_range) ||
!TrackingTable::TryGetPageRange(address, size, tracking_range)) {
return {};
}
MembershipList candidates;
for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) {
if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) {
AppendUnique(candidates, *owners);
}
}
std::vector<OwnerT> result;
for (const Registration* registration: candidates) {
if ((!strict_bytes || Overlaps(registration->ranges, address, size)) &&
HasTrackingMembership(registration, tracking_range) &&
predicate(registration->owner)) {
result.push_back(registration->owner);
}
}
return result;
}
struct Registration final {
OwnerT owner;
std::vector<ByteRange> ranges;
};
using RegistrationIterator = typename std::list<Registration>::iterator;
using ConstRegistrationIterator = typename std::list<Registration>::const_iterator;
[[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) {
return std::find_if(m_registrations.begin(), m_registrations.end(),
[&](const Registration& item) { return item.owner == owner; });
}
[[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const {
return std::find_if(m_registrations.begin(), m_registrations.end(),
[&](const Registration& item) { return item.owner == owner; });
}
[[nodiscard]] static std::vector<ByteRange> Normalize(const std::vector<ByteRange>& ranges) {
std::vector<ByteRange> sorted;
for (const auto& range: ranges) {
typename CoarseTable::PageRange ignored {};
if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) {
return {};
}
sorted.push_back(range);
}
std::sort(sorted.begin(), sorted.end(), [](const ByteRange& lhs, const ByteRange& rhs) {
return lhs.address < rhs.address;
});
std::vector<ByteRange> merged;
for (const auto& range: sorted) {
if (merged.empty() || range.address > merged.back().address + merged.back().size) {
merged.push_back(range);
} else {
const uint64_t end = std::max(merged.back().address + merged.back().size,
range.address + range.size);
merged.back().size = end - merged.back().address;
}
}
return merged;
}
template <size_t Bits>
[[nodiscard]] static std::vector<size_t> CollectPages(const std::vector<ByteRange>& ranges) {
std::vector<size_t> pages;
for (const auto& range: ranges) {
const size_t first = static_cast<size_t>(range.address >> Bits);
const size_t last = static_cast<size_t>((range.address + range.size - 1) >> Bits);
for (size_t page = first; page <= last; ++page) {
pages.push_back(page);
}
}
std::sort(pages.begin(), pages.end());
pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
return pages;
}
template <typename Table>
[[nodiscard]] static bool HasAllMemberships(const Table& table,
const std::vector<size_t>& pages,
const Registration* registration) {
for (const size_t page: pages) {
const auto* owners = table.Find(page);
if (owners == nullptr ||
std::find(owners->begin(), owners->end(), registration) == owners->end()) {
return false;
}
}
return true;
}
[[nodiscard]] bool HasTrackingMembership(const Registration* registration,
const typename TrackingTable::PageRange& range) const {
for (size_t page = range.first; page < range.last_exclusive; ++page) {
const auto* owners = m_tracking_pages.Find(page);
if (owners != nullptr &&
std::find(owners->begin(), owners->end(), registration) != owners->end()) {
return true;
}
}
return false;
}
[[nodiscard]] static bool Overlaps(const std::vector<ByteRange>& ranges, uint64_t address,
uint64_t size) {
const uint64_t end = address + size;
return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) {
return range.address < end && address < range.address + range.size;
});
}
static void AppendUnique(MembershipList& destination, const MembershipList& source) {
for (const Registration* registration: source) {
if (std::find(destination.begin(), destination.end(), registration) ==
destination.end()) {
destination.push_back(registration);
}
}
}
[[nodiscard]] static std::vector<ByteRange>
CoalesceTrackingPages(const std::vector<size_t>& pages) {
std::vector<ByteRange> result;
for (const size_t page: pages) {
const uint64_t address = static_cast<uint64_t>(page) << 12;
if (!result.empty() && result.back().address + result.back().size == address) {
result.back().size += uint64_t {1} << 12;
} else {
result.push_back({address, uint64_t {1} << 12});
}
}
return result;
}
CoarseTable m_coarse_pages;
TrackingTable m_tracking_pages;
std::list<Registration> m_registrations;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
src/graphics/host_gpu/renderer/cache/resourceMutex.cpp
Vendored
+93
View File
@@ -0,0 +1,93 @@
#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
#include "common/assert.h"
namespace Libs::Graphics {
ResourceMutex::FaultScope::FaultScope(ResourceMutex& mutex): m_mutex(mutex) {
m_resource_preowned = m_mutex.BeginFault();
}
ResourceMutex::FaultScope::~FaultScope() {
m_mutex.EndFault(m_resource_preowned);
}
ResourceMutex::~ResourceMutex() {
std::lock_guard state(m_state);
if (m_resource_owner != std::thread::id {} || m_fault_owner != std::thread::id {}) {
EXIT("ResourceMutex destroyed with an active owner or fault transaction\n");
}
}
void ResourceMutex::lock() {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_resource_owner == current) {
EXIT("recursive resource transaction\n");
}
if (m_fault_owner == current) {
EXIT("resource transaction re-entered from a page-fault callback\n");
}
}
m_resource.lock();
std::lock_guard state(m_state);
if (m_resource_owner != std::thread::id {}) {
EXIT("resource mutex acquired with a stale owner\n");
}
m_resource_owner = current;
}
void ResourceMutex::unlock() {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_resource_owner != current) {
EXIT("resource transaction released without ownership\n");
}
m_resource_owner = {};
}
m_resource.unlock();
}
bool ResourceMutex::IsOwnedByCurrentThread() {
std::lock_guard state(m_state);
return m_resource_owner == std::this_thread::get_id();
}
bool ResourceMutex::BeginFault() {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_fault_owner == current) {
EXIT("nested resource page-fault transaction\n");
}
if (m_resource_owner == current) {
m_fault_owner = current;
return true;
}
}
lock();
std::lock_guard state(m_state);
if (m_resource_owner != current || m_fault_owner != std::thread::id {}) {
EXIT("resource fault transaction acquired inconsistent ownership\n");
}
m_fault_owner = current;
return false;
}
void ResourceMutex::EndFault(bool resource_preowned) {
const auto current = std::this_thread::get_id();
{
std::lock_guard state(m_state);
if (m_fault_owner != current || m_resource_owner != current) {
EXIT("resource page-fault transaction ended with inconsistent ownership\n");
}
m_fault_owner = {};
}
if (!resource_preowned) {
unlock();
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/resourceMutex.h
Vendored
+48
View File
@@ -0,0 +1,48 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RESOURCEMUTEX_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RESOURCEMUTEX_H_
#include "common/common.h"
#include <mutex>
#include <thread>
namespace Libs::Graphics {
// Owner-tracked shared buffer/image transaction. External faults pause GPU submissions first;
// command-processor faults drain pending guest processors before entering this transaction.
class ResourceMutex final {
public:
class FaultScope final {
public:
explicit FaultScope(ResourceMutex& mutex);
~FaultScope();
KYTY_CLASS_NO_COPY(FaultScope);
private:
ResourceMutex& m_mutex;
bool m_resource_preowned = false;
};
ResourceMutex() = default;
~ResourceMutex();
KYTY_CLASS_NO_COPY(ResourceMutex);
void lock();
void unlock();
[[nodiscard]] bool IsOwnedByCurrentThread();
private:
friend class FaultScope;
[[nodiscard]] bool BeginFault();
void EndFault(bool resource_preowned);
std::mutex m_resource;
std::mutex m_state;
std::thread::id m_resource_owner;
std::thread::id m_fault_owner;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RESOURCEMUTEX_H_
src/graphics/host_gpu/renderer/cache/samplerCache.cpp
Vendored
+160
View File
@@ -0,0 +1,160 @@
#include "graphics/host_gpu/renderer/cache/samplerCache.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/host_gpu/renderer/renderContext.h"
namespace Libs::Graphics {
SamplerCache::~SamplerCache() {
for (const auto& [key, sampler]: m_samplers) {
(void)key;
m_graphics.device.destroySampler(sampler, nullptr);
}
}
vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
Common::LockGuard lock(m_mutex);
const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]};
if (auto iter = m_samplers.find(key); iter != m_samplers.end()) {
return iter->second;
}
float aniso_ratio = 1.0f;
const auto mag_filter = r.XyMagFilter();
const auto min_filter = r.XyMinFilter();
auto is_aniso_filter = [](uint8_t filter) {
switch (static_cast<Prospero::SamplerFilter>(filter)) {
case Prospero::SamplerFilter::kAnisoPoint:
case Prospero::SamplerFilter::kAnisoLinear: return true;
case Prospero::SamplerFilter::kPoint:
case Prospero::SamplerFilter::kBilinear: return false;
default: EXIT("unknown sampler filter: %u\n", filter);
}
return false;
};
auto to_vk_filter = [](uint8_t filter) {
switch (static_cast<Prospero::SamplerFilter>(filter)) {
case Prospero::SamplerFilter::kPoint:
case Prospero::SamplerFilter::kAnisoPoint: return vk::Filter::eNearest;
case Prospero::SamplerFilter::kBilinear:
case Prospero::SamplerFilter::kAnisoLinear: return vk::Filter::eLinear;
default: EXIT("unknown sampler filter: %u\n", filter);
}
return vk::Filter::eNearest;
};
const bool aniso = is_aniso_filter(mag_filter) || is_aniso_filter(min_filter);
if (aniso) {
switch (static_cast<Prospero::SamplerAnisoRatio>(r.MaxAnisoRatio())) {
case Prospero::SamplerAnisoRatio::kOne: aniso_ratio = 1.0f; break;
case Prospero::SamplerAnisoRatio::kTwo: aniso_ratio = 2.0f; break;
case Prospero::SamplerAnisoRatio::kFour: aniso_ratio = 4.0f; break;
case Prospero::SamplerAnisoRatio::kEight: aniso_ratio = 8.0f; break;
case Prospero::SamplerAnisoRatio::kSixteen: aniso_ratio = 16.0f; break;
default: EXIT("unknown ratio: %d\n", static_cast<int>(r.MaxAnisoRatio()));
}
}
const auto mip_filter = r.MipFilter();
float min_lod = 0.0f;
float max_lod = 0.0f;
if (static_cast<Prospero::SamplerMipFilter>(mip_filter) != Prospero::SamplerMipFilter::kNone) {
min_lod = static_cast<float>(r.MinLod()) / 256.0f;
max_lod = static_cast<float>(r.MaxLod()) / 256.0f;
}
vk::SamplerCreateInfo sampler_info {};
auto to_vk_address_mode = [](uint8_t clamp) {
switch (static_cast<Prospero::SamplerClampMode>(clamp)) {
case Prospero::SamplerClampMode::kWrap: return vk::SamplerAddressMode::eRepeat;
case Prospero::SamplerClampMode::kMirror:
return vk::SamplerAddressMode::eMirroredRepeat;
case Prospero::SamplerClampMode::kClampLastTexel:
return vk::SamplerAddressMode::eClampToEdge;
case Prospero::SamplerClampMode::kMirrorOnceLastTexel:
return vk::SamplerAddressMode::eMirrorClampToEdge;
case Prospero::SamplerClampMode::kClampHalfBorder:
return vk::SamplerAddressMode::eClampToBorder;
case Prospero::SamplerClampMode::kMirrorOnceHalfBorder:
return vk::SamplerAddressMode::eMirrorClampToEdge;
case Prospero::SamplerClampMode::kClampBorder:
return vk::SamplerAddressMode::eClampToBorder;
case Prospero::SamplerClampMode::kMirrorOnceBorder:
return vk::SamplerAddressMode::eMirrorClampToEdge;
default: EXIT("unknown clamp: %u\n", clamp);
}
return vk::SamplerAddressMode::eClampToBorder;
};
vk::BorderColor border = vk::BorderColor::eIntTransparentBlack;
switch (static_cast<Prospero::SamplerBorderColor>(r.BorderColorType())) {
case Prospero::SamplerBorderColor::kTransBlack:
border = vk::BorderColor::eIntTransparentBlack;
break;
case Prospero::SamplerBorderColor::kOpaqueBlack:
border = vk::BorderColor::eIntOpaqueBlack;
break;
case Prospero::SamplerBorderColor::kOpaqueWhite:
border = vk::BorderColor::eIntOpaqueWhite;
break;
case Prospero::SamplerBorderColor::kFromTable:
LOGF(
"temporary: approximating table border color as transparent black, index = %" PRIu16
"\n",
r.BorderColorPtr());
border = vk::BorderColor::eIntTransparentBlack;
break;
default: EXIT("unknown border color: %d", static_cast<int>(r.BorderColorType()));
}
sampler_info.sType = vk::StructureType::eSamplerCreateInfo;
sampler_info.pNext = nullptr;
sampler_info.flags = {};
sampler_info.magFilter = to_vk_filter(mag_filter);
sampler_info.minFilter = to_vk_filter(min_filter);
sampler_info.mipmapMode =
(static_cast<Prospero::SamplerMipFilter>(mip_filter) == Prospero::SamplerMipFilter::kLinear
? vk::SamplerMipmapMode::eLinear
: vk::SamplerMipmapMode::eNearest);
sampler_info.addressModeU = to_vk_address_mode(r.ClampX());
sampler_info.addressModeV = to_vk_address_mode(r.ClampY());
sampler_info.addressModeW = to_vk_address_mode(r.ClampZ());
sampler_info.mipLodBias =
static_cast<float>(static_cast<int16_t>((r.LodBias() ^ 0x2000u) - 0x2000u)) / 256.0f;
sampler_info.anisotropyEnable = (aniso ? VK_TRUE : VK_FALSE);
sampler_info.maxAnisotropy = aniso_ratio;
sampler_info.compareEnable = (r.DepthCompareFunc() != 0 ? VK_TRUE : VK_FALSE);
sampler_info.compareOp = static_cast<vk::CompareOp>(r.DepthCompareFunc());
sampler_info.minLod = min_lod;
sampler_info.maxLod = max_lod;
sampler_info.borderColor = border;
sampler_info.unnormalizedCoordinates = (r.ForceUnormCoords() ? VK_TRUE : VK_FALSE);
if (r.ForceUnormCoords()) {
sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge;
sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge;
sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge;
sampler_info.mipmapMode = vk::SamplerMipmapMode::eNearest;
sampler_info.minLod = 0.0f;
sampler_info.maxLod = 0.0f;
sampler_info.anisotropyEnable = VK_FALSE;
sampler_info.maxAnisotropy = 1.0f;
sampler_info.compareEnable = VK_FALSE;
sampler_info.mipLodBias = 0.0f;
}
vk::Sampler vk_sampler = nullptr;
const auto result = m_graphics.device.createSampler(&sampler_info, nullptr, &vk_sampler);
EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || vk_sampler == nullptr);
m_samplers.emplace(key, vk_sampler);
return vk_sampler;
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/samplerCache.h
Vendored
+51
View File
@@ -0,0 +1,51 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SAMPLERCACHE_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SAMPLERCACHE_H_
#include "common/abi.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/threads.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include "graphics/shader/shaderBindings.h"
#include <array>
#include <cstddef>
#include <unordered_map>
namespace Libs::Graphics {
struct GraphicContext;
class SamplerCache {
public:
explicit SamplerCache(GraphicContext& graphics): m_graphics(graphics) {
EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
}
~SamplerCache();
KYTY_CLASS_NO_COPY(SamplerCache);
vk::Sampler GetSampler(const ShaderSamplerResource& r);
private:
using SamplerKey = std::array<uint32_t, 4>;
struct SamplerKeyHash {
std::size_t operator()(const SamplerKey& key) const {
std::size_t hash = 0;
for (auto value: key) {
hash ^= static_cast<std::size_t>(value) +
static_cast<std::size_t>(0x9e3779b97f4a7c15ull) + (hash << 6u) +
(hash >> 2u);
}
return hash;
}
};
GraphicContext& m_graphics;
Common::Mutex m_mutex;
std::unordered_map<SamplerKey, vk::Sampler, SamplerKeyHash> m_samplers;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SAMPLERCACHE_H_
src/graphics/host_gpu/renderer/cache/streamBuffer.cpp
Vendored
+360
View File
@@ -0,0 +1,360 @@
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include "common/assert.h"
#include "common/profiler.h"
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/vma.h"
#include <cstring>
#include <limits>
#include <numeric>
#include <vk_mem_alloc.h>
namespace Libs::Graphics {
namespace {
constexpr size_t WATCHES_INITIAL_RESERVE = 0x4000;
constexpr size_t WATCHES_RESERVE_CHUNK = 0x1000;
[[nodiscard]] VmaAllocationCreateFlags AllocationFlags(MemoryUsage usage) {
switch (usage) {
case MemoryUsage::Upload:
case MemoryUsage::Stream:
return VMA_ALLOCATION_CREATE_MAPPED_BIT |
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT;
case MemoryUsage::Download:
return VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT;
case MemoryUsage::DeviceLocal: return {};
}
return {};
}
[[nodiscard]] VmaMemoryUsage AllocationUsage(MemoryUsage usage) {
switch (usage) {
case MemoryUsage::DeviceLocal:
case MemoryUsage::Stream: return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
case MemoryUsage::Upload:
case MemoryUsage::Download: return VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
}
return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
}
[[nodiscard]] bool AlignUp(uint64_t value, uint64_t alignment, uint64_t& result) {
if (alignment == 0) {
result = value;
return true;
}
const auto remainder = value % alignment;
if (remainder == 0) {
result = value;
return true;
}
const auto increment = alignment - remainder;
if (value > std::numeric_limits<uint64_t>::max() - increment) {
return false;
}
result = value + increment;
return true;
}
} // namespace
Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
uint64_t cpu_address, vk::BufferUsageFlags flags, uint64_t size)
: m_graphics(&graphics), m_scheduler(&scheduler), m_usage(usage), m_cpu_address(cpu_address),
m_size(size), m_buffer(std::make_unique<VulkanBuffer>()) {
KYTY_PROFILER_FUNCTION();
EXIT_IF(graphics.allocator == nullptr || size == 0);
vk::BufferCreateInfo buffer_info {};
buffer_info.size = size;
buffer_info.usage = flags;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
VmaAllocationCreateInfo allocation_info {};
allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
allocation_info.usage = AllocationUsage(usage);
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
? VkMemoryPropertyFlags {}
: VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
VmaAllocationInfo allocation_result {};
VkBuffer native_buffer = VK_NULL_HANDLE;
const auto result = static_cast<vk::Result>(vmaCreateBuffer(
graphics.allocator, static_cast<const VkBufferCreateInfo*>(buffer_info), &allocation_info,
&native_buffer, &m_buffer->memory.allocation, &allocation_result));
if (result != vk::Result::eSuccess) {
graphics.LogMemoryBudget();
}
EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
m_buffer->buffer = native_buffer;
m_buffer->usage = flags;
m_buffer->buffer_size = size;
m_buffer->memory.allocation_info = allocation_result;
m_buffer->memory.memory = allocation_result.deviceMemory;
m_buffer->memory.offset = allocation_result.offset;
m_buffer->memory.type = allocation_result.memoryType;
m_buffer->memory.unique_id = VulkanNextMemoryUniqueId();
graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
VkMemoryPropertyFlags properties = 0;
vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
m_buffer->memory.property = vk::MemoryPropertyFlags(properties);
m_is_coherent = (properties & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
if (allocation_result.pMappedData != nullptr) {
m_mapped = {static_cast<uint8_t*>(allocation_result.pMappedData),
static_cast<size_t>(size)};
}
VulkanTrackAllocation(m_buffer->memory);
}
Buffer::~Buffer() {
if (m_buffer->buffer != nullptr) {
VulkanUntrackAllocation(m_buffer->memory);
vmaDestroyBuffer(m_graphics->allocator, m_buffer->buffer, m_buffer->memory.allocation);
}
}
vk::Buffer Buffer::Handle() const noexcept {
return m_buffer->buffer;
}
bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
}
void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
EXIT_IF(source == nullptr || m_mapped.empty() || offset > m_size || size > m_size - offset);
std::memcpy(m_mapped.data() + offset, source, static_cast<size_t>(size));
Flush(offset, size);
}
void Buffer::Flush(uint64_t offset, uint64_t size) {
EXIT_IF(m_mapped.empty() || offset > m_size || size > m_size - offset);
if (!m_is_coherent && size != 0) {
const auto result = vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation,
offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
}
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
vk::AccessFlags destination) const {
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64
" size=0x%016" PRIx64 " capacity=0x%016" PRIx64 "\n",
static_cast<const void*>(Handle()), offset, size, m_size);
}
vk::BufferMemoryBarrier barrier {};
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
barrier.srcAccessMask = source;
barrier.dstAccessMask = destination;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.buffer = Handle();
barrier.offset = offset;
barrier.size = size;
return barrier;
}
void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before,
vk::AccessFlags destination_before, vk::AccessFlags source_after,
vk::AccessFlags destination_after) {
if (size == 0 || source_offset > source.m_size || size > source.m_size - source_offset ||
destination_offset > m_size || size > m_size - destination_offset) {
EXIT("Buffer: invalid copy range\n");
}
if (source.Handle() == Handle() && source_offset < destination_offset + size &&
destination_offset < source_offset + size) {
EXIT("Buffer: overlapping self-copy\n");
}
command.EndRendering();
const vk::BufferMemoryBarrier before[] = {
source.Barrier(source_offset, size, source_before, vk::AccessFlagBits::eTransferRead),
Barrier(destination_offset, size, destination_before,
vk::AccessFlagBits::eTransferWrite),
};
const auto host_access = vk::AccessFlagBits::eHostRead | vk::AccessFlagBits::eHostWrite;
auto before_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands};
if (static_cast<bool>((source_before | destination_before) & host_access)) {
before_stage |= vk::PipelineStageFlagBits::eHost;
}
const auto native = command.Handle();
native.pipelineBarrier(before_stage, vk::PipelineStageFlagBits::eTransfer,
vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, before, 0, nullptr);
const vk::BufferCopy copy {source_offset, destination_offset, size};
native.copyBuffer(source.Handle(), Handle(), 1, &copy);
const vk::BufferMemoryBarrier after[] = {
source.Barrier(source_offset, size, vk::AccessFlagBits::eTransferRead, source_after),
Barrier(destination_offset, size, vk::AccessFlagBits::eTransferWrite, destination_after),
};
auto after_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands};
if (static_cast<bool>((source_after | destination_after) & host_access)) {
after_stage |= vk::PipelineStageFlagBits::eHost;
}
native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, after_stage,
vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, after, 0, nullptr);
}
void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) {
if (((offset | size) & 3u) != 0) {
EXIT("Buffer: fill range must be dword aligned\n");
}
auto& command = Scheduler().Current();
command.EndRendering();
const auto before =
Barrier(offset, size, vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlagBits::eTransferWrite);
const auto native = command.Handle();
native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
0, nullptr, 1, &before, 0, nullptr);
native.fillBuffer(Handle(), offset, size, value);
const auto after =
Barrier(offset, size, vk::AccessFlagBits::eTransferWrite,
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
vk::PipelineStageFlagBits::eAllCommands,
vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr);
}
StreamBuffer::StreamBuffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
uint64_t size)
: Buffer(graphics, scheduler, usage, 0, AllFlags, size) {
ReserveWatches(m_current_watches, WATCHES_INITIAL_RESERVE);
ReserveWatches(m_previous_watches, WATCHES_INITIAL_RESERVE);
}
bool StreamBuffer::NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size,
uint64_t& alignment) {
if (coherent) {
return true;
}
if (!AlignUp(size, atom, size)) {
return false;
}
const auto divisor = std::gcd(alignment, atom);
if (alignment != 0 && alignment / divisor > UINT64_MAX / atom) {
return false;
}
alignment = alignment == 0 ? atom : alignment / divisor * atom;
return true;
}
std::pair<uint8_t*, uint64_t> StreamBuffer::Map(uint64_t size, uint64_t alignment,
bool allow_wait) {
if (Mapped().empty()) {
return {nullptr, 0};
}
uint64_t mapped_size = size;
const auto atom = Graphics().physical_device_properties.limits.nonCoherentAtomSize;
if (!NormalizeReservation(IsCoherent(), atom, mapped_size, alignment)) {
return {nullptr, 0};
}
if (mapped_size > Size()) {
return {nullptr, 0};
}
uint64_t aligned_offset = 0;
if (!AlignUp(m_offset, alignment, aligned_offset)) {
return {nullptr, 0};
}
const bool wrap = aligned_offset > Size() - mapped_size;
if (wrap) {
aligned_offset = 0;
}
auto wait_cursor = wrap ? size_t {0} : m_wait_cursor;
auto wait_bound = wrap ? uint64_t {0} : m_wait_bound;
auto invalidation_mark =
wrap ? std::optional<size_t> {m_current_watch_cursor} : m_invalidation_mark;
auto& pending_watches = wrap ? m_current_watches : m_previous_watches;
if (!WaitPendingOperations(pending_watches, invalidation_mark, aligned_offset + mapped_size,
allow_wait, wait_cursor, wait_bound)) {
return {nullptr, 0};
}
if (wrap) {
m_invalidation_mark = invalidation_mark;
m_current_watch_cursor = 0;
std::swap(m_previous_watches, m_current_watches);
}
m_wait_cursor = wait_cursor;
m_wait_bound = wait_bound;
m_offset = aligned_offset;
m_mapped_size = mapped_size;
return {Mapped().data() + m_offset, m_offset};
}
void StreamBuffer::Commit() {
if (!IsCoherent() && Usage() != MemoryUsage::Download && m_mapped_size != 0) {
const auto result = vmaFlushAllocation(
Graphics().allocator, NativeBuffer().memory.allocation, m_offset, m_mapped_size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
m_offset += m_mapped_size;
const auto tick = Scheduler().CurrentTick();
if (m_current_watch_cursor != 0 && m_current_watches[m_current_watch_cursor - 1].tick == tick) {
m_current_watches[m_current_watch_cursor - 1].upper_bound = m_offset;
return;
}
if (m_current_watch_cursor + 1 >= m_current_watches.size()) {
ReserveWatches(m_current_watches, WATCHES_RESERVE_CHUNK);
}
auto& watch = m_current_watches[m_current_watch_cursor++];
watch.upper_bound = m_offset;
watch.tick = tick;
}
void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset);
if (IsCoherent() || size == 0) {
return;
}
const auto result = vmaInvalidateAllocation(Graphics().allocator,
NativeBuffer().memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
}
uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
EXIT_IF(source == nullptr);
const auto [data, offset] = Map(size, alignment);
EXIT_IF(data == nullptr);
std::memcpy(data, source, static_cast<size_t>(size));
Commit();
return offset;
}
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, size_t grow_size) {
watches.resize(watches.size() + grow_size);
}
bool StreamBuffer::WaitPendingOperations(const std::vector<Watch>& watches,
std::optional<size_t> invalidation_mark,
uint64_t requested_upper_bound, bool allow_wait,
size_t& wait_cursor, uint64_t& wait_bound) {
if (!invalidation_mark.has_value()) {
return true;
}
while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) {
const auto& watch = watches[wait_cursor];
if (!Scheduler().IsFree(watch.tick) && !allow_wait) {
return false;
}
Scheduler().Wait(watch.tick);
if (Usage() == MemoryUsage::Download) {
Scheduler().WaitPriorityOperations(watch.tick);
}
wait_bound = watch.upper_bound;
++wait_cursor;
}
return true;
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/streamBuffer.h
Vendored
+130
View File
@@ -0,0 +1,130 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_
#include "common/abi.h"
#include "common/common.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <cstdint>
#include <memory>
#include <optional>
#include <span>
#include <utility>
#include <vector>
namespace Libs::Graphics {
class CommandBuffer;
class CommandScheduler;
struct StreamBufferTestAccess;
struct GraphicContext;
struct VulkanBuffer;
enum class MemoryUsage : uint8_t {
DeviceLocal,
Upload,
Download,
Stream,
};
inline constexpr vk::BufferUsageFlags ReadFlags =
vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eUniformBuffer |
vk::BufferUsageFlagBits::eIndexBuffer | vk::BufferUsageFlagBits::eVertexBuffer |
vk::BufferUsageFlagBits::eIndirectBuffer;
inline constexpr vk::BufferUsageFlags AllFlags =
ReadFlags | vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer;
class Buffer {
public:
Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
uint64_t cpu_address, vk::BufferUsageFlags flags, uint64_t size);
~Buffer();
KYTY_CLASS_NO_COPY(Buffer);
[[nodiscard]] vk::Buffer Handle() const noexcept;
[[nodiscard]] uint64_t Size() const noexcept { return m_size; }
[[nodiscard]] std::span<uint8_t> Mapped() const noexcept { return m_mapped; }
[[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; }
[[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; }
[[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; }
[[nodiscard]] uint64_t Offset(uint64_t address) const noexcept {
return address - m_cpu_address;
}
[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
void Write(uint64_t offset, const void* source, uint64_t size);
void Flush(uint64_t offset, uint64_t size);
void CopyFrom(
CommandBuffer& command, const Buffer& source, uint64_t source_offset,
uint64_t destination_offset, uint64_t size,
vk::AccessFlags source_before = vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlags destination_before =
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlags source_after =
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
vk::AccessFlags destination_after =
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
void Fill(uint64_t offset, uint64_t size, uint32_t value);
protected:
[[nodiscard]] GraphicContext& Graphics() const noexcept { return *m_graphics; }
[[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; }
[[nodiscard]] VulkanBuffer& NativeBuffer() noexcept { return *m_buffer; }
private:
[[nodiscard]] vk::BufferMemoryBarrier Barrier(uint64_t offset, uint64_t size,
vk::AccessFlags source,
vk::AccessFlags destination) const;
GraphicContext* m_graphics = nullptr;
CommandScheduler* m_scheduler = nullptr;
MemoryUsage m_usage = MemoryUsage::DeviceLocal;
uint64_t m_cpu_address = 0;
uint64_t m_size = 0;
std::unique_ptr<VulkanBuffer> m_buffer;
std::span<uint8_t> m_mapped;
bool m_is_coherent = false;
};
class StreamBuffer final: public Buffer {
public:
StreamBuffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
uint64_t size);
[[nodiscard]] std::pair<uint8_t*, uint64_t> Map(uint64_t size, uint64_t alignment = 0,
bool allow_wait = true);
void Commit();
// Download mappings become visible to the CPU only after their GPU completion tick is free.
// Call this from the scheduler's deferred completion operation before reading Mapped().
void Invalidate(uint64_t offset, uint64_t size);
[[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0);
private:
friend struct StreamBufferTestAccess;
struct Watch {
uint64_t tick = 0;
uint64_t upper_bound = 0;
};
void ReserveWatches(std::vector<Watch>& watches, size_t grow_size);
[[nodiscard]] static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size,
uint64_t& alignment);
[[nodiscard]] bool WaitPendingOperations(const std::vector<Watch>& watches,
std::optional<size_t> invalidation_mark,
uint64_t requested_upper_bound, bool allow_wait,
size_t& wait_cursor, uint64_t& wait_bound);
uint64_t m_offset = 0;
uint64_t m_mapped_size = 0;
std::vector<Watch> m_current_watches;
size_t m_current_watch_cursor = 0;
std::optional<size_t> m_invalidation_mark;
std::vector<Watch> m_previous_watches;
size_t m_wait_cursor = 0;
uint64_t m_wait_bound = 0;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_STREAMBUFFER_H_
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
+1883
View File
File diff suppressed because it is too large Load Diff
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
+188
View File
@@ -0,0 +1,188 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TEXTURECACHE_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TEXTURECACHE_H_
#include "common/abi.h"
#include "common/common.h"
#include "common/lruCache.h"
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/renderer/image/blitHelper.h"
#include "graphics/host_gpu/renderer/image/image.h"
#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
#include <compare>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <span>
#include <utility>
#include <vector>
namespace Libs::Graphics {
struct GraphicContext;
class Buffer;
class BufferCache;
class CommandBuffer;
class CommandScheduler;
class ResourceMutex;
class RenderExecutor;
class StreamBuffer;
class TileManager;
struct TextureCacheTestAccess;
class TextureCache {
public:
enum class BindingType : uint8_t { Texture, Storage, RenderTarget, DepthTarget, VideoOut };
struct ImageDesc {
ImageInfo info;
ImageViewInfo view_info;
BindingType type = BindingType::Texture;
};
struct RegionInfo {
bool image_pages = false;
bool image_bytes = false;
bool gpu_image_bytes = false;
};
TextureCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
BufferCache& buffer_cache, ResourceMutex& resource_mutex);
~TextureCache();
KYTY_CLASS_NO_COPY(TextureCache);
[[nodiscard]] ImageId FindImage(ImageDesc& desc, bool exact_format = false);
[[nodiscard]] ImageId FindImageFromRange(uint64_t address, uint64_t size,
bool ensure_valid = true);
[[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc);
[[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc);
[[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc);
[[nodiscard]] Image& GetImage(ImageId id);
[[nodiscard]] const Image& GetImage(ImageId id) const;
void MarkGpuWritten(ImageId id);
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
uint32_t packed_clear);
void PrepareHostWrite(uint64_t address, uint64_t size);
[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
bool formatted_buffer_write = false);
[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
[[nodiscard]] bool IsMeta(uint64_t address);
[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);
[[nodiscard]] bool ClearMeta(uint64_t address);
[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size,
PageFaultPhase phase) noexcept;
void UnmapMemory(uint64_t address, uint64_t size);
void ProcessDownloadImages();
void RunGarbageCollector();
private:
enum class TransferDirection { Upload, Download };
struct ColorTransferPlan;
struct DownloadPlan;
struct Slot {
std::shared_ptr<Image> image;
uint32_t generation = 1;
};
struct MetaDataInfo {
uint32_t clear_mask = 0;
};
struct OverlapResult {
ImageId image;
int32_t mip = -1;
int32_t layer = -1;
};
using ImageOwnerIndex = MultiRangePageOwnerIndex<ImageId>;
[[nodiscard]] Image& ResolveImage(ImageId id);
[[nodiscard]] const Image& ResolveImage(ImageId id) const;
[[nodiscard]] std::shared_ptr<Image> ResolveOwner(ImageId id) const;
[[nodiscard]] ImageId InsertImage(const ImageInfo& info);
[[nodiscard]] ImageId GetNullImage(const ImageDesc& desc);
void RegisterImage(ImageId id);
void UnregisterImage(ImageId id, bool release_tracking);
void DeleteImage(ImageId id, bool release_tracking = true);
void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
void RetainImage(CommandBuffer& command, ImageId id);
void TouchImage(Image& image);
void TrackImageDownload(ImageId id);
void TrackImageDownloadLocked(ImageId id, Image& image);
[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
bool exact_format);
[[nodiscard]] static BindingType UploadBinding(const Image& image);
[[nodiscard]] bool SafeToDownload(const Image& image);
[[nodiscard]] std::vector<ImageId> FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const;
[[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding,
ImageId cached, ImageId merged);
[[nodiscard]] ImageId ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
ImageId cached);
[[nodiscard]] ImageId ExpandImage(const ImageInfo& info, ImageId source);
void RefreshImage(ImageId id, const ImageDesc& desc);
void InitializeImage(ImageId id, const ImageDesc& desc);
[[nodiscard]] ColorTransferPlan BuildColorTransfer(const Image& image, BindingType binding,
TransferDirection direction) const;
[[nodiscard]] DownloadPlan BuildDownload(const Image& image) const;
void UploadImage(Image& image, const ImageDesc& desc, Buffer& source, uint64_t source_offset);
void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
DownloadPlan plan);
void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
void CommitGpuWrite(Image& image);
void PrepareImageCopy(Image& image);
void RefreshCopySource(ImageId id);
[[nodiscard]] bool CopyD16(Image& destination, Image& source);
void CopyImage(ImageId destination, ImageId source);
void AssociateStencil(ImageId depth, GuestRange stencil);
void AssociateStencilLocked(ImageId depth, GuestRange stencil);
void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer);
void ValidateImageDesc(const ImageDesc& desc) const;
void InvalidateCpuAliases(uint64_t address, uint64_t size);
void RestoreGpuTracking(const Image& image);
void ReleaseGpuTracking(ImageId id);
[[nodiscard]] bool SynchronizeImageToBuffer(ImageId id);
void DownloadImage(ImageId id);
[[nodiscard]] bool TryDownloadImage(ImageId id);
[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset);
GraphicContext& m_graphics;
CommandScheduler& m_scheduler;
TrackingSpinLock m_lock;
MemoryTracker m_memory_tracker;
BlitHelper m_blit_helper;
std::unique_ptr<TileManager> m_tiler;
BufferCache& m_buffer_cache;
ResourceMutex& m_resource_mutex;
std::vector<Slot> m_slots;
std::vector<uint32_t> m_free_slots;
ImageOwnerIndex m_image_owner_index;
std::map<vk::Format, ImageId> m_null_images;
Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
std::set<ImageId> m_download_images;
std::map<uint64_t, MetaDataInfo> m_surface_metas;
uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 0;
uint64_t m_pressure_gc_memory = 1536ull * 1024 * 1024;
uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0;
bool m_readback_linear_images = false;
friend struct TextureCacheTestAccess;
friend class RenderExecutor;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_TEXTURECACHE_H_