raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 02:47:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: reduce texture registration and lookup overhead

Browse Source
This commit is contained in:
nmzik
2026-08-03 06:56:28 +02:00
parent 2e9e2ae566
commit 26f0c7b0bf
6 changed files with 500 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/cache/multiLevelPageTable.h
Vendored
+154 -242
View File
@@ -7,8 +7,9 @@
#include <array>
#include <cstddef>
#include <cstdint>
#include <list>
#include <cstring>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>
@@ -103,6 +104,158 @@ private:
size_t m_allocated_buckets = 0;
};
// Inline owner storage for page-table entries. Texture pages normally have only a
// handful of owners, so avoid a heap allocation on the first 16 registrations.
//
// Owners are packed into live uint64_t objects instead of being constructed in
// raw storage. This keeps sparse bucket construction cheap without relying on
// subtle implicit-lifetime or pointer-provenance rules.
template <typename OwnerT, size_t InlineCapacity = 16>
class InlinePageOwnerList final {
public:
using value_type = OwnerT;
using size_type = size_t;
class const_iterator final {
public:
[[nodiscard]] OwnerT operator*() const noexcept { return m_owners->At(m_index); }
const_iterator& operator++() noexcept {
++m_index;
return *this;
}
bool operator==(const const_iterator&) const = default;
private:
friend class InlinePageOwnerList;
const_iterator(const InlinePageOwnerList* owners, size_type index) noexcept
: m_owners(owners), m_index(index) {}
const InlinePageOwnerList* m_owners = nullptr;
size_type m_index = 0;
};
static_assert(InlineCapacity > 0);
static_assert(std::is_default_constructible_v<OwnerT>);
static_assert(std::is_trivially_copyable_v<OwnerT>);
static_assert(std::has_unique_object_representations_v<OwnerT>);
static_assert(sizeof(OwnerT) <= sizeof(uint64_t));
InlinePageOwnerList() noexcept {}
InlinePageOwnerList(const InlinePageOwnerList&) = delete;
InlinePageOwnerList& operator=(const InlinePageOwnerList&) = delete;
InlinePageOwnerList(InlinePageOwnerList&& other) noexcept
: m_overflow(std::move(other.m_overflow)),
m_inline_size(std::exchange(other.m_inline_size, 0)) {
if (m_overflow == nullptr) {
std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
}
}
InlinePageOwnerList& operator=(InlinePageOwnerList&& other) noexcept {
if (this != &other) {
m_overflow = std::move(other.m_overflow);
m_inline_size = std::exchange(other.m_inline_size, 0);
if (m_overflow == nullptr) {
std::copy_n(other.m_inline.begin(), m_inline_size, m_inline.begin());
}
}
return *this;
}
[[nodiscard]] const_iterator begin() const noexcept { return {this, 0}; }
[[nodiscard]] const_iterator end() const noexcept { return {this, size()}; }
[[nodiscard]] bool empty() const noexcept { return size() == 0; }
[[nodiscard]] size_type size() const noexcept {
return m_overflow == nullptr ? m_inline_size : m_overflow->size();
}
[[nodiscard]] OwnerT front() const noexcept { return At(0); }
[[nodiscard]] OwnerT operator[](size_type index) const noexcept { return At(index); }
void push_back(const OwnerT& owner) {
const uint64_t packed = Pack(owner);
if (m_overflow != nullptr) {
m_overflow->push_back(packed);
return;
}
if (m_inline_size < InlineCapacity) {
m_inline[m_inline_size] = packed;
++m_inline_size;
return;
}
auto overflow = std::make_unique<std::vector<uint64_t>>();
overflow->reserve(InlineCapacity * 2);
overflow->assign(m_inline.begin(), m_inline.end());
overflow->push_back(packed);
m_overflow = std::move(overflow);
}
[[nodiscard]] bool Contains(const OwnerT& owner) const noexcept {
const uint64_t packed = Pack(owner);
if (m_overflow != nullptr) {
return std::find(m_overflow->begin(), m_overflow->end(), packed) != m_overflow->end();
}
return std::find(m_inline.begin(), m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size),
packed) != m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
}
[[nodiscard]] bool Erase(const OwnerT& owner) noexcept {
const uint64_t packed = Pack(owner);
if (m_overflow != nullptr) {
const auto found = std::find(m_overflow->begin(), m_overflow->end(), packed);
if (found == m_overflow->end()) {
return false;
}
m_overflow->erase(found);
if (m_overflow->size() == InlineCapacity) {
std::copy(m_overflow->begin(), m_overflow->end(), m_inline.begin());
m_inline_size = InlineCapacity;
m_overflow.reset();
}
return true;
}
const auto end = m_inline.begin() + static_cast<ptrdiff_t>(m_inline_size);
const auto found = std::find(m_inline.begin(), end, packed);
if (found == end) {
return false;
}
std::move(found + 1, end, found);
--m_inline_size;
return true;
}
template <typename Func>
void ForEach(Func&& func) const {
if (m_overflow != nullptr) {
for (const uint64_t owner: *m_overflow) {
func(Unpack(owner));
}
return;
}
for (size_type index = 0; index < m_inline_size; ++index) {
func(Unpack(m_inline[index]));
}
}
private:
[[nodiscard]] OwnerT At(size_type index) const noexcept {
return Unpack(m_overflow == nullptr ? m_inline[index] : (*m_overflow)[index]);
}
[[nodiscard]] static uint64_t Pack(const OwnerT& owner) noexcept {
uint64_t packed = 0;
std::memcpy(&packed, &owner, sizeof(owner));
return packed;
}
[[nodiscard]] static OwnerT Unpack(uint64_t packed) noexcept {
OwnerT owner {};
std::memcpy(&owner, &packed, sizeof(owner));
return owner;
}
std::array<uint64_t, InlineCapacity> m_inline;
std::unique_ptr<std::vector<uint64_t>> m_overflow;
size_type m_inline_size = 0;
};
// Removes only the requested owner. Other owners in the same page entry remain intact.
template <typename Container, typename Value>
[[nodiscard]] bool EraseExact(Container& owners, const Value& owner) {
@@ -114,247 +267,6 @@ template <typename Container, typename Value>
return true;
}
// Multi-range ownership with 1 MiB candidate buckets and precise 4 KiB
// lifetime accounting. OwnerT only needs equality.
template <typename OwnerT>
class MultiRangePageOwnerIndex final {
private:
struct Registration;
using MembershipList = std::vector<const Registration*>;
public:
struct ByteRange final {
uint64_t address = 0;
uint64_t size = 0;
};
using CoarseTable = MultiLevelPageTable<MembershipList, 20, 40, 10>;
using TrackingTable = MultiLevelPageTable<MembershipList, 12, 40, 18>;
[[nodiscard]] bool Register(const OwnerT& owner, const std::vector<ByteRange>& ranges) {
if (FindRegistration(owner) != m_registrations.end()) {
return false;
}
auto normalized = Normalize(ranges);
if (normalized.empty()) {
return false;
}
m_registrations.push_back({owner, std::move(normalized)});
const Registration* registration = &m_registrations.back();
for (const size_t page: CollectPages<20>(registration->ranges)) {
m_coarse_pages[page].push_back(registration);
}
for (const size_t page: CollectPages<12>(registration->ranges)) {
m_tracking_pages[page].push_back(registration);
}
return true;
}
// No state changes if an expected membership is absent. Final releases are
// sorted and coalesced 4 KiB spans whose final owner disappeared.
[[nodiscard]] bool Unregister(const OwnerT& owner, std::vector<ByteRange>& final_releases) {
final_releases.clear();
auto registration = FindRegistration(owner);
if (registration == m_registrations.end()) {
return false;
}
const auto coarse_pages = CollectPages<20>(registration->ranges);
const auto tracking_pages = CollectPages<12>(registration->ranges);
const Registration* registration_ptr = &*registration;
if (!HasAllMemberships(m_coarse_pages, coarse_pages, registration_ptr) ||
!HasAllMemberships(m_tracking_pages, tracking_pages, registration_ptr)) {
return false;
}
std::vector<size_t> final_pages;
for (const size_t page: coarse_pages) {
(void)EraseExact(*m_coarse_pages.Find(page), registration_ptr);
}
for (const size_t page: tracking_pages) {
auto& owners = *m_tracking_pages.Find(page);
if (owners.size() == 1) {
final_pages.push_back(page);
}
(void)EraseExact(owners, registration_ptr);
}
m_registrations.erase(registration);
final_releases = CoalesceTrackingPages(final_pages);
return true;
}
[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size) const {
return Query(address, size, [](const OwnerT&) { return true; });
}
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> Query(uint64_t address, uint64_t size,
Predicate&& predicate) const {
return QueryImpl(address, size, true, std::forward<Predicate>(predicate));
}
// Fault paths use page candidates: exact byte-disjoint owners sharing a
// touched 4 KiB page are intentionally retained.
[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size) const {
return QueryCandidates(address, size, [](const OwnerT&) { return true; });
}
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> QueryCandidates(uint64_t address, uint64_t size,
Predicate&& predicate) const {
return QueryImpl(address, size, false, std::forward<Predicate>(predicate));
}
[[nodiscard]] size_t CoarseMembershipCount(size_t page) const {
const auto* owners = m_coarse_pages.Find(page);
return owners == nullptr ? 0 : owners->size();
}
[[nodiscard]] size_t TrackingMembershipCount(size_t page) const {
const auto* owners = m_tracking_pages.Find(page);
return owners == nullptr ? 0 : owners->size();
}
private:
template <typename Predicate>
[[nodiscard]] std::vector<OwnerT> QueryImpl(uint64_t address, uint64_t size, bool strict_bytes,
Predicate&& predicate) const {
typename CoarseTable::PageRange coarse_range {};
typename TrackingTable::PageRange tracking_range {};
if (!CoarseTable::TryGetPageRange(address, size, coarse_range) ||
(!strict_bytes && !TrackingTable::TryGetPageRange(address, size, tracking_range))) {
return {};
}
MembershipList candidates;
for (size_t page = coarse_range.first; page < coarse_range.last_exclusive; ++page) {
if (const auto* owners = m_coarse_pages.Find(page); owners != nullptr) {
AppendUnique(candidates, *owners);
}
}
std::vector<OwnerT> result;
for (const Registration* registration: candidates) {
if ((strict_bytes ? Overlaps(registration->ranges, address, size)
: HasTrackingMembership(registration, tracking_range)) &&
predicate(registration->owner)) {
result.push_back(registration->owner);
}
}
return result;
}
struct Registration final {
OwnerT owner;
std::vector<ByteRange> ranges;
};
using RegistrationIterator = typename std::list<Registration>::iterator;
using ConstRegistrationIterator = typename std::list<Registration>::const_iterator;
[[nodiscard]] RegistrationIterator FindRegistration(const OwnerT& owner) {
return std::find_if(m_registrations.begin(), m_registrations.end(),
[&](const Registration& item) { return item.owner == owner; });
}
[[nodiscard]] ConstRegistrationIterator FindRegistration(const OwnerT& owner) const {
return std::find_if(m_registrations.begin(), m_registrations.end(),
[&](const Registration& item) { return item.owner == owner; });
}
[[nodiscard]] static std::vector<ByteRange> Normalize(const std::vector<ByteRange>& ranges) {
std::vector<ByteRange> sorted;
for (const auto& range: ranges) {
typename CoarseTable::PageRange ignored {};
if (!CoarseTable::TryGetPageRange(range.address, range.size, ignored)) {
return {};
}
sorted.push_back(range);
}
std::sort(sorted.begin(), sorted.end(), [](const ByteRange& lhs, const ByteRange& rhs) {
return lhs.address < rhs.address;
});
std::vector<ByteRange> merged;
for (const auto& range: sorted) {
if (merged.empty() || range.address > merged.back().address + merged.back().size) {
merged.push_back(range);
} else {
const uint64_t end = std::max(merged.back().address + merged.back().size,
range.address + range.size);
merged.back().size = end - merged.back().address;
}
}
return merged;
}
template <size_t Bits>
[[nodiscard]] static std::vector<size_t> CollectPages(const std::vector<ByteRange>& ranges) {
std::vector<size_t> pages;
for (const auto& range: ranges) {
const size_t first = static_cast<size_t>(range.address >> Bits);
const size_t last = static_cast<size_t>((range.address + range.size - 1) >> Bits);
for (size_t page = first; page <= last; ++page) {
pages.push_back(page);
}
}
std::sort(pages.begin(), pages.end());
pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
return pages;
}
template <typename Table>
[[nodiscard]] static bool HasAllMemberships(const Table& table,
const std::vector<size_t>& pages,
const Registration* registration) {
for (const size_t page: pages) {
const auto* owners = table.Find(page);
if (owners == nullptr ||
std::find(owners->begin(), owners->end(), registration) == owners->end()) {
return false;
}
}
return true;
}
[[nodiscard]] bool HasTrackingMembership(const Registration* registration,
const typename TrackingTable::PageRange& range) const {
for (size_t page = range.first; page < range.last_exclusive; ++page) {
const auto* owners = m_tracking_pages.Find(page);
if (owners != nullptr &&
std::find(owners->begin(), owners->end(), registration) != owners->end()) {
return true;
}
}
return false;
}
[[nodiscard]] static bool Overlaps(const std::vector<ByteRange>& ranges, uint64_t address,
uint64_t size) {
const uint64_t end = address + size;
return std::any_of(ranges.begin(), ranges.end(), [&](const ByteRange& range) {
return range.address < end && address < range.address + range.size;
});
}
static void AppendUnique(MembershipList& destination, const MembershipList& source) {
for (const Registration* registration: source) {
if (std::find(destination.begin(), destination.end(), registration) ==
destination.end()) {
destination.push_back(registration);
}
}
}
[[nodiscard]] static std::vector<ByteRange>
CoalesceTrackingPages(const std::vector<size_t>& pages) {
std::vector<ByteRange> result;
for (const size_t page: pages) {
const uint64_t address = static_cast<uint64_t>(page) << 12;
if (!result.empty() && result.back().address + result.back().size == address) {
result.back().size += uint64_t {1} << 12;
} else {
result.push_back({address, uint64_t {1} << 12});
}
}
return result;
}
CoarseTable m_coarse_pages;
TrackingTable m_tracking_pages;
std::list<Registration> m_registrations;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_MULTILEVELPAGETABLE_H_
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
+60 -14
View File
@@ -195,10 +195,12 @@ void TextureCache::RegisterImage(ImageId id) {
if (image.registered || image.info.data.Empty()) {
EXIT("TextureCache: invalid image registration\n");
}
std::vector<ImageOwnerIndex::ByteRange> ranges;
ranges.push_back({image.info.data.address, image.info.data.size});
if (!m_image_owner_index.Register(id, ranges)) {
EXIT("TextureCache: duplicate or invalid image registration\n");
ImagePageTable::PageRange pages {};
if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
EXIT("TextureCache: image registration is outside the guest address space\n");
}
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
m_image_page_table[page].push_back(id);
}
image.registered = true;
image.lru_id = m_lru_cache.Insert(id, m_gc_tick);
@@ -211,9 +213,15 @@ void TextureCache::UnregisterImage(ImageId id) {
return;
}
UntrackImage(id);
std::vector<ImageOwnerIndex::ByteRange> releases;
if (!m_image_owner_index.Unregister(id, releases)) {
EXIT("TextureCache: image missing from owner index\n");
ImagePageTable::PageRange pages {};
if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
EXIT("TextureCache: registered image is outside the guest address space\n");
}
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
auto* owners = m_image_page_table.Find(page);
if (owners == nullptr || !owners->Erase(id)) {
EXIT("TextureCache: image missing from page owner index\n");
}
}
m_lru_cache.Free(image.lru_id);
const auto accounted = image.AccountedSize();
@@ -451,10 +459,48 @@ const Image& TextureCache::GetImage(ImageId id) const {
return ResolveImage(id);
}
std::vector<ImageId> TextureCache::FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const {
return page_overlap ? m_image_owner_index.QueryCandidates(address, size)
: m_image_owner_index.Query(address, size);
TextureCache::ImageIds TextureCache::FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const {
ImagePageTable::PageRange pages {};
if (!ImagePageTable::TryGetPageRange(address, size, pages)) {
return {};
}
uint32_t query_epoch = ++m_image_query_epoch;
if (query_epoch == 0) {
for (const auto& slot: m_slots) {
if (slot.image != nullptr) {
slot.image->query_epoch = 0;
}
}
query_epoch = ++m_image_query_epoch;
}
ImageIds result;
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
const auto* owners = m_image_page_table.Find(page);
if (owners == nullptr) {
continue;
}
owners->ForEach([&](ImageId id) {
if (!id || id.index >= m_slots.size()) {
return;
}
const auto& slot = m_slots[id.index];
if (slot.generation != id.generation || slot.image == nullptr) {
return;
}
auto& image = *slot.image;
if (image.query_epoch == query_epoch) {
return;
}
image.query_epoch = query_epoch;
if (image.Overlaps(address, size, page_overlap)) {
result.push_back(id);
}
});
}
return result;
}
ImageId TextureCache::GetNullImage(const ImageDesc& desc) {
@@ -1152,9 +1198,9 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
ImageId result {};
bool inserted_new = false;
{
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
const std::vector<ImageId> candidates =
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
const auto candidates =
FindImagesInRegion(desc.info.data.address, desc.info.data.size, false);
for (const auto id: candidates) {
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
+10 -7
View File
@@ -99,7 +99,8 @@ private:
int32_t layer = -1;
};
using ImageOwnerIndex = MultiRangePageOwnerIndex<ImageId>;
using ImageIds = InlinePageOwnerList<ImageId, 16>;
using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
[[nodiscard]] Image& ResolveImage(ImageId id);
[[nodiscard]] const Image& ResolveImage(ImageId id) const;
@@ -125,8 +126,9 @@ private:
[[nodiscard]] static BindingType UploadBinding(const Image& image);
[[nodiscard]] bool SafeToDownload(const Image& image);
[[nodiscard]] std::vector<ImageId> FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const;
// Caller holds m_lock; it also serializes the per-image query epoch.
[[nodiscard]] ImageIds FindImagesInRegion(uint64_t address, uint64_t size,
bool page_overlap) const;
[[nodiscard]] OverlapResult ResolveOverlap(const ImageInfo& requested, BindingType binding,
ImageId cached, ImageId merged);
[[nodiscard]] ImageId ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
@@ -169,7 +171,7 @@ private:
ResourceMutex& m_resource_mutex;
std::vector<Slot> m_slots;
std::vector<uint32_t> m_free_slots;
ImageOwnerIndex m_image_owner_index;
ImagePageTable m_image_page_table;
std::map<vk::Format, ImageId> m_null_images;
Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
std::set<ImageId> m_download_images;
@@ -177,9 +179,10 @@ private:
uint64_t m_total_used_memory = 0;
uint64_t m_trigger_gc_memory = 0;
uint64_t m_pressure_gc_memory = 1536ull * 1024 * 1024;
uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0;
bool m_readback_linear_images = false;
uint64_t m_critical_gc_memory = 3ull * 1024 * 1024 * 1024;
uint64_t m_gc_tick = 0;
mutable uint32_t m_image_query_epoch = 0;
bool m_readback_linear_images = false;
friend struct TextureCacheTestAccess;
friend class BufferCache;
src/graphics/host_gpu/renderer/image/image.h
+12 -11
View File
@@ -154,17 +154,18 @@ public:
}
[[nodiscard]] uint64_t HashGuestEdges() const;
ImageInfo info;
VulkanImage backing;
ImageViewCache views;
ImageUsage usage;
ImageBinding binding;
bool registered = false;
uint64_t track_addr = 0;
uint64_t track_addr_end = 0;
ImageId depth_id {};
uint64_t tick_accessed_last = 0;
size_t lru_id = 0;
ImageInfo info;
VulkanImage backing;
ImageViewCache views;
ImageUsage usage;
ImageBinding binding;
bool registered = false;
mutable uint32_t query_epoch = 0;
uint64_t track_addr = 0;
uint64_t track_addr_end = 0;
ImageId depth_id {};
uint64_t tick_accessed_last = 0;
size_t lru_id = 0;
private:
friend struct ImageTestAccess;