raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 02:47:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

graphics: perf - batch GPU page watcher updates at 4 MiB granularity

Browse Source
This commit is contained in:
nmzik
2026-08-02 08:39:50 +02:00
parent 9da7fc5dd6
commit b877b4be9c
16 changed files with 1909 additions and 1142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/CMakeLists.txt
+7
View File
@@ -345,6 +345,11 @@ add_executable(page_manager_tests EXCLUDE_FROM_ALL
)
target_include_directories(page_manager_tests PRIVATE ${inc_headers})
add_executable(bit_array_tests EXCLUDE_FROM_ALL
../tests/BitArrayTests.cpp
)
target_include_directories(bit_array_tests PRIVATE ${inc_headers})
add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
../tests/MemoryTrackerTests.cpp
graphics/host_gpu/pageManager.cpp
@@ -457,6 +462,7 @@ if(BUILD_TESTING)
add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
add_test(NAME shader_stage_runtime COMMAND $<TARGET_FILE:shader_stage_runtime_tests>)
add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
@@ -502,6 +508,7 @@ if(BUILD_TESTING)
image_page_table_tests
memory_tracker_tests
page_manager_tests
bit_array_tests
shader_vertex_metadata_tests
shader_stage_runtime_tests
resource_tracking_tests
src/common/bitArray.h
+260
View File
@@ -0,0 +1,260 @@
#ifndef EMULATOR_SRC_COMMON_BITARRAY_H_
#define EMULATOR_SRC_COMMON_BITARRAY_H_
#include <array>
#include <bit>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <utility>
namespace Common {
template <size_t N>
class BitArray final {
static_assert(N != 0, "BitArray size must be nonzero");
static_assert(N % 64 == 0, "BitArray size must be a multiple of 64 bits");
static constexpr size_t BITS_PER_WORD = 64;
static constexpr size_t WORD_COUNT = N / BITS_PER_WORD;
public:
using Range = std::pair<size_t, size_t>;
class Iterator final {
public:
using iterator_category = std::forward_iterator_tag;
using value_type = Range;
using difference_type = std::ptrdiff_t;
using pointer = const Range*;
using reference = const Range&;
Iterator(const BitArray& bits, size_t start)
: m_bits(bits), m_range(bits.FirstRangeFrom(start)) {}
Iterator& operator++() {
m_range = m_bits.FirstRangeFrom(m_range.second);
return *this;
}
[[nodiscard]] bool operator==(const Iterator& other) const {
return &m_bits == &other.m_bits && m_range == other.m_range;
}
[[nodiscard]] bool operator!=(const Iterator& other) const { return !(*this == other); }
[[nodiscard]] reference operator*() const { return m_range; }
[[nodiscard]] pointer operator->() const { return &m_range; }
private:
const BitArray& m_bits;
Range m_range;
};
using const_iterator = Iterator;
constexpr BitArray() = default;
constexpr BitArray(const BitArray& other, size_t start, size_t end) {
if (start >= end || end > N) {
return;
}
const auto first_word = start / BITS_PER_WORD;
const auto last_word = (end - 1) / BITS_PER_WORD;
const auto start_bit = start % BITS_PER_WORD;
const auto end_bit = (end - 1) % BITS_PER_WORD;
const auto start_mask = ~uint64_t {0} << start_bit;
const auto end_mask =
end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
if (first_word == last_word) {
m_data[first_word] = other.m_data[first_word] & start_mask & end_mask;
return;
}
m_data[first_word] = other.m_data[first_word] & start_mask;
for (auto word = first_word + 1; word < last_word; word++) {
m_data[word] = other.m_data[word];
}
m_data[last_word] = other.m_data[last_word] & end_mask;
}
[[nodiscard]] constexpr bool Get(size_t index) const {
return (m_data[index / BITS_PER_WORD] & (uint64_t {1} << (index % BITS_PER_WORD))) != 0;
}
constexpr void Set(size_t index) {
m_data[index / BITS_PER_WORD] |= uint64_t {1} << (index % BITS_PER_WORD);
}
constexpr void Unset(size_t index) {
m_data[index / BITS_PER_WORD] &= ~(uint64_t {1} << (index % BITS_PER_WORD));
}
constexpr void SetRange(size_t start, size_t end) {
if (start >= end || end > N) {
return;
}
const auto first_word = start / BITS_PER_WORD;
const auto last_word = (end - 1) / BITS_PER_WORD;
const auto start_bit = start % BITS_PER_WORD;
const auto end_bit = (end - 1) % BITS_PER_WORD;
const auto start_mask = ~uint64_t {0} << start_bit;
const auto end_mask =
end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
if (first_word == last_word) {
m_data[first_word] |= start_mask & end_mask;
return;
}
m_data[first_word] |= start_mask;
for (auto word = first_word + 1; word < last_word; word++) {
m_data[word] = ~uint64_t {0};
}
m_data[last_word] |= end_mask;
}
constexpr void UnsetRange(size_t start, size_t end) {
if (start >= end || end > N) {
return;
}
const auto first_word = start / BITS_PER_WORD;
const auto last_word = (end - 1) / BITS_PER_WORD;
const auto start_bit = start % BITS_PER_WORD;
const auto end_bit = (end - 1) % BITS_PER_WORD;
const auto start_mask = (uint64_t {1} << start_bit) - 1;
const auto end_mask =
end_bit == BITS_PER_WORD - 1 ? uint64_t {0} : ~((uint64_t {1} << (end_bit + 1)) - 1);
if (first_word == last_word) {
m_data[first_word] &= start_mask | end_mask;
return;
}
m_data[first_word] &= start_mask;
for (auto word = first_word + 1; word < last_word; word++) {
m_data[word] = 0;
}
m_data[last_word] &= end_mask;
}
constexpr void Clear() { m_data.fill(0); }
constexpr void Fill() { m_data.fill(~uint64_t {0}); }
[[nodiscard]] constexpr bool None() const {
uint64_t combined = 0;
for (const auto word: m_data) {
combined |= word;
}
return combined == 0;
}
[[nodiscard]] constexpr bool Any() const { return !None(); }
[[nodiscard]] constexpr Range FirstRangeFrom(size_t start) const {
if (start >= N) {
return {N, N};
}
auto word_index = start / BITS_PER_WORD;
auto word = m_data[word_index] & (~uint64_t {0} << (start % BITS_PER_WORD));
while (word == 0) {
word_index++;
if (word_index == WORD_COUNT) {
return {N, N};
}
word = m_data[word_index];
}
const auto first = word_index * BITS_PER_WORD + std::countr_zero(word);
const auto first_bit = first % BITS_PER_WORD;
const auto first_ones =
static_cast<size_t>(std::countr_one(m_data[word_index] >> first_bit));
if (first_bit + first_ones < BITS_PER_WORD) {
return {first, first + first_ones};
}
for (word_index++; word_index < WORD_COUNT; word_index++) {
word = m_data[word_index];
if (word != ~uint64_t {0}) {
return {first, word_index * BITS_PER_WORD + std::countr_one(word)};
}
}
return {first, N};
}
[[nodiscard]] constexpr Range FirstRange() const { return FirstRangeFrom(0); }
[[nodiscard]] constexpr Range LastRangeFrom(size_t end) const {
if (end == 0) {
return {0, 0};
}
if (end > N) {
end = N;
}
auto word_index = (end - 1) / BITS_PER_WORD;
const auto end_bit = (end - 1) % BITS_PER_WORD;
const auto end_mask =
end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1;
auto word = m_data[word_index] & end_mask;
while (word == 0) {
if (word_index == 0) {
return {0, 0};
}
word = m_data[--word_index];
}
const auto empty_bits = static_cast<size_t>(std::countl_zero(word));
const auto ones = static_cast<size_t>(std::countl_one(word << empty_bits));
const auto last = (word_index + 1) * BITS_PER_WORD - empty_bits;
if (empty_bits + ones < BITS_PER_WORD) {
return {last - ones, last};
}
while (word_index != 0) {
word = m_data[--word_index];
if (word != ~uint64_t {0}) {
return {(word_index + 1) * BITS_PER_WORD - std::countl_one(word), last};
}
}
return {0, last};
}
[[nodiscard]] constexpr Range LastRange() const { return LastRangeFrom(N); }
[[nodiscard]] const_iterator begin() const { return Iterator(*this, 0); }
[[nodiscard]] const_iterator end() const { return Iterator(*this, N); }
constexpr BitArray& operator^=(const BitArray& other) {
for (size_t word = 0; word < WORD_COUNT; word++) {
m_data[word] ^= other.m_data[word];
}
return *this;
}
[[nodiscard]] constexpr BitArray operator^(const BitArray& other) const {
auto result = *this;
result ^= other;
return result;
}
[[nodiscard]] constexpr BitArray operator~() const {
auto result = *this;
for (auto& word: result.m_data) {
word = ~word;
}
return result;
}
private:
std::array<uint64_t, WORD_COUNT> m_data {};
};
} // namespace Common
#endif // EMULATOR_SRC_COMMON_BITARRAY_H_
src/graphics/host_gpu/memoryTracker.cpp
+7 -24
View File
@@ -6,13 +6,7 @@ namespace Libs::Graphics {
static_assert(std::atomic<void*>::is_always_lock_free);
MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
: m_page_manager(page_manager), m_gpu_watch_mode(gpu_watch_mode) {
switch (m_gpu_watch_mode) {
case PageWatchMode::Write:
case PageWatchMode::ReadWrite: break;
default: EXIT("unsupported memory tracker GPU page-watch mode\n");
}
MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) {
m_regions = std::make_unique<std::atomic<RegionManager*>[]>(REGION_COUNT);
for (size_t i = 0; i < REGION_COUNT; i++) {
m_regions[i].store(nullptr, std::memory_order_relaxed);
@@ -106,34 +100,25 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
std::lock_guard access(m_access_mutex);
Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
manager->ApplyProtection(changed, false);
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
});
}
void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
manager->ChangeState<DirtySource::Gpu, true>(manager->GetCpuAddr() + offset, bytes);
manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode);
manager->ChangeState<DirtySource::Gpu, true>(manager->GetCpuAddr() + offset, bytes);
});
}
void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
EXIT("cannot clear partially GPU-dirty tracking range\n");
}
const auto changed =
manager->ChangeState<DirtySource::Gpu, false>(manager->GetCpuAddr() + offset, bytes);
manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode);
manager->ChangeState<DirtySource::Gpu, false>(manager->GetCpuAddr() + offset, bytes);
});
}
@@ -156,9 +141,7 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
EXIT("cannot untrack GPU-dirty memory\n");
}
Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto changed =
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
manager->ApplyProtection(changed, false);
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
});
locks.clear();
}
src/graphics/host_gpu/memoryTracker.h
+10 -16
View File
@@ -18,8 +18,7 @@ namespace Libs::Graphics {
class MemoryTracker final {
public:
explicit MemoryTracker(PageManager& page_manager,
PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite);
explicit MemoryTracker(PageManager& page_manager);
~MemoryTracker();
KYTY_CLASS_NO_COPY(MemoryTracker);
@@ -56,9 +55,8 @@ public:
}
Iterate<false>(vaddr, size,
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto changed = manager->ChangeState<DirtySource::Cpu, true>(
manager->ChangeState<DirtySource::Cpu, true>(
manager->GetCpuAddr() + offset, bytes);
manager->ApplyProtection(changed, false);
});
return false;
};
@@ -109,10 +107,8 @@ public:
Iterate<false>(vaddr, size,
[&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto address = manager->GetCpuAddr() + offset;
const auto changed =
manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
address, bytes, [](uint64_t, uint64_t) noexcept {});
manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode);
manager->template ForEachModifiedRange<DirtySource::Gpu, true>(
address, bytes, [](uint64_t, uint64_t) noexcept {});
});
}
}
@@ -142,13 +138,12 @@ public:
});
upload_func();
if (is_written) {
Iterate<false>(
vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto changed = manager->template ChangeState<DirtySource::Gpu, true>(
manager->GetCpuAddr() + offset, bytes);
manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode);
manager->lock.unlock();
});
Iterate<false>(vaddr, size,
[](RegionManager* manager, uint64_t offset, uint64_t bytes) {
manager->template ChangeState<DirtySource::Gpu, true>(
manager->GetCpuAddr() + offset, bytes);
manager->lock.unlock();
});
}
s_upload_owner = previous_upload_owner;
}
@@ -202,7 +197,6 @@ private:
std::mutex m_region_mutex;
std::mutex m_access_mutex;
PageManager& m_page_manager;
PageWatchMode m_gpu_watch_mode = PageWatchMode::ReadWrite;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/pageManager.cpp
+156 -303
View File
@@ -9,7 +9,6 @@
#include <cstdarg>
#include <cstdio>
#include <cstdlib>
#include <limits>
#include <memory>
#include <mutex>
#include <vector>
@@ -22,13 +21,9 @@
#undef min
#undef max
#elif defined(__APPLE__)
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#else
#include <execinfo.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif
@@ -41,9 +36,8 @@ constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE;
constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE;
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
// The tracker reuses Win32 memory-protection tags as internal page-state values (on
// Windows they come from <windows.h> and are what VirtualQuery returns). Mirror the
// canonical Win32 numeric values so the shared state-machine logic is identical.
// The tracker reuses Win32 memory-protection tags as internal page-state values.
// Mirror their canonical numeric values so the shared state-machine logic is identical.
constexpr uint32_t PAGE_NOACCESS = 0x01;
constexpr uint32_t PAGE_READONLY = 0x02;
constexpr uint32_t PAGE_READWRITE = 0x04;
@@ -53,8 +47,6 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS;
constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY;
constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
// Zero is the unknown protection sentinel.
constexpr uint32_t UNKNOWN_PROTECTION = 0;
[[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
std::fputs("PageManager fail-fast: ", stderr);
@@ -102,25 +94,6 @@ Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
}
}
uint32_t CurrentThread() noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
return GetCurrentThreadId();
#elif defined(__APPLE__)
return static_cast<uint32_t>(pthread_mach_thread_np(pthread_self()));
#elif defined(__linux__)
static thread_local const uint32_t tid = [] {
const auto raw = static_cast<uint32_t>(::syscall(SYS_gettid));
if (raw == 0) {
FailFast("gettid returned the reserved zero owner token");
}
return raw;
}();
return tid;
#else
FailFast("page tracking thread identity is unsupported on this platform");
#endif
}
class SpinGuard final {
public:
explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -154,38 +127,58 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
struct PageManager::Impl {
struct PageState {
std::atomic_flag lock = ATOMIC_FLAG_INIT;
uint32_t write_watchers = 0;
uint32_t access_watchers = 0;
uint32_t original_protection = 0;
uint32_t backing_writer = 0;
// Shadow the protection applied through Protect().
uint32_t current_protection = UNKNOWN_PROTECTION;
bool resolving = false;
};
uint8_t write_watchers : 7 = 0;
uint8_t access_watchers : 1 = 0;
struct Region {
std::array<PageState, REGION_PAGES> pages;
};
[[nodiscard]] uint32_t Perms() const noexcept {
if (access_watchers != 0) {
return NO_ACCESS_PROTECTION;
}
if (write_watchers != 0) {
return READ_ONLY_PROTECTION;
}
return READ_WRITE_PROTECTION;
}
class PageRangeGuard final {
public:
explicit PageRangeGuard(std::span<PageState*> pages): m_pages(pages) {
for (auto* page: m_pages) {
while (page->lock.test_and_set(std::memory_order_acquire)) {
std::atomic_signal_fence(std::memory_order_seq_cst);
template <int delta, bool is_read>
uint32_t AddDelta(uint64_t address) {
static_assert(delta >= -1 && delta <= 1);
if constexpr (is_read) {
if constexpr (delta == 1) {
if (access_watchers != 0) {
Fatal("read-watcher overflow at 0x%016" PRIx64, address);
}
return ++access_watchers;
} else if constexpr (delta == -1) {
if (access_watchers == 0) {
Fatal("read-watcher underflow at 0x%016" PRIx64, address);
}
return --access_watchers;
} else {
return access_watchers;
}
} else {
if constexpr (delta == 1) {
if (write_watchers == 0x7f) {
Fatal("write-watcher overflow at 0x%016" PRIx64, address);
}
return ++write_watchers;
} else if constexpr (delta == -1) {
if (write_watchers == 0) {
Fatal("write-watcher underflow at 0x%016" PRIx64, address);
}
return --write_watchers;
} else {
return write_watchers;
}
}
}
~PageRangeGuard() {
for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) {
(*it)->lock.clear(std::memory_order_release);
}
}
KYTY_CLASS_NO_COPY(PageRangeGuard);
};
static_assert(sizeof(PageState) == 1);
private:
std::span<PageState*> m_pages;
struct Region {
std::atomic_flag lock = ATOMIC_FLAG_INIT;
std::array<PageState, REGION_PAGES> pages;
};
Impl() {
@@ -215,10 +208,9 @@ struct PageManager::Impl {
~Impl() {
for (const auto& region: region_storage) {
SpinGuard lock(region->lock);
for (auto& page: region->pages) {
SpinGuard lock(page.lock);
if (page.write_watchers != 0 || page.access_watchers != 0 ||
page.backing_writer != 0 || page.resolving) {
if (page.write_watchers != 0 || page.access_watchers != 0) {
FailFast("PageManager destroyed with live page state");
}
}
@@ -246,56 +238,81 @@ struct PageManager::Impl {
return ptr;
}
PageState& GetPage(Region& region, uint64_t vaddr) const {
return region.pages[(vaddr % REGION_SIZE) / PAGE_SIZE];
}
static uint32_t WatcherProtection(const PageState& page) {
if (page.access_watchers != 0) {
return NO_ACCESS_PROTECTION;
}
if (page.write_watchers != 0) {
return READ_ONLY_PROTECTION;
}
return page.original_protection;
}
static void InitializeProtection(std::span<PageState*> pages) {
for (auto* page: pages) {
page->original_protection = READ_WRITE_PROTECTION;
page->current_protection = READ_WRITE_PROTECTION;
}
}
void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old) noexcept {
const auto size = pages.size() * PAGE_SIZE;
if (pages.size() != expected_old.size()) {
FailFast("protection range state size mismatch");
}
for (size_t i = 0; i < pages.size(); i++) {
const auto actual = pages[i]->current_protection;
if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
}
}
void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept {
if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
ToMemoryMode(protection))) {
Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
protection);
}
for (auto* page: pages) {
page->current_protection = protection;
}
}
void Protect(PageState& page, uint64_t vaddr, uint32_t protection,
uint32_t expected_old) noexcept {
PageState* pages[] = {&page};
uint32_t expected[] = {expected_old};
ProtectRange(pages, vaddr, protection, expected);
template <bool track, bool is_read, bool masked>
void UpdateRegionWatchers(Region& region, uint64_t base_addr, size_t first, size_t last,
const RegionBits* mask = nullptr) {
SpinGuard lock(region.lock);
auto perms = region.pages[first].Perms();
uint64_t range_begin = 0;
uint64_t range_bytes = 0;
uint64_t potential_range_bytes = 0;
const auto release_pending = [&] {
if (range_bytes != 0) {
Protect(base_addr + range_begin * PAGE_SIZE, range_bytes, perms);
range_bytes = 0;
potential_range_bytes = 0;
}
};
for (size_t page_index = first; page_index < last; page_index++) {
auto& page = region.pages[page_index];
const auto address = base_addr + page_index * PAGE_SIZE;
const bool update = !masked || mask->Get(page_index);
const auto old_perms = page.Perms();
const auto new_count = update ? page.AddDelta<track ? 1 : -1, is_read>(address)
: page.AddDelta<0, is_read>(address);
const auto new_perms = page.Perms();
if (new_perms != perms) [[unlikely]] {
release_pending();
perms = new_perms;
} else if (range_bytes != 0) {
potential_range_bytes += PAGE_SIZE;
}
if (!update) {
continue;
}
const bool watcher_edge = (track && new_count == 1) || (!track && new_count == 0);
if (watcher_edge && old_perms != new_perms) {
if (range_bytes == 0) {
range_begin = page_index;
potential_range_bytes = PAGE_SIZE;
}
range_bytes = potential_range_bytes;
}
}
release_pending();
}
template <bool track, bool is_read>
void UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
const auto begin = PageStart(vaddr);
const auto end = PageEnd(vaddr, size);
for (auto chunk_begin = begin; chunk_begin < end;) {
const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE;
auto* region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin);
if (region == nullptr) {
Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
}
const auto first = static_cast<size_t>((chunk_begin - region_base) / PAGE_SIZE);
const auto last = static_cast<size_t>((chunk_end - region_base) / PAGE_SIZE);
UpdateRegionWatchers<track, is_read, false>(*region, region_base, first, last);
chunk_begin = chunk_end;
}
}
std::unique_ptr<std::atomic<Region*>[]> regions;
@@ -313,212 +330,48 @@ uint64_t PageManager::GetPageSize() const {
return PAGE_SIZE;
}
void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode) {
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
Fatal("invalid watcher mode");
}
const auto begin = PageStart(vaddr);
const auto end = PageEnd(vaddr, size);
for (auto chunk_begin = begin; chunk_begin < end;) {
const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
auto* region =
track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin);
if (region == nullptr) {
Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
}
const auto page_count = static_cast<size_t>((chunk_end - chunk_begin) / PAGE_SIZE);
std::vector<Impl::PageState*> pages;
pages.reserve(page_count);
for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) {
pages.push_back(&m_impl->GetPage(*region, address));
}
Impl::PageRangeGuard lock(pages);
std::vector<uint8_t> first_watchers(page_count);
for (size_t i = 0; i < page_count; i++) {
auto& page = *pages[i];
const auto address = chunk_begin + i * PAGE_SIZE;
if (page.resolving && track) {
FailFast("new page watcher raced active fault resolution");
}
auto& watchers =
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
if (track) {
if (watchers == std::numeric_limits<uint32_t>::max()) {
Fatal("watcher overflow at 0x%016" PRIx64, address);
}
first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0;
} else {
if (watchers == 0) {
Fatal("watcher underflow at 0x%016" PRIx64, address);
}
if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
Fatal("backing write ownership changed at 0x%016" PRIx64, address);
}
}
}
if (track) {
for (size_t first = 0; first < page_count;) {
while (first < page_count && first_watchers[first] == 0) {
first++;
}
auto last = first;
while (last < page_count && first_watchers[last] != 0) {
last++;
}
if (first != last) {
Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
}
first = last;
}
}
std::vector<uint32_t> old_protections(page_count);
std::vector<uint32_t> new_protections(page_count);
std::vector<uint8_t> transitions(page_count);
for (size_t i = 0; i < page_count; i++) {
auto& page = *pages[i];
auto& watchers =
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
const auto old_protection = Impl::WatcherProtection(page);
if (track) {
watchers++;
} else {
watchers--;
}
const auto new_protection = Impl::WatcherProtection(page);
old_protections[i] = old_protection;
new_protections[i] = new_protection;
if (new_protection != old_protection && (track || page.backing_writer == 0)) {
transitions[i] = 1;
}
}
for (size_t first = 0; first < page_count;) {
while (first < page_count && transitions[first] == 0) {
first++;
}
if (first == page_count) {
break;
}
const auto protection = new_protections[first];
auto current = first + 1;
auto last = current;
for (; current < page_count && new_protections[current] == protection; current++) {
if (old_protections[current] != new_protections[current] &&
transitions[current] == 0) {
break;
}
if (transitions[current] != 0) {
last = current + 1;
}
}
m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first));
first = current;
}
for (auto* page: pages) {
if (!track && page->backing_writer == 0 && page->write_watchers == 0 &&
page->access_watchers == 0) {
page->original_protection = 0;
}
}
chunk_begin = chunk_end;
}
template <bool track>
void PageManager::UpdatePageWatchers(uint64_t vaddr, uint64_t size) {
m_impl->UpdatePageWatchers<track, false>(vaddr, size);
}
template void PageManager::UpdatePageWatchers<true>(uint64_t, uint64_t);
template void PageManager::UpdatePageWatchers<false>(uint64_t, uint64_t);
template <bool track, bool is_read>
void PageManager::UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask) {
if (base_addr % REGION_SIZE != 0 || base_addr >= ADDRESS_SIZE ||
REGION_SIZE > ADDRESS_SIZE - base_addr) {
Fatal("invalid tracking region base 0x%016" PRIx64, base_addr);
}
const auto start_range = mask.FirstRange();
const auto end_range = mask.LastRange();
if (start_range.first == REGION_PAGES) {
FailFast("empty region watcher mask");
}
const auto first = start_range.first;
const auto last = end_range.second;
if (start_range.second == end_range.second) {
m_impl->UpdatePageWatchers<track, is_read>(base_addr + first * PAGE_SIZE,
(last - first) * PAGE_SIZE);
return;
}
auto* region = track ? m_impl->GetOrCreateRegion(base_addr) : m_impl->FindRegion(base_addr);
if (region == nullptr) {
Fatal("untracking unknown region 0x%016" PRIx64, base_addr);
}
m_impl->UpdateRegionWatchers<track, is_read, true>(*region, base_addr, first, last, &mask);
}
template void PageManager::UpdatePageWatchersForRegion<true, true>(uint64_t, RegionBits&);
template void PageManager::UpdatePageWatchersForRegion<true, false>(uint64_t, RegionBits&);
template void PageManager::UpdatePageWatchersForRegion<false, true>(uint64_t, RegionBits&);
template void PageManager::UpdatePageWatchersForRegion<false, false>(uint64_t, RegionBits&);
void PageManager::OnGpuMap(uint64_t, uint64_t) {}
void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
uint64_t size) noexcept
: m_manager(manager), m_vaddr(vaddr), m_size(size) {
m_manager.BeginBackingWrite(vaddr, size);
}
PageManager::BackingWrite::~BackingWrite() {
m_manager.EndBackingWrite(m_vaddr, m_size);
}
std::vector<std::unique_ptr<PageManager::BackingWrite>>
PageManager::ReserveBackingWrites(std::span<const RangeSet::Range> ranges) {
if (ranges.empty()) {
Fatal("cannot reserve empty backing-write ranges");
}
std::vector<std::unique_ptr<BackingWrite>> writes;
writes.reserve(ranges.size());
uint64_t begin = 0;
uint64_t end = 0;
for (const auto& range: ranges) {
if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address ||
range.address + range.size > UINT64_MAX - (PAGE_SIZE - 1)) {
Fatal("invalid backing-write range");
}
const auto page_begin = PageStart(range.address);
const auto page_end = PageStart(range.address + range.size + PAGE_SIZE - 1);
if (begin != 0 && page_begin > end) {
writes.push_back(std::make_unique<BackingWrite>(*this, begin, end - begin));
begin = 0;
}
if (begin == 0) {
begin = page_begin;
end = page_end;
} else {
end = std::max(end, page_end);
}
}
writes.push_back(std::make_unique<BackingWrite>(*this, begin, end - begin));
return writes;
}
void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
const auto end = PageEnd(vaddr, size);
const auto writer = CurrentThread();
for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
auto* region = m_impl->FindRegion(address);
if (region == nullptr) {
Fatal("backing write reserves an unknown page at 0x%016" PRIx64, address);
}
auto& page = m_impl->GetPage(*region, address);
SpinGuard lock(page.lock);
if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
Fatal("backing write races page resolution at 0x%016" PRIx64, address);
}
page.resolving = true;
page.backing_writer = writer;
}
}
void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
const auto end = PageEnd(vaddr, size);
const auto writer = CurrentThread();
for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
auto* region = m_impl->FindRegion(address);
if (region == nullptr) {
FailFast("backing write ended for an unknown page");
}
auto& page = m_impl->GetPage(*region, address);
SpinGuard lock(page.lock);
if (!page.resolving || page.backing_writer != writer) {
FailFast("backing write ended without matching owner and resolving state");
}
const auto old_protection = NO_ACCESS_PROTECTION;
const auto new_protection = Impl::WatcherProtection(page);
if (new_protection != old_protection) {
m_impl->Protect(page, address, new_protection, old_protection);
}
if (page.write_watchers == 0 && page.access_watchers == 0) {
page.original_protection = 0;
}
page.backing_writer = 0;
page.resolving = false;
}
}
} // namespace Libs::Graphics
src/graphics/host_gpu/pageManager.h
+5 -24
View File
@@ -2,31 +2,16 @@
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_
#include "common/common.h"
#include "graphics/host_gpu/rangeSet.h"
#include "graphics/host_gpu/regionDefinitions.h"
#include <memory>
#include <span>
#include <vector>
namespace Libs::Graphics {
enum class PageFaultAccess { Read, Write, Execute, Unknown };
enum class PageWatchMode { Write, ReadWrite };
class PageManager final {
public:
class BackingWrite final {
public:
BackingWrite(PageManager& manager, uint64_t vaddr, uint64_t size) noexcept;
~BackingWrite();
KYTY_CLASS_NO_COPY(BackingWrite);
private:
PageManager& m_manager;
uint64_t m_vaddr = 0;
uint64_t m_size = 0;
};
PageManager();
// The owner must stop all PageManager callers before destruction.
~PageManager();
@@ -35,18 +20,14 @@ public:
[[nodiscard]] uint64_t GetPageSize() const;
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode = PageWatchMode::Write);
template <bool track>
void UpdatePageWatchers(uint64_t vaddr, uint64_t size);
template <bool track, bool is_read = false>
void UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask);
void OnGpuMap(uint64_t vaddr, uint64_t size);
void OnGpuUnmap(uint64_t vaddr, uint64_t size);
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
private:
void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
void EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept;
struct Impl;
std::unique_ptr<Impl> m_impl;
};
src/graphics/host_gpu/regionDefinitions.h
+3 -3
View File
@@ -1,10 +1,9 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_
#include "common/bitArray.h"
#include "common/common.h"
#include <bitset>
namespace Libs::Graphics {
constexpr uint64_t TRACKER_PAGE_SIZE = 4ull * 1024ull;
@@ -13,7 +12,8 @@ constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u;
constexpr size_t TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE;
enum class DirtySource { Cpu, Gpu };
using RegionBits = std::bitset<TRACKER_REGION_PAGES>;
using RegionBits = Common::BitArray<TRACKER_REGION_PAGES>;
static_assert(sizeof(RegionBits) == TRACKER_REGION_PAGES / 8);
} // namespace Libs::Graphics
src/graphics/host_gpu/regionManager.h
+53 -86
View File
@@ -76,8 +76,9 @@ public:
if (m_cpu_addr % TRACKER_REGION_SIZE != 0) {
EXIT("invalid region tracking manager construction\n");
}
m_cpu_dirty.set();
m_writable.set();
m_cpu_dirty.Fill();
m_writable.Fill();
m_readable.Fill();
}
KYTY_CLASS_NO_COPY(RegionManager);
@@ -87,106 +88,81 @@ public:
[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
const auto& bits = GetBits<source>();
for (auto page = start; page < end; page++) {
if (bits.test(page)) {
return true;
}
}
return false;
}
template <DirtySource source>
[[nodiscard]] bool IsFullyModified(uint64_t offset, uint64_t size) const {
const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
const auto& bits = GetBits<source>();
for (auto page = start; page < end; page++) {
if (!bits.test(page)) {
return false;
}
}
return true;
return RegionBits(bits, start, end).Any();
}
template <DirtySource source, bool enable>
RegionBits ChangeState(uint64_t vaddr, uint64_t size) {
void ChangeState(uint64_t vaddr, uint64_t size) {
const auto [start, end] = GetPageRange(vaddr, size);
if constexpr (source == DirtySource::Cpu && enable) {
for (auto page = start; page < end; page++) {
if (m_gpu_dirty.test(page)) {
EXIT("CPU dirty state conflicts with GPU dirty state\n");
}
if (RegionBits(m_gpu_dirty, start, end).Any()) {
EXIT("CPU dirty state conflicts with GPU dirty state\n");
}
}
if constexpr (source == DirtySource::Gpu && enable) {
for (auto page = start; page < end; page++) {
if (m_cpu_dirty.test(page)) {
EXIT("GPU dirty state conflicts with CPU dirty state\n");
}
if (RegionBits(m_cpu_dirty, start, end).Any()) {
EXIT("GPU dirty state conflicts with CPU dirty state\n");
}
}
auto& bits = GetBits<source>();
auto changed = bits;
for (auto page = start; page < end; page++) {
bits.set(page, enable);
auto& bits = GetBits<source>();
if constexpr (enable) {
bits.SetRange(start, end);
} else {
bits.UnsetRange(start, end);
}
changed ^= bits;
if constexpr (source == DirtySource::Cpu) {
changed = m_cpu_dirty ^ m_writable;
m_writable = m_cpu_dirty;
UpdateCpuProtection<!enable>();
} else {
UpdateGpuProtection<enable>();
}
return changed;
}
template <DirtySource source, bool clear, typename Func>
RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
const auto [start, end] = GetPageRange(vaddr, size);
auto mask = GetBits<source>();
for (auto page = 0u; page < start; page++) {
mask.reset(page);
}
for (auto page = end; page < TRACKER_REGION_PAGES; page++) {
mask.reset(page);
}
RegionBits mask(GetBits<source>(), start, end);
if constexpr (clear) {
auto& bits = GetBits<source>();
for (auto page = start; page < end; page++) {
if (mask.test(page)) {
bits.reset(page);
}
}
GetBits<source>().UnsetRange(start, end);
}
if constexpr (source == DirtySource::Cpu && clear) {
auto changed = m_cpu_dirty ^ m_writable;
m_writable = m_cpu_dirty;
ApplyProtection(changed, true);
UpdateCpuProtection<true>();
ForEachRange(mask, std::forward<Func>(func));
return changed;
return;
}
if constexpr (source == DirtySource::Gpu && clear) {
UpdateGpuProtection<false>();
}
ForEachRange(mask, std::forward<Func>(func));
if constexpr (clear) {
return mask;
}
return {};
}
void ApplyProtection(const RegionBits& changed, bool track) {
ForEachRange(changed, [this, track](uint64_t vaddr, uint64_t size) {
m_page_manager.UpdatePageWatchers(track, vaddr, size);
});
}
void ApplyGpuProtection(const RegionBits& changed, bool track, PageWatchMode mode) {
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
EXIT("unsupported GPU page-watch mode\n");
}
ForEachRange(changed, [this, track, mode](uint64_t vaddr, uint64_t size) {
m_page_manager.UpdatePageWatchers(track, vaddr, size, mode);
});
}
TrackingSpinLock lock;
private:
template <bool track>
void UpdateCpuProtection() {
auto mask = m_cpu_dirty ^ m_writable;
m_writable = m_cpu_dirty;
if (mask.None()) {
return;
}
m_page_manager.UpdatePageWatchersForRegion<track>(m_cpu_addr, mask);
}
template <bool track>
void UpdateGpuProtection() {
auto readable = ~m_gpu_dirty;
auto mask = readable ^ m_readable;
m_readable = readable;
if (mask.None()) {
return;
}
if constexpr (track) {
m_page_manager.UpdatePageWatchersForRegion<true, true>(m_cpu_addr, mask);
} else {
m_page_manager.UpdatePageWatchersForRegion<false, true>(m_cpu_addr, mask);
}
}
template <DirtySource source>
RegionBits& GetBits() {
if constexpr (source == DirtySource::Cpu) {
@@ -217,18 +193,8 @@ private:
template <typename Func>
void ForEachRange(const RegionBits& bits, Func&& func) const {
size_t page = 0;
while (page < TRACKER_REGION_PAGES) {
while (page < TRACKER_REGION_PAGES && !bits.test(page)) {
page++;
}
const auto start = page;
while (page < TRACKER_REGION_PAGES && bits.test(page)) {
page++;
}
if (start != page) {
func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (page - start) * TRACKER_PAGE_SIZE);
}
for (const auto [start, end]: bits) {
func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE);
}
}
@@ -237,6 +203,7 @@ private:
RegionBits m_cpu_dirty;
RegionBits m_gpu_dirty;
RegionBits m_writable;
RegionBits m_readable;
};
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+51 -314
View File
@@ -123,12 +123,6 @@ struct BufferCache::RetiredBuffer {
std::shared_ptr<Buffer> owner;
};
struct BufferCache::PendingBackingPublication {
uint64_t address = 0;
uint64_t size = 0;
uint64_t tick = 0;
};
std::pair<uint64_t, uint64_t> BufferCache::DownloadEnvelope(const DownloadCopy& copy) {
if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() ||
copy.size > copy.owner->Size() - copy.source_offset) {
@@ -200,34 +194,26 @@ void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, Ret
if (copies.empty()) {
return;
}
auto downloads = RecordDownloads(copies);
const auto tick = m_scheduler.CurrentTick();
BeginBackingPublication(retire.address, retire.size, tick);
m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire),
tick]() mutable {
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
m_memory_tracker.ForEachDownloadRange<true>(
retire.address, retire.size,
[&](uint64_t address, uint64_t size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
"asynchronous garbage retirement");
},
[](uint64_t, uint64_t) noexcept {});
}
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
}
m_memory_tracker.UntrackMemory(retire.address, retire.size);
}
CompleteBackingPublication(retire.address, retire.size, tick);
});
auto downloads = RecordDownloads(copies);
m_scheduler.DeferOperation(
[this, downloads = std::move(downloads), retire = std::move(retire)]() mutable {
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
// ForEachDownloadRange reports full tracker pages, and every exact GPU-owned
// interval on those pages was downloaded and removed. Clearing the original
// query therefore cannot orphan a dirty sibling on an edge page.
m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size);
if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
!m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
}
m_memory_tracker.UntrackMemory(retire.address, retire.size);
}
});
}
BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -240,8 +226,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB),
m_page_manager(page_manager), m_texture_cache(texture_cache),
m_resource_mutex(resource_mutex) {
m_texture_cache(texture_cache), m_resource_mutex(resource_mutex) {
std::memset(m_gds_buffer.Mapped().data(), 0, static_cast<size_t>(m_gds_buffer.Size()));
m_gds_buffer.Flush(0, m_gds_buffer.Size());
if (!m_graphics.CanReportMemoryUsage()) {
@@ -262,9 +247,6 @@ BufferCache::~BufferCache() {
if (!m_gpu_modified_ranges.Empty()) {
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
}
if (!m_pending_backing_publications.empty()) {
EXIT("BufferCache: destroyed with pending backing publications\n");
}
for (const auto& [vaddr, cached]: m_buffers) {
(void)vaddr;
if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) {
@@ -274,68 +256,6 @@ BufferCache::~BufferCache() {
m_buffers.clear();
}
bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
bool waited = false;
for (;;) {
uint64_t tick = 0;
const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
CacheRange affected {.address = page_begin, .size = page_end - page_begin};
{
FaultSafeCacheLock lock(this, m_mutex);
bool changed = true;
while (changed) {
changed = false;
for (const auto& [address, cached]: m_buffers) {
const CacheRange previous = affected;
if (ResolveOverlap(affected, {address, cached->size}) &&
(previous.address != affected.address || previous.size != affected.size)) {
changed = true;
}
}
}
}
{
std::lock_guard lock(m_publication_mutex);
for (const auto& publication: m_pending_backing_publications) {
if (publication.address < affected.address + affected.size &&
affected.address < publication.address + publication.size) {
tick = std::max(tick, publication.tick);
}
}
}
if (tick == 0) {
return waited;
}
waited = true;
m_scheduler.Wait(tick);
m_scheduler.WaitPriorityOperations(tick);
}
}
void BufferCache::RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached,
uint64_t vaddr, uint64_t size, bool upload) {
const auto invalidated = m_image_invalidated_ranges.Intersections(vaddr, size);
if (upload) {
std::array<uint8_t, 64 * 1024> bytes;
for (const auto& range: invalidated) {
for (uint64_t copied = 0; copied < range.size;) {
const auto chunk = std::min<uint64_t>(range.size - copied, bytes.size());
if (!Libs::LibKernel::Memory::TryReadBacking(range.address + copied, bytes.data(),
chunk)) {
EXIT("BufferCache: failed to refresh an invalidated image alias\n");
}
Upload(command, *cached.buffer, cached.buffer->Offset(range.address + copied),
bytes.data(), chunk);
copied += chunk;
}
}
}
if (!invalidated.empty()) {
m_image_invalidated_ranges.Subtract(vaddr, size);
}
}
StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept {
switch (usage) {
case MemoryUsage::Upload: return m_staging_buffer;
@@ -364,7 +284,6 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid memory-invalidation range\n");
}
(void)SynchronizeBacking(vaddr, size);
if (!HasPageOverlap(vaddr, size)) {
return;
}
@@ -373,7 +292,6 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
}
void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
(void)SynchronizeBacking(vaddr, size);
std::vector<DownloadCopy> copies;
{
FaultSafeCacheLock lock(this, m_mutex);
@@ -413,16 +331,11 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
PublishDownloads(downloads);
{
FaultSafeCacheLock lock(this, m_mutex);
m_memory_tracker.ForEachDownloadRange<true>(
vaddr, size,
[&](uint64_t address, uint64_t bytes) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
"memory invalidation completion");
},
[](uint64_t, uint64_t) noexcept {});
for (const auto& range: downloads) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
// The enumeration above covered whole dirty pages and every exact interval on them.
m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size);
}
}
@@ -430,13 +343,9 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("BufferCache: invalid unmap range\n");
}
(void)SynchronizeBacking(vaddr, size);
std::vector<DownloadCopy> copies;
std::vector<RangeSet::Range> dirty_ranges;
std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
std::vector<std::unique_ptr<PageManager::BackingWrite>> backing_writes;
std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
std::vector<DownloadCopy> copies;
std::vector<std::pair<uint64_t, uint64_t>> modified_buffers;
std::vector<std::pair<uint64_t, uint64_t>> retired_buffers;
{
FaultSafeCacheLock lock(this, m_mutex);
for (const auto& [begin, cached]: m_buffers) {
@@ -453,12 +362,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
if (dirty.empty()) {
EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n");
}
dirty_ranges.insert(dirty_ranges.end(), dirty.begin(), dirty.end());
modified_buffers.emplace_back(begin, cached->size);
}
if (!dirty_ranges.empty()) {
backing_writes = m_page_manager.ReserveBackingWrites(dirty_ranges);
}
for (const auto& [begin, bytes]: modified_buffers) {
auto owner = m_buffers.find(begin);
if (owner == m_buffers.end() || owner->second->size != bytes) {
@@ -488,24 +393,11 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
// command stream before removing such backing.
m_scheduler.FinishCurrent();
}
backing_writes.clear();
{
FaultSafeCacheLock lock(this, m_mutex);
for (const auto& [begin, bytes]: modified_buffers) {
if (!m_memory_tracker.IsRegionGpuModified(begin, bytes)) {
continue;
}
m_memory_tracker.ForEachDownloadRange<true>(
begin, bytes,
[&](uint64_t address, uint64_t download_size) noexcept {
m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address,
download_size, "unmap retirement");
},
[](uint64_t, uint64_t) noexcept {});
}
for (const auto& [begin, bytes]: modified_buffers) {
m_gpu_modified_ranges.Subtract(begin, bytes);
m_memory_tracker.UnmarkRegionAsGpuModified(begin, bytes);
}
for (const auto& [begin, bytes]: retired_buffers) {
m_memory_tracker.MarkRegionAsCpuModified(begin, bytes);
@@ -513,7 +405,6 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
EXIT("BufferCache: unmap retained dirty byte ranges\n");
}
m_image_invalidated_ranges.Subtract(vaddr, size);
m_memory_tracker.UntrackMemory(vaddr, size);
for (auto it = m_buffers.begin(); it != m_buffers.end();) {
if (vaddr < it->first + it->second->size && it->first < vaddr + size) {
@@ -607,7 +498,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
std::lock_guard transaction(m_resource_mutex);
(void)SynchronizeBacking(vaddr, size);
if (is_read && !is_written && size <= CACHING_PAGE_SIZE &&
!m_memory_tracker.IsRegionGpuModified(vaddr, size) &&
@@ -630,9 +520,7 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
}
}
if (is_formatted && is_read && !is_written) {
(void)m_texture_cache.SynchronizeImageToBuffer(vaddr, size);
} else if (is_formatted && is_written) {
if (is_formatted && is_written) {
(void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true);
}
@@ -648,10 +536,12 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
reinterpret_cast<const void*>(address), bytes);
}
});
RefreshInvalidatedRanges(command, cached, vaddr, size, is_read);
if (is_written) {
m_gpu_modified_ranges.Add(vaddr, size);
}
if (is_formatted && is_read && !is_written) {
(void)SynchronizeBufferFromImage(*cached.buffer, vaddr, size);
}
return {cached.buffer, cached.buffer->Handle(), cached.buffer->Offset(vaddr)};
}
@@ -676,7 +566,6 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid image source\n");
}
(void)SynchronizeBacking(vaddr, size);
auto find_owner = [&]() {
auto owner = m_buffers.upper_bound(vaddr);
if (owner == m_buffers.begin()) {
@@ -691,13 +580,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool requested_gpu_owned = !dirty.empty();
const bool has_dirty_buffer_source = !dirty.empty();
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
"image source");
auto owner = find_owner();
if (requested_gpu_owned && owner == m_buffers.end()) {
if (has_dirty_buffer_source && owner == m_buffers.end()) {
CacheRange merged {.address = AlignDown(vaddr),
.size = AlignUp(vaddr + size) - AlignDown(vaddr)};
using Iterator = decltype(m_buffers.begin());
@@ -747,42 +635,32 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
EXIT("BufferCache: merged image source does not contain the requested range\n");
}
}
if (owner != m_buffers.end() && !cpu_modified && !invalidated &&
(!gpu_modified || requested_gpu_owned)) {
DiscardGpuDirtyBytesLocked(vaddr, size, "image source transfer");
if (owner != m_buffers.end() && !cpu_modified &&
(!gpu_modified || has_dirty_buffer_source)) {
owner->second->tick_accessed_last = m_gc_tick;
return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr),
requested_gpu_owned};
return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr)};
}
if (requested_gpu_owned && owner == m_buffers.end()) {
if (has_dirty_buffer_source && owner == m_buffers.end()) {
EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
}
}
// Direct-memory backing remains readable while PageManager protects the guest mapping. The
// fallback exists for plain host mappings used by standalone renderer tests and is deliberately
// performed outside the cache lock so a page fault cannot recurse into BufferCache.
const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto stage_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
const auto stage_size = stage_end - stage_address;
(void)SynchronizeBacking(stage_address, stage_size);
std::vector<uint8_t> bytes(stage_size);
if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) {
auto [staging, stage_offset] = m_staging_buffer.Map(size, 16);
if (staging == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size)) {
EXIT("BufferCache: failed to read mapped guest image backing\n");
}
m_staging_buffer.Commit();
FaultSafeCacheLock lock(this, m_mutex);
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
const bool requested_gpu_owned = !dirty.empty();
auto owner = find_owner();
if (requested_gpu_owned && owner == m_buffers.end()) {
const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size);
const bool has_dirty_buffer_source = !dirty.empty();
auto owner = find_owner();
if (has_dirty_buffer_source && owner == m_buffers.end()) {
EXIT("BufferCache: GPU-dirty image source lost its native owner\n");
}
const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16);
if (owner == m_buffers.end() || invalidated ||
(m_memory_tracker.IsRegionGpuModified(vaddr, size) && !requested_gpu_owned)) {
return {&m_staging_buffer, stage_offset + vaddr - stage_address, false};
if (owner == m_buffers.end() ||
(m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) {
return {&m_staging_buffer, stage_offset};
}
auto& cached = *owner->second;
@@ -796,42 +674,17 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
}
});
DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
return {cached.buffer.get(), cached.buffer->Offset(vaddr), requested_gpu_owned};
}
void BufferCache::DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation) {
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation);
m_gpu_modified_ranges.Subtract(vaddr, size);
const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
for (auto page = page_begin; page < page_end; page += TRACKER_PAGE_SIZE) {
if (m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE).empty() &&
m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE)) {
m_memory_tracker.UnmarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
}
}
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation);
}
void BufferCache::DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid dirty-byte discard range\n");
}
FaultSafeCacheLock lock(this, m_mutex);
DiscardGpuDirtyBytesLocked(vaddr, size, "image output supersession");
return {cached.buffer.get(), cached.buffer->Offset(vaddr)};
}
void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data) {
if (vaddr == 0 || data.empty() || data.size() > UINT64_MAX - vaddr) {
EXIT("BufferCache: invalid host DMA write\n");
}
(void)SynchronizeBacking(vaddr, data.size());
Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size());
FaultSafeCacheLock lock(this, m_mutex);
@@ -847,45 +700,6 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data)
data.data() + begin - vaddr, range_end - begin);
cached->tick_accessed_last = m_gc_tick;
}
m_image_invalidated_ranges.Subtract(vaddr, data.size());
}
std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWrite(uint64_t vaddr,
uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid image destination\n");
}
const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto stage_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
const auto stage_size = stage_end - stage_address;
(void)SynchronizeBacking(stage_address, stage_size);
std::vector<uint8_t> bytes(stage_size);
if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) {
EXIT("BufferCache: failed to preserve guest bytes around an image mirror\n");
}
FaultSafeCacheLock lock(this, m_mutex);
auto& cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size);
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
"image destination");
if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
EXIT("BufferCache: image destination aliases GPU-owned buffer bytes\n");
}
const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16);
std::vector<std::pair<uint64_t, uint64_t>> uploads;
m_memory_tracker.ForEachUploadRange(
vaddr, size, false,
[&](uint64_t address, uint64_t upload_size) noexcept {
uploads.emplace_back(address, upload_size);
},
[&]() noexcept {
for (const auto& [address, upload_size]: uploads) {
cached.buffer->CopyFrom(
m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
}
});
return {cached.buffer, cached.buffer->Offset(vaddr)};
}
void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds) {
@@ -945,17 +759,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
}
if (src_memory || dst_memory) {
std::lock_guard transaction(m_resource_mutex);
if (src_memory) {
(void)SynchronizeBacking(src_vaddr, size);
}
const auto src_region =
const auto src_region =
src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
const auto dst_region =
dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
if (src_memory && src_region.gpu_image_bytes &&
!m_texture_cache.SynchronizeImageToBuffer(src_vaddr, size)) {
EXIT("BufferCache: GPU copy source image could not be synchronized\n");
}
if (src_memory && dst_memory && !HasGpuDirtyBytes(src_vaddr, size) &&
!HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
!dst_region.gpu_image_bytes) {
@@ -977,7 +784,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
}
auto& command = m_scheduler.Current();
auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true)
auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true)
: BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr};
auto dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true)
: BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr};
@@ -1030,75 +837,6 @@ bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
return m_memory_tracker.IsRegionCpuModified(vaddr, size);
}
void BufferCache::InvalidateImageAliases(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid image-alias invalidation\n");
}
FaultSafeCacheLock lock(this, m_mutex);
const auto end = vaddr + size;
for (const auto& [address, cached]: m_buffers) {
const auto cached_end = address + cached->size;
const auto begin = std::max(vaddr, address);
const auto range_end = std::min(end, cached_end);
if (begin >= range_end) {
continue;
}
const auto bytes = range_end - begin;
if (!m_gpu_modified_ranges.Intersections(begin, bytes).empty()) {
EXIT("BufferCache: image ownership overlaps exact dirty buffer bytes\n");
}
m_image_invalidated_ranges.Add(begin, bytes);
}
}
void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
if (vaddr == 0 || size == 0 || tick == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
size > TRACKER_ADDRESS_SIZE - vaddr) {
EXIT("BufferCache: invalid pending backing publication\n");
}
std::lock_guard lock(m_publication_mutex);
m_pending_backing_publications.push_back({vaddr, size, tick});
}
void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
std::lock_guard lock(m_publication_mutex);
const auto publication =
std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) {
return pending.address == vaddr && pending.size == size && pending.tick == tick;
});
if (publication == m_pending_backing_publications.end()) {
EXIT("BufferCache: completed an unknown backing publication\n");
}
m_pending_backing_publications.erase(publication);
}
void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
FaultSafeCacheLock lock(this, m_mutex);
auto owner = m_buffers.end();
for (auto it = m_buffers.begin(); it != m_buffers.end(); ++it) {
if (!PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) {
continue;
}
if (owner != m_buffers.end() || !it->second->buffer->IsInBounds(vaddr, size)) {
EXIT("BufferCache: image destination aliases a non-containing cached buffer\n");
}
owner = it;
}
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
"image destination publication");
if (owner == m_buffers.end() || m_memory_tracker.IsRegionCpuModified(vaddr, size) ||
!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) {
EXIT("BufferCache: image destination requires clean buffer ownership\n");
}
m_memory_tracker.MarkRegionAsGpuModified(vaddr, size);
m_gpu_modified_ranges.Add(vaddr, size);
m_image_invalidated_ranges.Subtract(vaddr, size);
m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
"published image destination");
owner->second->tick_accessed_last = m_gc_tick;
}
void BufferCache::RunGarbageCollector() {
std::lock_guard transaction(m_resource_mutex);
const auto tick = m_gc_tick++;
@@ -1174,7 +912,6 @@ void BufferCache::RunGarbageCollector() {
if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
m_memory_tracker.UntrackMemory(retire.address, retire.size);
}
m_image_invalidated_ranges.Subtract(retire.address, retire.size);
if (retire.size > m_total_used_memory) {
EXIT("BufferCache: allocation accounting underflow\n");
}
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+3 -20
View File
@@ -10,7 +10,6 @@
#include <map>
#include <memory>
#include <mutex>
#include <span>
#include <utility>
#include <vector>
@@ -30,9 +29,8 @@ struct BufferBinding {
};
struct ImageBufferSource {
Buffer* buffer = nullptr;
uint64_t offset = 0;
bool gpu_owned = false;
Buffer* buffer = nullptr;
uint64_t offset = 0;
};
class BufferCache {
@@ -60,9 +58,6 @@ public:
uint64_t alignment);
[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
[[nodiscard]] ImageBufferSource ObtainBufferForImage(uint64_t vaddr, uint64_t size);
[[nodiscard]] std::pair<std::shared_ptr<Buffer>, uint64_t>
ObtainBufferForImageWrite(uint64_t vaddr, uint64_t size);
void DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size);
void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
bool src_gds = false);
@@ -70,11 +65,6 @@ public:
[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
void InvalidateImageAliases(uint64_t vaddr, uint64_t size);
void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
@@ -88,7 +78,6 @@ private:
struct DownloadCopy;
struct DownloadRange;
struct RetiredBuffer;
struct PendingBackingPublication;
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
[[nodiscard]] static uint64_t AlignUp(uint64_t value);
@@ -103,12 +92,10 @@ private:
const void* source, uint64_t size);
[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
uint64_t size);
[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
void PublishDownloads(std::span<const DownloadRange> downloads);
void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
uint64_t size, bool upload);
void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation);
void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
GraphicContext& m_graphics;
@@ -118,15 +105,11 @@ private:
std::shared_ptr<Buffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
RangeSet m_gpu_modified_ranges;
RangeSet m_image_invalidated_ranges;
std::mutex m_publication_mutex;
std::vector<PendingBackingPublication> m_pending_backing_publications;
MemoryTracker m_memory_tracker;
StreamBuffer m_staging_buffer;
StreamBuffer m_stream_buffer;
StreamBuffer m_download_buffer;
StreamBuffer m_device_buffer;
PageManager& m_page_manager;
TextureCache& m_texture_cache;
ResourceMutex& m_resource_mutex;
uint64_t m_total_used_memory = 0;
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
+119 -101
View File
@@ -323,7 +323,7 @@ void TextureCache::TrackImage(ImageId id) {
if (!image.IsTracked()) {
image.track_addr = image_begin;
image.track_addr_end = image_end;
m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size);
m_page_manager.UpdatePageWatchers<true>(image_begin, image.info.data.size);
return;
}
if (image_begin < image.track_addr) {
@@ -348,7 +348,7 @@ void TextureCache::TrackImageHead(ImageId id) {
}
const auto size = image.track_addr - image_begin;
image.track_addr = image_begin;
m_page_manager.UpdatePageWatchers(true, image_begin, size);
m_page_manager.UpdatePageWatchers<true>(image_begin, size);
}
void TextureCache::TrackImageTail(ImageId id) {
@@ -366,7 +366,7 @@ void TextureCache::TrackImageTail(ImageId id) {
const auto address = image.track_addr_end;
const auto size = image_end - address;
image.track_addr_end = image_end;
m_page_manager.UpdatePageWatchers(true, address, size);
m_page_manager.UpdatePageWatchers<true>(address, size);
}
void TextureCache::UntrackImage(ImageId id) {
@@ -379,7 +379,7 @@ void TextureCache::UntrackImage(ImageId id) {
image.track_addr = 0;
image.track_addr_end = 0;
if (size != 0) {
m_page_manager.UpdatePageWatchers(false, address, size);
m_page_manager.UpdatePageWatchers<false>(address, size);
}
}
@@ -400,7 +400,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
UntrackImage(id);
}
if (size != 0) {
m_page_manager.UpdatePageWatchers(false, begin, size);
m_page_manager.UpdatePageWatchers<false>(begin, size);
}
}
@@ -421,7 +421,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
UntrackImage(id);
}
if (size != 0) {
m_page_manager.UpdatePageWatchers(false, address, size);
m_page_manager.UpdatePageWatchers<false>(address, size);
}
}
@@ -1038,25 +1038,20 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
if (image.info.samples > 1) {
return;
}
bool data_gpu_owned = false;
bool data_imported = false;
const bool upload = image.IsBufferModified() || image.IsCpuDirty();
bool data_imported = false;
const bool upload = image.IsBufferModified() || image.IsCpuDirty();
if (upload) {
const auto source =
m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
if (source.buffer == nullptr) {
EXIT("TextureCache: failed to obtain image upload source\n");
}
data_gpu_owned |= source.gpu_owned;
data_imported = true;
UploadImage(image, desc, *source.buffer, source.offset);
}
if (data_imported) {
image.ClearBufferModified();
}
if (data_gpu_owned) {
image.MarkGpuModified();
}
if (image.IsCpuDirty()) {
image.RefreshComplete();
}
@@ -1131,8 +1126,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
}
ImageId result {};
bool replacement_buffer = false;
bool inserted_new = false;
bool inserted_new = false;
{
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
@@ -1174,23 +1168,15 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) {
result = {};
} else if (resolved.info.resources < desc.info.resources) {
ImageDesc refresh {
.info = resolved.info, .view_info = {}, .type = UploadBinding(resolved)};
RefreshImage(result, refresh);
if (resolved.IsGpuModified() && !SynchronizeImageToBuffer(result)) {
EXIT("TextureCache: cannot preserve an unsupported replacement image\n");
}
replacement_buffer = resolved.IsBufferModified();
DeleteImage(result);
result = {};
result = ExpandImage(desc.info, result);
}
}
if (!result) {
result = InsertImage(desc.info);
inserted_new = true;
auto& inserted = ResolveImage(result);
if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
inserted.info.data.size)) {
if (m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
inserted.info.data.size)) {
inserted.MarkBufferModified();
}
}
@@ -1360,11 +1346,6 @@ void TextureCache::CommitGpuWrite(Image& image) {
if (image.depth_id || image.backing.image == nullptr) {
EXIT("TextureCache: stencil association cannot own image contents\n");
}
const auto range = image.info.data;
if (m_buffer_cache.HasGpuDirtyBytes(range.address, range.size)) {
m_buffer_cache.DiscardGpuDirtyBytes(range.address, range.size);
}
m_buffer_cache.InvalidateImageAliases(range.address, range.size);
image.ClearBufferModified();
if (image.IsCpuDirty()) {
image.RefreshComplete();
@@ -1429,9 +1410,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
return false;
}
}
if (m_buffer_cache.HasGpuDirtyBytes(address, size)) {
m_buffer_cache.DiscardGpuDirtyBytes(address, size);
}
if (image.IsBufferModified() || image.IsCpuDirty()) {
ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)};
InitializeImage(selected, refresh);
@@ -1551,11 +1529,14 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des
}
void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
DownloadPlan plan) {
uint64_t destination_size, DownloadPlan plan) {
if (!plan.valid) {
EXIT("TextureCache: invalid image download plan\n");
}
if (plan.depth) {
if (destination_size != image.info.data.size) {
EXIT("TextureCache: partial depth image download is unsupported\n");
}
DownloadDepth(image, destination, destination_offset);
return;
}
@@ -1565,22 +1546,118 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t
: TileManager::ColorTransform::None;
if (!color.tiled) {
if (transform == TileManager::ColorTransform::SwapBgra16) {
auto linear = m_tiler->GetScratchBuffer(image.info.data.size);
auto linear = m_tiler->GetScratchBuffer(destination_size);
image.Download(color.regions, linear.buffer, 0, linear.size);
m_tiler->SwapBgra16(linear,
{destination.Handle(), destination_offset, image.info.data.size});
{destination.Handle(), destination_offset, destination_size});
return;
}
for (auto& copy: color.regions) {
copy.bufferOffset += destination_offset;
}
image.Download(color.regions, destination.Handle(), destination_offset,
image.info.data.size);
image.Download(color.regions, destination.Handle(), destination_offset, destination_size);
return;
}
m_tiler->TileImage(image, color.regions, destination.Handle(), destination_offset,
image.info.data.size, image.info.data.size, color.tiles, transform);
destination_size, destination_size, color.tiles, transform);
}
bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size) {
CacheLock lock(m_texture_cache, m_texture_cache.m_lock);
std::vector<ImageId> matches;
for (const auto id: m_texture_cache.FindImagesInRegion(vaddr, size, false)) {
auto owner = m_texture_cache.ResolveOwner(id);
if (owner == nullptr || owner->info.data.address != vaddr) {
continue;
}
if (owner->depth_id) {
owner = m_texture_cache.ResolveOwner(owner->depth_id);
}
if (owner != nullptr && owner->SafeToDownload()) {
matches.push_back(id);
}
}
ImageId selected {};
if (matches.size() == 1) {
selected = matches.front();
} else {
for (const auto id: matches) {
const auto& image = m_texture_cache.ResolveImage(id);
if (image.info.data.size == size) {
selected = id;
break;
}
}
}
if (!selected) {
return false;
}
if (const auto owner = m_texture_cache.ResolveOwner(selected);
owner != nullptr && owner->depth_id) {
selected = owner->depth_id;
}
auto& image = m_texture_cache.ResolveImage(selected);
if (!buffer.IsInBounds(image.info.data.address, 1)) {
return false;
}
const auto buf_offset = buffer.Offset(image.info.data.address);
const auto available = buffer.Size() - buf_offset;
uint32_t levels = 0;
uint64_t copy_size = 0;
if (image.info.IsVolume()) {
// Volume mips contain strided block slices, so a mip's linear span cannot prove that
// every retained slice fits. Keep volume synchronization whole-image only.
if (!buffer.IsInBounds(image.info.data.address, image.info.data.size)) {
return false;
}
levels = image.info.resources.levels;
copy_size = image.info.data.size;
} else {
for (; levels < image.info.resources.levels; ++levels) {
const auto& mip = image.info.mip_layout[levels];
if (mip.size == 0 || mip.offset > available || mip.size > available - mip.offset) {
break;
}
copy_size = std::max(copy_size, mip.offset + mip.size);
}
}
if (copy_size == 0) {
return false;
}
auto plan = m_texture_cache.BuildDownload(image);
if (!plan.valid) {
return false;
}
if (plan.depth && copy_size != image.info.data.size) {
return false;
}
if (!plan.depth && levels < image.info.resources.levels) {
auto& color = plan.color;
std::erase_if(color.regions, [levels](const vk::BufferImageCopy& region) {
return region.imageSubresource.mipLevel >= levels;
});
if (color.regions.empty()) {
return false;
}
if (color.tiled) {
const auto binding = m_texture_cache.UploadBinding(image);
const auto format =
binding == TextureCache::BindingType::RenderTarget
? ImageOps::RenderTargetTransferFormat(image.info.bytes_per_block)
: image.info.guest_format;
color.tiles.clear();
if (!TextureBuildGpuTileInfos(copy_size, color.regions, color.layout, format,
image.info.TransferLayers(), levels, color.tiles)) {
return false;
}
}
}
m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan));
m_texture_cache.RetainImage(m_scheduler.Current(), selected);
return true;
}
std::pair<uint8_t*, uint64_t> TextureCache::MapDownload(uint64_t size, uint64_t alignment) {
@@ -1612,12 +1689,9 @@ void TextureCache::QueueDownload(GuestRange range, StreamBuffer& download, uint8
m_scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr,
1, &barrier, 0, nullptr);
const auto tick = m_scheduler.CurrentTick();
m_buffer_cache.BeginBackingPublication(range.address, range.size, tick);
m_scheduler.DeferPriorityOperation([this, &download, range, mapped, offset, tick] {
m_scheduler.DeferPriorityOperation([&download, range, mapped, offset] {
download.Invalidate(offset, range.size);
LibKernel::Memory::WriteBacking(range.address, mapped, range.size);
m_buffer_cache.CompleteBackingPublication(range.address, range.size, tick);
});
}
@@ -1638,7 +1712,7 @@ bool TextureCache::TryDownloadImage(ImageId id) {
}
download.Flush(offset, range.size);
DownloadImageData(image, download, offset, std::move(plan));
DownloadImageData(image, download, offset, range.size, std::move(plan));
QueueDownload(range, download, mapped, offset);
return true;
@@ -1652,62 +1726,6 @@ void TextureCache::DownloadImage(ImageId id) {
m_scheduler.DrainPriorityOperations();
}
bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
auto& image = ResolveImage(id);
if (image.depth_id) {
return true;
}
auto plan = BuildDownload(image);
if (!plan.valid) {
return false;
}
const auto range = image.info.data;
if (image.IsCpuDirty()) {
RefreshImage(id,
ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)});
}
if (!image.IsGpuModified()) {
return true;
}
if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) {
EXIT("TextureCache: image mirror source is not native-current\n");
}
auto [destination, offset] =
m_buffer_cache.ObtainBufferForImageWrite(range.address, range.size);
if (destination == nullptr) {
EXIT("TextureCache: failed to allocate image mirror\n");
}
DownloadImageData(image, *destination, offset, std::move(plan));
m_scheduler.Current().RetainResourceUntilFence(destination);
m_buffer_cache.PublishImageBuffer(range.address, range.size);
image.MarkBufferModified();
RetainImage(m_scheduler.Current(), id);
ClearGpuModified(id);
return true;
}
bool TextureCache::SynchronizeImageToBuffer(uint64_t address, uint64_t size) {
if (!GuestRange {address, size}.Valid()) {
return false;
}
CacheLock lock(*this, m_lock);
ImageId selected {};
for (const auto id: FindImagesInRegion(address, size, true)) {
auto owner = ResolveOwner(id);
if (owner == nullptr || !owner->GpuOverlaps(address, size)) {
continue;
}
if (selected) {
EXIT("TextureCache: ambiguous image-to-buffer synchronization\n");
}
selected = id;
}
if (!selected) {
return false;
}
return SynchronizeImageToBuffer(selected);
}
bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
bool formatted_buffer_write) {
if (!GuestRange {address, size}.Valid()) {
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
+2 -3
View File
@@ -66,7 +66,6 @@ public:
[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
uint32_t packed_clear);
void InvalidateMemory(uint64_t address, uint64_t size);
[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
bool formatted_buffer_write = false);
[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
@@ -140,7 +139,7 @@ private:
[[nodiscard]] DownloadPlan BuildDownload(const Image& image) const;
void UploadImage(Image& image, const ImageDesc& desc, Buffer& source, uint64_t source_offset);
void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset,
DownloadPlan plan);
uint64_t destination_size, DownloadPlan plan);
void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset);
void CommitGpuWrite(Image& image);
void PrepareImageCopy(Image& image);
@@ -155,7 +154,6 @@ private:
void InvalidateCpuAliases(uint64_t address, uint64_t size);
void ClearGpuModified(ImageId id);
[[nodiscard]] bool SynchronizeImageToBuffer(ImageId id);
void DownloadImage(ImageId id);
[[nodiscard]] bool TryDownloadImage(ImageId id);
[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
@@ -184,6 +182,7 @@ private:
bool m_readback_linear_images = false;
friend struct TextureCacheTestAccess;
friend class BufferCache;
friend class RenderExecutor;
};