raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 19:07:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Rework guest memory tracking/virtual address space/direct and flexible memory (#135)

Browse Source 
* Rework guest memory tracking

* add unknwon flag

* Fix macOS guest address-space reservation
This commit is contained in:
nmzik
2026-07-31 03:07:17 +02:00
committed by GitHub
parent ecb48f90bb
commit 2f5396c6a5
28 changed files with 3155 additions and 2887 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/memoryTracker.cpp
-33
View File
@@ -4,16 +4,6 @@
namespace Libs::Graphics {
#if defined(KYTY_MEMORY_TRACKER_TESTS)
namespace {
std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
}
void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
g_unmap_contention_hook.store(hook, std::memory_order_release);
}
#endif
static_assert(std::atomic<void*>::is_always_lock_free);
MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
@@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
@@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
@@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
@@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
}
void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
RequireMapped(vaddr, size);
std::vector<RegionManager*> managers;
managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
TRACKER_REGION_SIZE);
@@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
UntrackMemoryLocked(vaddr, size);
}
void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::unique_lock access(m_access_mutex, std::try_to_lock);
if (!access.owns_lock()) {
#if defined(KYTY_MEMORY_TRACKER_TESTS)
if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
hook != nullptr) {
hook();
}
#endif
access.lock();
}
UntrackMemoryLocked(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
}
bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
switch (phase) {
case PageFaultPhase::Release: return true;
src/graphics/host_gpu/memoryTracker.h
+3 -19
View File
@@ -30,7 +30,6 @@ public:
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UntrackMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] CpuFaultAction
BeginCpuFault(uint64_t vaddr, uint64_t size,
PageFaultAccess access = PageFaultAccess::Write) noexcept;
@@ -91,8 +90,7 @@ public:
static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
std::lock_guard access(m_access_mutex);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
@@ -132,11 +130,6 @@ public:
vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
}
#if defined(KYTY_MEMORY_TRACKER_TESTS)
using UnmapContentionHook = void (*)() noexcept;
static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
#endif
template <typename RangeFunc, typename UploadFunc>
void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
UploadFunc&& upload_func) {
@@ -144,7 +137,6 @@ public:
static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
CheckNotInUploadCallback();
std::unique_lock access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -209,16 +201,8 @@ private:
return false;
}
static void ValidateRange(uint64_t vaddr, uint64_t size);
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
void RequireMapped(uint64_t vaddr, uint64_t size) const {
ValidateRange(vaddr, size);
if (!m_page_manager.IsMapped(vaddr, size)) {
EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
static_cast<unsigned long long>(vaddr),
static_cast<unsigned long long>(vaddr + size));
}
}
static void ValidateRange(uint64_t vaddr, uint64_t size);
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
RegionManager* GetOrCreateRegion(uint64_t index);
std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
src/graphics/host_gpu/pageManager.cpp
+43 -464
View File
@@ -1,6 +1,7 @@
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/regionDefinitions.h"
#include "kernel/memory.h"
#include <algorithm>
#include <array>
@@ -21,16 +22,11 @@
#undef min
#undef max
#elif defined(__APPLE__)
#include <mach/mach.h>
#include <mach/mach_vm.h>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#else
#include <cerrno>
#include <cstring>
#include <execinfo.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
@@ -57,45 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS;
constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY;
constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
#if defined(__APPLE__)
// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
static int PageProtToPosix(uint32_t protection) {
switch (protection) {
case PAGE_NOACCESS: return PROT_NONE;
case PAGE_READONLY: return PROT_READ;
case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
default: return PROT_NONE;
}
}
// Query the current protection of the page containing vaddr via the Mach VM map and
// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
static uint32_t MachQueryPageProt(uint64_t vaddr) {
auto region_addr = static_cast<mach_vm_address_t>(vaddr);
mach_vm_size_t region_size = 0;
vm_region_basic_info_data_64_t info {};
mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64;
mach_port_t object_name = MACH_PORT_NULL;
kern_return_t kr =
mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
if (kr != KERN_SUCCESS || region_addr > vaddr) {
return PAGE_NOACCESS; // no region covering vaddr
}
if ((info.protection & VM_PROT_WRITE) != 0) {
return PAGE_READWRITE;
}
if ((info.protection & VM_PROT_READ) != 0) {
return PAGE_READONLY;
}
return PAGE_NOACCESS;
}
#elif defined(__linux__)
// Zero is the unknown protection sentinel.
constexpr uint32_t UNKNOWN_PROTECTION = 0;
#endif
thread_local bool g_in_fault_resolution = false;
@@ -136,6 +95,15 @@ thread_local bool g_in_fault_resolution = false;
std::_Exit(322);
}
Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
switch (protection) {
case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
default: Fatal("unmappable protection 0x%08" PRIx32, protection);
}
}
uint32_t CurrentThread() noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
return GetCurrentThreadId();
@@ -155,130 +123,6 @@ uint32_t CurrentThread() noexcept {
#endif
}
#if defined(__linux__)
int ToHostProtection(uint32_t protection) {
switch (protection) {
case NO_ACCESS_PROTECTION: return PROT_NONE;
case READ_ONLY_PROTECTION: return PROT_READ;
case READ_WRITE_PROTECTION: return PROT_READ | PROT_WRITE;
default: Fatal("unmappable protection 0x%08" PRIx32, protection);
}
}
struct HostMapping {
uint64_t end = 0;
uint32_t protection = UNKNOWN_PROTECTION;
};
// Async-signal-safe lookup in the address-ordered /proc/self/maps.
HostMapping QueryHostMapping(uint64_t vaddr) noexcept {
int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT
if (fd < 0) {
return {};
}
enum class Field { Start, End, Perms, Rest };
HostMapping result {};
auto field = Field::Start;
uint64_t start = 0;
uint64_t end = 0;
char perms[4] = {};
uint32_t perms_len = 0;
bool line_valid = true;
char buffer[8192];
for (bool done = false; !done;) {
const auto got = ::read(fd, buffer, sizeof(buffer));
if (got < 0) {
if (errno == EINTR) {
continue;
}
break;
}
if (got == 0) {
break;
}
for (ssize_t i = 0; i < got && !done; i++) {
const char c = buffer[i];
if (c == '\n') {
field = Field::Start;
start = 0;
end = 0;
perms_len = 0;
line_valid = true;
continue;
}
if (!line_valid) {
continue;
}
switch (field) {
case Field::Start:
case Field::End: {
uint64_t digit = 0;
if (c >= '0' && c <= '9') {
digit = static_cast<uint64_t>(c - '0');
} else if (c >= 'a' && c <= 'f') {
digit = static_cast<uint64_t>(c - 'a') + 10;
} else if (c == '-' && field == Field::Start) {
field = Field::End;
break;
} else if (c == ' ' && field == Field::End) {
field = Field::Perms;
perms_len = 0;
break;
} else {
line_valid = false;
break;
}
auto& value = (field == Field::Start ? start : end);
value = (value << 4u) | digit;
break;
}
case Field::Perms: {
if (c != ' ') {
if (perms_len < sizeof(perms)) {
perms[perms_len] = c;
}
perms_len++;
break;
}
if (vaddr < start) {
done = true;
} else if (vaddr < end && perms_len >= 2) {
result.end = end;
result.protection = perms[1] == 'w' ? READ_WRITE_PROTECTION
: perms[0] == 'r' ? READ_ONLY_PROTECTION
: NO_ACCESS_PROTECTION;
done = true;
} else {
field = Field::Rest;
}
break;
}
case Field::Rest: break;
}
}
}
::close(fd);
return result;
}
uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
return QueryHostMapping(vaddr).protection;
}
#endif
class SpinGuard final {
public:
explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -313,21 +157,16 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
struct PageManager::Impl {
struct PageState {
std::atomic_flag lock = ATOMIC_FLAG_INIT;
uint32_t mappings = 0;
uint32_t gpu_read_mappings = 0;
uint32_t gpu_write_mappings = 0;
uint32_t write_watchers = 0;
uint32_t access_watchers = 0;
uint32_t original_protection = 0;
uint32_t backing_writer = 0;
#if defined(__linux__)
// Shadow the protection applied through Protect().
uint32_t current_protection = UNKNOWN_PROTECTION;
#endif
bool resolving = false;
bool resolving_read_write = false;
bool late_read_pending = false;
bool late_write_pending = false;
uint32_t current_protection = UNKNOWN_PROTECTION;
bool resolving = false;
bool resolving_read_write = false;
bool late_read_pending = false;
bool late_write_pending = false;
};
struct Region {
@@ -356,7 +195,7 @@ struct PageManager::Impl {
Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
if (fault_handler == nullptr) {
Fatal("null fault handler");
Fatal("null page-manager fault callback");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
SYSTEM_INFO info {};
@@ -386,9 +225,8 @@ struct PageManager::Impl {
for (const auto& region: region_storage) {
for (auto& page: region->pages) {
SpinGuard lock(page.lock);
if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
if (page.write_watchers != 0 || page.access_watchers != 0 ||
page.backing_writer != 0 || page.resolving) {
FailFast("PageManager destroyed with live page state");
}
}
@@ -441,179 +279,30 @@ struct PageManager::Impl {
}
}
static void ValidateInitialProtection(std::span<PageState*> pages, uint64_t vaddr) {
const auto end = vaddr + pages.size() * PAGE_SIZE;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
for (auto address = vaddr; address < end;) {
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
", protection=0x%08" PRIx32 ")",
address, static_cast<uint32_t>(info.State),
static_cast<uint32_t>(info.Protect));
}
const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
if (region_end <= address) {
Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
}
address = std::min(end, region_end);
}
#elif defined(__APPLE__)
for (auto address = vaddr; address < end; address += PAGE_SIZE) {
const uint32_t protection = MachQueryPageProt(address);
if (protection != PAGE_READWRITE) {
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64
" (protection=0x%08" PRIx32 ")",
address, protection);
}
}
#else
for (auto address = vaddr; address < end;) {
const auto mapping = QueryHostMapping(address);
if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) {
Fatal("basic path requires a read/write mapping at 0x%016" PRIx64
" (protection=0x%08" PRIx32 ")",
address, mapping.protection);
}
address = std::min(end, mapping.end);
}
for (auto* page: pages) {
page->current_protection = READ_WRITE_PROTECTION;
}
#endif
static void InitializeProtection(std::span<PageState*> pages) {
for (auto* page: pages) {
page->original_protection = READ_WRITE_PROTECTION;
page->current_protection = READ_WRITE_PROTECTION;
}
}
static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr,
static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
PageFaultAccess access) noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT) {
return false;
}
switch (access) {
case PageFaultAccess::Read:
return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
return page.current_protection == READ_ONLY_PROTECTION ||
page.current_protection == READ_WRITE_PROTECTION;
case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
default: return false;
}
#elif defined(__APPLE__)
const uint32_t protection = MachQueryPageProt(vaddr);
switch (access) {
case PageFaultAccess::Read:
return protection == PAGE_READONLY || protection == PAGE_READWRITE;
case PageFaultAccess::Write: return protection == PAGE_READWRITE;
default: return false;
}
#else
const auto permitted = [](uint32_t protection, PageFaultAccess wanted) {
switch (wanted) {
case PageFaultAccess::Read:
return protection == READ_ONLY_PROTECTION ||
protection == READ_WRITE_PROTECTION;
case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION;
default: return false;
}
};
if (!permitted(page.current_protection, access)) {
return false;
}
return permitted(QueryHostProtection(vaddr), access);
#endif
}
static void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old, bool fault_path) noexcept {
void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old, bool fault_path) noexcept {
const auto size = pages.size() * PAGE_SIZE;
if (pages.size() != expected_old.size()) {
FailFast("protection range state size mismatch");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
struct HostRange {
uint64_t begin = 0;
uint64_t end = 0;
};
std::vector<HostRange> host_ranges;
const auto end = vaddr + size;
for (auto address = vaddr; address < end;) {
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT) {
if (fault_path) {
FailFast("VirtualProtect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32
", new=0x%08" PRIx32,
address, static_cast<uint32_t>(info.State), protection);
}
const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
const auto query_end = std::min(end, region_end);
if (query_end <= address) {
if (fault_path) {
FailFast("VirtualQuery returned an invalid fault transition region");
}
Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
}
const auto first_page = static_cast<size_t>((address - vaddr) / PAGE_SIZE);
const auto last_page =
static_cast<size_t>((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE);
for (auto page = first_page; page < last_page; page++) {
if (info.Protect != expected_old[page]) {
if (fault_path) {
FailFast(
"VirtualProtect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
vaddr + page * PAGE_SIZE, static_cast<uint32_t>(info.Protect),
expected_old[page], protection);
}
}
const auto allocation = reinterpret_cast<uint64_t>(info.AllocationBase);
if (host_ranges.empty() || allocation != host_ranges.back().begin) {
host_ranges.push_back({allocation, query_end});
} else {
host_ranges.back().end = query_end;
}
address = query_end;
}
for (auto range: host_ranges) {
range.begin = std::max(range.begin, vaddr);
DWORD old_protection = 0;
const auto first_page = static_cast<size_t>((range.begin - vaddr) / PAGE_SIZE);
if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(range.begin)),
range.end - range.begin, protection, &old_protection) == 0 ||
old_protection != expected_old[first_page]) {
if (fault_path) {
FailFast("VirtualProtect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
range.begin, static_cast<uint32_t>(old_protection), expected_old[first_page],
protection);
}
}
#elif defined(__APPLE__)
// mprotect cannot report the previous protection, so the expected_old comparison
// is dropped; the tracker is the sole mutator of these pages and drives the
// transition from its own shadow state.
(void)expected_old;
if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
PageProtToPosix(protection)) != 0) {
if (fault_path) {
FailFast("mprotect fault transition failed");
}
Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
}
#else
for (size_t i = 0; i < pages.size(); i++) {
const auto actual = pages[i]->current_protection;
if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
@@ -625,22 +314,21 @@ struct PageManager::Impl {
vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
}
}
if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
ToHostProtection(protection)) != 0) {
if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
ToMemoryMode(protection))) {
if (fault_path) {
FailFast("mprotect failed on the fault path");
FailFast("address-space fault protection transition failed");
}
Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr,
protection, std::strerror(errno));
Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
protection);
}
for (auto* page: pages) {
page->current_protection = protection;
}
#endif
}
static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
bool fault_path) noexcept {
void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
bool fault_path) noexcept {
PageState* pages[] = {&page};
uint32_t expected[] = {expected_old};
ProtectRange(pages, vaddr, protection, expected, fault_path);
@@ -680,50 +368,6 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
return page.write_watchers != 0 || page.access_watchers != 0;
}
bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.mappings == 0) {
return false;
}
}
return true;
}
bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("HasGpuAccess received an invalid GPU access mode");
}
const bool need_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(addr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, addr);
SpinGuard lock(page.lock);
if ((need_read && page.gpu_read_mappings == 0) ||
(need_write && page.gpu_write_mappings == 0)) {
return false;
}
}
return true;
}
void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode) {
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
@@ -754,9 +398,6 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
if (page.resolving && track) {
FailFast("new page watcher raced active fault resolution");
}
if (page.mappings == 0) {
Fatal("watching unmapped page 0x%016" PRIx64, address);
}
auto& watchers =
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
if (track) {
@@ -784,8 +425,7 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
last++;
}
if (first != last) {
Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE);
Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
}
first = last;
}
@@ -831,9 +471,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
last = current + 1;
}
}
Impl::ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first), false);
m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first), false);
first = current;
}
@@ -860,70 +500,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
}
}
void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (g_in_fault_resolution) {
FailFast("GPU mapping changed during fault resolution");
}
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("GPU map received an invalid access mode");
}
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
const auto end = PageEnd(vaddr, size);
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
auto& page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
SpinGuard lock(page.lock);
if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
(gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
(gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
Fatal("invalid map state at 0x%016" PRIx64, addr);
}
page.mappings++;
page.gpu_read_mappings += gpu_read ? 1u : 0u;
page.gpu_write_mappings += gpu_write ? 1u : 0u;
#if defined(__linux__)
// New guest mappings start read/write.
if (page.current_protection == UNKNOWN_PROTECTION) {
page.current_protection = READ_WRITE_PROTECTION;
}
#endif
}
}
void PageManager::OnGpuMap(uint64_t, uint64_t) {}
void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (g_in_fault_resolution) {
FailFast("GPU unmapping changed during fault resolution");
}
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("GPU unmap received an invalid access mode");
}
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
(gpu_write && page.gpu_write_mappings == 0) ||
(page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
}
page.mappings--;
page.gpu_read_mappings -= gpu_read ? 1u : 0u;
page.gpu_write_mappings -= gpu_write ? 1u : 0u;
if (page.mappings == 0) {
if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
FailFast("GPU unmap left nonzero GPU mapping counts");
}
page.late_read_pending = false;
page.late_write_pending = false;
}
}
}
void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
uint64_t size) noexcept
@@ -979,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
}
auto& page = m_impl->GetPage(*region, address);
SpinGuard lock(page.lock);
if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
page.access_watchers == 0) {
if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
Fatal("backing write races page resolution at 0x%016" PRIx64, address);
}
page.resolving = true;
@@ -1008,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
const auto old_protection = NO_ACCESS_PROTECTION;
const auto new_protection = Impl::WatcherProtection(page);
if (new_protection != old_protection) {
Impl::Protect(page, address, new_protection, old_protection, false);
m_impl->Protect(page, address, new_protection, old_protection, false);
}
Impl::PublishDelayedFaults(page, old_protection, new_protection);
if (page.write_watchers == 0 && page.access_watchers == 0) {
@@ -1109,7 +687,8 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
page.write_watchers = 0;
}
const auto restored_protection = Impl::WatcherProtection(page);
Impl::Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, true);
m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
true);
if (page.write_watchers == 0) {
page.original_protection = 0;
}
src/graphics/host_gpu/pageManager.h
+2 -6
View File
@@ -13,11 +13,9 @@ namespace Libs::Graphics {
enum class PageFaultAccess { Read, Write, Execute, Unknown };
enum class PageFaultPhase { Invalidate, Complete, Release };
enum class PageWatchMode { Write, ReadWrite };
enum class GpuAccess { Read, Write, ReadWrite };
using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
class PageManager final {
public:
class BackingWrite final {
@@ -40,13 +38,11 @@ public:
[[nodiscard]] uint64_t GetPageSize() const;
[[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode = PageWatchMode::Write);
void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
void OnGpuMap(uint64_t vaddr, uint64_t size);
void OnGpuUnmap(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
-21
View File
@@ -714,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
if (command.IsInvalid() || command.IsExecute()) {
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
ValidateGpuAccess(vaddr, size, is_read, is_written);
std::lock_guard transaction(m_resource_mutex);
(void)SynchronizeBacking(vaddr, size);
@@ -999,7 +998,6 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
if (vaddr == 0) {
EXIT("BufferCache: invalid fill memory address\n");
}
ValidateGpuAccess(vaddr, size, false, true);
(void)m_texture_cache.ClearMeta(vaddr);
{
std::lock_guard transaction(m_resource_mutex);
@@ -1041,12 +1039,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
(src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
EXIT("BufferCache: invalid or overlapping copy range\n");
}
if (src_memory) {
ValidateGpuAccess(src_vaddr, size, true, false);
}
if (dst_memory) {
ValidateGpuAccess(dst_vaddr, size, false, true);
}
if (src_memory || dst_memory) {
std::lock_guard transaction(m_resource_mutex);
if (src_memory) {
@@ -1203,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
owner->second->tick_accessed_last = m_gc_tick;
}
void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
bool is_written) const {
if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("BufferCache: invalid GPU access request\n");
}
if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
EXIT("BufferCache: GPU-read access denied\n");
}
if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
EXIT("BufferCache: GPU-write access denied\n");
}
}
void BufferCache::RunGarbageCollector() {
std::lock_guard transaction(m_resource_mutex);
const auto tick = m_gc_tick++;
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+1 -2
View File
@@ -77,8 +77,7 @@ public:
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
void RunGarbageCollector();
void RunGarbageCollector();
private:
friend struct BufferCacheTestAccess;
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+6 -10
View File
@@ -4,7 +4,6 @@
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
namespace Libs::Graphics {
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
@@ -115,22 +114,19 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
return m_mapped_ranges.Contains(vaddr, size);
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
{
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Add(vaddr, size);
}
m_page_manager.OnGpuMap(vaddr, size, access);
m_page_manager.OnGpuMap(vaddr, size);
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (!IsMapped(vaddr, size)) {
EXIT("cannot unmap an unmapped GPU resource range\n");
}
const auto unmap = [this, vaddr, size, access] {
m_texture_cache.UnmapMemory(vaddr, size);
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
const auto unmap = [this, vaddr, size] {
m_buffer_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size, access);
m_texture_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Subtract(vaddr, size);
};
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+2 -2
View File
@@ -29,8 +29,8 @@ public:
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void MapMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
-1
View File
@@ -1377,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
EXIT("TextureCache: invalid image clear\n");
}
m_buffer_cache.ValidateGpuAccess(address, size, false, true);
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
ImageId selected {};
src/graphics/host_gpu/renderer/pipeline/descriptors.cpp
-2
View File
@@ -616,8 +616,6 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
(address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
if (storage) {
ValidateStorageTexture(resource, descriptor, size.size);
m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
resource.written);
}
const auto pixel_format = TextureGetFormat(format);