mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-19 18:36:53 +02:00
renderer: reduce synchronization and allocation overhead
Performance changes from #1495: - Sync changed BDA ranges instead of scanning all mappings. - Let VMA suballocate BDA buffers. - Remove the renderer lock from presentation. - Keep compatible pipeline caches across builds. - Fold masked shifts that always produce zero.
This commit is contained in:
shadowbeat070
@@ -66,6 +66,18 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void MemoryTracker::QueueBdaSync(uint64_t vaddr, uint64_t size) {
|
||||
const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE);
|
||||
const auto end = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE);
|
||||
std::lock_guard lock(m_bda_sync_mutex);
|
||||
m_bda_sync.Add(begin, end - begin);
|
||||
}
|
||||
|
||||
RangeSet MemoryTracker::TakeBdaSync() {
|
||||
std::lock_guard lock(m_bda_sync_mutex);
|
||||
return std::exchange(m_bda_sync, RangeSet {});
|
||||
}
|
||||
|
||||
bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
|
||||
CheckNotInUploadCallback();
|
||||
return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
@@ -88,6 +100,7 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
|
||||
std::scoped_lock lock(manager->lock);
|
||||
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
|
||||
});
|
||||
QueueBdaSync(vaddr, size);
|
||||
}
|
||||
|
||||
void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
|
||||
@@ -128,6 +141,8 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
|
||||
Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
|
||||
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
|
||||
});
|
||||
locks.clear();
|
||||
QueueBdaSync(vaddr, size);
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
@@ -29,6 +29,10 @@ public:
|
||||
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
|
||||
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
|
||||
void UntrackMemory(uint64_t vaddr, uint64_t size);
|
||||
// Queue after changing ownership; a concurrent drain must not consume an event
|
||||
// before the dirty pages it describes become visible.
|
||||
void QueueBdaSync(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] RangeSet TakeBdaSync();
|
||||
// Removes protection from a range and flushes GPU-owned data when required.
|
||||
template <typename Flush>
|
||||
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
|
||||
@@ -51,6 +55,7 @@ public:
|
||||
on_flush();
|
||||
}
|
||||
});
|
||||
QueueBdaSync(vaddr, size);
|
||||
}
|
||||
#if KYTY_BUILD == KYTY_BUILD_DEBUG
|
||||
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
|
||||
@@ -149,6 +154,8 @@ private:
|
||||
std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
|
||||
std::vector<std::unique_ptr<RegionManager>> m_region_storage;
|
||||
std::mutex m_region_mutex;
|
||||
std::mutex m_bda_sync_mutex;
|
||||
RangeSet m_bda_sync;
|
||||
PageManager& m_page_manager;
|
||||
};
|
||||
|
||||
|
||||
@@ -126,6 +126,7 @@ void BufferCache::ChangeRegister(BufferId id) {
|
||||
}
|
||||
WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
|
||||
addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
|
||||
m_memory_tracker.QueueBdaSync(buffer.CpuAddress(), buffer.Size());
|
||||
} else {
|
||||
const auto found = m_buffers.find(buffer.CpuAddress());
|
||||
EXIT_IF(found == m_buffers.end() || found->second != id);
|
||||
@@ -685,6 +686,16 @@ void BufferCache::ProcessFaultBuffer() {
|
||||
m_fault_manager.ProcessFaultBuffer();
|
||||
}
|
||||
|
||||
void BufferCache::SynchronizeBda(const RangeSet& mapped) {
|
||||
// Take before walking: writes queued during synchronization remain pending.
|
||||
const auto pending = m_memory_tracker.TakeBdaSync();
|
||||
pending.ForEach([&](uint64_t begin, uint64_t end) {
|
||||
mapped.ForEachInRange(begin, end - begin, [this](uint64_t start, uint64_t finish) {
|
||||
SynchronizeBuffersInRange(start, finish - start);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) {
|
||||
const auto end = vaddr + size;
|
||||
auto it = m_buffers.upper_bound(vaddr);
|
||||
|
||||
+5
-1
@@ -80,7 +80,10 @@ public:
|
||||
[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
|
||||
void ProcessFaultBuffer();
|
||||
void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
|
||||
void SynchronizeBda(const RangeSet& mapped);
|
||||
void QueueBdaSync(uint64_t vaddr, uint64_t size) {
|
||||
m_memory_tracker.QueueBdaSync(vaddr, size);
|
||||
}
|
||||
void RunGarbageCollector();
|
||||
|
||||
private:
|
||||
@@ -114,6 +117,7 @@ private:
|
||||
void DeleteBuffer(BufferId id);
|
||||
[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
|
||||
bool is_written, bool is_texel_buffer);
|
||||
void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
|
||||
[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
|
||||
uint64_t total_size);
|
||||
[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
|
||||
|
||||
+1
-3
@@ -67,11 +67,9 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
|
||||
buffer_info.usage = flags;
|
||||
|
||||
const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
|
||||
const VmaAllocationCreateFlags bda_flag =
|
||||
with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
|
||||
VmaAllocationCreateInfo allocation_info {};
|
||||
allocation_info.flags =
|
||||
VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
|
||||
VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
|
||||
allocation_info.usage = AllocationUsage(usage);
|
||||
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
|
||||
? VkMemoryPropertyFlags {}
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
#include "graphics/shader/recompiler/ShaderRecompiler.h"
|
||||
#include "graphics/shader/shaderCompiler.h"
|
||||
#include "kernel/memory.h"
|
||||
#include "kytyGitVersion.h"
|
||||
#include "loader/systemContent.h"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -81,7 +80,8 @@ std::string DriverCacheSignature(const vk::PhysicalDeviceProperties& properties)
|
||||
uuid[i * 2] = hex[properties.pipelineCacheUUID[i] >> 4u];
|
||||
uuid[i * 2 + 1] = hex[properties.pipelineCacheUUID[i] & 0xfu];
|
||||
}
|
||||
return fmt::format("KytyPC1:{}:{:08x}:{:08x}:{:08x}:{}\n", KYTY_GIT_REVISION,
|
||||
// Driver cache entries already distinguish shader code and pipeline state across builds.
|
||||
return fmt::format("KytyPC2:{:08x}:{:08x}:{:08x}:{}\n",
|
||||
properties.vendorID, properties.deviceID, properties.driverVersion, uuid);
|
||||
}
|
||||
|
||||
@@ -516,17 +516,6 @@ void PipelineCache::InitializeDriverCache() {
|
||||
PipelineCacheLog("Vulkan pipeline cache: disabled (non-Release build)");
|
||||
return;
|
||||
}
|
||||
const std::string_view git_hash = KYTY_GIT_HASH;
|
||||
const std::string_view git_revision = KYTY_GIT_REVISION;
|
||||
if (git_hash == "unknown" || git_revision == "unknown") {
|
||||
PipelineCacheLog("Vulkan pipeline cache: disabled (unknown git revision)");
|
||||
return;
|
||||
}
|
||||
if (git_hash.ends_with("-dirty")) {
|
||||
PipelineCacheLog("Vulkan pipeline cache: disabled (dirty build)");
|
||||
return;
|
||||
}
|
||||
|
||||
m_driver_cache_path = std::filesystem::path("_PipelineCache") / (title_id + ".bin");
|
||||
const auto path = Common::PathToString(m_driver_cache_path);
|
||||
const bool cache_exists = Common::File::IsFileExisting(m_driver_cache_path);
|
||||
@@ -559,7 +548,7 @@ void PipelineCache::InitializeDriverCache() {
|
||||
XXH3_64bits(initial_data.data(), initial_data.size()) != payload_hash) {
|
||||
initial_data.clear();
|
||||
PipelineCacheLog(
|
||||
"Vulkan pipeline cache: invalidating {} (driver, emulator, or data mismatch)",
|
||||
"Vulkan pipeline cache: invalidating {} (driver or data mismatch)",
|
||||
path);
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -90,6 +90,7 @@ bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
|
||||
void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) {
|
||||
std::lock_guard lock(m_mapped_ranges_mutex);
|
||||
m_mapped_ranges.Add(vaddr, size);
|
||||
m_buffer_cache.QueueBdaSync(vaddr, size);
|
||||
}
|
||||
|
||||
void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
|
||||
@@ -129,9 +130,7 @@ void RenderContext::PrepareBda() {
|
||||
m_bda_logged = true;
|
||||
}
|
||||
std::shared_lock lock(m_mapped_ranges_mutex);
|
||||
m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
|
||||
m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
|
||||
});
|
||||
m_buffer_cache.SynchronizeBda(m_mapped_ranges);
|
||||
m_fault_process_pending = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -945,7 +945,6 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
|
||||
auto* frame = m_impl->frames.Acquire({width, height}, format);
|
||||
frame->hdr = false;
|
||||
frame->blank = true;
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
frame->Configure(m_impl->window.graphic_ctx, {width, height}, format);
|
||||
vk::ClearColorValue clear {};
|
||||
clear.float32[3] = opaque ? 1.0f : 0.0f;
|
||||
@@ -953,6 +952,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
|
||||
EXIT_IF(producer->IsInvalid());
|
||||
frame->Clear(*producer, clear);
|
||||
} else {
|
||||
Common::LockGuard lock(m_impl->present_mutex);
|
||||
auto& command = m_impl->present_scheduler.BeginCommand();
|
||||
frame->Clear(command, clear);
|
||||
frame->present_tick = m_impl->present_scheduler.Submit();
|
||||
@@ -1036,18 +1036,17 @@ void Presenter::Impl::Present() {
|
||||
RecoverSwapchain(status);
|
||||
continue;
|
||||
}
|
||||
{
|
||||
Common::LockGuard render_lock(renderer.GetMutex());
|
||||
auto& command = present_scheduler.BeginCommand();
|
||||
const bool draw_system_overlay =
|
||||
overlay_visual.active && swapchain.PrepareSystemOverlay();
|
||||
swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
|
||||
draw_system_overlay);
|
||||
const auto tick = swapchain.Submit(present_scheduler);
|
||||
for (const auto& layer: layers) {
|
||||
if (layer.frame != nullptr) {
|
||||
layer.frame->present_tick = tick;
|
||||
}
|
||||
// present_mutex owns the prepared frames and this scheduler. Only queue submission
|
||||
// is shared with rendering, and it has its own queue_mutex.
|
||||
auto& command = present_scheduler.BeginCommand();
|
||||
const bool draw_system_overlay =
|
||||
overlay_visual.active && swapchain.PrepareSystemOverlay();
|
||||
swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
|
||||
draw_system_overlay);
|
||||
const auto tick = swapchain.Submit(present_scheduler);
|
||||
for (const auto& layer: layers) {
|
||||
if (layer.frame != nullptr) {
|
||||
layer.frame->present_tick = tick;
|
||||
}
|
||||
}
|
||||
status = swapchain.Present();
|
||||
|
||||
@@ -608,13 +608,23 @@ void FoldInstruction(Block& block, Block::iterator instruction,
|
||||
ReplaceBinaryIdentity(inst, Type::U32, UINT32_MAX)) return;
|
||||
for (uint32_t arm = 0; arm < 2u; ++arm) {
|
||||
const auto mask = Arg(inst, arm);
|
||||
if (!IsImmediate(mask, Type::U32)) continue;
|
||||
if (mask.U32() == 0u) {
|
||||
Replace(inst, Value(0u));
|
||||
return;
|
||||
}
|
||||
const auto* shifted = Arg(inst, arm ^ 1u).TryInstruction();
|
||||
if (!IsImmediate(mask, Type::U32) || shifted == nullptr ||
|
||||
shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32 || !HasOneUse(*shifted)) continue;
|
||||
if (shifted == nullptr || shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32) continue;
|
||||
const auto shift = Arg(*shifted, 1);
|
||||
if (!IsImmediate(shift, Type::U32) || shift.U32() >= 32u) continue;
|
||||
if (!IsImmediate(shift, Type::U32)) continue;
|
||||
const auto shifted_mask = mask.U32() >> (shift.U32() & 31u);
|
||||
if (shifted_mask == 0u) {
|
||||
Replace(inst, Value(0u));
|
||||
return;
|
||||
}
|
||||
if (!HasOneUse(*shifted) || shift.U32() >= 32u) continue;
|
||||
const auto masked = worklist.Prepend(block, instruction, ValueOpcode::BitwiseAnd32,
|
||||
{Arg(*shifted, 0), Value(mask.U32() >> shift.U32())});
|
||||
{Arg(*shifted, 0), Value(shifted_mask)});
|
||||
Replace(inst, Value(&*worklist.Prepend(block, instruction, ValueOpcode::ShiftLeftLogical32,
|
||||
{Value(&*masked), shift})));
|
||||
return;
|
||||
|
||||
@@ -213,6 +213,12 @@ struct ProtectionCall {
|
||||
std::vector<ProtectionCall> g_protection_log;
|
||||
std::mutex g_protection_log_mutex;
|
||||
|
||||
struct ProtectionGate {
|
||||
std::binary_semaphore entered{0};
|
||||
std::binary_semaphore resume{0};
|
||||
};
|
||||
ProtectionGate *g_protection_gate = nullptr;
|
||||
|
||||
void ResetProtectionLog() {
|
||||
g_protection_calls = 0;
|
||||
g_protection_log.clear();
|
||||
@@ -220,6 +226,10 @@ void ResetProtectionLog() {
|
||||
|
||||
bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
|
||||
Common::VirtualMemory::Mode mode) {
|
||||
if (g_protection_gate != nullptr && mode == Common::VirtualMemory::Mode::ReadWrite) {
|
||||
g_protection_gate->entered.release();
|
||||
g_protection_gate->resume.acquire();
|
||||
}
|
||||
uint32_t protection = PAGE_NOACCESS;
|
||||
if (mode == Common::VirtualMemory::Mode::Read) {
|
||||
protection = PAGE_READONLY;
|
||||
@@ -407,6 +417,74 @@ void TestCpuDirtyUpload() {
|
||||
Release(memory);
|
||||
}
|
||||
|
||||
void TestBdaSyncDirtiness() {
|
||||
constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
|
||||
TrackerHarness harness;
|
||||
auto &tracker = harness.tracker;
|
||||
auto *memory = Allocate(harness.page_manager, 3);
|
||||
const auto address = reinterpret_cast<uint64_t>(memory);
|
||||
Check(tracker.TakeBdaSync().Empty(), "new tracker queued unowned memory");
|
||||
tracker.QueueBdaSync(address, 3 * page);
|
||||
Check(tracker.IsRegionCpuModified(address, page), "new region was clean");
|
||||
auto pending = tracker.TakeBdaSync();
|
||||
Check(pending.Contains(address, 3 * page) && !pending.Intersects(address + 3 * page, page),
|
||||
"mapped or registered range did not bound initial BDA synchronization");
|
||||
tracker.ForEachUploadRange(address, 3 * page, false,
|
||||
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
Check(tracker.TakeBdaSync().Empty(), "clean upload queued redundant BDA work");
|
||||
|
||||
tracker.MarkRegionAsCpuModified(address + 17, 1);
|
||||
pending = tracker.TakeBdaSync();
|
||||
Check(pending.Contains(address, page) && !pending.Intersects(address + page, page) &&
|
||||
tracker.TakeBdaSync().Empty(),
|
||||
"byte dirtiness did not drain exactly its whole tracker page");
|
||||
// A write after Take belongs to the next drain, even when its page was just taken.
|
||||
tracker.MarkRegionAsCpuModified(address + 31, 1);
|
||||
Check(tracker.TakeBdaSync().Contains(address, page), "re-dirty event was lost");
|
||||
tracker.ForEachUploadRange(address, page, true,
|
||||
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
Check(tracker.TakeBdaSync().Empty(), "GPU ownership queued an unnecessary upload");
|
||||
tracker.InvalidateRegion(address + 7, 1, [&] {
|
||||
tracker.UnmarkRegionAsGpuModified(address, page);
|
||||
tracker.MarkRegionAsCpuModified(address, page);
|
||||
});
|
||||
Check(tracker.TakeBdaSync().Contains(address, page),
|
||||
"GPU-to-CPU invalidation did not queue BDA synchronization");
|
||||
tracker.InvalidateRegion(address + page + 7, 1, [] {});
|
||||
Check(tracker.TakeBdaSync().Contains(address + page, page),
|
||||
"CPU invalidation did not queue BDA synchronization");
|
||||
tracker.UntrackMemory(address, 3 * page);
|
||||
Check(tracker.TakeBdaSync().Contains(address, 3 * page),
|
||||
"untracked memory did not queue its newly dirty pages");
|
||||
Release(memory);
|
||||
}
|
||||
|
||||
void TestBdaSyncConcurrentDrain() {
|
||||
constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
|
||||
TrackerHarness harness;
|
||||
auto &tracker = harness.tracker;
|
||||
auto *memory = Allocate(harness.page_manager, 1);
|
||||
const auto address = reinterpret_cast<uint64_t>(memory);
|
||||
tracker.ForEachUploadRange(address, page, false,
|
||||
[](uint64_t, uint64_t) noexcept {}, []() noexcept {});
|
||||
(void)tracker.TakeBdaSync();
|
||||
ProtectionGate gate;
|
||||
g_protection_gate = &gate;
|
||||
std::jthread writer([&] { tracker.MarkRegionAsCpuModified(address + 17, 1); });
|
||||
gate.entered.acquire();
|
||||
// Ownership is still changing. Publishing its event before this point would let
|
||||
// a drain consume it before the writer has finished releasing page protection.
|
||||
Check(tracker.TakeBdaSync().Empty(), "BDA event preceded its state transition");
|
||||
gate.resume.release();
|
||||
writer.join();
|
||||
g_protection_gate = nullptr;
|
||||
Check(tracker.TakeBdaSync().Contains(address, page) &&
|
||||
tracker.IsRegionCpuModified(address, page),
|
||||
"concurrent drain lost a completed CPU dirty transition");
|
||||
tracker.UntrackMemory(address, page);
|
||||
Release(memory);
|
||||
}
|
||||
|
||||
void TestCleanUploadPreservesOwnership() {
|
||||
constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
|
||||
constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
|
||||
@@ -1202,6 +1280,8 @@ int main(int argc, char **argv) {
|
||||
TestQueriesDoNotRequireMappedOwnership();
|
||||
TestConcurrentRegionPublication();
|
||||
TestCpuDirtyUpload();
|
||||
TestBdaSyncDirtiness();
|
||||
TestBdaSyncConcurrentDrain();
|
||||
TestCleanUploadPreservesOwnership();
|
||||
TestRangeInvalidation();
|
||||
TestGpuReacquisitionAfterInvalidation();
|
||||
|
||||
@@ -5281,6 +5281,85 @@ public:
|
||||
std::printf("[gpu] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckBdaSyncUploads() {
|
||||
constexpr const char *name = "BdaSyncUploads";
|
||||
constexpr auto page = BufferCache::CACHING_PAGESIZE;
|
||||
EnsureRuntimeContext();
|
||||
const auto base = LibKernel::Memory::AllocateRuntimeMemory(
|
||||
0, 3 * page, Common::VirtualMemory::Mode::ReadWrite, "bda_sync_uploads");
|
||||
Require(name, "allocation", base != 0, "guest allocation failed");
|
||||
auto *words = reinterpret_cast<u32 *>(base);
|
||||
std::fill_n(words, 3 * page / sizeof(u32), 0x12345678u);
|
||||
RenderContext context(m_runtime_context);
|
||||
auto &scheduler = context.GetCommandScheduler();
|
||||
auto &cache = context.GetBufferCache();
|
||||
HW::Context registers{};
|
||||
HW::UserConfig user_config{};
|
||||
HW::Shader shaders{};
|
||||
scheduler.Begin(registers, user_config, shaders);
|
||||
context.MapMemory(base, page);
|
||||
context.MapMemory(base + 2 * page, page);
|
||||
const auto id = cache.FindBuffer(base, 3 * page);
|
||||
auto &buffer = cache.GetBuffer(id);
|
||||
auto readback = CreateHostBuffer(name, 6 * sizeof(u32),
|
||||
vk::BufferUsageFlagBits::eTransferDst, {});
|
||||
const auto snapshot = [&](uint64_t offset, uint32_t index) {
|
||||
const vk::BufferCopy copy{offset, index * sizeof(u32), sizeof(u32)};
|
||||
scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer, 1, ©);
|
||||
};
|
||||
context.PrepareBda();
|
||||
Require(name, "mapped intersection",
|
||||
!cache.IsRegionCpuModified(base, page) &&
|
||||
cache.IsRegionCpuModified(base + page, page) &&
|
||||
!cache.IsRegionCpuModified(base + 2 * page, page),
|
||||
"BDA upload missed a mapped range or uploaded an unmapped hole");
|
||||
snapshot(0, 0);
|
||||
snapshot(2 * page, 1);
|
||||
|
||||
cache.InvalidateMemory(base + 17, 1);
|
||||
words[4] = 0x87654321u;
|
||||
words[TRACKER_PAGE_SIZE / sizeof(u32) - 1] = 0xabcdef01u;
|
||||
context.PrepareBda();
|
||||
snapshot(16, 2);
|
||||
snapshot(TRACKER_PAGE_SIZE - sizeof(u32), 3);
|
||||
Require(name, "whole-page upload", !cache.IsRegionCpuModified(base, TRACKER_PAGE_SIZE),
|
||||
"one-byte invalidation did not upload and consume its whole page");
|
||||
|
||||
context.MapMemory(base + page, page);
|
||||
context.PrepareBda();
|
||||
snapshot(page, 4);
|
||||
Require(name, "new mapping", !cache.IsRegionCpuModified(base + page, page),
|
||||
"mapping into an existing buffer did not queue its dirty pages");
|
||||
context.UnmapMemory(base + 2 * page, page);
|
||||
words[2 * page / sizeof(u32)] = 0xfedcba98u;
|
||||
context.PrepareBda();
|
||||
Require(name, "unmapped dirty page", cache.IsRegionCpuModified(base + 2 * page, page),
|
||||
"BDA synchronization consumed dirtiness outside mapped memory");
|
||||
context.MapMemory(base + 2 * page, page);
|
||||
context.PrepareBda();
|
||||
snapshot(2 * page, 5);
|
||||
|
||||
vk::BufferMemoryBarrier barrier{};
|
||||
barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
|
||||
barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
|
||||
barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
barrier.buffer = readback.buffer;
|
||||
barrier.size = readback.size;
|
||||
scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
|
||||
vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
|
||||
scheduler.Finish();
|
||||
Require(name, "GPU contents", ReadBuffer(name, readback, 6) ==
|
||||
std::vector<u32>{0x12345678u, 0x12345678u, 0x87654321u, 0xabcdef01u,
|
||||
0x12345678u, 0xfedcba98u},
|
||||
"initial, re-dirtied, newly mapped or remapped bytes were stale on the GPU");
|
||||
context.UnmapMemory(base, 3 * page);
|
||||
scheduler.Finish();
|
||||
DestroyBuffer(&readback);
|
||||
Require(name, "release", LibKernel::Memory::FreeGuestMemory(base, 3 * page),
|
||||
"guest allocation release failed");
|
||||
std::printf("[gpu] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckBdaPageTableUploads() {
|
||||
constexpr const char *name = "BdaPageTableUploads";
|
||||
constexpr uintptr_t base = 0x0000000207b00000ull;
|
||||
@@ -48472,6 +48551,7 @@ int main(int argc, char **argv) {
|
||||
if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
vulkan.CheckBdaPageTableUploads();
|
||||
vulkan.CheckBdaSyncUploads();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
|
||||
@@ -48702,6 +48782,7 @@ int main(int argc, char **argv) {
|
||||
CheckComputeLdsLimit(vulkan);
|
||||
vulkan.CheckNullTextureDescriptors();
|
||||
vulkan.CheckBdaPageTableUploads();
|
||||
vulkan.CheckBdaSyncUploads();
|
||||
vulkan.CheckLargeSparseImageUpload();
|
||||
vulkan.CheckUnifiedTextureCacheFlow();
|
||||
vulkan.CheckUnifiedImageViewCache();
|
||||
|
||||
@@ -12874,6 +12874,51 @@ void TestLateConstantPropagation() {
|
||||
"an ungrounded Phi cycle acquired a constant");
|
||||
}
|
||||
|
||||
void TestMaskedShiftConstantPropagation() {
|
||||
using namespace ShaderRecompiler::IR;
|
||||
for (const uint32_t count : {0u, 1u, 8u, 31u, 32u, 33u, 40u, 63u, 64u, UINT32_MAX}) {
|
||||
for (const uint32_t mask : {0u, 1u, 0xffu, 0x7fffffffu, 0x80000000u, UINT32_MAX}) {
|
||||
for (const bool swapped : {false, true}) {
|
||||
for (const bool shared : {false, true}) {
|
||||
Block block;
|
||||
IREmitter ir(&block);
|
||||
const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
|
||||
const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, Value(count)});
|
||||
const auto result = ir.Emit(ValueOpcode::BitwiseAnd32,
|
||||
swapped ? std::initializer_list<Value>{Value(mask), shifted}
|
||||
: std::initializer_list<Value>{shifted, Value(mask)});
|
||||
const auto other = shared ? ir.Emit(ValueOpcode::IAdd32, {shifted, Value(1u)}) : Value{};
|
||||
ConstantPropagationPass({&block});
|
||||
const bool zero = ((UINT32_MAX << (count & 31u)) & mask) == 0u;
|
||||
Check(zero ? result.Resolve() == Value(0u) : !result.Resolve().IsImmediate(),
|
||||
"masked shift retained impossible bits or discarded possible bits");
|
||||
if (shared) {
|
||||
Check(other.Resolve().Instruction()->Arg(0).Resolve() == shifted.Resolve(),
|
||||
"folding a masked shift changed its other consumer");
|
||||
}
|
||||
constexpr uint32_t source = 0xa5f03c69u;
|
||||
input.Instruction()->ReplaceUsesWith(Value(source));
|
||||
ConstantPropagationPass({&block});
|
||||
const uint32_t expected = source << (count & 31u);
|
||||
Check(result.Resolve() == Value(expected & mask) &&
|
||||
(!shared || other.Resolve() == Value(expected + 1u)),
|
||||
"masked shift folding changed a runtime result");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Block block;
|
||||
IREmitter ir(&block);
|
||||
const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
|
||||
const auto count = ir.GetUserData(static_cast<ScalarReg>(1));
|
||||
const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, count});
|
||||
const auto result = ir.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(0xffu)});
|
||||
ConstantPropagationPass({&block});
|
||||
Check(result.Resolve().Instruction()->GetOpcode() == ValueOpcode::BitwiseAnd32,
|
||||
"a variable shift count was treated as a constant");
|
||||
}
|
||||
|
||||
void TestU64ShiftConstantPropagation() {
|
||||
using namespace ShaderRecompiler::IR;
|
||||
|
||||
@@ -16364,6 +16409,7 @@ int main() {
|
||||
TestValuePhiValidation();
|
||||
TestLaneMaskProjection();
|
||||
TestLateConstantPropagation();
|
||||
TestMaskedShiftConstantPropagation();
|
||||
TestU64ShiftConstantPropagation();
|
||||
TestU64RegisterPairRoundTrip();
|
||||
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
|
||||
|
||||
Reference in New Issue
Block a user