diff --git a/src/graphics/host_gpu/memoryTracker.cpp b/src/graphics/host_gpu/memoryTracker.cpp index 4c45702b8..a3a070e9e 100644 --- a/src/graphics/host_gpu/memoryTracker.cpp +++ b/src/graphics/host_gpu/memoryTracker.cpp @@ -66,6 +66,18 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) { return ptr; } +void MemoryTracker::QueueBdaSync(uint64_t vaddr, uint64_t size) { + const auto begin = Common::AlignDown(vaddr, TRACKER_PAGE_SIZE); + const auto end = Common::AlignUp(vaddr + size, TRACKER_PAGE_SIZE); + std::lock_guard lock(m_bda_sync_mutex); + m_bda_sync.Add(begin, end - begin); +} + +RangeSet MemoryTracker::TakeBdaSync() { + std::lock_guard lock(m_bda_sync_mutex); + return std::exchange(m_bda_sync, RangeSet {}); +} + bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); return Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { @@ -88,6 +100,7 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) { std::scoped_lock lock(manager->lock); manager->ChangeState(manager->GetCpuAddr() + offset, bytes); }); + QueueBdaSync(vaddr, size); } void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { @@ -128,6 +141,8 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) { Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { manager->ChangeState(manager->GetCpuAddr() + offset, bytes); }); + locks.clear(); + QueueBdaSync(vaddr, size); } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index 0bc9fe547..ff81021f2 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -29,6 +29,10 @@ public: void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UntrackMemory(uint64_t vaddr, uint64_t size); + // Queue after changing ownership; a concurrent drain must not consume an event + // before the dirty pages it describes become visible. + void QueueBdaSync(uint64_t vaddr, uint64_t size); + [[nodiscard]] RangeSet TakeBdaSync(); // Removes protection from a range and flushes GPU-owned data when required. template void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept { @@ -51,6 +55,7 @@ public: on_flush(); } }); + QueueBdaSync(vaddr, size); } #if KYTY_BUILD == KYTY_BUILD_DEBUG void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, @@ -149,6 +154,8 @@ private: std::unique_ptr[]> m_regions; std::vector> m_region_storage; std::mutex m_region_mutex; + std::mutex m_bda_sync_mutex; + RangeSet m_bda_sync; PageManager& m_page_manager; }; diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 91376ea26..e2c3802f3 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -126,6 +126,7 @@ void BufferCache::ChangeRegister(BufferId id) { } WriteDataBuffer(m_bda_pagetable_buffer, table_offset, addresses.data(), addresses.size() * sizeof(vk::DeviceAddress)); + m_memory_tracker.QueueBdaSync(buffer.CpuAddress(), buffer.Size()); } else { const auto found = m_buffers.find(buffer.CpuAddress()); EXIT_IF(found == m_buffers.end() || found->second != id); @@ -685,6 +686,16 @@ void BufferCache::ProcessFaultBuffer() { m_fault_manager.ProcessFaultBuffer(); } +void BufferCache::SynchronizeBda(const RangeSet& mapped) { + // Take before walking: writes queued during synchronization remain pending. + const auto pending = m_memory_tracker.TakeBdaSync(); + pending.ForEach([&](uint64_t begin, uint64_t end) { + mapped.ForEachInRange(begin, end - begin, [this](uint64_t start, uint64_t finish) { + SynchronizeBuffersInRange(start, finish - start); + }); + }); +} + void BufferCache::SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size) { const auto end = vaddr + size; auto it = m_buffers.upper_bound(vaddr); diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 1212bd535..27bf987c6 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -80,7 +80,10 @@ public: [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); void ProcessFaultBuffer(); - void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size); + void SynchronizeBda(const RangeSet& mapped); + void QueueBdaSync(uint64_t vaddr, uint64_t size) { + m_memory_tracker.QueueBdaSync(vaddr, size); + } void RunGarbageCollector(); private: @@ -114,6 +117,7 @@ private: void DeleteBuffer(BufferId id); [[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size, bool is_written, bool is_texel_buffer); + void SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size); [[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span copies, uint64_t total_size); [[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size); diff --git a/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp b/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp index b0fc5fb41..76ae015b2 100644 --- a/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp +++ b/src/graphics/host_gpu/renderer/cache/streamBuffer.cpp @@ -67,11 +67,9 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag buffer_info.usage = flags; const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress); - const VmaAllocationCreateFlags bda_flag = - with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; VmaAllocationCreateInfo allocation_info {}; allocation_info.flags = - VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage); + VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage); allocation_info.usage = AllocationUsage(usage); allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal ? VkMemoryPropertyFlags {} diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index d2c4a7d96..5aca239d6 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -18,7 +18,6 @@ #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/shaderCompiler.h" #include "kernel/memory.h" -#include "kytyGitVersion.h" #include "loader/systemContent.h" #include @@ -81,7 +80,8 @@ std::string DriverCacheSignature(const vk::PhysicalDeviceProperties& properties) uuid[i * 2] = hex[properties.pipelineCacheUUID[i] >> 4u]; uuid[i * 2 + 1] = hex[properties.pipelineCacheUUID[i] & 0xfu]; } - return fmt::format("KytyPC1:{}:{:08x}:{:08x}:{:08x}:{}\n", KYTY_GIT_REVISION, + // Driver cache entries already distinguish shader code and pipeline state across builds. + return fmt::format("KytyPC2:{:08x}:{:08x}:{:08x}:{}\n", properties.vendorID, properties.deviceID, properties.driverVersion, uuid); } @@ -516,17 +516,6 @@ void PipelineCache::InitializeDriverCache() { PipelineCacheLog("Vulkan pipeline cache: disabled (non-Release build)"); return; } - const std::string_view git_hash = KYTY_GIT_HASH; - const std::string_view git_revision = KYTY_GIT_REVISION; - if (git_hash == "unknown" || git_revision == "unknown") { - PipelineCacheLog("Vulkan pipeline cache: disabled (unknown git revision)"); - return; - } - if (git_hash.ends_with("-dirty")) { - PipelineCacheLog("Vulkan pipeline cache: disabled (dirty build)"); - return; - } - m_driver_cache_path = std::filesystem::path("_PipelineCache") / (title_id + ".bin"); const auto path = Common::PathToString(m_driver_cache_path); const bool cache_exists = Common::File::IsFileExisting(m_driver_cache_path); @@ -559,7 +548,7 @@ void PipelineCache::InitializeDriverCache() { XXH3_64bits(initial_data.data(), initial_data.size()) != payload_hash) { initial_data.clear(); PipelineCacheLog( - "Vulkan pipeline cache: invalidating {} (driver, emulator, or data mismatch)", + "Vulkan pipeline cache: invalidating {} (driver or data mismatch)", path); } } else { diff --git a/src/graphics/host_gpu/renderer/renderContext.cpp b/src/graphics/host_gpu/renderer/renderContext.cpp index 431a56f4b..e256b8ae4 100644 --- a/src/graphics/host_gpu/renderer/renderContext.cpp +++ b/src/graphics/host_gpu/renderer/renderContext.cpp @@ -90,6 +90,7 @@ bool RenderContext::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { void RenderContext::MapMemory(uint64_t vaddr, uint64_t size) { std::lock_guard lock(m_mapped_ranges_mutex); m_mapped_ranges.Add(vaddr, size); + m_buffer_cache.QueueBdaSync(vaddr, size); } void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) { @@ -129,9 +130,7 @@ void RenderContext::PrepareBda() { m_bda_logged = true; } std::shared_lock lock(m_mapped_ranges_mutex); - m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) { - m_buffer_cache.SynchronizeBuffersInRange(start, end - start); - }); + m_buffer_cache.SynchronizeBda(m_mapped_ranges); m_fault_process_pending = true; } diff --git a/src/graphics/presentation/window/swapchain.cpp b/src/graphics/presentation/window/swapchain.cpp index 07c740125..2a67094eb 100644 --- a/src/graphics/presentation/window/swapchain.cpp +++ b/src/graphics/presentation/window/swapchain.cpp @@ -945,7 +945,6 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height, auto* frame = m_impl->frames.Acquire({width, height}, format); frame->hdr = false; frame->blank = true; - Common::LockGuard render_lock(m_impl->renderer.GetMutex()); frame->Configure(m_impl->window.graphic_ctx, {width, height}, format); vk::ClearColorValue clear {}; clear.float32[3] = opaque ? 1.0f : 0.0f; @@ -953,6 +952,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height, EXIT_IF(producer->IsInvalid()); frame->Clear(*producer, clear); } else { + Common::LockGuard lock(m_impl->present_mutex); auto& command = m_impl->present_scheduler.BeginCommand(); frame->Clear(command, clear); frame->present_tick = m_impl->present_scheduler.Submit(); @@ -1036,18 +1036,17 @@ void Presenter::Impl::Present() { RecoverSwapchain(status); continue; } - { - Common::LockGuard render_lock(renderer.GetMutex()); - auto& command = present_scheduler.BeginCommand(); - const bool draw_system_overlay = - overlay_visual.active && swapchain.PrepareSystemOverlay(); - swapchain.RecordPresentCommands(command, layers[0].frame, layers[1], - draw_system_overlay); - const auto tick = swapchain.Submit(present_scheduler); - for (const auto& layer: layers) { - if (layer.frame != nullptr) { - layer.frame->present_tick = tick; - } + // present_mutex owns the prepared frames and this scheduler. Only queue submission + // is shared with rendering, and it has its own queue_mutex. + auto& command = present_scheduler.BeginCommand(); + const bool draw_system_overlay = + overlay_visual.active && swapchain.PrepareSystemOverlay(); + swapchain.RecordPresentCommands(command, layers[0].frame, layers[1], + draw_system_overlay); + const auto tick = swapchain.Submit(present_scheduler); + for (const auto& layer: layers) { + if (layer.frame != nullptr) { + layer.frame->present_tick = tick; } } status = swapchain.Present(); diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp index 5ad68358f..413676da3 100644 --- a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -608,13 +608,23 @@ void FoldInstruction(Block& block, Block::iterator instruction, ReplaceBinaryIdentity(inst, Type::U32, UINT32_MAX)) return; for (uint32_t arm = 0; arm < 2u; ++arm) { const auto mask = Arg(inst, arm); + if (!IsImmediate(mask, Type::U32)) continue; + if (mask.U32() == 0u) { + Replace(inst, Value(0u)); + return; + } const auto* shifted = Arg(inst, arm ^ 1u).TryInstruction(); - if (!IsImmediate(mask, Type::U32) || shifted == nullptr || - shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32 || !HasOneUse(*shifted)) continue; + if (shifted == nullptr || shifted->GetOpcode() != ValueOpcode::ShiftLeftLogical32) continue; const auto shift = Arg(*shifted, 1); - if (!IsImmediate(shift, Type::U32) || shift.U32() >= 32u) continue; + if (!IsImmediate(shift, Type::U32)) continue; + const auto shifted_mask = mask.U32() >> (shift.U32() & 31u); + if (shifted_mask == 0u) { + Replace(inst, Value(0u)); + return; + } + if (!HasOneUse(*shifted) || shift.U32() >= 32u) continue; const auto masked = worklist.Prepend(block, instruction, ValueOpcode::BitwiseAnd32, - {Arg(*shifted, 0), Value(mask.U32() >> shift.U32())}); + {Arg(*shifted, 0), Value(shifted_mask)}); Replace(inst, Value(&*worklist.Prepend(block, instruction, ValueOpcode::ShiftLeftLogical32, {Value(&*masked), shift}))); return; diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index d859a492f..63bbc11c9 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -213,6 +213,12 @@ struct ProtectionCall { std::vector g_protection_log; std::mutex g_protection_log_mutex; +struct ProtectionGate { + std::binary_semaphore entered{0}; + std::binary_semaphore resume{0}; +}; +ProtectionGate *g_protection_gate = nullptr; + void ResetProtectionLog() { g_protection_calls = 0; g_protection_log.clear(); @@ -220,6 +226,10 @@ void ResetProtectionLog() { bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { + if (g_protection_gate != nullptr && mode == Common::VirtualMemory::Mode::ReadWrite) { + g_protection_gate->entered.release(); + g_protection_gate->resume.acquire(); + } uint32_t protection = PAGE_NOACCESS; if (mode == Common::VirtualMemory::Mode::Read) { protection = PAGE_READONLY; @@ -407,6 +417,74 @@ void TestCpuDirtyUpload() { Release(memory); } +void TestBdaSyncDirtiness() { + constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto *memory = Allocate(harness.page_manager, 3); + const auto address = reinterpret_cast(memory); + Check(tracker.TakeBdaSync().Empty(), "new tracker queued unowned memory"); + tracker.QueueBdaSync(address, 3 * page); + Check(tracker.IsRegionCpuModified(address, page), "new region was clean"); + auto pending = tracker.TakeBdaSync(); + Check(pending.Contains(address, 3 * page) && !pending.Intersects(address + 3 * page, page), + "mapped or registered range did not bound initial BDA synchronization"); + tracker.ForEachUploadRange(address, 3 * page, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + Check(tracker.TakeBdaSync().Empty(), "clean upload queued redundant BDA work"); + + tracker.MarkRegionAsCpuModified(address + 17, 1); + pending = tracker.TakeBdaSync(); + Check(pending.Contains(address, page) && !pending.Intersects(address + page, page) && + tracker.TakeBdaSync().Empty(), + "byte dirtiness did not drain exactly its whole tracker page"); + // A write after Take belongs to the next drain, even when its page was just taken. + tracker.MarkRegionAsCpuModified(address + 31, 1); + Check(tracker.TakeBdaSync().Contains(address, page), "re-dirty event was lost"); + tracker.ForEachUploadRange(address, page, true, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + Check(tracker.TakeBdaSync().Empty(), "GPU ownership queued an unnecessary upload"); + tracker.InvalidateRegion(address + 7, 1, [&] { + tracker.UnmarkRegionAsGpuModified(address, page); + tracker.MarkRegionAsCpuModified(address, page); + }); + Check(tracker.TakeBdaSync().Contains(address, page), + "GPU-to-CPU invalidation did not queue BDA synchronization"); + tracker.InvalidateRegion(address + page + 7, 1, [] {}); + Check(tracker.TakeBdaSync().Contains(address + page, page), + "CPU invalidation did not queue BDA synchronization"); + tracker.UntrackMemory(address, 3 * page); + Check(tracker.TakeBdaSync().Contains(address, 3 * page), + "untracked memory did not queue its newly dirty pages"); + Release(memory); +} + +void TestBdaSyncConcurrentDrain() { + constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto *memory = Allocate(harness.page_manager, 1); + const auto address = reinterpret_cast(memory); + tracker.ForEachUploadRange(address, page, false, + [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + (void)tracker.TakeBdaSync(); + ProtectionGate gate; + g_protection_gate = &gate; + std::jthread writer([&] { tracker.MarkRegionAsCpuModified(address + 17, 1); }); + gate.entered.acquire(); + // Ownership is still changing. Publishing its event before this point would let + // a drain consume it before the writer has finished releasing page protection. + Check(tracker.TakeBdaSync().Empty(), "BDA event preceded its state transition"); + gate.resume.release(); + writer.join(); + g_protection_gate = nullptr; + Check(tracker.TakeBdaSync().Contains(address, page) && + tracker.IsRegionCpuModified(address, page), + "concurrent drain lost a completed CPU dirty transition"); + tracker.UntrackMemory(address, page); + Release(memory); +} + void TestCleanUploadPreservesOwnership() { constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE; constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE; @@ -1202,6 +1280,8 @@ int main(int argc, char **argv) { TestQueriesDoNotRequireMappedOwnership(); TestConcurrentRegionPublication(); TestCpuDirtyUpload(); + TestBdaSyncDirtiness(); + TestBdaSyncConcurrentDrain(); TestCleanUploadPreservesOwnership(); TestRangeInvalidation(); TestGpuReacquisitionAfterInvalidation(); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index ce37e2208..627a81127 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -5281,6 +5281,85 @@ public: std::printf("[gpu] %-32s ok\n", name); } + void CheckBdaSyncUploads() { + constexpr const char *name = "BdaSyncUploads"; + constexpr auto page = BufferCache::CACHING_PAGESIZE; + EnsureRuntimeContext(); + const auto base = LibKernel::Memory::AllocateRuntimeMemory( + 0, 3 * page, Common::VirtualMemory::Mode::ReadWrite, "bda_sync_uploads"); + Require(name, "allocation", base != 0, "guest allocation failed"); + auto *words = reinterpret_cast(base); + std::fill_n(words, 3 * page / sizeof(u32), 0x12345678u); + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + auto &cache = context.GetBufferCache(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + context.MapMemory(base, page); + context.MapMemory(base + 2 * page, page); + const auto id = cache.FindBuffer(base, 3 * page); + auto &buffer = cache.GetBuffer(id); + auto readback = CreateHostBuffer(name, 6 * sizeof(u32), + vk::BufferUsageFlagBits::eTransferDst, {}); + const auto snapshot = [&](uint64_t offset, uint32_t index) { + const vk::BufferCopy copy{offset, index * sizeof(u32), sizeof(u32)}; + scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer, 1, ©); + }; + context.PrepareBda(); + Require(name, "mapped intersection", + !cache.IsRegionCpuModified(base, page) && + cache.IsRegionCpuModified(base + page, page) && + !cache.IsRegionCpuModified(base + 2 * page, page), + "BDA upload missed a mapped range or uploaded an unmapped hole"); + snapshot(0, 0); + snapshot(2 * page, 1); + + cache.InvalidateMemory(base + 17, 1); + words[4] = 0x87654321u; + words[TRACKER_PAGE_SIZE / sizeof(u32) - 1] = 0xabcdef01u; + context.PrepareBda(); + snapshot(16, 2); + snapshot(TRACKER_PAGE_SIZE - sizeof(u32), 3); + Require(name, "whole-page upload", !cache.IsRegionCpuModified(base, TRACKER_PAGE_SIZE), + "one-byte invalidation did not upload and consume its whole page"); + + context.MapMemory(base + page, page); + context.PrepareBda(); + snapshot(page, 4); + Require(name, "new mapping", !cache.IsRegionCpuModified(base + page, page), + "mapping into an existing buffer did not queue its dirty pages"); + context.UnmapMemory(base + 2 * page, page); + words[2 * page / sizeof(u32)] = 0xfedcba98u; + context.PrepareBda(); + Require(name, "unmapped dirty page", cache.IsRegionCpuModified(base + 2 * page, page), + "BDA synchronization consumed dirtiness outside mapped memory"); + context.MapMemory(base + 2 * page, page); + context.PrepareBda(); + snapshot(2 * page, 5); + + vk::BufferMemoryBarrier barrier{}; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = readback.buffer; + barrier.size = readback.size; + scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); + scheduler.Finish(); + Require(name, "GPU contents", ReadBuffer(name, readback, 6) == + std::vector{0x12345678u, 0x12345678u, 0x87654321u, 0xabcdef01u, + 0x12345678u, 0xfedcba98u}, + "initial, re-dirtied, newly mapped or remapped bytes were stale on the GPU"); + context.UnmapMemory(base, 3 * page); + scheduler.Finish(); + DestroyBuffer(&readback); + Require(name, "release", LibKernel::Memory::FreeGuestMemory(base, 3 * page), + "guest allocation release failed"); + std::printf("[gpu] %-32s ok\n", name); + } + void CheckBdaPageTableUploads() { constexpr const char *name = "BdaPageTableUploads"; constexpr uintptr_t base = 0x0000000207b00000ull; @@ -48472,6 +48551,7 @@ int main(int argc, char **argv) { if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) { VulkanHarness vulkan; vulkan.CheckBdaPageTableUploads(); + vulkan.CheckBdaSyncUploads(); return 0; } if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) { @@ -48702,6 +48782,7 @@ int main(int argc, char **argv) { CheckComputeLdsLimit(vulkan); vulkan.CheckNullTextureDescriptors(); vulkan.CheckBdaPageTableUploads(); + vulkan.CheckBdaSyncUploads(); vulkan.CheckLargeSparseImageUpload(); vulkan.CheckUnifiedTextureCacheFlow(); vulkan.CheckUnifiedImageViewCache(); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index b7089f6c3..b8e0b75fc 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -12874,6 +12874,51 @@ void TestLateConstantPropagation() { "an ungrounded Phi cycle acquired a constant"); } +void TestMaskedShiftConstantPropagation() { + using namespace ShaderRecompiler::IR; + for (const uint32_t count : {0u, 1u, 8u, 31u, 32u, 33u, 40u, 63u, 64u, UINT32_MAX}) { + for (const uint32_t mask : {0u, 1u, 0xffu, 0x7fffffffu, 0x80000000u, UINT32_MAX}) { + for (const bool swapped : {false, true}) { + for (const bool shared : {false, true}) { + Block block; + IREmitter ir(&block); + const auto input = ir.GetUserData(static_cast(0)); + const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, Value(count)}); + const auto result = ir.Emit(ValueOpcode::BitwiseAnd32, + swapped ? std::initializer_list{Value(mask), shifted} + : std::initializer_list{shifted, Value(mask)}); + const auto other = shared ? ir.Emit(ValueOpcode::IAdd32, {shifted, Value(1u)}) : Value{}; + ConstantPropagationPass({&block}); + const bool zero = ((UINT32_MAX << (count & 31u)) & mask) == 0u; + Check(zero ? result.Resolve() == Value(0u) : !result.Resolve().IsImmediate(), + "masked shift retained impossible bits or discarded possible bits"); + if (shared) { + Check(other.Resolve().Instruction()->Arg(0).Resolve() == shifted.Resolve(), + "folding a masked shift changed its other consumer"); + } + constexpr uint32_t source = 0xa5f03c69u; + input.Instruction()->ReplaceUsesWith(Value(source)); + ConstantPropagationPass({&block}); + const uint32_t expected = source << (count & 31u); + Check(result.Resolve() == Value(expected & mask) && + (!shared || other.Resolve() == Value(expected + 1u)), + "masked shift folding changed a runtime result"); + } + } + } + } + + Block block; + IREmitter ir(&block); + const auto input = ir.GetUserData(static_cast(0)); + const auto count = ir.GetUserData(static_cast(1)); + const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, count}); + const auto result = ir.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(0xffu)}); + ConstantPropagationPass({&block}); + Check(result.Resolve().Instruction()->GetOpcode() == ValueOpcode::BitwiseAnd32, + "a variable shift count was treated as a constant"); +} + void TestU64ShiftConstantPropagation() { using namespace ShaderRecompiler::IR; @@ -16364,6 +16409,7 @@ int main() { TestValuePhiValidation(); TestLaneMaskProjection(); TestLateConstantPropagation(); + TestMaskedShiftConstantPropagation(); TestU64ShiftConstantPropagation(); TestU64RegisterPairRoundTrip(); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS