renderer: reduce synchronization and allocation overhead

Performance changes from #1495: - Sync changed BDA ranges instead of scanning all mappings. - Let VMA suballocate BDA buffers. - Remove the renderer lock from presentation. - Keep compatible pipeline caches across builds. - Fold masked shifts that always produce zero.
2026-10-19 18:36:53 +02:00 · 2026-10-16 16:58:07 +02:00
parent 4c6be6cd1d
commit d58aefde49
12 changed files with 277 additions and 38 deletions
@@ -213,6 +213,12 @@ struct ProtectionCall {
 std::vector<ProtectionCall> g_protection_log;
 std::mutex g_protection_log_mutex;

+struct ProtectionGate {
+  std::binary_semaphore entered{0};
+  std::binary_semaphore resume{0};
+};
+ProtectionGate *g_protection_gate = nullptr;
+
 void ResetProtectionLog() {
  g_protection_calls = 0;
  g_protection_log.clear();
@@ -220,6 +226,10 @@ void ResetProtectionLog() {

 bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
                         Common::VirtualMemory::Mode mode) {
+  if (g_protection_gate != nullptr && mode == Common::VirtualMemory::Mode::ReadWrite) {
+    g_protection_gate->entered.release();
+    g_protection_gate->resume.acquire();
+  }
  uint32_t protection = PAGE_NOACCESS;
  if (mode == Common::VirtualMemory::Mode::Read) {
    protection = PAGE_READONLY;
@@ -407,6 +417,74 @@ void TestCpuDirtyUpload() {
  Release(memory);
 }

+void TestBdaSyncDirtiness() {
+  constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto *memory = Allocate(harness.page_manager, 3);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  Check(tracker.TakeBdaSync().Empty(), "new tracker queued unowned memory");
+  tracker.QueueBdaSync(address, 3 * page);
+  Check(tracker.IsRegionCpuModified(address, page), "new region was clean");
+  auto pending = tracker.TakeBdaSync();
+  Check(pending.Contains(address, 3 * page) && !pending.Intersects(address + 3 * page, page),
+        "mapped or registered range did not bound initial BDA synchronization");
+  tracker.ForEachUploadRange(address, 3 * page, false,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  Check(tracker.TakeBdaSync().Empty(), "clean upload queued redundant BDA work");
+
+  tracker.MarkRegionAsCpuModified(address + 17, 1);
+  pending = tracker.TakeBdaSync();
+  Check(pending.Contains(address, page) && !pending.Intersects(address + page, page) &&
+            tracker.TakeBdaSync().Empty(),
+        "byte dirtiness did not drain exactly its whole tracker page");
+  // A write after Take belongs to the next drain, even when its page was just taken.
+  tracker.MarkRegionAsCpuModified(address + 31, 1);
+  Check(tracker.TakeBdaSync().Contains(address, page), "re-dirty event was lost");
+  tracker.ForEachUploadRange(address, page, true,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  Check(tracker.TakeBdaSync().Empty(), "GPU ownership queued an unnecessary upload");
+  tracker.InvalidateRegion(address + 7, 1, [&] {
+    tracker.UnmarkRegionAsGpuModified(address, page);
+    tracker.MarkRegionAsCpuModified(address, page);
+  });
+  Check(tracker.TakeBdaSync().Contains(address, page),
+        "GPU-to-CPU invalidation did not queue BDA synchronization");
+  tracker.InvalidateRegion(address + page + 7, 1, [] {});
+  Check(tracker.TakeBdaSync().Contains(address + page, page),
+        "CPU invalidation did not queue BDA synchronization");
+  tracker.UntrackMemory(address, 3 * page);
+  Check(tracker.TakeBdaSync().Contains(address, 3 * page),
+        "untracked memory did not queue its newly dirty pages");
+  Release(memory);
+}
+
+void TestBdaSyncConcurrentDrain() {
+  constexpr auto page = Libs::Graphics::TRACKER_PAGE_SIZE;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto *memory = Allocate(harness.page_manager, 1);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  tracker.ForEachUploadRange(address, page, false,
+                             [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
+  (void)tracker.TakeBdaSync();
+  ProtectionGate gate;
+  g_protection_gate = &gate;
+  std::jthread writer([&] { tracker.MarkRegionAsCpuModified(address + 17, 1); });
+  gate.entered.acquire();
+  // Ownership is still changing. Publishing its event before this point would let
+  // a drain consume it before the writer has finished releasing page protection.
+  Check(tracker.TakeBdaSync().Empty(), "BDA event preceded its state transition");
+  gate.resume.release();
+  writer.join();
+  g_protection_gate = nullptr;
+  Check(tracker.TakeBdaSync().Contains(address, page) &&
+            tracker.IsRegionCpuModified(address, page),
+        "concurrent drain lost a completed CPU dirty transition");
+  tracker.UntrackMemory(address, page);
+  Release(memory);
+}
+
 void TestCleanUploadPreservesOwnership() {
  constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE;
  constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE;
@@ -1202,6 +1280,8 @@ int main(int argc, char **argv) {
  TestQueriesDoNotRequireMappedOwnership();
  TestConcurrentRegionPublication();
  TestCpuDirtyUpload();
+  TestBdaSyncDirtiness();
+  TestBdaSyncConcurrentDrain();
  TestCleanUploadPreservesOwnership();
  TestRangeInvalidation();
  TestGpuReacquisitionAfterInvalidation();
@@ -5281,6 +5281,85 @@ public:
    std::printf("[gpu]     %-32s ok\n", name);
  }

+  void CheckBdaSyncUploads() {
+    constexpr const char *name = "BdaSyncUploads";
+    constexpr auto page = BufferCache::CACHING_PAGESIZE;
+    EnsureRuntimeContext();
+    const auto base = LibKernel::Memory::AllocateRuntimeMemory(
+        0, 3 * page, Common::VirtualMemory::Mode::ReadWrite, "bda_sync_uploads");
+    Require(name, "allocation", base != 0, "guest allocation failed");
+    auto *words = reinterpret_cast<u32 *>(base);
+    std::fill_n(words, 3 * page / sizeof(u32), 0x12345678u);
+    RenderContext context(m_runtime_context);
+    auto &scheduler = context.GetCommandScheduler();
+    auto &cache = context.GetBufferCache();
+    HW::Context registers{};
+    HW::UserConfig user_config{};
+    HW::Shader shaders{};
+    scheduler.Begin(registers, user_config, shaders);
+    context.MapMemory(base, page);
+    context.MapMemory(base + 2 * page, page);
+    const auto id = cache.FindBuffer(base, 3 * page);
+    auto &buffer = cache.GetBuffer(id);
+    auto readback = CreateHostBuffer(name, 6 * sizeof(u32),
+                                     vk::BufferUsageFlagBits::eTransferDst, {});
+    const auto snapshot = [&](uint64_t offset, uint32_t index) {
+      const vk::BufferCopy copy{offset, index * sizeof(u32), sizeof(u32)};
+      scheduler.Current().Handle().copyBuffer(buffer.Handle(), readback.buffer, 1, &copy);
+    };
+    context.PrepareBda();
+    Require(name, "mapped intersection",
+            !cache.IsRegionCpuModified(base, page) &&
+                cache.IsRegionCpuModified(base + page, page) &&
+                !cache.IsRegionCpuModified(base + 2 * page, page),
+            "BDA upload missed a mapped range or uploaded an unmapped hole");
+    snapshot(0, 0);
+    snapshot(2 * page, 1);
+
+    cache.InvalidateMemory(base + 17, 1);
+    words[4] = 0x87654321u;
+    words[TRACKER_PAGE_SIZE / sizeof(u32) - 1] = 0xabcdef01u;
+    context.PrepareBda();
+    snapshot(16, 2);
+    snapshot(TRACKER_PAGE_SIZE - sizeof(u32), 3);
+    Require(name, "whole-page upload", !cache.IsRegionCpuModified(base, TRACKER_PAGE_SIZE),
+            "one-byte invalidation did not upload and consume its whole page");
+
+    context.MapMemory(base + page, page);
+    context.PrepareBda();
+    snapshot(page, 4);
+    Require(name, "new mapping", !cache.IsRegionCpuModified(base + page, page),
+            "mapping into an existing buffer did not queue its dirty pages");
+    context.UnmapMemory(base + 2 * page, page);
+    words[2 * page / sizeof(u32)] = 0xfedcba98u;
+    context.PrepareBda();
+    Require(name, "unmapped dirty page", cache.IsRegionCpuModified(base + 2 * page, page),
+            "BDA synchronization consumed dirtiness outside mapped memory");
+    context.MapMemory(base + 2 * page, page);
+    context.PrepareBda();
+    snapshot(2 * page, 5);
+
+    vk::BufferMemoryBarrier barrier{};
+    barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
+    barrier.dstAccessMask = vk::AccessFlagBits::eHostRead;
+    barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+    barrier.buffer = readback.buffer;
+    barrier.size = readback.size;
+    scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
+        vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr);
+    scheduler.Finish();
+    Require(name, "GPU contents", ReadBuffer(name, readback, 6) ==
+        std::vector<u32>{0x12345678u, 0x12345678u, 0x87654321u, 0xabcdef01u,
+                        0x12345678u, 0xfedcba98u},
+        "initial, re-dirtied, newly mapped or remapped bytes were stale on the GPU");
+    context.UnmapMemory(base, 3 * page);
+    scheduler.Finish();
+    DestroyBuffer(&readback);
+    Require(name, "release", LibKernel::Memory::FreeGuestMemory(base, 3 * page),
+            "guest allocation release failed");
+    std::printf("[gpu]     %-32s ok\n", name);
+  }
+
  void CheckBdaPageTableUploads() {
    constexpr const char *name = "BdaPageTableUploads";
    constexpr uintptr_t base = 0x0000000207b00000ull;
@@ -48472,6 +48551,7 @@ int main(int argc, char **argv) {
  if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) {
    VulkanHarness vulkan;
    vulkan.CheckBdaPageTableUploads();
+    vulkan.CheckBdaSyncUploads();
    return 0;
  }
  if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) {
@@ -48702,6 +48782,7 @@ int main(int argc, char **argv) {
  CheckComputeLdsLimit(vulkan);
  vulkan.CheckNullTextureDescriptors();
  vulkan.CheckBdaPageTableUploads();
+  vulkan.CheckBdaSyncUploads();
  vulkan.CheckLargeSparseImageUpload();
  vulkan.CheckUnifiedTextureCacheFlow();
  vulkan.CheckUnifiedImageViewCache();
@@ -12874,6 +12874,51 @@ void TestLateConstantPropagation() {
        "an ungrounded Phi cycle acquired a constant");
 }

+void TestMaskedShiftConstantPropagation() {
+  using namespace ShaderRecompiler::IR;
+  for (const uint32_t count : {0u, 1u, 8u, 31u, 32u, 33u, 40u, 63u, 64u, UINT32_MAX}) {
+    for (const uint32_t mask : {0u, 1u, 0xffu, 0x7fffffffu, 0x80000000u, UINT32_MAX}) {
+      for (const bool swapped : {false, true}) {
+        for (const bool shared : {false, true}) {
+          Block block;
+          IREmitter ir(&block);
+          const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
+          const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, Value(count)});
+          const auto result = ir.Emit(ValueOpcode::BitwiseAnd32,
+              swapped ? std::initializer_list<Value>{Value(mask), shifted}
+                      : std::initializer_list<Value>{shifted, Value(mask)});
+          const auto other = shared ? ir.Emit(ValueOpcode::IAdd32, {shifted, Value(1u)}) : Value{};
+          ConstantPropagationPass({&block});
+          const bool zero = ((UINT32_MAX << (count & 31u)) & mask) == 0u;
+          Check(zero ? result.Resolve() == Value(0u) : !result.Resolve().IsImmediate(),
+                "masked shift retained impossible bits or discarded possible bits");
+          if (shared) {
+            Check(other.Resolve().Instruction()->Arg(0).Resolve() == shifted.Resolve(),
+                  "folding a masked shift changed its other consumer");
+          }
+          constexpr uint32_t source = 0xa5f03c69u;
+          input.Instruction()->ReplaceUsesWith(Value(source));
+          ConstantPropagationPass({&block});
+          const uint32_t expected = source << (count & 31u);
+          Check(result.Resolve() == Value(expected & mask) &&
+                    (!shared || other.Resolve() == Value(expected + 1u)),
+                "masked shift folding changed a runtime result");
+        }
+      }
+    }
+  }
+
+  Block block;
+  IREmitter ir(&block);
+  const auto input = ir.GetUserData(static_cast<ScalarReg>(0));
+  const auto count = ir.GetUserData(static_cast<ScalarReg>(1));
+  const auto shifted = ir.Emit(ValueOpcode::ShiftLeftLogical32, {input, count});
+  const auto result = ir.Emit(ValueOpcode::BitwiseAnd32, {shifted, Value(0xffu)});
+  ConstantPropagationPass({&block});
+  Check(result.Resolve().Instruction()->GetOpcode() == ValueOpcode::BitwiseAnd32,
+        "a variable shift count was treated as a constant");
+}
+
 void TestU64ShiftConstantPropagation() {
  using namespace ShaderRecompiler::IR;

@@ -16364,6 +16409,7 @@ int main() {
  TestValuePhiValidation();
  TestLaneMaskProjection();
  TestLateConstantPropagation();
+  TestMaskedShiftConstantPropagation();
  TestU64ShiftConstantPropagation();
  TestU64RegisterPairRoundTrip();
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS