diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index ba0f2af75..165e32e37 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -34,9 +34,36 @@ void BufferCache::WriteDataBuffer(Buffer& buffer, uint64_t address, const void* auto* bytes = static_cast(source); while (size != 0) { const auto chunk = std::min(size, m_staging_buffer.Size()); + // Copy waits for ring reuse and flushes the fresh host data before submission. const auto offset = m_staging_buffer.Copy(bytes, chunk, 4); - buffer.CopyFrom(m_scheduler.Current(), m_staging_buffer, offset, buffer.Offset(address), - chunk, vk::AccessFlagBits::eHostWrite); + const auto destination_offset = buffer.Offset(address); + EXIT_IF(destination_offset > buffer.Size() || chunk > buffer.Size() - destination_offset); + m_scheduler.EndRendering(); + const auto command = m_scheduler.Current().Handle(); + vk::BufferMemoryBarrier2 before {}; + before.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands; + before.srcAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; + before.dstStageMask = vk::PipelineStageFlagBits2::eTransfer; + before.dstAccessMask = vk::AccessFlagBits2::eTransferWrite; + before.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + before.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + before.buffer = buffer.Handle(); + before.offset = destination_offset; + before.size = chunk; + vk::DependencyInfo dependency {}; + dependency.dependencyFlags = vk::DependencyFlagBits::eByRegion; + dependency.bufferMemoryBarrierCount = 1; + dependency.pBufferMemoryBarriers = &before; + command.pipelineBarrier2(dependency); + const vk::BufferCopy copy {offset, destination_offset, chunk}; + command.copyBuffer(m_staging_buffer.Handle(), buffer.Handle(), 1, ©); + auto after = before; + after.srcStageMask = vk::PipelineStageFlagBits2::eTransfer; + after.srcAccessMask = vk::AccessFlagBits2::eTransferWrite; + after.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands; + after.dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite; + dependency.pBufferMemoryBarriers = &after; + command.pipelineBarrier2(dependency); bytes += chunk; address += chunk; size -= chunk; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 2bc68b910..3748322e3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -164,6 +164,13 @@ struct BufferCacheTestAccess { return cache.m_download_buffer; } + static void SetUnusedStagingBufferSize(BufferCache &cache, uint64_t size) { + // Called immediately after cache construction, before any staging reservation. + std::destroy_at(&cache.m_staging_buffer); + std::construct_at(&cache.m_staging_buffer, cache.m_graphics, cache.m_scheduler, + MemoryUsage::Upload, size); + } + static bool SynchronizeBufferFromImage(BufferCache &cache, Buffer &buffer, uint64_t address, uint64_t size) { return cache.SynchronizeBufferFromImage(buffer, address, size); @@ -5109,6 +5116,67 @@ public: std::printf("[host] %-32s ok\n", name); } + void CheckBdaPageTableUploads() { + constexpr const char *name = "BdaPageTableUploads"; + constexpr uintptr_t base = 0x0000000207b00000ull; + constexpr uint64_t page = BufferCache::CACHING_PAGESIZE; + constexpr std::array page_counts{2, 521, 525}; + constexpr uint64_t result_entries = page_counts[0] + page_counts[1] + page_counts[2]; + EnsureRuntimeContext(); + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + auto &cache = context.GetBufferCache(); + // A small real ring exercises chunking and waits for same-command-buffer reuse. + BufferCacheTestAccess::SetUnusedStagingBufferSize(cache, 4096); + auto &table = *cache.GetBdaPageTableBuffer(); + const auto table_offset = BufferCache::PageIndex(base) * sizeof(vk::DeviceAddress); + auto readback = CreateHostBuffer(name, result_entries * sizeof(vk::DeviceAddress), + vk::BufferUsageFlagBits::eTransferDst, {}); + std::vector expected; + expected.reserve(result_entries); + for (const auto pages : page_counts) { + // The destination has a preceding GPU write and, after the first iteration, + // a preceding read. Registration must preserve both ordering requirements. + table.Fill(table_offset, pages * sizeof(vk::DeviceAddress), 0xa5a5a5a5u); + const auto previous_tick = scheduler.CurrentTick(); + const auto id = cache.FindBuffer(base, pages * page); + const auto &buffer = cache.GetBuffer(id); + const auto destination_offset = expected.size() * sizeof(vk::DeviceAddress); + for (uint64_t index = 0; index < pages; ++index) { + expected.push_back(buffer.BufferDeviceAddress() + index * page); + } + const vk::BufferCopy copy{table_offset, destination_offset, + pages * sizeof(vk::DeviceAddress)}; + scheduler.Current().Handle().copyBuffer(table.Handle(), readback.buffer, 1, ©); + Require(name, "chunked registration", + buffer.CpuAddress() == base && buffer.Size() == pages * page && + (pages == page_counts.front() || scheduler.CurrentTick() > previous_tick), + "BDA registration failed to replace its owner or wait for staging-ring reuse"); + } + vk::BufferMemoryBarrier barrier{}; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = readback.buffer; + barrier.size = readback.size; + scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); + scheduler.Finish(); + const auto words = ReadBuffer(name, readback, result_entries * 2); + for (size_t index = 0; index < expected.size(); ++index) { + const auto actual = uint64_t{words[index * 2]} | (uint64_t{words[index * 2 + 1]} << 32); + Require(name, "GPU page-table snapshot", actual == expected[index], + "staged BDA updates were stale, overwritten before their read, or incompletely flushed"); + } + DestroyBuffer(&readback); + std::printf("[gpu] %-32s ok\n", name); + } + void CheckUnifiedTextureCacheFlow() { constexpr const char *name = "UnifiedTextureCacheFlow"; constexpr uintptr_t base = 0x0000000200600000ull; @@ -41922,6 +41990,11 @@ int main(int argc, char **argv) { CheckDepthTargetFootprints(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--bda-page-table-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckBdaPageTableUploads(); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) { VulkanHarness vulkan; vulkan.CheckUnifiedTextureCacheFlow(); @@ -42123,6 +42196,7 @@ int main(int argc, char **argv) { vulkan.CheckGpuTilerCpuParity(); vulkan.CheckNativeIndirectDispatch(); CheckComputeLdsLimit(vulkan); + vulkan.CheckBdaPageTableUploads(); vulkan.CheckUnifiedTextureCacheFlow(); vulkan.CheckUnifiedImageViewCache(); vulkan.CheckPackedTextureComponents();