graphics: retain oversized readbacks through GPU completion

Co-authored-by: Mac <me@itsmemac.com>
This commit is contained in:
nmzik
2026-09-29 21:28:16 +04:00
parent da9b38af2b
commit 18a5f04d05
2 changed files with 61 additions and 54 deletions
+12 -7
View File
@@ -129,11 +129,16 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
return false; return false;
} }
const auto [mapped, offset] = m_download_buffer.Map(total_size, 64); auto [mapped, offset] = m_download_buffer.Map(total_size, 64);
std::unique_ptr<Buffer> temporary;
if (mapped == nullptr) { if (mapped == nullptr) {
EXIT("BufferCache: download exceeds 64 MiB staging buffer capacity\n"); temporary = std::make_unique<Buffer>(m_graphics, m_scheduler, MemoryUsage::Download, 0,
vk::BufferUsageFlagBits::eTransferDst, total_size);
mapped = temporary->Mapped().data();
} else {
m_download_buffer.Commit();
} }
m_download_buffer.Commit(); const auto& download = temporary ? *temporary : m_download_buffer;
for (auto& copy: copies) { for (auto& copy: copies) {
copy.dstOffset += offset; copy.dstOffset += offset;
} }
@@ -152,13 +157,13 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &before, 0, vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &before, 0,
nullptr); nullptr);
native.copyBuffer(buffer.Handle(), m_download_buffer.Handle(), native.copyBuffer(buffer.Handle(), download.Handle(),
static_cast<uint32_t>(copies.size()), copies.data()); static_cast<uint32_t>(copies.size()), copies.data());
auto after = before; auto after = before;
after.srcAccessMask = vk::AccessFlagBits::eTransferWrite; after.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
after.dstAccessMask = vk::AccessFlagBits::eHostRead; after.dstAccessMask = vk::AccessFlagBits::eHostRead;
after.buffer = m_download_buffer.Handle(); after.buffer = download.Handle();
after.offset = offset; after.offset = offset;
after.size = total_size; after.size = total_size;
native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
@@ -166,8 +171,8 @@ bool BufferCache::DownloadBufferMemory(Buffer& buffer, uint64_t vaddr, uint64_t
vk::PipelineStageFlagBits::eHost, vk::PipelineStageFlagBits::eHost,
{}, 0, nullptr, 1, &after, 0, nullptr); {}, 0, nullptr, 1, &after, 0, nullptr);
m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address, m_scheduler.DeferPriorityOperation([this, mapped, offset, total_size, buffer_address,
copies = std::move(copies)] { copies = std::move(copies), owner = std::move(temporary)] {
m_download_buffer.Invalidate(offset, total_size); (owner ? *owner : m_download_buffer).Invalidate(offset, total_size);
for (const auto& copy: copies) { for (const auto& copy: copies) {
Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset, Libs::LibKernel::Memory::WriteBacking(buffer_address + copy.srcOffset,
mapped + (copy.dstOffset - offset), copy.size); mapped + (copy.dstOffset - offset), copy.size);
+49 -47
View File
@@ -4261,54 +4261,56 @@ public:
partial_unmap_survivor_value, partial_unmap_survivor_value,
"partial invalidation lost disjoint native bytes"); "partial invalidation lost disjoint native bytes");
constexpr uint64_t large_offset = 0x10000; for (const uint64_t large_size : {64ull << 20, 68ull << 20}) {
constexpr uint64_t large_size = 64ull << 20; constexpr uint64_t large_offset = 0x10000;
constexpr uint32_t large_value = 0x5aa55aa5u; const uint32_t large_value =
constexpr uint32_t large_stale = 0x12345678u; large_size == (64ull << 20) ? 0x5aa55aa5u : 0xa55aa55au;
std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale)); constexpr uint32_t large_stale = 0x12345678u;
std::memcpy(memory + large_offset + large_size - sizeof(large_stale), std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale));
&large_stale, sizeof(large_stale)); std::memcpy(memory + large_offset + large_size - sizeof(large_stale),
auto large_allocation = &large_stale, sizeof(large_stale));
cache.ObtainBuffer(base + large_offset, large_size, true, false); auto large_allocation =
Require(name, "near-capacity dirty allocation", cache.ObtainBuffer(base + large_offset, large_size, true, false);
large_allocation.first != nullptr, Require(name, "large dirty allocation",
"failed to allocate the near-capacity dirty native buffer"); large_allocation.first != nullptr,
cache.FillBuffer(base + large_offset, large_size, large_value, false); "failed to allocate the large dirty native buffer");
const auto large_submission_tick = scheduler.CurrentTick(); cache.FillBuffer(base + large_offset, large_size, large_value, false);
for (uint32_t tick = 0; tick <= 160; tick++) { const auto large_submission_tick = scheduler.CurrentTick();
cache.RunGarbageCollector(); for (uint32_t tick = 0; tick <= 160; tick++) {
cache.RunGarbageCollector();
}
// The capacity-sized case can wrap the ring; the oversized case must
// keep its separate staging allocation alive through publication.
Require(name, "large synchronized retirement",
!cache.IsRegionRegistered(base + large_offset, large_size) &&
scheduler.CurrentTick() > large_submission_tick &&
scheduler.CurrentTick() <= large_submission_tick + 2,
"large dirty Buffer exceeded the GC synchronization budget");
uint32_t large_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(base + large_offset,
&large_before_completion,
sizeof(large_before_completion));
Require(name, "large backing publication",
large_before_completion == large_value,
"large Buffer GC retired ownership before publication");
const auto large_image_source =
cache.ObtainBufferForImage(base + large_offset, sizeof(large_value));
Require(name, "fixed download after Buffer retirement",
large_image_source.first != nullptr &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
fixed_download->Handle() == fixed_download_handle &&
fixed_download->Size() == (64ull << 20),
"image acquisition replaced the shared Buffer download stream");
std::vector<uint32_t> large_published(large_size / sizeof(uint32_t));
Require(name, "large Buffer publication contents",
Libs::LibKernel::Memory::TryReadBacking(
base + large_offset, large_published.data(), large_size) &&
std::ranges::all_of(large_published, [large_value](uint32_t value) {
return value == large_value;
}),
"large Buffer GC did not publish its complete transfer");
} }
// Earlier dirty owners in this GC pass may require one ring-wrap drain
// before the full-capacity download can reserve the stream.
Require(name, "near-capacity synchronized retirement",
!cache.IsRegionRegistered(base + large_offset, large_size) &&
scheduler.CurrentTick() > large_submission_tick &&
scheduler.CurrentTick() <= large_submission_tick + 2,
"capacity-sized dirty Buffer exceeded one ring-wrap drain");
uint32_t large_before_completion = 0;
Libs::LibKernel::Memory::TryReadBacking(base + large_offset,
&large_before_completion,
sizeof(large_before_completion));
Require(name, "near-capacity backing publication",
large_before_completion == large_value,
"near-capacity Buffer GC retired ownership before publication");
const auto large_image_source =
cache.ObtainBufferForImage(base + large_offset, sizeof(large_value));
Require(name, "fixed download after Buffer retirement",
large_image_source.first != nullptr &&
&BufferCacheTestAccess::DownloadBuffer(cache) ==
fixed_download &&
fixed_download->Handle() == fixed_download_handle &&
fixed_download->Size() == (64ull << 20),
"image acquisition replaced the shared Buffer download stream");
std::vector<uint32_t> large_published(large_size / sizeof(uint32_t));
Require(name, "near-capacity Buffer publication contents",
Libs::LibKernel::Memory::TryReadBacking(
base + large_offset, large_published.data(), large_size) &&
std::ranges::all_of(large_published, [](uint32_t value) {
return value == large_value;
}),
"near-capacity Buffer GC did not publish its complete transfer");
constexpr uint64_t grouped_first_offset = 0x10000; constexpr uint64_t grouped_first_offset = 0x10000;
constexpr uint64_t grouped_second_offset = 0x2300000; constexpr uint64_t grouped_second_offset = 0x2300000;