render: simplify buffer and memory tracking ownership

This commit is contained in:
nmzik
2026-09-06 03:46:34 +04:00
parent ccd19b3e56
commit 4c769dfbf9
13 changed files with 50 additions and 179 deletions
-7
View File
@@ -93,11 +93,8 @@ struct GraphicContext {
[[nodiscard]] bool CanReportMemoryUsage() const noexcept { return memory_budget_ext_enabled; }
[[nodiscard]] uint64_t GetDeviceMemoryUsage() const;
[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
void CreateBuffer(uint64_t size, VulkanBuffer& buffer);
[[nodiscard]] bool CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
void DeleteImage(VulkanImage& image);
void MapMemory(VulkanMemory& memory, void*& data);
void UnmapMemory(VulkanMemory& memory);
void AppendHardwareRayTracingDeviceExtensions(
const std::vector<vk::ExtensionProperties>& available_extensions,
std::vector<const char*>& device_extensions);
@@ -120,12 +117,8 @@ struct VulkanMemory {
vk::MemoryRequirements requirements = {};
vk::MemoryPropertyFlags property = {};
vk::MemoryPropertyFlags preferred_property = {};
vk::DeviceMemory memory = nullptr;
VmaAllocation allocation = nullptr;
VmaAllocationInfo allocation_info = {};
vk::DeviceSize offset = 0;
uint32_t type = 0;
uint64_t unique_id = 0;
};
struct VulkanImageState {
+4 -14
View File
@@ -23,10 +23,7 @@ void MemoryTracker::ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr,
EXIT("MemoryTracker: invalid dirty-page validation range\n");
}
for (auto page = vaddr; page < vaddr + size; page += TRACKER_PAGE_SIZE) {
bool found = false;
dirty.ForEachIntersection(page, TRACKER_PAGE_SIZE,
[&found](RangeSet::Range) { found = true; });
if (!found) {
if (!dirty.Intersects(page, TRACKER_PAGE_SIZE)) {
EXIT("MemoryTracker: GPU-dirty tracker page has no dirty bytes, operation=%s "
"addr=0x%016" PRIx64 "\n",
operation, page);
@@ -40,9 +37,7 @@ void MemoryTracker::ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t va
const auto begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
const auto end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
bool has_dirty_bytes = false;
dirty.ForEachIntersection(page, TRACKER_PAGE_SIZE,
[&has_dirty_bytes](RangeSet::Range) { has_dirty_bytes = true; });
const bool has_dirty_bytes = dirty.Intersects(page, TRACKER_PAGE_SIZE);
if (IsRegionGpuModified(page, TRACKER_PAGE_SIZE) != has_dirty_bytes) {
EXIT("MemoryTracker: tracker and byte ownership disagree, operation=%s "
"addr=0x%016" PRIx64 "\n",
@@ -113,7 +108,8 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
});
}
void MemoryTracker::UntrackMemoryImpl(uint64_t vaddr, uint64_t size) {
void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::vector<RegionManager*> managers;
managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
TRACKER_REGION_SIZE);
@@ -134,12 +130,6 @@ void MemoryTracker::UntrackMemoryImpl(uint64_t vaddr, uint64_t size) {
Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
});
locks.clear();
}
void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
UntrackMemoryImpl(vaddr, size);
}
} // namespace Libs::Graphics
-2
View File
@@ -34,7 +34,6 @@ public:
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) noexcept {
static_assert(std::is_invocable_v<Flush&>);
CheckNotInUploadCallback();
ValidateRange(vaddr, size);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const bool should_flush = [&] {
@@ -171,7 +170,6 @@ private:
}
static void ValidateRange(uint64_t vaddr, uint64_t size);
void UntrackMemoryImpl(uint64_t vaddr, uint64_t size);
RegionManager* GetOrCreateRegion(uint64_t index);
std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
-4
View File
@@ -371,8 +371,4 @@ template void PageManager::UpdatePageWatchersForRegion<true, false>(uint64_t, Re
template void PageManager::UpdatePageWatchersForRegion<false, true>(uint64_t, RegionBits&);
template void PageManager::UpdatePageWatchersForRegion<false, false>(uint64_t, RegionBits&);
void PageManager::OnGpuMap(uint64_t, uint64_t) {}
void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
} // namespace Libs::Graphics
-2
View File
@@ -24,8 +24,6 @@ public:
void UpdatePageWatchers(uint64_t vaddr, uint64_t size);
template <bool track, bool is_read = false>
void UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask);
void OnGpuMap(uint64_t vaddr, uint64_t size);
void OnGpuUnmap(uint64_t vaddr, uint64_t size);
private:
struct Impl;
@@ -47,7 +47,6 @@ void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Add(vaddr, size);
}
m_page_manager.OnGpuMap(vaddr, size);
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
@@ -64,7 +63,6 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
}
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Subtract(vaddr, size);
};
+24 -29
View File
@@ -64,7 +64,7 @@ constexpr size_t WATCHES_RESERVE_CHUNK = 0x1000;
Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
uint64_t cpu_address, vk::BufferUsageFlags flags, uint64_t size)
: m_graphics(&graphics), m_scheduler(&scheduler), m_usage(usage), m_cpu_address(cpu_address),
m_size(size), m_buffer(std::make_unique<VulkanBuffer>()) {
m_buffer(std::make_unique<VulkanBuffer>()) {
KYTY_PROFILER_FUNCTION();
EXIT_IF(graphics.allocator == nullptr || size == 0);
@@ -96,11 +96,7 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
m_buffer->buffer = native_buffer;
m_buffer->usage = flags;
m_buffer->buffer_size = size;
m_buffer->memory.allocation_info = allocation_result;
m_buffer->memory.memory = allocation_result.deviceMemory;
m_buffer->memory.offset = allocation_result.offset;
m_buffer->memory.type = allocation_result.memoryType;
m_buffer->memory.unique_id = VulkanNextMemoryUniqueId();
graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
vk::BufferDeviceAddressInfo address_info {};
@@ -112,7 +108,6 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
VkMemoryPropertyFlags properties = 0;
vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
m_buffer->memory.property = vk::MemoryPropertyFlags(properties);
m_is_coherent = (properties & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
if (allocation_result.pMappedData != nullptr) {
m_mapped = {static_cast<uint8_t*>(allocation_result.pMappedData),
static_cast<size_t>(size)};
@@ -131,24 +126,32 @@ vk::Buffer Buffer::Handle() const noexcept {
return m_buffer->buffer;
}
uint64_t Buffer::Size() const noexcept {
return m_buffer->buffer_size;
}
bool Buffer::IsCoherent() const noexcept {
return bool(m_buffer->memory.property & vk::MemoryPropertyFlagBits::eHostCoherent);
}
vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
EXIT_IF(m_device_address == 0);
return m_device_address;
}
bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
return address >= m_cpu_address && size <= Size() && address - m_cpu_address <= Size() - size;
}
void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
EXIT_IF(source == nullptr || m_mapped.empty() || offset > m_size || size > m_size - offset);
EXIT_IF(source == nullptr || m_mapped.empty() || offset > Size() || size > Size() - offset);
std::memcpy(m_mapped.data() + offset, source, static_cast<size_t>(size));
Flush(offset, size);
}
void Buffer::Flush(uint64_t offset, uint64_t size) {
EXIT_IF(m_mapped.empty() || offset > m_size || size > m_size - offset);
if (!m_is_coherent && size != 0) {
EXIT_IF(m_mapped.empty() || offset > Size() || size > Size() - offset);
if (!IsCoherent() && size != 0) {
const auto result =
vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
@@ -156,8 +159,8 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
}
void Buffer::Invalidate(uint64_t offset, uint64_t size) {
EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
if (!m_is_coherent && size != 0) {
EXIT_IF(m_usage != MemoryUsage::Download || offset > Size() || size > Size() - offset);
if (!IsCoherent() && size != 0) {
const auto result =
vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
@@ -166,10 +169,10 @@ void Buffer::Invalidate(uint64_t offset, uint64_t size) {
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
vk::AccessFlags destination) const {
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
if (Handle() == nullptr || size == 0 || offset > Size() || size > Size() - offset) {
EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64 " size=0x%016" PRIx64
" capacity=0x%016" PRIx64 "\n",
static_cast<const void*>(Handle()), offset, size, m_size);
static_cast<const void*>(Handle()), offset, size, Size());
}
vk::BufferMemoryBarrier barrier {};
barrier.srcAccessMask = source;
@@ -186,8 +189,8 @@ void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t sou
uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before,
vk::AccessFlags destination_before, vk::AccessFlags source_after,
vk::AccessFlags destination_after) {
if (size == 0 || source_offset > source.m_size || size > source.m_size - source_offset ||
destination_offset > m_size || size > m_size - destination_offset) {
if (size == 0 || source_offset > source.Size() || size > source.Size() - source_offset ||
destination_offset > Size() || size > Size() - destination_offset) {
EXIT("Buffer: invalid copy range\n");
}
if (source.Handle() == Handle() && source_offset < destination_offset + size &&
@@ -244,10 +247,8 @@ void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) {
StreamBuffer::StreamBuffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
uint64_t size)
: Buffer(graphics, scheduler, usage, 0, AllFlags, size) {
ReserveWatches(m_current_watches, WATCHES_INITIAL_RESERVE);
ReserveWatches(m_previous_watches, WATCHES_INITIAL_RESERVE);
}
: Buffer(graphics, scheduler, usage, 0, AllFlags, size),
m_current_watches(WATCHES_INITIAL_RESERVE), m_previous_watches(WATCHES_INITIAL_RESERVE) {}
bool StreamBuffer::NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size,
uint64_t& alignment) {
@@ -312,10 +313,8 @@ std::pair<uint8_t*, uint64_t> StreamBuffer::Map(uint64_t size, uint64_t alignmen
}
void StreamBuffer::Commit() {
if (!IsCoherent() && Usage() != MemoryUsage::Download && m_mapped_size != 0) {
const auto result = vmaFlushAllocation(
Graphics().allocator, NativeBuffer().memory.allocation, m_offset, m_mapped_size);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
if (Usage() != MemoryUsage::Download && m_mapped_size != 0) {
Flush(m_offset, m_mapped_size);
}
m_offset += m_mapped_size;
@@ -325,7 +324,7 @@ void StreamBuffer::Commit() {
return;
}
if (m_current_watch_cursor + 1 >= m_current_watches.size()) {
ReserveWatches(m_current_watches, WATCHES_RESERVE_CHUNK);
m_current_watches.resize(m_current_watches.size() + WATCHES_RESERVE_CHUNK);
}
auto& watch = m_current_watches[m_current_watch_cursor++];
watch.upper_bound = m_offset;
@@ -341,10 +340,6 @@ uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignmen
return offset;
}
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, size_t grow_size) {
watches.resize(watches.size() + grow_size);
}
bool StreamBuffer::WaitPendingOperations(const std::vector<Watch>& watches,
std::optional<size_t> invalidation_mark,
uint64_t requested_upper_bound, bool allow_wait,
+2 -6
View File
@@ -43,9 +43,9 @@ public:
KYTY_CLASS_NO_COPY(Buffer);
[[nodiscard]] vk::Buffer Handle() const noexcept;
[[nodiscard]] uint64_t Size() const noexcept { return m_size; }
[[nodiscard]] uint64_t Size() const noexcept;
[[nodiscard]] std::span<uint8_t> Mapped() const noexcept { return m_mapped; }
[[nodiscard]] bool IsCoherent() const noexcept { return m_is_coherent; }
[[nodiscard]] bool IsCoherent() const noexcept;
[[nodiscard]] MemoryUsage Usage() const noexcept { return m_usage; }
[[nodiscard]] uint64_t CpuAddress() const noexcept { return m_cpu_address; }
[[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept;
@@ -77,7 +77,6 @@ public:
protected:
[[nodiscard]] GraphicContext& Graphics() const noexcept { return *m_graphics; }
[[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; }
[[nodiscard]] VulkanBuffer& NativeBuffer() noexcept { return *m_buffer; }
private:
[[nodiscard]] vk::BufferMemoryBarrier Barrier(uint64_t offset, uint64_t size,
@@ -88,11 +87,9 @@ private:
CommandScheduler* m_scheduler = nullptr;
MemoryUsage m_usage = MemoryUsage::DeviceLocal;
uint64_t m_cpu_address = 0;
uint64_t m_size = 0;
vk::DeviceAddress m_device_address = 0;
std::unique_ptr<VulkanBuffer> m_buffer;
std::span<uint8_t> m_mapped;
bool m_is_coherent = false;
};
class StreamBuffer final: public Buffer {
@@ -113,7 +110,6 @@ private:
uint64_t upper_bound = 0;
};
void ReserveWatches(std::vector<Watch>& watches, size_t grow_size);
[[nodiscard]] static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size,
uint64_t& alignment);
[[nodiscard]] bool WaitPendingOperations(const std::vector<Watch>& watches,
+7 -74
View File
@@ -35,28 +35,20 @@ struct MemoryStats {
MemoryStats g_memory_stats;
void TrackAllocationImpl(const VulkanMemory& memory) {
} // namespace
void VulkanTrackAllocation(const VulkanMemory& memory) {
g_memory_stats.allocated[memory.type] += memory.requirements.size;
g_memory_stats.count[memory.type]++;
}
void UntrackAllocationImpl(const VulkanMemory& memory) {
void VulkanUntrackAllocation(const VulkanMemory& memory) {
EXIT_IF(g_memory_stats.allocated[memory.type] < memory.requirements.size);
EXIT_IF(g_memory_stats.count[memory.type] == 0);
g_memory_stats.allocated[memory.type] -= memory.requirements.size;
g_memory_stats.count[memory.type]--;
}
} // namespace
void VulkanTrackAllocation(const VulkanMemory& memory) {
TrackAllocationImpl(memory);
}
void VulkanUntrackAllocation(const VulkanMemory& memory) {
UntrackAllocationImpl(memory);
}
bool GraphicContext::CreateAllocator() {
KYTY_PROFILER_FUNCTION();
EXIT_IF(instance == nullptr || physical_device == nullptr || device == nullptr ||
@@ -93,11 +85,6 @@ void GraphicContext::DestroyAllocator() {
allocator = nullptr;
}
uint64_t VulkanNextMemoryUniqueId() {
static std::atomic_uint64_t sequence = 0;
return ++sequence;
}
void GraphicContext::LogMemoryBudget() const {
if (allocator == nullptr || physical_device == nullptr) {
return;
@@ -166,42 +153,6 @@ uint64_t GraphicContext::GetTotalMemoryBudget() const {
return std::max(local, available > system_reserve ? available - system_reserve : uint64_t {0});
}
void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) {
KYTY_PROFILER_FUNCTION();
EXIT_IF(allocator == nullptr || buffer.buffer != nullptr ||
buffer.memory.allocation != nullptr || size == 0);
vk::BufferCreateInfo buffer_info {};
buffer_info.sType = vk::StructureType::eBufferCreateInfo;
buffer_info.size = size;
buffer_info.usage = buffer.usage;
buffer_info.sharingMode = vk::SharingMode::eExclusive;
VmaAllocationCreateInfo alloc_info {};
alloc_info.requiredFlags =
static_cast<vk::MemoryPropertyFlags::MaskType>(buffer.memory.property);
alloc_info.preferredFlags =
static_cast<vk::MemoryPropertyFlags::MaskType>(buffer.memory.preferred_property);
vk::Buffer::CType native_buffer = VK_NULL_HANDLE;
const auto result = static_cast<vk::Result>(vmaCreateBuffer(
allocator, static_cast<const vk::BufferCreateInfo::NativeType*>(buffer_info), &alloc_info,
&native_buffer, &buffer.memory.allocation, &buffer.memory.allocation_info));
buffer.buffer = native_buffer;
if (result != vk::Result::eSuccess) {
LogMemoryBudget();
}
EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
device.getBufferMemoryRequirements(buffer.buffer, &buffer.memory.requirements);
buffer.memory.type = buffer.memory.allocation_info.memoryType;
buffer.memory.memory = buffer.memory.allocation_info.deviceMemory;
buffer.memory.offset = buffer.memory.allocation_info.offset;
buffer.memory.unique_id = VulkanNextMemoryUniqueId();
buffer.buffer_size = size;
VulkanTrackAllocation(buffer.memory);
}
bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanImage& image) {
KYTY_PROFILER_FUNCTION();
EXIT_IF(allocator == nullptr || image.image != nullptr || image.memory.allocation != nullptr);
@@ -212,10 +163,11 @@ bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanIm
alloc_info.preferredFlags =
static_cast<vk::MemoryPropertyFlags::MaskType>(memory.preferred_property);
VmaAllocationInfo allocation_result {};
vk::Image::CType native_image = VK_NULL_HANDLE;
const auto result = static_cast<vk::Result>(
vmaCreateImage(allocator, static_cast<const vk::ImageCreateInfo::NativeType*>(image_info),
&alloc_info, &native_image, &memory.allocation, &memory.allocation_info));
&alloc_info, &native_image, &memory.allocation, &allocation_result));
image.image = native_image;
if (result != vk::Result::eSuccess) {
LogMemoryBudget();
@@ -223,10 +175,7 @@ bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanIm
}
device.getImageMemoryRequirements(image.image, &memory.requirements);
memory.type = memory.allocation_info.memoryType;
memory.memory = memory.allocation_info.deviceMemory;
memory.offset = memory.allocation_info.offset;
memory.unique_id = VulkanNextMemoryUniqueId();
memory.type = allocation_result.memoryType;
VulkanTrackAllocation(memory);
return true;
}
@@ -239,23 +188,7 @@ void GraphicContext::DeleteImage(VulkanImage& image) {
VulkanUntrackAllocation(memory);
vmaDestroyImage(allocator, image.image, memory.allocation);
image.image = nullptr;
memory.memory = nullptr;
memory.allocation = nullptr;
memory.allocation_info = {};
memory.offset = 0;
}
void GraphicContext::MapMemory(VulkanMemory& memory, void*& data) {
KYTY_PROFILER_FUNCTION();
EXIT_IF(allocator == nullptr || memory.allocation == nullptr);
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(vmaMapMemory(allocator, memory.allocation,
&data)) != vk::Result::eSuccess);
}
void GraphicContext::UnmapMemory(VulkanMemory& memory) {
KYTY_PROFILER_FUNCTION();
EXIT_IF(allocator == nullptr || memory.allocation == nullptr);
vmaUnmapMemory(allocator, memory.allocation);
}
} // namespace Libs::Graphics
-1
View File
@@ -6,7 +6,6 @@
namespace Libs::Graphics {
uint64_t VulkanNextMemoryUniqueId();
void VulkanTrackAllocation(const VulkanMemory& memory);
void VulkanUntrackAllocation(const VulkanMemory& memory);