renderer: grow repeatedly joined buffer ranges (perf)

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -311,38 +311,81 @@ BufferId BufferCache::FindBuffer(uint64_t vaddr, uint64_t size) {
 	return CreateBuffer(vaddr, size);
 }
 
-BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
+BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t size) {
-	auto& command = m_scheduler.Current();
+	static constexpr int      StreamLeapThreshold = 16;
-	EXIT_IF(command.IsInvalid());
+	static constexpr uint64_t StreamLeapSize      = CACHING_PAGESIZE * 128;
-	auto       begin = vaddr & ~(CACHING_PAGESIZE - 1);
+
-	auto       end   = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
+	auto       begin      = vaddr;
-	auto       first = m_buffers.lower_bound(begin);
+	auto       end        = vaddr + size;
+	const auto find_first = [&](uint64_t address) {
+		auto first = m_buffers.lower_bound(address);
 		if (first != m_buffers.begin()) {
 			const auto  previous = std::prev(first);
-		if (const auto& buffer = m_slot_buffers[previous->second];
+			const auto& buffer   = m_slot_buffers[previous->second];
-		    buffer.CpuAddress() + buffer.Size() > begin) {
+			if (buffer.CpuAddress() + buffer.Size() > address) {
 				first = previous;
 			}
 		}
+		return first;
+	};
+	auto first           = find_first(begin);
 	auto last            = first;
+	int  stream_score    = 0;
+	bool has_stream_leap = false;
 	for (; last != m_buffers.end() && last->first < end; ++last) {
 		const auto& buffer        = m_slot_buffers[last->second];
-		begin              = std::min(begin, buffer.CpuAddress());
+		const auto  buffer_begin  = buffer.CpuAddress();
-		end                = std::max(end, buffer.CpuAddress() + buffer.Size());
+		const auto  buffer_end    = buffer_begin + buffer.Size();
+		const bool  expands_left  = buffer_begin < begin;
+		const bool  expands_right = buffer_end > end;
+		begin                     = std::min(begin, buffer_begin);
+		end                       = std::max(end, buffer_end);
+		if (!has_stream_leap && (stream_score += buffer.StreamScore()) > StreamLeapThreshold) {
+			has_stream_leap = true;
+			if (expands_right) {
+				end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
 			}
+			if (expands_left) {
+				const auto minimum = CACHING_PAGESIZE * 2;
+				if (begin > minimum) {
+					begin -= std::min(StreamLeapSize, begin - minimum);
+				}
+				first = find_first(begin);
+				begin = std::min(begin, first->first);
+			}
+		}
+	}
+	return {first, last, begin, end, has_stream_leap};
+}
+
+void BufferCache::JoinOverlap(BufferId new_id, BufferId overlap_id, bool accumulate_stream_score) {
+	auto& new_buffer = m_slot_buffers[new_id];
+	auto& overlap    = m_slot_buffers[overlap_id];
+	if (accumulate_stream_score) {
+		new_buffer.IncreaseStreamScore(overlap.StreamScore() + 1);
+	}
+	new_buffer.CopyFrom(m_scheduler.Current(), overlap, 0,
+	                    overlap.CpuAddress() - new_buffer.CpuAddress(), overlap.Size());
+	DeleteBuffer(overlap_id);
+}
+
+BufferId BufferCache::CreateBuffer(uint64_t vaddr, uint64_t size) {
+	EXIT_IF(m_scheduler.Current().IsInvalid());
+	const auto end = (vaddr + size + CACHING_PAGESIZE - 1) & ~(CACHING_PAGESIZE - 1);
+	vaddr &= ~(CACHING_PAGESIZE - 1);
+	size               = end - vaddr;
+	const auto overlap = ResolveOverlaps(vaddr, size);
 
 	const auto id = m_slot_buffers.insert(
-	    m_graphics, m_scheduler, MemoryUsage::DeviceLocal, begin,
+	    m_graphics, m_scheduler, MemoryUsage::DeviceLocal, overlap.begin,
-	    AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, end - begin);
+	    AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, overlap.end - overlap.begin);
-	auto&      buffer = m_slot_buffers[id];
+	const auto& buffer = m_slot_buffers[id];
 	SetVulkanObjectNameF(m_graphics.device, buffer.Handle(),
-	                     "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", begin, end - begin);
+	                     "Kyty.GameBuffer[guest=0x{:016x} size=0x{:x}]", overlap.begin,
-	for (auto overlap = first; overlap != last;) {
+	                     overlap.end - overlap.begin);
-		const auto current = overlap++;
+	for (auto it = overlap.first; it != overlap.last;) {
-		const auto old_id  = current->second;
+		const auto old_id = (it++)->second;
-		const auto& old    = m_slot_buffers[old_id];
+		JoinOverlap(id, old_id, !overlap.has_stream_leap);
-		buffer.CopyFrom(command, old, 0, old.CpuAddress() - begin, old.Size());
-		DeleteBuffer(old_id);
 	}
 	Register(id);
 	return id;

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -74,6 +74,15 @@ public:
 private:
 	friend struct BufferCacheTestAccess;
 
+	using BufferMap = std::map<uint64_t, BufferId>;
+	struct OverlapResult {
+		BufferMap::iterator first;
+		BufferMap::iterator last;
+		uint64_t            begin;
+		uint64_t            end;
+		bool                has_stream_leap;
+	};
+
 	struct DownloadCopy;
 	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
 	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
@@ -84,6 +93,8 @@ private:
 	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
 	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
 	void TouchBuffer(const Buffer& buffer);
+	[[nodiscard]] OverlapResult ResolveOverlaps(uint64_t vaddr, uint64_t size);
+	void JoinOverlap(BufferId new_id, BufferId overlap_id, bool accumulate_stream_score);
 	[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
 	void                   Register(BufferId id);
 	void Unregister(BufferId id);
@@ -106,7 +117,7 @@ private:
 	Buffer                                            m_bda_pagetable_buffer;
 	Common::SlotVector<Buffer>                        m_slot_buffers;
 	Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
-	std::map<uint64_t, BufferId>                      m_buffers;
+	BufferMap                                         m_buffers;
 	PageTable                                         m_page_table;
 	RangeSet                                          m_gpu_modified_ranges;
 	MemoryTracker                                     m_memory_tracker;

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -53,6 +53,8 @@ public:
 		return address - m_cpu_address;
 	}
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
+	void               IncreaseStreamScore(int score) noexcept { stream_score += score; }
+	[[nodiscard]] int  StreamScore() const noexcept { return stream_score; }
 	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
 	void               Invalidate(uint64_t offset, uint64_t size);
@@ -69,6 +71,7 @@ public:
 
 	// BufferCache state lives directly on the resource.
 	bool   is_deleted   = false;
+	int    stream_score = 0;
 	size_t lru_id       = 0;
 
 protected:

tests/ShaderRecompilerComputeTests.cpp

@@ -3310,6 +3310,35 @@ public:
                   resolved_right == resolved_left &&
                   resolved_left_offset != resolved_right_offset,
               "saved descriptor IDs did not resolve through the merged owner");
+
+      constexpr uint64_t stream_begin = base + 0x1800000;
+      constexpr uint64_t stream_page = BufferCache::CACHING_PAGESIZE;
+      constexpr uint64_t stream_leap = stream_page * 128;
+      const auto caught =
+          cache.FindBuffer(stream_begin - stream_leap / 2, stream_page);
+      const auto anchor = cache.FindBuffer(stream_begin, stream_page * 2);
+      cache.GetBuffer(anchor).IncreaseStreamScore(16);
+      const auto scored =
+          cache.FindBuffer(stream_begin + stream_page * 2 - 4, 8);
+      Require(name, "stream growth threshold",
+              cache.GetBuffer(scored).CpuAddress() == stream_begin &&
+                  cache.GetBuffer(scored).Size() == stream_page * 3 &&
+                  cache.GetBuffer(scored).StreamScore() == 17,
+              "buffer range grew before the shadPS4 stream threshold");
+      const auto grown =
+          cache.FindBuffer(stream_begin + stream_page * 3 - 4, 8);
+      const auto &grown_buffer = cache.GetBuffer(grown);
+      Require(name, "stream range leap",
+              grown_buffer.CpuAddress() == stream_begin - stream_leap &&
+                  grown_buffer.Size() == stream_leap + stream_page * 4 &&
+                  grown_buffer.StreamScore() == 0 &&
+                  cache.GetBuffer(caught).is_deleted &&
+                  cache.GetBuffer(scored).is_deleted &&
+                  BufferCacheTestAccess::PageOwner(
+                      cache, stream_begin - stream_leap / 2) == grown,
+              "buffer stream leap changed its direction, size, reset, or "
+              "overlap capture");
+
       resources.UnmapMemory(index_begin, index_span);
       resources.UnmapMemory(base + recycled_offset, index_page);
       Require(name, "invalidation retains owners",