gpu: decode RELEASE_MEM directly from the driver's native packet

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -83,16 +83,10 @@ public:
 	void DrawIndirectMulti(uint32_t data_offset, uint32_t max_count_or_count,
 	                       const volatile uint32_t* count_addr, uint32_t stride_in_bytes,
 	                       IndirectDrawRegisters registers, uint32_t draw_initiator, bool indexed);
-	void WriteAtEndOfPipe32(uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index,
+	void ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector, void* address, uint64_t data,
-	                        uint32_t event_write_source, void* dst_gpu_addr, uint32_t value,
+	                uint32_t interrupt_context_id);
-	                        uint32_t interrupt_selector, uint32_t interrupt_context_id);
-	void WriteAtEndOfPipe64(uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index,
-	                        uint32_t event_write_source, void* dst_gpu_addr, uint64_t value,
-	                        uint32_t interrupt_selector, uint32_t interrupt_context_id);
 	void Flip(int handle, int index, int flip_mode, int64_t flip_arg);
 	void PrepareCpuFlip(uint64_t request_id);
-	void SynchronizeGpu();
-	void TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id);
 	void DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y, uint32_t thread_group_z,
 	                    uint32_t mode);
 	void DispatchIndirect(uint64_t args_addr, uint32_t mode);
@@ -121,10 +115,6 @@ public:
 	[[nodiscard]] bool     IsAsyncComputeQueue() const { return m_interrupt_event_id >= 0x20; }
 
 private:
-	template <typename T>
-	void WriteAtEndOfPipe(uint32_t eop_event_type, uint32_t cache_action, uint32_t event_index,
-	                      uint32_t event_write_source, void* dst_gpu_addr, T value,
-	                      uint32_t interrupt_selector, uint32_t interrupt_context_id);
 	void ProcessPm4(Pm4Execution& execution);
 	void SuspendPm4();
 	uint32_t            NumInstances();

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -201,6 +201,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_EVENT_WRITE]               = CpOpEventWrite;
 	g_cp_op_func[Pm4::IT_DMA_DATA]                  = CpOpDmaData;
 	g_cp_op_func[Pm4::IT_ACQUIRE_MEM]               = CpOpAcquireMem;
+	g_cp_op_func[Pm4::IT_RELEASE_MEM]               = CpOpReleaseMem;
 	g_cp_op_func[Pm4::IT_REWIND]                    = CpOpRewind;
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG]           = CpOpSetContextReg;
 	g_cp_op_func[Pm4::IT_SET_SH_REG]                = CpOpSetShaderReg;
@@ -223,7 +224,6 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	g_cp_op_custom_func[Pm4::R_PUSH_MARKER]    = CpOpPushMarker;
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
-	g_cp_op_custom_func[Pm4::R_RELEASE_MEM]    = CpOpReleaseMem;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;
 
 	return g_cp_op_custom_func;

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -135,6 +135,7 @@ uint32_t CpOpCopyData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, ui
 uint32_t CpOpEventWrite(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDmaData(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpAcquireMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpReleaseMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetContextReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetShaderReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpSetUconfigReg(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
@@ -149,7 +150,6 @@ uint32_t CpOpPushMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t,
 uint32_t CpOpPopMarker(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpWaitRegMem64(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpFlip(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
-uint32_t CpOpReleaseMem(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 
 } // namespace Libs::Graphics
 

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -54,34 +54,12 @@ namespace Libs::Graphics {
 
 namespace {
 
-constexpr uint32_t GcrGl2MetadataInvalidate = 1u << 1u;
-constexpr uint32_t GcrGl0VectorInvalidate   = 1u << 2u;
-constexpr uint32_t GcrGl1Invalidate         = 1u << 3u;
-constexpr uint32_t GcrGl2Unshared           = 1u << 4u;
-constexpr uint32_t GcrGl2Invalidate         = 1u << 8u;
-constexpr uint32_t GcrGl2Writeback          = 1u << 9u;
-constexpr uint32_t GcrOrder012              = 1u << 10u;
-constexpr uint32_t GcrOrder210              = 2u << 10u;
-constexpr uint32_t GcrKnownMask             = GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate |
-                                              GcrGl1Invalidate | GcrGl2Unshared | GcrGl2Invalidate |
-                                              GcrGl2Writeback | GcrOrder012 | GcrOrder210;
 constexpr uint32_t RegisterSelectorMask = 0x70000000u;
 
 constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
 	return raw_offset & ~RegisterSelectorMask;
 }
 
-uint32_t ReleaseMemCacheActionFromGcr(uint32_t gcr_cntl) {
-	return ((gcr_cntl & GcrGl2Writeback) != 0 ? 0x38u : 0x00u);
-}
-
-void LogUnknownReleaseMemGcr(uint32_t gcr_cntl) {
-	const uint32_t unknown = (gcr_cntl & ~GcrKnownMask);
-	if (unknown != 0) {
-		LOGF("\t warning: release_mem uses unknown GCR bits: 0x%04" PRIx32 "\n", unknown);
-	}
-}
-
 } // namespace
 
 static HW::BlendControl DecodeBlendControl(uint32_t value) {
@@ -2068,105 +2046,14 @@ KYTY_CP_OP_PARSER(CpOpPushMarker) {
 KYTY_CP_OP_PARSER(CpOpReleaseMem) {
 	KYTY_PROFILER_FUNCTION();
 
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0061060);
+	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0064900);
 
-	uint32_t eop_event_type     = buffer[0] & 0x3fu;
+	// The event type, cache actions and destination cache only order GPU work, which the renderer's
-	uint32_t event_index        = (buffer[0] >> 8u) & 0x7u;
+	// resource barriers already order.
-	uint32_t gcr_cntl           = (buffer[0] >> 12u) & 0xfffu;
+	cp.ReleaseMem(
-	uint32_t release_dst        = (buffer[1] >> 16u) & 0x3u;
+	    (buffer[1] >> 29u) & 0x7u, (buffer[1] >> 24u) & 0x7u,
-	uint32_t data_sel           = (buffer[1] >> 29u) & 0x7u;
+	    reinterpret_cast<void*>((buffer[2] & ~0x3u) | (static_cast<uint64_t>(buffer[3]) << 32u)),
-	uint32_t interrupt_selector = (buffer[1] >> 24u) & 0x7u;
+	    buffer[4] | (static_cast<uint64_t>(buffer[5]) << 32u), buffer[6] & 0x07ffffffu);
-	auto*    dst_gpu_addr =
-	    reinterpret_cast<void*>(buffer[2] | (static_cast<uint64_t>(buffer[3]) << 32u));
-	uint64_t value                = buffer[4] | (static_cast<uint64_t>(buffer[5]) << 32u);
-	uint32_t interrupt_context_id = buffer[6] & 0x07ffffffu;
-
-	constexpr uint32_t ReleaseMemDstMemory = 0;
-	constexpr uint32_t ReleaseMemDstTcL2   = 1;
-	EXIT_NOT_IMPLEMENTED(release_dst != ReleaseMemDstMemory && release_dst != ReleaseMemDstTcL2);
-	EXIT_NOT_IMPLEMENTED(data_sel != 0 && data_sel != 1 && data_sel != 2 && data_sel != 3 &&
-	                     data_sel != 5);
-
-	LogUnknownReleaseMemGcr(gcr_cntl);
-
-	const bool gl2_writeback = ((gcr_cntl & GcrGl2Writeback) != 0);
-
-	auto trigger_interrupt = [&]() {
-		switch (interrupt_selector) {
-			case 0x00:
-			case 0x03: break;
-			case 0x01:
-			case 0x02:
-			case 0x04:
-				cp.TriggerEopEventAtEndOfPipe(interrupt_context_id);
-				cp.BufferFlush();
-				break;
-			default: EXIT("unknown release_mem interrupt selector\n");
-		}
-	};
-
-	// Cache actions only order GPU work, which the renderer's resource barriers already order.
-	if (data_sel == 0 || interrupt_selector == 4 ||
-	    (release_dst == ReleaseMemDstMemory && dst_gpu_addr == nullptr)) {
-		trigger_interrupt();
-
-		return 7;
-	}
-
-	auto cache_action = ReleaseMemCacheActionFromGcr(gcr_cntl);
-
-	if (data_sel == 1) {
-		eop_event_type    = 0x2f;
-		event_index       = 6;
-		auto event_source = 2u;
-
-		cp.WriteAtEndOfPipe32(eop_event_type, cache_action, event_index, event_source, dst_gpu_addr,
-		                      static_cast<uint32_t>(value), interrupt_selector,
-		                      interrupt_context_id);
-		if (interrupt_selector == 0x01 || interrupt_selector == 0x02) {
-			cp.BufferFlush();
-		}
-
-		return 7;
-	}
-
-	if (data_sel == 5) {
-		if (gl2_writeback) {
-			cache_action = 0;
-		}
-
-		eop_event_type    = 0x2f;
-		event_index       = 6;
-		auto event_source = 1u;
-
-		cp.WriteAtEndOfPipe32(eop_event_type, cache_action, event_index, event_source, dst_gpu_addr,
-		                      static_cast<uint32_t>(value), interrupt_selector,
-		                      interrupt_context_id);
-		if (interrupt_selector == 0x01) {
-			cp.BufferFlush();
-		}
-
-		return 7;
-	}
-
-	const bool keep_native_event_index =
-	    (eop_event_type == 0x04 && cache_action == 0x00) ||
-	    (eop_event_type == 0x2f && event_index == 0x06 && cache_action == 0x38);
-
-	if (data_sel == 2) {
-		if (!keep_native_event_index) {
-			event_index = 0;
-		}
-	} else {
-		EXIT_IF(data_sel != 3);
-		if (!keep_native_event_index) {
-			event_index = 0;
-		}
-		data_sel = 4;
-	}
-
-	cp.WriteAtEndOfPipe64(eop_event_type, cache_action, event_index, data_sel, dst_gpu_addr, value,
-	                      interrupt_selector, interrupt_context_id);
 
 	return 7;
 }

src/graphics/guest_gpu/graphicsRun.cpp

@@ -631,18 +631,6 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution) {
 				     opcode, KYTY_PM4_R(packet_header), packet_dw,
 				     reinterpret_cast<uint64_t>(packet), packet_header);
 			}
-			if (opcode == Pm4::IT_NOP && KYTY_PM4_R(packet_header) == Pm4::R_RELEASE_MEM &&
-			    packet_dw >= 7) {
-				static std::atomic<uint32_t> log_count {0};
-				if (log_count.fetch_add(1) < 128) {
-					const auto dst = packet[3] | (static_cast<uint64_t>(packet[4]) << 32u);
-					const auto val = packet[5] | (static_cast<uint64_t>(packet[6]) << 32u);
-					LOGF("\t predicated skip: R_RELEASE_MEM dst=0x%016" PRIx64
-					     ", value=0x%016" PRIx64 ", action=0x%08" PRIx32
-					     ", gcr/data/int=0x%08" PRIx32 "\n",
-					     dst, val, packet[1], packet[2]);
-				}
-			}
 			cursor.offset_dw += packet_dw;
 			execution.m_made_progress = true;
 			continue;
@@ -948,192 +936,65 @@ void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_
 	}
 }
 
-template <typename T>
+void CommandProcessor::ReleaseMem(uint32_t data_sel, uint32_t interrupt_selector, void* address,
-void CommandProcessor::WriteAtEndOfPipe(uint32_t eop_event_type, uint32_t cache_action,
+                                  uint64_t data, uint32_t interrupt_context_id) {
-                                        uint32_t event_index, uint32_t event_write_source,
-                                        void* dst_gpu_addr, T value, uint32_t interrupt_selector,
-                                        uint32_t interrupt_context_id) {
-	static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
-
-	auto& command = CurrentBuffer();
-
 	if (GraphicsRunDebugDumpEnabled()) {
-		const auto bits      = static_cast<unsigned>(sizeof(T) * 8u);
+		LOGF("CommandProcessor::ReleaseMem()\n"
-		const auto log_width = static_cast<int>(sizeof(T) * 2u);
+		     "\t data_sel           = %" PRIu32 "\n"
+		     "\t interrupt_selector = %" PRIu32 "\n"
+		     "\t address            = 0x%016" PRIx64 "\n"
+		     "\t data               = 0x%016" PRIx64 "\n"
+		     "\t interrupt_context  = 0x%08" PRIx32 "\n",
+		     data_sel, interrupt_selector, reinterpret_cast<uint64_t>(address), data,
+		     interrupt_context_id);
+	}
 
-		LOGF("CommandProcessor::WriteAtEndOfPipe%u()\n"
+	// QueuedInterrupt: kNone, kOnly, kAfterWriteConfirm, kDataAfterWriteConfirm, kSendIntCtx.
-		     "\t eop_event_type      = 0x%08" PRIx32 "\n"
+	EXIT_NOT_IMPLEMENTED(interrupt_selector > 4);
-		     "\t cache_action        = 0x%08" PRIx32 "\n"
+	const bool interrupt = (interrupt_selector != 0 && interrupt_selector != 3);
-		     "\t event_index         = 0x%08" PRIx32 "\n"
+	// kSendIntCtx has no label, and kOnly writes none on graphics queues.
-		     "\t event_write_source  = 0x%08" PRIx32 "\n"
+	if (address == nullptr || interrupt_selector == 4 ||
-		     "\t interrupt_selector  = 0x%08" PRIx32 "\n"
+	    (interrupt_selector == 1 && !IsAsyncComputeQueue())) {
-		     "\t interrupt_context   = 0x%08" PRIx32 "\n"
+		data_sel = 0;
-		     "\t dst_gpu_addr        = 0x%016" PRIx64 "\n"
+	}
-		     "\t value               = 0x%0*" PRIx64 "\n",
+	if (data_sel == 0 && !interrupt) {
-		     bits, eop_event_type, cache_action, event_index, event_write_source,
+		return;
-		     interrupt_selector, interrupt_context_id, reinterpret_cast<uint64_t>(dst_gpu_addr),
-		     log_width, static_cast<uint64_t>(value));
 	}
 
 	// Dumped occlusion counters are written before any end-of-pipe signal reaches the guest.
 	m_renderer.GetOcclusionQueries().Publish();
 
-	bool with_interrupt = false;
+	// EndOfPipeData: kNone, kImmediate32b, kImmediate64b, kRefClk; 5 reads GDS.
-	switch (interrupt_selector) {
+	switch (data_sel) {
-		case 0x00:
+		case 0: break;
-		case 0x03: with_interrupt = false; break;
+		case 1: {
-		case 0x01:
+			const auto value = static_cast<uint32_t>(data);
-			if (!IsAsyncComputeQueue()) {
+			std::memcpy(address, &value, sizeof(value));
-				TriggerEopEventAtEndOfPipe(interrupt_context_id);
-				return;
-			}
-			with_interrupt = true;
 			break;
-		case 0x02: with_interrupt = true; break;
-		default: EXIT("unknown interrupt_selector\n");
 		}
-
+		case 3: data = Sync::ReadReferenceClock(); [[fallthrough]];
-	auto write32 = [&] {
+		case 2: std::memcpy(address, &data, sizeof(data)); break;
-		auto* dst  = static_cast<uint32_t*>(dst_gpu_addr);
+		case 5: {
-		auto  data = static_cast<uint32_t>(value);
+			GetScheduler().Finish();
-		EXIT_IF(dst == nullptr);
+			const auto gds = static_cast<uint32_t>(data);
-		(void)command.Handle();
+			Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(),
-		std::memcpy(dst, &data, sizeof(data));
+			              static_cast<uint32_t*>(address), gds & 0xffffu, gds >> 16u);
-
+			if (interrupt) {
-		if (with_interrupt) {
+				// The GPU is idle, so the interrupt needs no fence.
-			TriggerEopEventAtEndOfPipe(interrupt_context_id);
-		}
-	};
-
-	switch (event_write_source) {
-		case 0x01:
-			if constexpr (sizeof(T) == sizeof(uint32_t)) {
-				if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
-					auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
-					SynchronizeGpu();
-					Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
-					              value >> 16u);
-					if (with_interrupt) {
 				m_renderer.TriggerInterrupt(m_interrupt_event_id, interrupt_context_id);
 			}
 			return;
 		}
-			}
+		default: EXIT("unknown release_mem data selector: %" PRIu32 "\n", data_sel);
-			break;
-		case 0x02:
-		case 0x04:
-			if constexpr (sizeof(T) == sizeof(uint32_t)) {
-				if (event_write_source == 0x02 && eop_event_type == 0x2f && event_index == 0x06) {
-					switch (cache_action) {
-						case 0x00:
-						case 0x38: write32(); return;
-						default: break;
-					}
-				}
-			} else {
-				if (event_write_source == 0x04) {
-					value = Sync::ReadReferenceClock();
-				}
-				auto write64 = [&] {
-					auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
-					EXIT_IF(dst == nullptr);
-					(void)command.Handle();
-					std::memcpy(dst, &value, sizeof(value));
-
-					if (with_interrupt) {
-						TriggerEopEventAtEndOfPipe(interrupt_context_id);
-					}
-				};
-
-				switch (cache_action) {
-					case 0x00:
-						switch (eop_event_type) {
-							case 0x04:
-								if (event_index == 0x05) {
-									write64();
-									return;
-								}
-								break;
-							case 0x14:
-							case 0x28:
-							case 0x2f:
-								if (event_index == 0x00) {
-									write64();
-									return;
-								}
-								break;
-							case 0x2b:
-							case 0x2d:
-							case 0x30:
-								if (event_index == 0x00 && !with_interrupt) {
-									write64();
-									return;
-								}
-								break;
-							default: break;
-						}
-						break;
-					case 0x38:
-						switch (eop_event_type) {
-							case 0x04:
-							case 0x14:
-							case 0x28:
-								if (event_index == 0x00) {
-									write64();
-									return;
-								}
-								break;
-							case 0x2b:
-							case 0x2d:
-								if (event_index == 0x00 && !with_interrupt) {
-									write64();
-									return;
-								}
-								break;
-							case 0x2f:
-								if (event_index == 0x06 && !with_interrupt) {
-									write64();
-									return;
-								}
-								break;
-							default: break;
-						}
-						break;
-					default: break;
-				}
-			}
-			break;
-		default: break;
 	}
 
-	EXIT("unknown event type\n");
+	if (interrupt) {
-}
-
-void CommandProcessor::WriteAtEndOfPipe32(uint32_t eop_event_type, uint32_t cache_action,
-                                          uint32_t event_index, uint32_t event_write_source,
-                                          void* dst_gpu_addr, uint32_t value,
-                                          uint32_t interrupt_selector,
-                                          uint32_t interrupt_context_id) {
-	WriteAtEndOfPipe(eop_event_type, cache_action, event_index, event_write_source, dst_gpu_addr,
-	                 value, interrupt_selector, interrupt_context_id);
-}
-
-void CommandProcessor::WriteAtEndOfPipe64(uint32_t eop_event_type, uint32_t cache_action,
-                                          uint32_t event_index, uint32_t event_write_source,
-                                          void* dst_gpu_addr, uint64_t value,
-                                          uint32_t interrupt_selector,
-                                          uint32_t interrupt_context_id) {
-	WriteAtEndOfPipe(eop_event_type, cache_action, event_index, event_write_source, dst_gpu_addr,
-	                 value, interrupt_selector, interrupt_context_id);
-}
-
-void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) {
-	m_renderer.GetOcclusionQueries().Publish();
-	(void)CurrentBuffer().Handle();
 		GetScheduler().DeferPriorityOperation(
 		    [&renderer = m_renderer, event_id = m_interrupt_event_id, interrupt_context_id] {
 			    renderer.TriggerInterrupt(event_id, interrupt_context_id);
 		    });
+		BufferFlush();
+	}
 }
 
 void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
@@ -1245,10 +1106,6 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 	GetScheduler().Flush();
 }
 
-void CommandProcessor::SynchronizeGpu() {
-	GetScheduler().Finish();
-}
-
 bool GuestGpu::IsGpuThread() noexcept {
 	return g_gpu_thread;
 }

src/graphics/guest_gpu/pm4.cpp

@@ -25,7 +25,6 @@ constexpr auto MakeRegisterNames() {
 	names[R_POP_MARKER]     = "R_POP_MARKER";
 	names[R_WRITE_DATA]     = "R_WRITE_DATA";
 	names[R_FLIP]           = "R_FLIP";
-	names[R_RELEASE_MEM]    = "R_RELEASE_MEM";
 	names[R_DMA_DATA]       = "R_DMA_DATA";
 	names[R_CONTEXT_STATE]  = "R_CONTEXT_STATE";
 	return names;

src/graphics/guest_gpu/pm4.h

@@ -79,7 +79,6 @@ constexpr uint32_t R_PUSH_MARKER    = 0x0B;
 constexpr uint32_t R_POP_MARKER     = 0x0C;
 constexpr uint32_t R_WRITE_DATA     = 0x15;
 constexpr uint32_t R_FLIP           = 0x17;
-constexpr uint32_t R_RELEASE_MEM    = 0x18;
 constexpr uint32_t R_DMA_DATA       = 0x19;
 constexpr uint32_t R_CONTEXT_STATE  = 0x1A;
 

src/libs/agc.cpp

@@ -2027,7 +2027,7 @@ uint32_t* KYTY_SYSV_ABI AgcCbReleaseMem(CommandBuffer* buf, uint8_t action, uint
 	const uint32_t packet_action      = static_cast<uint32_t>(action) & 0x3fu;
 	const uint32_t packet_event_index = (action >= 0x2fu ? 6u : 5u);
 
-	cmd[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
+	cmd[0] = KYTY_PM4(8, Pm4::IT_RELEASE_MEM, 0);
 	cmd[1] = packet_action | (packet_event_index << 8u) | ((packet_gcr_cntl & 0xfffu) << 12u) |
 	         ((static_cast<uint32_t>(cache_policy) & 0x3u) << 25u);
 	cmd[2] = ((static_cast<uint32_t>(dst) & 0x3u) << 16u) |
@@ -3772,24 +3772,21 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchAddress(uint32_t*             cmd,
                                                       const volatile Label* address) {
 	PRINT_NAME();
 
-	// Not sure
-
 	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
 	         "\t address = 0x%016" PRIx64 "\n",
 	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
-	auto vaddr = reinterpret_cast<uint64_t>(address);
+	if (((cmd[0] >> 8u) & 0xffu) != Pm4::IT_RELEASE_MEM) {
-	auto op    = (cmd[0] >> 8u) & 0xffu;
+		return GRAPHICS5_ERROR_INVALID_PACKET;
+	}
 
-	if ((op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM) ||
+	// A kSendIntCtx release has no address.
-	    op == Pm4::IT_RELEASE_MEM) {
+	if (((cmd[2] >> 24u) & 0x7u) != 4u) {
-		cmd[3] = static_cast<uint32_t>(vaddr & 0xffffffffu);
+		const auto vaddr = reinterpret_cast<uint64_t>(address);
-		cmd[4] = static_cast<uint32_t>((vaddr >> 32u) & 0xffffffffu);
+		cmd[3]           = (cmd[3] & 0x3u) | (static_cast<uint32_t>(vaddr) & ~0x3u);
-	} else {
+		cmd[4]           = static_cast<uint32_t>(vaddr >> 32u);
-		EXIT("unsupported queueEndOfPipeAction packet for address patch: 0x%08" PRIx32 "\n",
-		     cmd[0]);
 	}
 
 	return OK;
@@ -3804,10 +3801,7 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchData(uint32_t* cmd, uint64_t data)
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
-	const auto op             = (cmd[0] >> 8u) & 0xffu;
+	if (((cmd[0] >> 8u) & 0xffu) != Pm4::IT_RELEASE_MEM) {
-	const bool is_release_mem = op == Pm4::IT_RELEASE_MEM ||
-	                            (op == Pm4::IT_NOP && KYTY_PM4_R(cmd[0]) == Pm4::R_RELEASE_MEM);
-	if (!is_release_mem) {
 		return GRAPHICS5_ERROR_INVALID_PACKET;
 	}
 

tests/ShaderRecompilerComputeTests.cpp

@@ -2943,7 +2943,7 @@ public:
             std::array<uint32_t, 8> packet{};
             const auto address = reinterpret_cast<uint64_t>(destination);
             const auto event_index = (action >= 0x2fu ? 6u : 5u);
-            packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
+            packet[0] = KYTY_PM4(8, Pm4::IT_RELEASE_MEM, 0);
             packet[1] = action | (event_index << 8u) | (gcr_cntl << 12u);
             packet[2] = (interrupt_selector << 24u) | (data_sel << 29u);
             packet[3] = static_cast<uint32_t>(address);
@@ -3075,16 +3075,34 @@ public:
       gpu.SendCommandSync([&] {
         processor->BufferInit();
 
+        // Every end-of-pipe and end-of-shader action, with or without GL2
+        // writeback, writes each label size and splits for its interrupt.
+        for (const uint32_t action : {0x04u, 0x14u, 0x28u, 0x2bu, 0x2du, 0x2fu, 0x30u}) {
+          for (const uint32_t gcr : {0u, 1u << 9u}) {
+            for (const uint32_t data_sel : {1u, 2u, 3u}) {
+              release_label = 0;
               const auto clock_before = Sync::ReadReferenceClock();
-        auto clock_release =
+              auto release = make_release_mem(data_sel, 2, &release_label,
-            make_release_mem(3, 2, &release_label, UINT64_MAX, 0x04u);
+                                              0x1122334455667788ull, action, gcr);
-        Pm4Execution clock_execution;
+              release[3] |= 1u; // ADDRESS_LO starts at bit 2.
-        Require("GpuCommandLane", "release reference clock with interrupt",
+              Pm4Execution release_execution;
-                processor->Process(clock_execution, clock_release) ==
+              const auto release_tick = gpu_scheduler.CurrentTick();
-                        Pm4ProcessResult::Complete &&
+              const auto release_result =
-                    release_label >= clock_before &&
+                  processor->Process(release_execution, release);
-                    release_label <= Sync::ReadReferenceClock(),
+              bool label_written =
-                "clock write with writeback and interrupt lost its data");
+                  release_label == (data_sel == 1 ? 0x55667788ull : 0x1122334455667788ull);
+              if (data_sel == 3) {
+                label_written = release_label >= clock_before &&
+                                release_label <= Sync::ReadReferenceClock();
+              }
+              Require("GpuCommandLane", "release with write-confirm interrupt",
+                      release_result == Pm4ProcessResult::Complete &&
+                          label_written &&
+                          gpu_scheduler.CurrentTick() == release_tick + 1,
+                      "an end-of-pipe action lost its label or interrupt split");
+            }
+          }
+        }
 
         for (const auto interrupt : {0u, 3u, 1u, 2u}) {
           release_label = 0;
@@ -3213,7 +3231,7 @@ public:
            void *destination, uint64_t value, uint32_t context_id) {
           std::array<uint32_t, 8> packet{};
           const auto address = reinterpret_cast<uint64_t>(destination);
-          packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM);
+          packet[0] = KYTY_PM4(8, Pm4::IT_RELEASE_MEM, 0);
           packet[1] = 0x28u | (5u << 8u);
           packet[2] = (interrupt_selector << 24u) | (data_selector << 29u);
           packet[3] = static_cast<uint32_t>(address);
@@ -49632,7 +49650,7 @@ void CheckPm4IndirectControlFlow(RenderContext &renderer) {
               disabled_commands[5] == KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO) &&
               disabled_commands[6] == 0x68750123u &&
               release[0] == KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_ZERO) &&
-              unpatched_commands[7] == KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM) &&
+              unpatched_commands[7] == 0xc0064900u &&
               Gen5::AgcGetPacketSize(disabled_commands.data()) == disabled_write.size() &&
               Gen5::AgcGetPacketSize(disabled_commands.data() + 5) == 2 &&
               Gen5::AgcGetPacketSize(release) == 8 &&
@@ -49647,6 +49665,27 @@ void CheckPm4IndirectControlFlow(RenderContext &renderer) {
           processor.Process(disabled_execution, disabled_commands) == Pm4ProcessResult::Complete &&
               returned == 0 && released == 0 && selected == 88,
           "a disabled packet still executed or hid the packet after it");
+  // Like the driver, the end-of-pipe patch helpers take only a native
+  // RELEASE_MEM, keep the address alignment bits and skip kSendIntCtx.
+  std::array<uint32_t, 8> patched{};
+  std::copy_n(unpatched_commands.begin() + 7, patched.size(), patched.begin());
+  patched[3] |= 1u;
+  const auto* label = reinterpret_cast<const volatile Gen5::Label*>(0x123456789abcdef7ull);
+  bool patch_helpers_match =
+      Gen5::AgcQueueEndOfPipeActionPatchAddress(patched.data(), label) == 0 &&
+      patched[3] == 0x9abcdef5u && patched[4] == 0x12345678u &&
+      Gen5::AgcQueueEndOfPipeActionPatchData(patched.data(), 0x1122334455667788ull) == 0 &&
+      patched[5] == 0x55667788u && patched[6] == 0x11223344u;
+  patched[2] |= 4u << 24u;
+  patch_helpers_match &= Gen5::AgcQueueEndOfPipeActionPatchAddress(patched.data(), nullptr) == 0 &&
+                         patched[3] == 0x9abcdef5u && patched[4] == 0x12345678u;
+  patched[0] = KYTY_PM4(8, Pm4::IT_NOP, 0x18);
+  patch_helpers_match &=
+      Gen5::AgcQueueEndOfPipeActionPatchAddress(patched.data(), label) ==
+          static_cast<int>(0x8a6c000cu) &&
+      Gen5::AgcQueueEndOfPipeActionPatchData(patched.data(), 0) == static_cast<int>(0x8a6c000cu);
+  Require(name, "end-of-pipe patch helpers", patch_helpers_match,
+          "RELEASE_MEM or its patch helpers diverged from the driver");
   const auto then_commands = write(&selected, 11);
   const auto else_commands = write(&selected, 33);
   const auto branch_suffix = write(&selected, 44);