shader: support V_CMPX_EQ_U16 (#816)

* shader: support V_CMPX_EQ_U16

Decode gfx11 VOPC 0xba and lower it as an unsigned 16-bit equality
compare with CMPX EXEC-mask semantics.

🤖 Generated with Codebuff 🤖
Co-Authored-By: Codebuff <noreply@codebuff.com>

* shader: correct CMPX EQ U16 runtime expectations

Remove the added CFG tests and correct two EXEC expectations in the compute regression case.

---------

Co-authored-by: Matthew Hurley <171001439+theantipopau@users.noreply.github.com>
Co-authored-by: Codebuff <noreply@codebuff.com>
Co-authored-by: nmzik <Nmzik@mail.ru>

src/graphics/presentation/videoOut.cpp

@@ -1556,7 +1556,9 @@ KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle) {
 	VideoOutFlipStatus status {};
 	DriverState().GetFlipQueue().GetFlipStatus(*ctx, status);
 
-	LOGF("\t flipPendingNum = %d\n", status.flipPendingNum);
+	if (Config::GraphicsDebugDumpEnabled()) {
+		LOGF("\t flipPendingNum = %d\n", status.flipPendingNum);
+	}
 
 	return status.flipPendingNum;
 }

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -404,6 +404,7 @@ enum class Opcode {
 	V_CMP_LE_U16,
 	V_CMP_GT_U16,
 	V_CMPX_LT_U16,
+	V_CMPX_EQ_U16,
 	V_CMPX_GT_U16,
 	V_CMP_NE_U16,
 	V_CMP_GE_U16,

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp

@@ -231,7 +231,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
     {0xa9u, Opcode::V_CMP_LT_U16},         {0xaau, Opcode::V_CMP_EQ_U16},
     {0xabu, Opcode::V_CMP_LE_U16},         {0xacu, Opcode::V_CMP_GT_U16},
     {0xadu, Opcode::V_CMP_NE_U16},         {0xaeu, Opcode::V_CMP_GE_U16},
-    {0xb9u, Opcode::V_CMPX_LT_U16, false},
+    {0xb9u, Opcode::V_CMPX_LT_U16, false}, {0xbau, Opcode::V_CMPX_EQ_U16, false},
     {0xbcu, Opcode::V_CMPX_GT_U16},        {0xc0u, Opcode::V_CMP_F_U32},
     {0xc1u, Opcode::V_CMP_LT_U32},         {0xc2u, Opcode::V_CMP_EQ_U32},
     {0xc3u, Opcode::V_CMP_LE_U32},         {0xc4u, Opcode::V_CMP_GT_U32},
@@ -1454,6 +1454,7 @@ bool IsVopcCompareExec(Opcode opcode) {
 		case Opcode::V_CMPX_NE_I64:
 		case Opcode::V_CMPX_NE_U64:
 		case Opcode::V_CMPX_LT_U16:
+		case Opcode::V_CMPX_EQ_U16:
 		case Opcode::V_CMPX_GT_U16:
 		case Opcode::V_CMPX_LT_F16:
 		case Opcode::V_CMPX_EQ_F16:

src/graphics/shader/recompiler/frontend/translate/Vector.cpp

@@ -141,6 +141,9 @@ bool Translator::EmitVector(const Decoder::Instruction& inst) {
 		case O::V_CMPX_LT_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, true);
 			return true;
+		case O::V_CMPX_EQ_U16:
+			EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, true);
+			return true;
 		case O::V_CMP_LE_U16:
 			EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false);
 			return true;

src/libs/agc.cpp

@@ -2110,7 +2110,7 @@ uint32_t KYTY_SYSV_ABI AgcCbQueueEndOfPipeActionGetSize() {
 uint32_t* KYTY_SYSV_ABI AgcAcbResetQueue(CommandBuffer* buf, uint32_t op) {
 	PRINT_NAME();
 
-	LOGF("\t op    = 0x%08" PRIx32 "\n", op);
+	AgcTrace("\t op    = 0x%08" PRIx32 "\n", op);
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 	EXIT_NOT_IMPLEMENTED((op & ~0x1c2u) != 0);
@@ -2130,9 +2130,9 @@ uint32_t* KYTY_SYSV_ABI AgcAcbResetQueue(CommandBuffer* buf, uint32_t op) {
 uint32_t* KYTY_SYSV_ABI AgcDcbResetQueue(CommandBuffer* buf, uint32_t op, uint32_t state) {
 	PRINT_NAME();
 
-	LOGF("\t op    = 0x%08" PRIx32 "\n"
-	     "\t state = 0x%08" PRIx32 "\n",
-	     op, state);
+	AgcTrace("\t op    = 0x%08" PRIx32 "\n"
+	         "\t state = 0x%08" PRIx32 "\n",
+	         op, state);
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 	EXIT_NOT_IMPLEMENTED((op & ~0xfffu) != 0);
@@ -2749,9 +2749,9 @@ uint32_t* KYTY_SYSV_ABI AgcDcbSetBaseIndirectArgs(CommandBuffer* buf, uint32_t s
                                                   const volatile void* indirect_base_addr) {
 	PRINT_NAME();
 
-	LOGF("\t shader_type        = %" PRIu32 "\n"
-	     "\t indirect_base_addr = 0x%016" PRIx64 "\n",
-	     shader_type, reinterpret_cast<uint64_t>(indirect_base_addr));
+	AgcTrace("\t shader_type        = %" PRIu32 "\n"
+	         "\t indirect_base_addr = 0x%016" PRIx64 "\n",
+	         shader_type, reinterpret_cast<uint64_t>(indirect_base_addr));
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 
@@ -2941,9 +2941,9 @@ uint32_t* KYTY_SYSV_ABI AgcDcbDispatchIndirect(CommandBuffer* buf, uint32_t data
                                                uint32_t flags) {
 	PRINT_NAME();
 
-	LOGF("\t data_offset = 0x%" PRIx32 "\n"
-	     "\t flags       = 0x%08" PRIx32 "\n",
-	     data_offset_in_bytes, flags);
+	AgcTrace("\t data_offset = 0x%" PRIx32 "\n"
+	         "\t flags       = 0x%08" PRIx32 "\n",
+	         data_offset_in_bytes, flags);
 
 	EXIT_NOT_IMPLEMENTED(buf == nullptr);
 
@@ -3294,9 +3294,9 @@ uint32_t* KYTY_SYSV_ABI AgcAcbDispatchIndirect(CommandBuffer*       buf,
                                                uint32_t             modifier) {
 	PRINT_NAME();
 
-	LOGF("\t indirect_args = 0x%016" PRIx64 "\n"
-	     "\t modifier      = 0x%08" PRIx32 "\n",
-	     reinterpret_cast<uint64_t>(indirect_args), modifier);
+	AgcTrace("\t indirect_args = 0x%016" PRIx64 "\n"
+	         "\t modifier      = 0x%08" PRIx32 "\n",
+	         reinterpret_cast<uint64_t>(indirect_args), modifier);
 
 	if (buf == nullptr) {
 		return nullptr;
@@ -3724,9 +3724,9 @@ int KYTY_SYSV_ABI AgcCondExecPatchSetCommandAddress(uint32_t*                cmd
                                                     const volatile uint32_t* command) {
 	PRINT_NAME();
 
-	LOGF("\t cmd     = 0x%016" PRIx64 "\n"
-	     "\t command = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(command));
+	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
+	         "\t command = 0x%016" PRIx64 "\n",
+	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(command));
 
 	if (cmd == nullptr || command == nullptr) {
 		return GRAPHICS5_ERROR_INVALID_PACKET;
@@ -3868,9 +3868,9 @@ static uint32_t* get_agc_wait_packet(uint32_t* cmd) {
 int KYTY_SYSV_ABI AgcWaitRegMemPatchAddress(uint32_t* cmd, const volatile void* address) {
 	PRINT_NAME();
 
-	LOGF("\t cmd     = 0x%016" PRIx64 "\n"
-	     "\t address = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
+	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
+	         "\t address = 0x%016" PRIx64 "\n",
+	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
 
 	auto* wait = get_agc_wait_packet(cmd);
 	if (wait == nullptr) {
@@ -3913,9 +3913,9 @@ int KYTY_SYSV_ABI AgcQueueEndOfPipeActionPatchAddress(uint32_t*             cmd,
 
 	// Not sure
 
-	LOGF("\t cmd     = 0x%016" PRIx64 "\n"
-	     "\t address = 0x%016" PRIx64 "\n",
-	     reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
+	AgcTrace("\t cmd     = 0x%016" PRIx64 "\n"
+	         "\t address = 0x%016" PRIx64 "\n",
+	         reinterpret_cast<uint64_t>(cmd), reinterpret_cast<uint64_t>(address));
 
 	EXIT_NOT_IMPLEMENTED(cmd == nullptr);
 
@@ -4246,7 +4246,7 @@ static void submit_acb(uint32_t queue, uint32_t* acb, uint32_t size_in_dwords) {
 	}
 
 	for (uint32_t i = 0; i < std::min<uint32_t>(size_in_dwords, 8); i++) {
-		LOGF("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]);
+		AgcTrace("\t acb[%u] = 0x%08" PRIx32 "\n", i, acb[i]);
 	}
 
 	GraphicsDbgDumpDcb("a", size_in_dwords, acb);
@@ -4322,17 +4322,17 @@ int KYTY_SYSV_ABI AgcDriverSubmitMultiCommandBuffers(void*            queue_cont
 int KYTY_SYSV_ABI AgcDriverSubmitAcb(uint32_t queue, const Packet* packet) {
 	PRINT_NAME();
 
-	LOGF("\t queue  = 0x%08" PRIx32 "\n"
-	     "\t packet = 0x%016" PRIx64 "\n",
-	     queue, reinterpret_cast<uint64_t>(packet));
+	AgcTrace("\t queue  = 0x%08" PRIx32 "\n"
+	         "\t packet = 0x%016" PRIx64 "\n",
+	         queue, reinterpret_cast<uint64_t>(packet));
 
 	if (packet == nullptr) {
 		return OK;
 	}
-	LOGF("\t acb   = 0x%016" PRIx64 "\n"
-	     "\t size  = 0x%08" PRIx32 "\n"
-	     "\t flags = 0x%02" PRIx8 "\n",
-	     reinterpret_cast<uint64_t>(packet->addr), packet->dw_num, packet->flags);
+	AgcTrace("\t acb   = 0x%016" PRIx64 "\n"
+	         "\t size  = 0x%08" PRIx32 "\n"
+	         "\t flags = 0x%02" PRIx8 "\n",
+	         reinterpret_cast<uint64_t>(packet->addr), packet->dw_num, packet->flags);
 
 	submit_acb(queue, packet->addr, packet->dw_num);
 	return OK;

src/libs/ajm.cpp

@@ -682,16 +682,9 @@ static AjmDecodeResult AjmControlInstance(uint32_t instance, uint64_t flags,
 
 static AjmDecodeResult AjmSetGaplessDecode(uint32_t instance, const void* gapless_decode,
                                            int reset) {
-	std::scoped_lock lock(g_ajm_instances_mutex);
-	auto*            state = AjmFindInstanceLocked(instance);
-	if (state == nullptr || state->decoder == nullptr || gapless_decode == nullptr) {
-		AjmDecodeResult result {};
-		result.result = AJM_RESULT_INVALID_PARAMETER;
-		return result;
-	}
-
-	state->gapless.Set(*static_cast<const AjmSidebandGaplessDecode*>(gapless_decode), reset != 0);
-	return state->decoder->MakeResult();
+	const auto flags = AJM_FLAG_SIDEBAND_GAPLESS | (reset != 0 ? AJM_FLAG_CONTROL_RESET : 0);
+	return AjmControlInstance(instance, flags, gapless_decode, sizeof(AjmSidebandGaplessDecode),
+	                          nullptr, nullptr);
 }
 
 static AjmDecodeResult AjmGetGaplessDecode(uint32_t                  instance,
@@ -930,18 +923,8 @@ int KYTY_SYSV_ABI AjmBatchJobClearContext(AjmBatchInfo* info, uint32_t instance,
 	PRINT_NAME();
 	EXIT_NOT_IMPLEMENTED(info == nullptr);
 
-	AjmDecodeResult decode_result {};
-	{
-		std::scoped_lock lock(g_ajm_instances_mutex);
-		auto*            state = AjmFindInstanceLocked(instance);
-		if (state != nullptr && state->decoder != nullptr) {
-			state->gapless.Reset();
-			state->decoder->Reset();
-			decode_result = state->decoder->MakeResult();
-		} else {
-			decode_result.result = AJM_RESULT_INVALID_PARAMETER;
-		}
-	}
+	const auto decode_result =
+	    AjmControlInstance(instance, AJM_FLAG_CONTROL_RESET, nullptr, 0, nullptr, nullptr);
 
 	AjmWriteResult(result, AJM_SIDEBAND_RESULT_SIZE, decode_result);
 	LOGF("\t instance = 0x%08" PRIx32 "\n", instance);

src/libs/libs.h

@@ -51,15 +51,7 @@
 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
 #define PRINT_NAME()                                                                               \
 	if (PRINT_NAME_ENABLED) {                                                                      \
-		if (Log::GetDirection() != Log::Direction::Silent) {                                       \
-			const auto elapsed_ms      = static_cast<uint64_t>(Loader::Timer::GetTimeMs());        \
-			const auto print_name_time = fmt::format(                                              \
-			    "{:02}:{:02}:{:02}.{:03}", elapsed_ms / 3600000, (elapsed_ms / 60000) % 60,        \
-			    (elapsed_ms / 1000) % 60, elapsed_ms % 1000);                                      \
-			LOGF_COLOR(Log::Color::Cyan, "[%d][%s] %s::%s::%s()\n",                                \
-			           Common::Thread::GetThreadIdUnique(), print_name_time.c_str(), g_library,    \
-			           g_module, __func__);                                                        \
-		}                                                                                          \
+		Libs::PrintName(g_library, g_module, __func__);                                              \
 	}
 
 namespace Loader {
@@ -68,6 +60,19 @@ class SymbolDatabase;
 
 namespace Libs {
 
+// Keep the formatting path from inflating fiber functions' stack frames under LTO.
+[[gnu::noinline]] inline void PrintName(const char* library, const char* module, const char* function) {
+	if (Log::GetDirection() != Log::Direction::Silent) {
+		const auto elapsed_ms      = static_cast<uint64_t>(Loader::Timer::GetTimeMs());
+		const auto print_name_time = fmt::format(
+		    "{:02}:{:02}:{:02}.{:03}", elapsed_ms / 3600000, (elapsed_ms / 60000) % 60,
+		    (elapsed_ms / 1000) % 60, elapsed_ms % 1000);
+		LOGF_COLOR(Log::Color::Cyan, "[%d][%s] %s::%s::%s()\n",
+		           Common::Thread::GetThreadIdUnique(), print_name_time.c_str(), library, module,
+		           function);
+	}
+}
+
 void InitAll(Loader::SymbolDatabase* s);
 
 } // namespace Libs

src/libs/ngs2.cpp

@@ -10,6 +10,7 @@
 #include <algorithm>
 #include <array>
 #include <atomic>
+#include <cmath>
 #include <cstring>
 #include <deque>
 #include <limits>
@@ -428,11 +429,19 @@ struct Ngs2VoiceInternal {
 	std::unique_ptr<Ajm::AjmAt9Decoder> decoder;
 	std::vector<float>                 decoded_frame;
 	bool                               accepts_blocks = true;
+	uint32_t                           sample_rate    = 0;
+	uint64_t                           sample_phase   = 0;
+	uint64_t                           sample_step    = 0;
 
 	void SetupSampler(const Ngs2WaveformFormat& format) {
-		EXIT_NOT_IMPLEMENTED(format.sample_rate != rack->ngs->option.sample_rate ||
-		                     format.frame_margin != 0 || format.frame_offset != 0);
+		EXIT_NOT_IMPLEMENTED(format.frame_margin != 0 || format.frame_offset != 0 ||
+		                     format.sample_rate == 0 ||
+		                     (format.waveform_type == NGS2_WAVEFORM_TYPE_ATRAC9 &&
+		                      format.sample_rate != rack->ngs->option.sample_rate));
 		channels = format.num_channels;
+		sample_rate = format.sample_rate;
+		sample_phase = 0;
+		sample_step = uint64_t(sample_rate) << 32u;
 		blocks.clear();
 		decoder.reset();
 		decoded_frame.clear();
@@ -563,6 +572,16 @@ struct Ngs2SamplerVoiceState {
 	const void*    waveform_data;
 };
 
+struct Ngs2CustomSamplerVoiceState {
+	Ngs2VoiceState voice_state;
+	const void*    waveform_data;
+	uint64_t       num_decoded_samples;
+	uint64_t       decoded_data_size;
+	uint64_t       user_data;
+	uint32_t       reserved;
+	uint32_t       reserved2;
+};
+
 static Ngs2Internal*        g_ngs_list     = nullptr;
 static Ngs2RackInternal*    g_racks_list   = nullptr;
 static std::atomic_uint32_t g_next_ngs_uid = 1;
@@ -578,6 +597,7 @@ static_assert(sizeof(Ngs2VoiceState) == 8);
 static_assert(sizeof(Ngs2SubmixerVoiceState) == 20);
 static_assert(sizeof(Ngs2CustomMasteringVoiceState) == 16);
 static_assert(sizeof(Ngs2SamplerVoiceState) == 56);
+static_assert(sizeof(Ngs2CustomSamplerVoiceState) == 48);
 static_assert(sizeof(Ngs2WaveformFormat) == 24);
 static_assert(sizeof(Ngs2WaveformBlock) == 40);
 static_assert(sizeof(Ngs2WaveformInfo) == 232);
@@ -1244,76 +1264,118 @@ static void Ngs2ApplyEvent(Ngs2VoiceInternal& voice) {
 	voice.event = Ngs2VoicePlayEvent::None;
 }
 
+static void Ngs2FinishBlock(Ngs2VoiceInternal& voice) {
+	const auto& block = voice.blocks.front();
+	struct CallbackInfo {
+		uintptr_t   data, voice;
+		uint32_t    flag, reserved;
+		uintptr_t   user;
+		const void* block_data;
+		size_t      size;
+		uint32_t    repeats, attributes;
+	} info {voice.callback_data,
+	        reinterpret_cast<uintptr_t>(&voice),
+	        1,
+	        0,
+	        block.info.user_data,
+	        block.data,
+	        block.info.data_size,
+	        0,
+	        0};
+	static_assert(sizeof(CallbackInfo) == 56);
+	voice.blocks.pop_front();
+	if (voice.blocks.empty() && !voice.accepts_blocks) {
+		voice.state = Ngs2VoicePlayState::Empty;
+	}
+	if (voice.callback != 0 && (voice.callback_flags & 1u) != 0) {
+		reinterpret_cast<void KYTY_SYSV_ABI (*)(const CallbackInfo*)>(voice.callback)(&info);
+	}
+}
+
+static void Ngs2AdvancePcm(Ngs2VoiceInternal& voice) {
+	const auto output_rate = uint64_t(voice.rack->ngs->option.sample_rate) << 32u;
+	while (!voice.blocks.empty() && voice.sample_phase >= output_rate) {
+		auto&      block     = voice.blocks.front();
+		const auto available = block.info.num_samples - block.cursor;
+		const auto advance   = std::min(voice.sample_phase / output_rate,
+		                                uint64_t(available));
+		block.cursor += static_cast<uint32_t>(advance);
+		voice.sample_phase -= advance * output_rate;
+		if (block.cursor == block.info.num_samples) {
+			Ngs2FinishBlock(voice);
+		}
+	}
+}
+
 static void Ngs2ConsumeSamples(Ngs2VoiceInternal& voice, uint32_t grain) {
 	uint32_t output = 0;
 	while (output < grain && !voice.blocks.empty()) {
+		if (voice.decoder == nullptr) {
+			Ngs2AdvancePcm(voice);
+			if (voice.blocks.empty()) {
+				break;
+			}
+			const auto& block = voice.blocks.front();
+			const auto* pcm = reinterpret_cast<const int16_t*>(block.data) +
+			                  (block.info.num_skip_samples + block.cursor) * voice.channels;
+			const auto* next = pcm;
+			if (block.cursor + 1 < block.info.num_samples) {
+				next += voice.channels;
+			} else if (voice.blocks.size() > 1) {
+				const auto& next_block = voice.blocks[1];
+				next = reinterpret_cast<const int16_t*>(next_block.data) +
+				       next_block.info.num_skip_samples * voice.channels;
+			}
+			const float fraction = static_cast<float>(voice.sample_phase) /
+			                       static_cast<float>(uint64_t(voice.rack->ngs->option.sample_rate)
+			                                          << 32u);
+			for (uint32_t c = 0; c < voice.channels; ++c) {
+				voice.samples[c * grain + output] =
+				    (static_cast<float>(pcm[c]) +
+				     (static_cast<float>(next[c]) - pcm[c]) * fraction) /
+				    32768.0f;
+			}
+			voice.sample_phase += voice.sample_step;
+			++output;
+			continue;
+		}
 		auto&          block   = voice.blocks.front();
 		auto           count   = std::min(grain - output, block.info.num_samples - block.cursor);
-		const int16_t* pcm     = nullptr;
-		const float*   decoded = nullptr;
-		if (voice.decoder != nullptr) {
-			const auto skip =
-			    std::min(block.skip_remaining,
-			             static_cast<uint32_t>(voice.decoded_frame.size() / voice.channels));
-			Ajm::AjmGaplessState gapless;
-			gapless.Set({block.info.num_samples - block.cursor, static_cast<uint16_t>(skip), 0},
-			            true);
-			const auto result = voice.decoder->Decode(
-			    block.data + block.data_cursor, block.info.data_size - block.data_cursor,
-			    voice.decoded_frame.data(), voice.decoded_frame.size() * sizeof(float), false,
-			    &gapless);
-			EXIT_NOT_IMPLEMENTED(result.result != OK || result.input_consumed == 0);
-			block.data_cursor += result.input_consumed;
-			block.skip_remaining -= skip - gapless.current.skip_samples;
-			const auto decoded_samples =
-			    static_cast<uint32_t>(result.output_written / (sizeof(float) * voice.channels));
-			if (decoded_samples == 0) {
-				continue;
-			}
-			EXIT_NOT_IMPLEMENTED(decoded_samples > count);
-			count   = decoded_samples;
-			decoded = voice.decoded_frame.data();
-		} else {
-			pcm = reinterpret_cast<const int16_t*>(block.data) +
-			      (block.info.num_skip_samples + block.cursor) * voice.channels;
+		const auto skip =
+		    std::min(block.skip_remaining,
+		             static_cast<uint32_t>(voice.decoded_frame.size() / voice.channels));
+		Ajm::AjmGaplessState gapless;
+		gapless.Set({block.info.num_samples - block.cursor, static_cast<uint16_t>(skip), 0},
+		            true);
+		const auto result = voice.decoder->Decode(
+		    block.data + block.data_cursor, block.info.data_size - block.data_cursor,
+		    voice.decoded_frame.data(), voice.decoded_frame.size() * sizeof(float), false,
+		    &gapless);
+		EXIT_NOT_IMPLEMENTED(result.result != OK || result.input_consumed == 0);
+		block.data_cursor += result.input_consumed;
+		block.skip_remaining -= skip - gapless.current.skip_samples;
+		const auto decoded_samples =
+		    static_cast<uint32_t>(result.output_written / (sizeof(float) * voice.channels));
+		if (decoded_samples == 0) {
+			continue;
 		}
+		EXIT_NOT_IMPLEMENTED(decoded_samples > count);
+		count = decoded_samples;
 		for (uint32_t c = 0; c < voice.channels; ++c) {
 			for (uint32_t i = 0; i < count; ++i) {
 				const auto index = i * voice.channels + c;
-				voice.samples[c * grain + output + i] =
-				    decoded != nullptr ? decoded[index] : pcm[index] / 32768.0f;
+				voice.samples[c * grain + output + i] = voice.decoded_frame[index];
 			}
 		}
 		output += count;
 		block.cursor += count;
 		if (block.cursor == block.info.num_samples) {
-			struct CallbackInfo {
-				uintptr_t   data, voice;
-				uint32_t    flag, reserved;
-				uintptr_t   user;
-				const void* block_data;
-				size_t      size;
-				uint32_t    repeats, attributes;
-			} info {voice.callback_data,
-			        reinterpret_cast<uintptr_t>(&voice),
-			        1,
-			        0,
-			        block.info.user_data,
-			        block.data,
-			        block.info.data_size,
-			        0,
-			        0};
-			static_assert(sizeof(CallbackInfo) == 56);
-			voice.blocks.pop_front();
-			if (voice.blocks.empty() && !voice.accepts_blocks) {
-				voice.state = Ngs2VoicePlayState::Empty;
-			}
-			if (voice.callback != 0 && (voice.callback_flags & 1u) != 0) {
-				reinterpret_cast<void KYTY_SYSV_ABI (*)(const CallbackInfo*)>(voice.callback)(
-				    &info);
-			}
+			Ngs2FinishBlock(voice);
 		}
 	}
+	if (voice.decoder == nullptr) {
+		Ngs2AdvancePcm(voice);
+	}
 	voice.has_samples = output != 0;
 }
 
@@ -1925,7 +1987,12 @@ int KYTY_SYSV_ABI Ngs2VoiceControl(uintptr_t voice_handle, const Ngs2VoiceParamH
 						};
 						const auto& blocks = *reinterpret_cast<const BlocksParam*>(param);
 						EXIT_NOT_IMPLEMENTED(!voice->accepts_blocks ||
-						                     (blocks.flags != 0 && blocks.flags != 0x11));
+						                     (blocks.flags != 0 && blocks.flags != 0x11 &&
+						                      blocks.flags != 0x4));
+						if (blocks.flags == 0x4) {
+							voice->blocks.clear();
+							voice->sample_phase = 0;
+						}
 						voice->accepts_blocks = (blocks.flags & 1u) != 0;
 						for (uint32_t i = 0; i < blocks.count; ++i) {
 							const auto& block = blocks.blocks[i];
@@ -1939,9 +2006,14 @@ int KYTY_SYSV_ABI Ngs2VoiceControl(uintptr_t voice_handle, const Ngs2VoiceParamH
 						}
 						break;
 					}
-					case 5:
-						EXIT_NOT_IMPLEMENTED(*reinterpret_cast<const float*>(param + 1) != 1.0f);
+					case 5: {
+						const float ratio = *reinterpret_cast<const float*>(param + 1);
+						EXIT_NOT_IMPLEMENTED(!std::isfinite(ratio) || ratio < 0.0f || ratio > 4.0f ||
+						                     (voice->decoder != nullptr && ratio != 1.0f));
+						voice->sample_step = static_cast<uint64_t>(std::llround(
+						    static_cast<double>(voice->sample_rate) * ratio * 4294967296.0));
 						break;
+					}
 					default: EXIT("unsupported sampler control: 0x%08" PRIx32 "\n", param->id);
 				}
 				break;
@@ -2041,8 +2113,17 @@ int KYTY_SYSV_ABI Ngs2VoiceGetState(uintptr_t voice_handle, Ngs2VoiceState* stat
 			mastering->voice_state.state_flags = Ngs2GetStateFlags(voice);
 			break;
 		}
-		case Ngs2RackType::Sampler:
 		case Ngs2RackType::CustomSampler: {
+			const auto configured_size =
+			    voice->rack->option.custom_sampler.custom_rack_option.state_size;
+			EXIT_NOT_IMPLEMENTED(configured_size < sizeof(Ngs2CustomSamplerVoiceState));
+			EXIT_NOT_IMPLEMENTED(state_size != configured_size);
+			std::memset(state, 0, state_size);
+			auto* sampler = reinterpret_cast<Ngs2CustomSamplerVoiceState*>(state);
+			sampler->voice_state.state_flags = Ngs2GetStateFlags(voice);
+			break;
+		}
+		case Ngs2RackType::Sampler: {
 			if (state_size != sizeof(Ngs2SamplerVoiceState)) {
 				LOGF("\t warning: sampler state_size = 0x%016" PRIx64 ", expected 0x%016" PRIx64
 				     "\n",

tests/ShaderRecompilerComputeTests.cpp

@@ -22492,6 +22492,74 @@ TestCase VectorVopcCmpxLtU16CapturedSdwaExecMask() {
   return test;
 }
 
+TestCase VectorVopcCmpxEqU16SdwaCompactVop3ExecMask() {
+  using O = ShaderOpcode;
+  struct CompareCase {
+    u32 lhs;
+    u32 rhs;
+    u32 incoming_exec;
+    u32 expected_exec;
+    u32 encoding = 0; // 0: SDWA, 1: compact, 2: VOP3, 3: SDWA sign-extend.
+  };
+  const std::array<CompareCase, 12> cases{{
+      {0x12340000u, 0x56780000u, 1, 1}, // Equal low half; ignore high half.
+      {0xffff0001u, 0xabcd0001u, 1, 1},
+      {0x00000000u, 0x12340000u, 1, 1},
+      {0xaaaa5555u, 0xbbbb5555u, 1, 1},
+      {0x00000000u, 0x00000001u, 1, 0},
+      {0x0000ffffu, 0x00000000u, 1, 0},
+      {0x7fff8000u, 0x00018000u, 1, 1}, // Equal low halves despite different high halves.
+      {0x00000000u, 0x00010000u, 1, 1},
+      {0x12340001u, 0x12340002u, 1, 0, 1},
+      {0x00008000u, 0x0000ffffu, 1, 0, 2},
+      {0x0001ffffu, 0x12340001u, 1, 0, 3}, // Sign-extended low halves differ.
+      {0x00000001u, 0x00010001u, 0, 0}, // CMPX cannot reactivate an inactive lane.
+  }};
+  constexpr u32 vcc_hi = 0x89abcdefu;
+  TestCase test;
+  test.name = "VectorVopcCmpxEqU16SdwaCompactVop3ExecMask";
+  for (const auto &entry : cases) {
+    test.initial.push_back(entry.lhs);
+  }
+  test.expected = test.initial;
+  auto &code = test.code;
+  for (u32 i = 0; i < cases.size(); ++i) {
+    const auto &entry = cases[i];
+    AppendVMovU32(&code, 30, i * 4u);
+    AppendBufferLoadDword(&code, 0, 30); // Runtime input prevents constant folding.
+    AppendVMovU32(&code, 1, entry.rhs);
+    AppendSMovLiteral(&code, 106, entry.rhs);
+    AppendSMovLiteral(&code, 107, vcc_hi);
+    code.push_back(EncodeSMovB32(126, InlineU32(entry.incoming_exec)));
+    switch (entry.encoding) {
+    case 1: code.push_back(0x7d740300u); break; // compact v0, v1
+    case 2: code.insert(code.end(), {0xd4ba007eu, 0x00020300u}); break;
+    case 3:
+      code.push_back(EncodeVopc(0xba, 249u, 1u));
+      code.push_back(EncodeVopcSdwa(0u, 0u, 0u, 6u, 6u, 1u, 1u));
+      break;
+    default:
+      code.push_back(EncodeVopc(0xba, 249u, 1u));
+      code.push_back(EncodeVopcSdwa(0u));
+      break;
+    }
+    code.push_back(EncodeSMovB32(20, 126)); // Snapshot EXEC before restoring it.
+    code.push_back(EncodeSMovB32(21, 127));
+    code.push_back(EncodeSMovB32(126, InlineU32(1u)));
+    const u32 out = static_cast<u32>(cases.size()) + i * 4u;
+    AppendStoreSgprPair(&code, 20, out);
+    AppendStoreSgprPair(&code, 106, out + 2u); // CMPX must not overwrite either VCC half.
+    test.expected.insert(test.expected.end(),
+                         {entry.expected_exec, 0u, entry.rhs, vcc_hi});
+  }
+  AppendEnd(&code);
+  test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::BUFFER_LOAD_DWORD,
+                  O::V_CMPX_EQ_U16, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  test.decoded_counts = {{"V_CMPX_EQ_U16", cases.size()}};
+  test.required_spirv = {"OpIEqual"};
+  return test;
+}
+
 TestCase VectorVopcCmpNgtF16CapturedSdwaAndEdges() {
   using O = ShaderOpcode;
   enum Encoding { Captured, Compact, Vop3NegAbs, HighWord, ScalarDst, SdwaNegAbs };
@@ -28588,6 +28656,7 @@ std::vector<TestCase> MakeCases() {
   AddCase(VectorVopcSdwaCmpxWritesExecMask);
   AddCase(VectorVopcCmpxGtU16CapturedSdwaExecMask);
   AddCase(VectorVopcCmpxLtU16CapturedSdwaExecMask);
+  AddCase(VectorVopcCmpxEqU16SdwaCompactVop3ExecMask);
   AddCase(VectorVopcCmpNgtF16CapturedSdwaAndEdges);
   AddCase(VectorVopcCmpNltF16CapturedSdwaAndEdges);
   AddCase(VectorVopcCmpxNgtF16CapturedSdwaExecMask);
@@ -33386,6 +33455,11 @@ int main(int argc, char **argv) {
     CheckEmbeddedFetchVertexOffset();
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--cmpx-eq-u16-only") == 0) {
+    VulkanHarness vulkan;
+    RunCase(&vulkan, VectorVopcCmpxEqU16SdwaCompactVop3ExecMask());
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--cmpx-lt-u16-only") == 0) {
     VulkanHarness vulkan;
     RunCase(&vulkan, VectorVopcCmpxLtU16CapturedSdwaExecMask());