libkernel: implement sceKernelAioWaitRequests (#1109)

kernel: implement batch AIO completion waits

Co-authored-by: nmzik <Nmzik@mail.ru>

src/graphics/presentation/window/hostInput.cpp

@@ -454,7 +454,7 @@ void HostInputToggleMouseToJoystick() {
 	LOGF("Mouse to right stick: enabled (F7 to release)\n");
 }
 
-bool HostInputWaitEvent(SDL_Event* event) {
+bool HostInputWaitEvent(SDL_Event* event, int max_wait_ms) {
 	int timeout = -1;
 	if (!g_mouse.enabled || SDL_GetKeyboardFocus() != g_mouse_window) {
 		g_mouse.next_poll = 0;
@@ -473,6 +473,7 @@ bool HostInputWaitEvent(SDL_Event* event) {
 		    now_ms < g_cursor_hide_at ? static_cast<int>(g_cursor_hide_at - now_ms) : 0;
 		timeout = timeout < 0 ? cursor_timeout : std::min(timeout, cursor_timeout);
 	}
+	timeout              = timeout < 0 ? max_wait_ms : std::min(timeout, max_wait_ms);
 	const bool has_event = SDL_WaitEventTimeout(event, timeout);
 
 	if (Config::HideCursorEnabled()) {

src/graphics/presentation/window/hostInput.h

@@ -13,7 +13,7 @@ void               HostInputShutdown();
 void               HostInputKey(int key_code, bool down);
 void               HostInputMouseButton(uint8_t mouse_button, bool down);
 void               HostInputToggleMouseToJoystick();
-[[nodiscard]] bool HostInputWaitEvent(SDL_Event* event);
+[[nodiscard]] bool HostInputWaitEvent(SDL_Event* event, int max_wait_ms);
 
 } // namespace Libs::Graphics
 

src/graphics/presentation/window/swapchain.cpp

@@ -998,7 +998,7 @@ void Presenter::Impl::Present() {
 		}
 
 		presented_overlay_revision.store(overlay_visual.revision, std::memory_order_release);
-		window.UpdateTitle();
+		window.loop.presented_frames.fetch_add(1, std::memory_order_relaxed);
 		return;
 	}
 	LOGF("Vulkan presentation retry exhausted; dropping frame\n");

src/graphics/presentation/window/window.cpp

@@ -722,6 +722,10 @@ void WindowContext::Run() {
 	loop.need_exit = false;
 	loop.paused.store(false, std::memory_order_release);
 
+	constexpr uint64_t title_interval_ms = 1000;
+	auto               title_time        = SDL_GetTicks();
+	auto               title_frames      = loop.presented_frames.load(std::memory_order_relaxed);
+
 	while (!loop.need_exit) {
 		if (loop.paused.load(std::memory_order_acquire)) {
 			if (!timer.IsPaused()) {
@@ -731,7 +735,20 @@ void WindowContext::Run() {
 			timer.Resume();
 		}
 
-		if (!HostInputWaitEvent(&loop.event)) {
+		// Refresh the title on the main thread without making presentation wait for it.
+		const auto now     = SDL_GetTicks();
+		const auto elapsed = now - title_time;
+		if (elapsed >= title_interval_ms) {
+			const auto frames = loop.presented_frames.load(std::memory_order_relaxed);
+			if (frames != 0) {
+				UpdateTitle(frames, static_cast<double>(frames - title_frames) * 1000.0 /
+				                        static_cast<double>(elapsed));
+			}
+			title_time   = now;
+			title_frames = frames;
+		}
+		const auto wait_ms = static_cast<int>(title_interval_ms - (now - title_time));
+		if (!HostInputWaitEvent(&loop.event, wait_ms)) {
 			continue;
 		}
 		ProcessEvent(timer.GetTimeS());
@@ -913,7 +930,7 @@ void WindowContext::UpdateIcon() {
 	}
 }
 
-void WindowContext::UpdateTitle() {
+void WindowContext::UpdateTitle(uint64_t frame_num, double current_fps) {
 	static char title[128];
 	static char title_id[12];
 	static char app_ver[12];
@@ -923,10 +940,6 @@ void WindowContext::UpdateTitle() {
 	static bool has_app_ver =
 	    Loader::SystemContentParamSfoGetString("APP_VER", app_ver, sizeof(app_ver));
 	static const std::string processor_name = Common::GetSystemInfo().ProcessorName;
-	static uint64_t fps_start   = Common::Timer::QueryPerformanceCounter();
-	static uint64_t frame_num   = 0;
-	static uint64_t fps_frames  = 0;
-	static double   current_fps = 0.0;
 
 #if KYTY_BUILD == KYTY_BUILD_DEBUG
 	static constexpr auto build_type = "Debug";
@@ -936,17 +949,6 @@ void WindowContext::UpdateTitle() {
 	static constexpr auto build_type = "Unknown";
 #endif
 
-	const auto now       = Common::Timer::QueryPerformanceCounter();
-	const auto frequency = Common::Timer::QueryPerformanceFrequency();
-	frame_num++;
-	fps_frames++;
-	if (now - fps_start >= frequency) {
-		current_fps = static_cast<double>(fps_frames) * static_cast<double>(frequency) /
-		              static_cast<double>(now - fps_start);
-		fps_start   = now;
-		fps_frames  = 0;
-	}
-
 	const auto* device_name = graphic_ctx.GetPhysicalDeviceProperties().deviceName.data();
 	auto text = fmt::format(
 	    "[{} | {}] {}{}{}{}{}{}[{}] [{}], frame: {}, fps: {:.0f}", KYTY_BUILD_LABEL, build_type,
@@ -954,16 +956,7 @@ void WindowContext::UpdateTitle() {
 	    (has_title_id ? ", " : ""), (has_app_ver ? app_ver : ""), (has_app_ver ? " " : ""),
 	    device_name, processor_name, frame_num, current_fps);
 
-	struct TitleUpdate {
-		SDL_Window*  window;
-		std::string* text;
-	} update {window, &text};
-	EXIT_IF(!SDL_RunOnMainThread(
-	    [](void* data) {
-		    auto& title = *static_cast<TitleUpdate*>(data);
-		    SDL_SetWindowTitle(title.window, title.text->c_str());
-	    },
-	    &update, true));
+	SDL_SetWindowTitle(window, text.c_str());
 }
 
 } // namespace Libs::Graphics

src/graphics/presentation/window/windowInternal.h

@@ -27,6 +27,7 @@ struct WindowLoopState {
 	SDL_Event        event {};
 	bool             need_exit = false;
 	std::atomic_bool paused    = false;
+	std::atomic<uint64_t> presented_frames {0};
 };
 
 struct WindowContext {
@@ -42,7 +43,7 @@ struct WindowContext {
 	void                                                    RecreateSurface();
 	void                                                    RefreshSurfaceCapabilities();
 	void                                                    UpdateIcon();
-	void                                                    UpdateTitle();
+	void UpdateTitle(uint64_t frame_num, double current_fps);
 	void                                                    Resize(int width, int height);
 	void ProcessWindowEvent(const SDL_WindowEvent& event);
 	void ProcessDisplayEvent(const SDL_DisplayEvent& event);

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -109,6 +109,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
     {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
     {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
     {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
+    {0x3fu, Opcode::DS_ORDERED_COUNT, 1, 32},
     {0x40u, Opcode::DS_ADD_U64, 2, 32},         {0x4au, Opcode::DS_OR_B64, 2, 32},
     {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
     {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
@@ -214,6 +215,7 @@ uint32_t DsSourceCount(Opcode opcode) {
 		case Opcode::DS_READ_ADDTID_B32:
 		case Opcode::DS_CONSUME:
 		case Opcode::DS_APPEND: return 0u;
+		case Opcode::DS_ORDERED_COUNT: return 1u;
 		default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u;
 	}
 }
@@ -412,6 +414,14 @@ void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (inst.opcode == Opcode::UNSUPPORTED) {
 		SetUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
 	}
+	if (inst.opcode == Opcode::DS_ORDERED_COUNT) {
+		// Keep OFFSET1's wave type as well as its add/swap selector for GDS addressing.
+		inst.offset           = offset0 & 0xfcu;
+		inst.secondary_offset = offset1;
+		if (!inst.gds || ((offset1 >> 4u) & 3u) > 1u || ((offset1 >> 6u) & 3u) != 0u) {
+			SetUnsupported(inst, Family::DS, opcode, "DS ordered count variant is not implemented");
+		}
+	}
 	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
 	}

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -580,6 +580,7 @@ enum class Opcode {
 	DS_BPERMUTE_B32,
 	DS_CONSUME,
 	DS_APPEND,
+	DS_ORDERED_COUNT,
 	DS_READ_I8,
 	DS_READ_U8,
 	DS_READ_I16,

src/graphics/shader/recompiler/frontend/translate/Memory.cpp

@@ -883,6 +883,50 @@ void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
 	                                ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()}));
 }
 
+void Translator::DS_ORDERED_COUNT(const Decoder::Instruction& inst) {
+	// Approximate ordered counting with one GDS atomic per wave. Wave-launch ordering and
+	// release/done synchronization are not emulated.
+	const auto memory = MemoryInfoFromDecoded(inst);
+	const auto m0 = ir.GetM0();
+	auto address = ir.BitwiseAnd(ir.ShiftRightLogical(m0, IR::U32(IR::Value(16u))),
+	                             IR::U32(IR::Value(0xfffcu)));
+	if (((inst.secondary_offset >> 2u) & 3u) == 1u) {
+		const auto packer = ir.BitwiseAnd(m0, IR::U32(IR::Value(0xfu)));
+		address = ir.IAdd(address, ir.ShiftLeftLogical(packer, IR::U32(IR::Value(2u))));
+	}
+	const auto exec   = ir.GetExec();
+	const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {exec});
+	const auto low = ir.CompositeExtract(ballot, 0u);
+	const auto high = ir.CompositeExtract(ballot, 1u);
+	const auto active = ir.INotEqual(ir.BitwiseOr(low, high), IR::U32(IR::Value(0u)));
+	const auto first = ir.Select(
+	    ir.INotEqual(low, IR::U32(IR::Value(0u))),
+	    IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {low})),
+	    ir.IAdd(IR::U32(IR::Value(32u)), IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {high}))));
+	// An empty EXEC still returns the counter. Elect lane zero and perform an add of zero.
+	const auto selected = ir.Select(active, first, IR::U32(IR::Value(0u)));
+	const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
+	const auto is_first = ir.IEqual(lane, selected);
+	const auto source = IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane,
+	                                   {ReadU32(inst.src0), is_first}));
+	const auto value = ir.Select(active, source, IR::U32(IR::Value(0u)));
+	const auto flags = AddMemoryInfo(memory, inst.pc);
+	IR::U32 result;
+	if (((inst.secondary_offset >> 4u) & 3u) == 1u) {
+		const auto swapped = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicSwap32,
+		    {address, value, ir.LogicalAnd(is_first, active)}, flags));
+		const auto unchanged = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32,
+		    {address, IR::Value(0u), ir.LogicalAnd(is_first, ir.LogicalNot(active))}, flags));
+		result = ir.Select(active, swapped, unchanged);
+	} else {
+		result = IR::U32(ir.Emit(IR::ValueOpcode::SharedAtomicIAdd32,
+		                        {address, value, is_first}, flags));
+	}
+	// DS_ORDERED_COUNT writes every destination lane, regardless of EXEC.
+	ir.SetVectorReg(static_cast<IR::VectorReg>(inst.dst.reg),
+	    IR::U32(ir.Emit(IR::ValueOpcode::ReadFirstLane, {result, is_first})));
+}
+
 void Translator::DS_PERMUTE(const Decoder::Instruction& inst, bool backward) {
 	const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset)));
 	WriteOperand(inst.dst, ir.Emit(backward ? IR::ValueOpcode::BpermuteU32
@@ -1114,6 +1158,7 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) {
 		case Decoder::Opcode::DS_SWIZZLE_B32: return DS_SWIZZLE_B32(inst);
 		case Decoder::Opcode::DS_PERMUTE_B32: return DS_PERMUTE(inst, false);
 		case Decoder::Opcode::DS_BPERMUTE_B32: return DS_PERMUTE(inst, true);
+		case Decoder::Opcode::DS_ORDERED_COUNT: return DS_ORDERED_COUNT(inst);
 		case Decoder::Opcode::DS_CONSUME:
 			return DS_APPEND_CONSUME(inst, IR::ValueOpcode::DataConsume);
 		case Decoder::Opcode::DS_APPEND:

src/graphics/shader/recompiler/frontend/translate/Translator.h

@@ -109,6 +109,7 @@ private:
 	void DS_ADDTID(const Decoder::Instruction& inst, bool write);
 	void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
 	void DS_PERMUTE(const Decoder::Instruction& inst, bool backward);
+	void DS_ORDERED_COUNT(const Decoder::Instruction& inst);
 
 	IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
 	IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,

src/libs/libKernel.cpp

@@ -2831,6 +2831,8 @@ constexpr int32_t KERNEL_AIO_STATE_ABORTED    = 4;
 constexpr int32_t KERNEL_AIO_STATE_NOTIFIED   = 0x10000;
 constexpr int32_t KERNEL_AIO_MAX_QUEUE        = 512;
 constexpr int32_t KERNEL_AIO_MAX_REQUESTS     = 128;
+constexpr uint32_t KERNEL_AIO_WAIT_AND        = 1;
+constexpr uint32_t KERNEL_AIO_WAIT_OR         = 2;
 
 struct KernelAioResult {
 	int64_t  return_value;
@@ -2933,26 +2935,28 @@ int KYTY_SYSV_ABI KernelAioSubmitWriteCommands(KernelAioRwRequest* req, int32_t
 	return OK;
 }
 
+static int32_t kernel_aio_get_state(int32_t id, bool notify) {
+	if (!kernel_aio_is_valid_id(id)) {
+		return LibKernel::KERNEL_ERROR_ESRCH;
+	}
+
+	auto current = g_kernel_aio_state[id].load(std::memory_order_acquire);
+	while (notify && (current == KERNEL_AIO_STATE_COMPLETED || current == KERNEL_AIO_STATE_ABORTED)) {
+		if (g_kernel_aio_state[id].compare_exchange_weak(current, current | KERNEL_AIO_STATE_NOTIFIED,
+		                                                std::memory_order_acq_rel)) {
+			break;
+		}
+	}
+	return current == 0 ? LibKernel::KERNEL_ERROR_ESRCH : current;
+}
+
 int KYTY_SYSV_ABI KernelAioPollRequest(int32_t id, int32_t* state) {
 	PRINT_NAME();
 
 	if (state == nullptr) {
 		return LibKernel::KERNEL_ERROR_EFAULT;
 	}
-
-	if (!kernel_aio_is_valid_id(id)) {
-		*state = LibKernel::KERNEL_ERROR_ESRCH;
-		return OK;
-	}
-
-	auto current = g_kernel_aio_state[id].load(std::memory_order_acquire);
-	while (current == KERNEL_AIO_STATE_COMPLETED || current == KERNEL_AIO_STATE_ABORTED) {
-		if (g_kernel_aio_state[id].compare_exchange_weak(current, current | KERNEL_AIO_STATE_NOTIFIED,
-		                                                std::memory_order_acq_rel)) {
-			break;
-		}
-	}
-	*state = current == 0 ? LibKernel::KERNEL_ERROR_ESRCH : current;
+	*state = kernel_aio_get_state(id, true);
 	return OK;
 }
 
@@ -2969,31 +2973,53 @@ int KYTY_SYSV_ABI KernelAioPollRequests(int32_t* ids, int32_t num, int32_t* stat
 	return OK;
 }
 
-int KYTY_SYSV_ABI KernelAioWaitRequest(int32_t id, int32_t* state, uint32_t* usec) {
+int KYTY_SYSV_ABI KernelAioWaitRequests(int32_t* ids, int32_t num, int32_t* states, uint32_t mode,
+                                        uint32_t* usec) {
 	PRINT_NAME();
 
-	if (state == nullptr) {
+	if (ids == nullptr || states == nullptr) {
 		return LibKernel::KERNEL_ERROR_EFAULT;
 	}
-
-	if (!kernel_aio_is_valid_id(id)) {
-		return KernelAioPollRequest(id, state);
+	if (num <= 0 || num > KERNEL_AIO_MAX_REQUESTS ||
+	    (num != 1 && mode != KERNEL_AIO_WAIT_AND && mode != KERNEL_AIO_WAIT_OR)) {
+		return LibKernel::KERNEL_ERROR_EINVAL;
 	}
 
-	uint32_t waited  = 0;
-	auto     current = g_kernel_aio_state[id].load(std::memory_order_acquire);
-	while (current == KERNEL_AIO_STATE_PROCESSING) {
-		if (usec != nullptr && *usec != 0 && waited >= *usec) {
-			*state = current;
-			return LibKernel::KERNEL_ERROR_ETIMEDOUT;
+	const auto collect_states = [&](bool notify) {
+		int32_t finished = 0;
+		bool newly_complete = false;
+		for (int32_t i = 0; i < num; i++) {
+			const auto current = kernel_aio_get_state(ids[i], notify);
+			states[i] = current;
+			finished += current != KERNEL_AIO_STATE_PROCESSING;
+			newly_complete |= current == KERNEL_AIO_STATE_COMPLETED || current == KERNEL_AIO_STATE_ABORTED;
+		}
+		return finished == num || (mode == KERNEL_AIO_WAIT_OR && newly_complete);
+	};
+	const auto start = std::chrono::steady_clock::now();
+	for (;;) {
+		const bool complete = collect_states(false);
+		const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(
+		                         std::chrono::steady_clock::now() - start).count();
+		const bool timed_out = usec != nullptr && elapsed >= *usec;
+		if (complete || timed_out) {
+			// A competing waiter can claim the newly completed request after the first scan.
+			if (collect_states(true)) {
+				if (usec != nullptr) {
+					*usec = elapsed < *usec ? *usec - static_cast<uint32_t>(elapsed) : 0;
+				}
+				return OK;
+			}
+			if (timed_out) {
+				return LibKernel::KERNEL_ERROR_ETIMEDOUT;
+			}
 		}
-
 		Common::Thread::SleepMicro(10);
-		waited += 10;
-		current = g_kernel_aio_state[id].load(std::memory_order_acquire);
 	}
+}
 
-	return KernelAioPollRequest(id, state);
+int KYTY_SYSV_ABI KernelAioWaitRequest(int32_t id, int32_t* state, uint32_t* usec) {
+	return KernelAioWaitRequests(&id, 1, state, KERNEL_AIO_WAIT_AND, usec);
 }
 
 int KYTY_SYSV_ABI KernelAioDeleteRequest(int32_t id, int32_t* ret) {
@@ -3376,6 +3402,7 @@ LIB_DEFINE(InitLibKernel_1) {
 	LIB_FUNC("2pOuoWoCxdk", KernelAioPollRequest);
 	LIB_FUNC("o7O4z3jwKzo", KernelAioPollRequests);
 	LIB_FUNC("KOF-oJbQVvc", KernelAioWaitRequest);
+	LIB_FUNC("lgK+oIWkJyA", KernelAioWaitRequests);
 	LIB_FUNC("XQ8C8y+de+E", KernelAioSubmitWriteCommands);
 	LIB_FUNC("nu4a0-arQis", KernelAioInitializeParam);
 	LIB_FUNC("il03nluKfMk", LibKernel::KernelRaiseException);

tests/KernelFileSystemTests.cpp

@@ -169,12 +169,14 @@ void TestAioBatches() {
   using Batch = int (KYTY_SYSV_ABI *)(int32_t *, int32_t, int32_t *);
   using Single = int (KYTY_SYSV_ABI *)(int32_t, int32_t *);
   using Wait = int (KYTY_SYSV_ABI *)(int32_t, int32_t *, uint32_t *);
+  using WaitBatch = int (KYTY_SYSV_ABI *)(int32_t *, int32_t, int32_t *, uint32_t, uint32_t *);
   const auto submit = reinterpret_cast<Submit>(find("HgX7+AORI58"));
   const auto poll = reinterpret_cast<Batch>(find("o7O4z3jwKzo"));
   const auto erase = reinterpret_cast<Batch>(find("Ft3EtsZzAoY"));
   const auto poll_one = reinterpret_cast<Single>(find("2pOuoWoCxdk"));
   const auto erase_one = reinterpret_cast<Single>(find("5TgME6AYty4"));
   const auto wait = reinterpret_cast<Wait>(find("KOF-oJbQVvc"));
+  const auto wait_batch = reinterpret_cast<WaitBatch>(find("lgK+oIWkJyA"));
   constexpr char Payload[] = "AIO payload";
   const int fd = FileSystem::KernelOpen("/savedata0/aio.dat", 0x602, 0777);
   Check(fd >= 3 && FileSystem::KernelWrite(fd, Payload, sizeof(Payload)) == sizeof(Payload),
@@ -191,14 +193,22 @@ void TestAioBatches() {
   }
   ids[2] = -1;
   std::array<int32_t, 3> states {-1, -1, -1};
-  Check(poll(ids.data(), ids.size(), states.data()) == OK &&
+  uint32_t timeout = 0;
+  Check(wait_batch(ids.data(), ids.size(), states.data(), 2, &timeout) == OK &&
             states[0] == 3 && states[1] == 3 && states[2] == Kernel::KERNEL_ERROR_ESRCH,
-        "batch poll writes every state and per-request invalid-ID error");
+        "OR wait observes completed reads and invalid-ID errors even with a zero timeout");
   Check(poll_one(ids[0], &states[0]) == OK && states[0] == (3 | 0x10000) &&
             poll(ids.data(), 2, states.data()) == OK &&
             states[0] == (3 | 0x10000) && states[1] == (3 | 0x10000) &&
             wait(ids[0], &states[0], nullptr) == OK && states[0] == (3 | 0x10000),
         "single and batch polls and waits share completion notification state");
+  timeout = 1000000;
+  Check(wait_batch(ids.data(), ids.size(), states.data(), 1, &timeout) == OK &&
+            states[0] == (3 | 0x10000) && states[1] == (3 | 0x10000) &&
+            states[2] == Kernel::KERNEL_ERROR_ESRCH && timeout <= 1000000 &&
+            wait_batch(ids.data(), ids.size(), states.data(), 2, nullptr) == OK &&
+            wait_batch(ids.data(), 1, states.data(), 0, nullptr) == OK,
+        "AND and OR waits return when all IDs are terminal and one-ID waits ignore mode");
   Check(erase(ids.data(), ids.size(), states.data()) == OK &&
             states[0] == OK && states[1] == OK && states[2] == Kernel::KERNEL_ERROR_ESRCH,
         "batch deletion writes per-request results");
@@ -221,6 +231,22 @@ void TestAioBatches() {
               }),
           "maximum-sized batch returns all invalid-ID errors");
   }
+  states.fill(42);
+  Check(wait_batch(nullptr, 1, states.data(), 1, nullptr) == Kernel::KERNEL_ERROR_EFAULT &&
+            wait_batch(ids.data(), 1, nullptr, 1, nullptr) == Kernel::KERNEL_ERROR_EFAULT &&
+            wait_batch(ids.data(), 0, states.data(), 1, nullptr) == Kernel::KERNEL_ERROR_EINVAL &&
+            wait_batch(ids.data(), 129, states.data(), 1, nullptr) == Kernel::KERNEL_ERROR_EINVAL &&
+            wait_batch(ids.data(), 2, states.data(), 0, nullptr) == Kernel::KERNEL_ERROR_EINVAL &&
+            wait_batch(ids.data(), 2, states.data(), 3, nullptr) == Kernel::KERNEL_ERROR_EINVAL &&
+            states == std::array<int32_t, 3> {42, 42, 42},
+        "invalid wait arguments do not modify outputs");
+  std::array<int32_t, 128> invalid_ids {};
+  std::array<int32_t, 128> errors {};
+  Check(wait_batch(invalid_ids.data(), invalid_ids.size(), errors.data(), 1, nullptr) == OK &&
+            std::all_of(errors.begin(), errors.end(), [](int32_t error) {
+              return error == Kernel::KERNEL_ERROR_ESRCH;
+            }),
+        "maximum-sized wait returns every invalid-ID error");
   Check(FileSystem::KernelClose(fd) == OK, "close AIO read fixture");
 }
 

tests/ShaderRecompilerComputeTests.cpp

@@ -32243,6 +32243,66 @@ TestCase DsAppendUsesEncodedGdsSelector() {
   return test;
 }
 
+TestCase DsOrderedCountAddressAndExec(u32 wave_size, bool pixel_counter) {
+  using O = ShaderOpcode;
+  TestCase test;
+  test.name = pixel_counter
+                  ? (wave_size == 64 ? "DsOrderedCountPixelAddressWave64"
+                                     : "DsOrderedCountPixelAddressWave32")
+                  : (wave_size == 64 ? "DsOrderedCountAddressAndExecWave64"
+                                     : "DsOrderedCountAddressAndExecWave32");
+  auto &code = test.code;
+  // Both the base and OFFSET0 are aligned independently; M0's low bits are
+  // a wave index, or a pixel packer ID, rather than a byte address or size.
+  AppendSMovLiteral(&code, 124, 0x01030003u);
+  code.push_back(EncodeSop1(0x04, 10, 126)); // Save the full EXEC mask.
+  code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); // ADDR value = lane + 1.
+  AppendVMovU32(&code, 9, 99); // DATA0 must not supply the count.
+  const u32 selected_lane = wave_size == 64 ? 47u : 7u;
+  const std::array<uint64_t, 3> masks = {
+      wave_size == 64 ? UINT64_MAX : uint64_t{UINT32_MAX},
+      uint64_t{1} << selected_lane, 0};
+  u32 counter = 1000;
+  for (bool exchange : {false, true}) {
+    for (size_t mode = 0; mode < masks.size(); ++mode) {
+      AppendVMovU32(&code, 3, 0xdeadbeefu);
+      AppendSMovLiteral(&code, 126, static_cast<u32>(masks[mode]));
+      AppendSMovLiteral(&code, 127, static_cast<u32>(masks[mode] >> 32u));
+      const u32 release_done = exchange && mode == 2 ? 3u : 0u;
+      const u32 offset1 = release_done | (pixel_counter ? 4u : 0u) |
+                          (exchange ? 0x10u : 0u);
+      code.push_back(EncodeDs0(0x3f, (offset1 << 8u) | 7u, true));
+      code.push_back(EncodeDs1(3, 9, 2));
+      code.push_back(EncodeSopp(0x0c, 0)); // Wait for the returned GDS value.
+      code.push_back(EncodeSop1(0x04, 126, 10)); // Restore EXEC before reading all lanes.
+      AppendStoreVgprAtLaneDwordOffset(&code, 3, 0,
+                                      static_cast<u32>(test.expected.size()));
+      test.expected.insert(test.expected.end(), wave_size, counter);
+      if (mode != 2) {
+        const u32 value = mode == 0 ? 1u : selected_lane + 1u;
+        counter = exchange ? value : counter + value;
+      }
+    }
+  }
+  AppendEnd(&code);
+  test.initial.assign(test.expected.size(), 0xdeadbeefu);
+  const u32 counter_address = pixel_counter ? 0x110u : 0x104u;
+  test.gds_initial.assign(0x120u / sizeof(u32), 100);
+  test.gds_initial[counter_address / sizeof(u32)] = 1000;
+  test.expected_gds = test.gds_initial;
+  test.expected_gds[counter_address / sizeof(u32)] = counter;
+  test.compute_info.threads_num[0] = wave_size;
+  test.compute_info.threads_num[1] = 1;
+  test.compute_info.threads_num[2] = 1;
+  test.compute_info.thread_ids_num = 1;
+  test.compute_info.wave_size = wave_size;
+  test.has_compute_info = true;
+  test.opcodes = {O::S_MOV_B32, O::S_MOV_B64, O::V_ADD_NC_U32, O::V_MOV_B32,
+                  O::DS_ORDERED_COUNT, O::S_WAITCNT, O::V_LSHLREV_B32,
+                  O::BUFFER_STORE_DWORD, O::S_ENDPGM};
+  return test;
+}
+
 TestCase DsAppendConsumeGdsRegionBounds() {
   using O = ShaderOpcode;
   struct Access {
@@ -35538,6 +35598,10 @@ std::vector<TestCase> MakeCases() {
   cases.push_back(DsAppendAllocatesAcrossWaves(32));
   cases.push_back(DsAppendAllocatesAcrossWaves(64));
   AddCase(DsAppendUsesEncodedGdsSelector);
+  for (u32 wave_size : {32, 64}) {
+    cases.push_back(DsOrderedCountAddressAndExec(wave_size, false));
+    cases.push_back(DsOrderedCountAddressAndExec(wave_size, true));
+  }
   AddCase(DsAppendConsumeGdsRegionBounds);
   AddCase(DsGdsSubdwordAndAtomicWrites);
   AddCase(DsReadWrite2Variants);
@@ -41059,6 +41123,14 @@ int main(int argc, char **argv) {
     }
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--ds-ordered-count-only") == 0) {
+    VulkanHarness vulkan;
+    for (u32 wave_size : {32, 64}) {
+      RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, false));
+      RunCase(&vulkan, DsOrderedCountAddressAndExec(wave_size, true));
+    }
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) {
     VulkanHarness vulkan;
     for (const auto &test : MakeCases()) {