launcher: compress game folders to ZAR

audio: submit AudioOut2 pushes outside the state mutex so contexts stop stalling each other

Co-authored-by: nmzik <Nmzik@mail.ru>

src/common/archive.h

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <functional>
 #include <memory>
 #include <vector>
 
@@ -45,6 +46,12 @@ private:
 // Accepts either a host archive filename or a virtual path inside the archive.
 [[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
 
+// Packs a host directory into a new ZArchive outside it. progress gets 0 < done <= total and
+// returns false to cancel. A failed or cancelled pack removes the partial archive.
+[[nodiscard]] bool PackArchive(const std::filesystem::path& directory,
+                               const std::filesystem::path& archive,
+                               const std::function<bool(uint64_t done, uint64_t total)>& progress);
+
 } // namespace Common
 
 #endif /* KYTY_COMMON_ARCHIVE_H_ */

src/common/archivePack.cpp

@@ -0,0 +1,84 @@
+// Kept out of archive.cpp so the statically linked emulator does not pull in the writer and the
+// zstd compressor.
+#include "common/archive.h"
+
+#include "common/stringUtils.h"
+
+#include <algorithm>
+#include <zarchive/zarchivewriter.h>
+
+namespace Common {
+
+bool PackArchive(const std::filesystem::path& directory, const std::filesystem::path& archive,
+                 const std::function<bool(uint64_t done, uint64_t total)>& progress) {
+	namespace fs = std::filesystem;
+	struct Output {
+		File file;
+		bool ok = true;
+	} output;
+	// Create the archive before walking, so a re-run started while a cancelled pack is still
+	// walking sees it exists instead of losing its archive to that pack's cleanup.
+	if (!output.file.Create(archive)) {
+		return false;
+	}
+	struct Entry {
+		fs::path path;
+		uint64_t size;
+		bool     is_file;
+	};
+	std::vector<Entry> entries;
+	uint64_t           total = 0;
+	std::error_code    error;
+	// Follow directory links, as the emulator does when it runs the folder.
+	fs::recursive_directory_iterator it(directory, fs::directory_options::follow_directory_symlink,
+	                                    error);
+	for (; !error && it != fs::end(it); it.increment(error)) {
+		const bool is_file = !it->is_directory(error);
+		const auto size    = is_file && !error ? it->file_size(error) : 0;
+		if (error) {
+			break;
+		}
+		entries.push_back({it->path(), size, is_file});
+		total += size;
+	}
+	output.ok = !error;
+	ZArchiveWriter writer(
+	    [](int32_t, void*) {},
+	    [](const void* data, size_t size, void* opaque) {
+		    auto&    out     = *static_cast<Output*>(opaque);
+		    uint32_t written = 0;
+		    out.file.Write(data, static_cast<uint32_t>(size), &written);
+		    out.ok = out.ok && written == size;
+	    },
+	    &output);
+	std::vector<uint8_t> buffer(1u << 20u);
+	uint64_t             done = 0;
+	for (size_t index = 0; output.ok && index < entries.size(); index++) {
+		const auto& entry = entries[index];
+		const auto  name  = PathToGenericString(entry.path.lexically_relative(directory));
+		File        input;
+		output.ok = entry.is_file ? writer.StartNewFile(name.c_str()) &&
+		                                input.Open(entry.path, File::Mode::Read)
+		                          : writer.MakeDir(name.c_str());
+		for (uint64_t left = entry.size; output.ok && left != 0;) {
+			uint32_t read = 0;
+			input.Read(buffer.data(), static_cast<uint32_t>(std::min<uint64_t>(left, buffer.size())),
+			           &read);
+			writer.AppendData(buffer.data(), read);
+			done += read;
+			left -= read;
+			output.ok = output.ok && read != 0 && progress(done, total);
+		}
+	}
+	if (output.ok) {
+		writer.Finalize();
+	}
+	output.ok = output.ok && output.file.Flush();
+	output.file.Close();
+	if (!output.ok) {
+		File::DeleteFile(archive);
+	}
+	return output.ok;
+}
+
+} // namespace Common

src/launcher/include/configurationListWidget.h

@@ -68,6 +68,7 @@ protected slots:
 	void list_itemDoubleClicked(QTreeWidgetItem* witem, int column);
 	void open_game_folder();
 	void remove_save_data();
+	void compress_game();
 	void filter_configurations(const QString& text);
 
 private:

src/launcher/src/configurationListWidget.cpp

@@ -27,6 +27,7 @@
 #include <QFile>
 #include <QFileDialog>
 #include <QFileInfo>
+#include <QFutureWatcher>
 #include <QHeaderView>
 #include <QIcon>
 #include <QInputDialog>
@@ -42,6 +43,8 @@
 #include <QPalette>
 #include <QPointer>
 #include <QProcess>
+#include <QProgressDialog>
+#include <QPromise>
 #include <QPushButton>
 #include <QRegularExpression>
 #include <QSaveFile>
@@ -54,6 +57,7 @@
 #include <QToolButton>
 #include <QTreeWidget>
 #include <QUrl>
+#include <QtConcurrentRun>
 #include <QtCore>
 
 #ifdef __linux__
@@ -62,6 +66,7 @@
 #include <QDBusPendingCallWatcher>
 #endif
 
+#include <atomic>
 #include <memory>
 #include <vector>
 
@@ -1171,6 +1176,71 @@ void ConfigurationListWidget::remove_save_data() {
 	}
 }
 
+void ConfigurationListWidget::compress_game() {
+	auto* item = static_cast<ConfigurationItem*>(m_ui->cfgs_list->currentItem());
+	if (item == nullptr) {
+		return;
+	}
+
+	const auto base    = item->GetInfo().basedir;
+	const auto archive = base + QStringLiteral(".zar");
+	const auto title   = tr("Compress to ZAR");
+	if (QFileInfo::exists(archive)) {
+		QMessageBox::warning(this, title,
+		                     tr("%1 already exists.").arg(QDir::toNativeSeparators(archive)));
+		return;
+	}
+
+	// Cancel through a flag, not the future, so a pack that already succeeded keeps its result.
+	auto  canceled = std::make_shared<std::atomic_bool>(false);
+	auto* progress = new QProgressDialog(
+	    tr("Compressing %1...").arg(QDir::toNativeSeparators(base)), tr("Cancel"), 0, 1000, this);
+	auto* watcher  = new QFutureWatcher<bool>(this);
+	// Stay modal through finalization and fsync, which follow the last progress step.
+	progress->setAutoReset(false);
+	connect(progress, &QProgressDialog::canceled, this, [canceled] { *canceled = true; });
+	connect(watcher, &QFutureWatcher<bool>::progressValueChanged, progress,
+	        &QProgressDialog::setValue);
+	connect(watcher, &QFutureWatcher<bool>::finished, this,
+	        [this, progress, watcher, canceled, base, archive, title] {
+		        // finished can arrive inside the modal dialog's setValue event processing.
+		        progress->deleteLater();
+		        watcher->deleteLater();
+		        if (!watcher->result()) {
+			        if (!*canceled) {
+				        QMessageBox::warning(
+				            this, title,
+				            tr("Could not compress %1.").arg(QDir::toNativeSeparators(base)));
+			        }
+			        return;
+		        }
+		        if (QMessageBox::Yes ==
+		                QMessageBox::question(this, title,
+		                                      tr("Created %1.\n\nDelete the original folder %2?")
+		                                          .arg(QDir::toNativeSeparators(archive),
+		                                               QDir::toNativeSeparators(base)),
+		                                      QMessageBox::Yes | QMessageBox::No, QMessageBox::No) &&
+		            !QDir(base).removeRecursively()) {
+			        QMessageBox::warning(
+			            this, title, tr("Could not remove %1.").arg(QDir::toNativeSeparators(base)));
+		        }
+		        ScanGameDirectory();
+	        });
+	progress->open();
+	watcher->setFuture(QtConcurrent::run(
+	    [directory = GameContent::ToPath(base), path = GameContent::ToPath(archive),
+	     canceled](QPromise<bool>& promise) {
+		    promise.setProgressRange(0, 1000);
+		    // A pack cancelled while queued must not touch a path a later pack may already own.
+		    promise.addResult(
+		        !*canceled &&
+		        Common::PackArchive(directory, path, [&](uint64_t done, uint64_t total) {
+			        promise.setProgressValue(static_cast<int>(done * 1000 / total));
+			        return !*canceled;
+		        }));
+	    }));
+}
+
 void ConfigurationListWidget::filter_configurations(const QString& text) {
 	const auto query              = text.trimmed();
 	const bool has_query          = !query.isEmpty();
@@ -1268,6 +1338,8 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		        }
 	        });
 	action_addcont->setEnabled(item != nullptr);
+	QAction* action_compress =
+	    menu.addAction(tr("Compress to ZAR..."), this, SLOT(compress_game()));
 	QAction* action_remove_save_data =
 	    menu.addAction(style()->standardIcon(QStyle::SP_DialogDiscardButton),
 	                   tr("Remove save data..."), this, SLOT(remove_save_data()));
@@ -1291,6 +1363,7 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		const auto& base = item->GetInfo().basedir;
 		action_open_folder->setDisabled(!QDir(base).exists() && !GameContent::IsArchive(base));
 		action_view_trophies->setDisabled(!has_trophy_data);
+		action_compress->setDisabled(item->IsRunning() || !QDir(base).exists());
 		action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty());
 		action_edit->setDisabled(item->IsRunning());
 		action_delete->setDisabled(item->IsRunning() || !item->GetInfo().custom_settings);
@@ -1298,6 +1371,7 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		action_run->setDisabled(true);
 		action_open_folder->setDisabled(true);
 		action_view_trophies->setDisabled(true);
+		action_compress->setDisabled(true);
 		action_remove_save_data->setDisabled(true);
 		action_edit->setDisabled(true);
 		action_delete->setDisabled(true);

src/libs/libAudio2.cpp

@@ -178,7 +178,6 @@ struct AudioOut2ContextState {
 	uint32_t               num_grains          = 512;
 	uint64_t               last_update         = 0;
 	int                    object_audio_handle = 0;
-	std::vector<float>     object_mix;
 };
 
 struct AudioOut2PortStateEntry {
@@ -212,8 +211,18 @@ struct AudioOut2LatencyState {
 	uint32_t latency_us = 0;
 };
 
+// Per-context copy of what a push hands the backend. The backend can sleep while pacing, so a push
+// holds only its context's mutex across it; closing the context's handles waits on that mutex.
+struct AudioOut2Submission {
+	Common::Mutex                                                   mutex;
+	std::array<std::vector<uint8_t>, AudioInternal::OUT_PORTS_MAX>  pcm;
+	std::array<std::array<float, 16>, AudioInternal::OUT_PORTS_MAX> gains;
+	std::vector<float>                                              object_mix;
+};
+
 static Common::Mutex                              g_audioout2_state_mutex;
 static std::array<AudioOut2ContextState, 16>      g_audioout2_contexts;
+static std::array<AudioOut2Submission, 16>        g_audioout2_submissions;
 static std::array<AudioOut2PortStateEntry, 256>   g_audioout2_ports;
 static std::vector<AudioOut2UserHandle>          g_audioout2_users;
 static Common::Mutex                              g_audioout2_speaker_array_mutex;
@@ -249,6 +258,10 @@ static AudioOut2ContextState* audioout2_find_context_locked(AudioOut2ContextHand
 	return nullptr;
 }
 
+static AudioOut2Submission& audioout2_submission(const AudioOut2ContextState& context) {
+	return g_audioout2_submissions[&context - g_audioout2_contexts.data()];
+}
+
 static uint32_t audioout2_grain_micros(uint32_t grains) {
 	const auto sample_count = (grains == 0 ? 512u : grains);
 	return std::max<uint32_t>((sample_count * 1000000u) / 48000u, 1000u);
@@ -372,11 +385,10 @@ static bool audioout2_context_has_queueable_device(const AudioOut2ContextState&
 	return false;
 }
 
-static void audioout2_mix_object(AudioOut2ContextState&         context,
+static void audioout2_mix_object(std::vector<float>& mix, uint32_t num_grains,
                                  const AudioOut2PortStateEntry& object) {
-	auto& mix = context.object_mix;
 	if (mix.empty()) {
-		mix.resize(static_cast<size_t>(context.num_grains) * 2, 0.0f);
+		mix.resize(static_cast<size_t>(num_grains) * 2, 0.0f);
 	}
 
 	// Center ordinary objects. First-order stereo ambisonics uses left/right-facing
@@ -393,7 +405,7 @@ static void audioout2_mix_object(AudioOut2ContextState&         context,
 	}
 	left *= object.gains[0];
 	right *= object.gains[0];
-	for (uint32_t frame = 0; frame < context.num_grains; frame++) {
+	for (uint32_t frame = 0; frame < num_grains; frame++) {
 		const float sample =
 		    object.audio_format == AudioInternal::Format::FloatMono
 		        ? reinterpret_cast<const float*>(object.pcm_data.data())[frame]
@@ -403,15 +415,16 @@ static void audioout2_mix_object(AudioOut2ContextState&         context,
 	}
 }
 
-static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, bool blocking) {
-	// The backend consumes the buffers synchronously, but can block while pacing SDL's queue.
-	// Keep both PCM storage and port handles alive until it returns.
-	Common::LockGuard lock(g_audioout2_state_mutex);
-	auto*             context = audioout2_find_context_locked(ctx);
+static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, AudioOut2Submission& out,
+                                          bool blocking) {
+	Common::LockGuard submit_lock(out.mutex);
+	g_audioout2_state_mutex.Lock();
+	auto* context = audioout2_find_context_locked(ctx);
 	if (context == nullptr) {
+		g_audioout2_state_mutex.Unlock();
 		return;
 	}
-	context->object_mix.clear();
+	out.object_mix.clear();
 	std::vector<AudioInternal::OutputParam> params;
 	params.reserve(AudioInternal::OUT_PORTS_MAX);
 	for (const auto& state: g_audioout2_ports) {
@@ -419,14 +432,18 @@ static void audioout2_queue_context_audio(AudioOut2ContextHandle ctx, bool block
 			continue;
 		}
 		if (audioout2_port_type_is_object(state.port_type)) {
-			audioout2_mix_object(*context, state);
+			audioout2_mix_object(out.object_mix, context->num_grains, state);
 		} else if (state.audio_handle > 0 && params.size() < AudioInternal::OUT_PORTS_MAX) {
-			params.push_back({state.audio_handle, state.pcm_data.data(), state.gains.data()});
+			const auto i = params.size();
+			out.pcm[i]   = state.pcm_data;
+			out.gains[i] = state.gains;
+			params.push_back({state.audio_handle, out.pcm[i].data(), out.gains[i].data()});
 		}
 	}
-	if (!context->object_mix.empty() && params.size() < AudioInternal::OUT_PORTS_MAX) {
-		params.push_back({context->object_audio_handle, context->object_mix.data()});
+	if (!out.object_mix.empty() && params.size() < AudioInternal::OUT_PORTS_MAX) {
+		params.push_back({context->object_audio_handle, out.object_mix.data()});
 	}
+	g_audioout2_state_mutex.Unlock();
 	if (!params.empty()) {
 		(void)AudioInternal::AudioOutOutputs(params.data(), static_cast<uint32_t>(params.size()),
 		                                     blocking);
@@ -528,7 +545,8 @@ int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) {
 	size_t audio_handles_num = 0;
 
 	g_audioout2_state_mutex.Lock();
-	if (auto* state = audioout2_find_context_locked(ctx); state != nullptr) {
+	auto* state = audioout2_find_context_locked(ctx);
+	if (state != nullptr) {
 		if (state->object_audio_handle > 0) {
 			audio_handles[audio_handles_num++] = state->object_audio_handle;
 		}
@@ -544,6 +562,10 @@ int KYTY_SYSV_ABI AudioOut2ContextDestroy(AudioOut2ContextHandle ctx) {
 	}
 	g_audioout2_state_mutex.Unlock();
 
+	if (state != nullptr) {
+		// Wait for an in-flight push that may still use the handles.
+		Common::LockGuard wait(audioout2_submission(*state).mutex);
+	}
 	for (size_t i = 0; i < audio_handles_num; i++) {
 		audioout2_close_audio_handle(audio_handles[i]);
 	}
@@ -595,8 +617,9 @@ int KYTY_SYSV_ABI AudioOut2ContextPush(AudioOut2ContextHandle ctx, uint32_t bloc
 				if (state->queued < state->queue_depth) {
 					state->queued++;
 				}
+				auto& submission = audioout2_submission(*state);
 				g_audioout2_state_mutex.Unlock();
-				audioout2_queue_context_audio(ctx, blocking != 0);
+				audioout2_queue_context_audio(ctx, submission, blocking != 0);
 				return OK;
 			}
 		}
@@ -729,14 +752,20 @@ int KYTY_SYSV_ABI AudioOut2PortDestroy(AudioOut2PortHandle port) {
 	PRINT_NAME();
 	LOGF("\t port = 0x%016" PRIx64 "\n", port);
 
-	int audio_handle = 0;
+	int                          audio_handle = 0;
+	const AudioOut2ContextState* context      = nullptr;
 	g_audioout2_state_mutex.Lock();
 	if (auto* state = audioout2_find_port_locked(port); state != nullptr) {
 		audio_handle = state->audio_handle;
+		context      = audioout2_find_context_locked(state->context);
 		*state       = AudioOut2PortStateEntry {};
 	}
 	g_audioout2_state_mutex.Unlock();
 
+	if (audio_handle > 0 && context != nullptr) {
+		// Wait for an in-flight push that may still use the handle.
+		Common::LockGuard wait(audioout2_submission(*context).mutex);
+	}
 	audioout2_close_audio_handle(audio_handle);
 
 	return OK;

src/libs/ngs2.cpp

@@ -541,7 +541,6 @@ struct Ngs2VoiceInternal {
 	struct Block {
 		const uint8_t*    data;
 		Ngs2WaveformBlock info;
-		uint32_t          cursor         = 0;
 		size_t            data_cursor    = 0;
 		uint32_t          repeated_count = 0;
 	};
@@ -595,10 +594,15 @@ struct Ngs2VoiceInternal {
 			return waveform_end;
 		}
 		const auto& block = blocks.front();
-		return block.data + (decoder != nullptr
-		                         ? block.data_cursor
-		                         : (uint64_t(block.info.num_skip_samples) + block.cursor) *
-		                               channels * sizeof(int16_t));
+		if (decoder != nullptr) {
+			return block.data + block.data_cursor;
+		}
+		// PCM never reads skipped samples, so the next read starts past a pending skip.
+		const auto skip = block.info.num_samples != 0 && block.data_cursor == 0
+		                      ? block.info.num_skip_samples
+		                      : skip_remaining;
+		return block.data + std::min(block.data_cursor + size_t(skip) * channels * sizeof(int16_t),
+		                             block.info.data_size);
 	}
 
 	// Custom racks keep num_buffers consecutive channel-major buffers; other racks keep one.
@@ -687,7 +691,6 @@ struct Ngs2VoiceInternal {
 		const bool clear     = (flags & 4u) != 0;
 		const bool only_data = (flags & 2u) != 0;
 		const bool join      = only_data && (flags & 0x10u) == 0;
-		EXIT_NOT_IMPLEMENTED(only_data && decoder == nullptr);
 		const auto retained = clear ? 0 : blocks.size();
 		Block      tail     = retained == 0 ? Block {} : blocks.back();
 		std::vector<Block> pending;
@@ -703,10 +706,6 @@ struct Ngs2VoiceInternal {
 				info.num_skip_samples = 0;
 				info.num_samples      = 0;
 			}
-			EXIT_NOT_IMPLEMENTED(decoder == nullptr &&
-			                     (uint64_t(info.num_skip_samples) + info.num_samples) *
-			                             channels * sizeof(int16_t) >
-			                         info.data_size);
 			Block block {data + info.data_offset, info};
 			auto* previous = pending.empty() ? (retained == 0 ? nullptr : &tail)
 			                                 : &pending.back();
@@ -1546,9 +1545,8 @@ static bool Ngs2FinishBlock(Ngs2VoiceInternal& voice) {
 			--block.info.num_repeats;
 		}
 		++block.repeated_count;
-		block.cursor = 0;
+		block.data_cursor = 0;
 		if (voice.decoder != nullptr) {
-			block.data_cursor = 0;
 			voice.compressed_input.clear();
 			voice.decoder->Reset();
 		}
@@ -1573,37 +1571,8 @@ static bool Ngs2FinishBlock(Ngs2VoiceInternal& voice) {
 	return voice.source_generation == generation;
 }
 
-static bool Ngs2DecodePcmFrame(Ngs2VoiceInternal& voice, std::vector<float>& frame,
-                               uint32_t& frame_samples) {
-	// PCM has no encoded frame size. Use one system grain as a bounded staging
-	// quantum, keeping source decoding separate from resampled output consumption.
-	const auto quantum = std::max(voice.rack->ngs->option.num_grain_samples, 1u);
-	frame.resize(size_t(quantum) * voice.channels);
-	frame_samples = 0;
-	while (voice.state == Ngs2VoicePlayState::Playing && !voice.blocks.empty() &&
-	       frame_samples < quantum) {
-		auto& block = voice.blocks.front();
-		const auto count = std::min(quantum - frame_samples, block.info.num_samples - block.cursor);
-		const auto* pcm = reinterpret_cast<const int16_t*>(voice.WaveformData());
-		for (size_t i = 0; i < size_t(count) * voice.channels; ++i) {
-			frame[size_t(frame_samples) * voice.channels + i] = pcm[i] / 32768.0f;
-		}
-		block.cursor += count;
-		frame_samples += count;
-		voice.decoded_samples += count;
-		voice.decoded_bytes += size_t(count) * voice.channels * sizeof(int16_t);
-		if (block.cursor == block.info.num_samples && !Ngs2FinishBlock(voice)) {
-			return false;
-		}
-	}
-	return frame_samples != 0;
-}
-
 static bool Ngs2DecodeFrame(Ngs2VoiceInternal& voice, std::vector<float>& frame,
                             uint32_t& frame_samples) {
-	if (voice.decoder == nullptr) {
-		return Ngs2DecodePcmFrame(voice, frame, frame_samples);
-	}
 	frame_samples = 0;
 	while (voice.state == Ngs2VoicePlayState::Playing && !voice.blocks.empty()) {
 		auto& block = voice.blocks.front();
@@ -1617,6 +1586,41 @@ static bool Ngs2DecodeFrame(Ngs2VoiceInternal& voice, std::vector<float>& frame,
 			}
 			continue;
 		}
+		if (voice.decoder == nullptr) {
+			// PCM has no encoded frame size, so stage at most one system grain.
+			const auto quantum     = std::max(voice.rack->ngs->option.num_grain_samples, 1u);
+			const auto frame_bytes = voice.channels * sizeof(int16_t);
+			const auto available   = (block.info.data_size - block.data_cursor) / frame_bytes;
+			const auto skip        = std::min<size_t>(voice.skip_remaining, available);
+			const auto count =
+			    std::min<size_t>({available - skip, voice.duration_remaining, quantum});
+			frame.resize(size_t(quantum) * voice.channels);
+			const auto* pcm = reinterpret_cast<const int16_t*>(block.data + block.data_cursor) +
+			                  skip * voice.channels;
+			for (size_t i = 0; i < count * voice.channels; ++i) {
+				frame[i] = pcm[i] / 32768.0f;
+			}
+			voice.skip_remaining -= skip;
+			voice.duration_remaining -= count;
+			block.data_cursor += (skip + count) * frame_bytes;
+			voice.decoded_bytes += (skip + count) * frame_bytes;
+			voice.decoded_samples += count;
+			frame_samples = static_cast<uint32_t>(count);
+			if (voice.duration_remaining == 0 ||
+			    block.info.data_size - block.data_cursor < frame_bytes) {
+				// A repeating block restarts its duration, so it must hold all of it.
+				EXIT_NOT_IMPLEMENTED(voice.duration_remaining != 0 &&
+				                     (block.data_cursor != block.info.data_size ||
+				                      block.info.num_repeats != 0));
+				if (!Ngs2FinishBlock(voice)) {
+					return false;
+				}
+			}
+			if (count != 0) {
+				return true;
+			}
+			continue;
+		}
 		const auto skip =
 		    std::min(voice.skip_remaining,
 		             static_cast<uint32_t>(frame.size() / voice.channels));

tests/ArchiveFileTests.cpp

@@ -12,6 +12,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
+#include <fstream>
 #include <limits>
 #include <string>
 #include <thread>
@@ -143,6 +144,72 @@ void CheckMalformedArchives(const std::filesystem::path &source,
         "recover by reading an unaffected compressed block");
 }
 
+void WriteHostFile(const std::filesystem::path &path, const void *data, size_t size) {
+  std::ofstream stream(path, std::ios::binary);
+  stream.write(static_cast<const char *>(data), static_cast<std::streamsize>(size));
+  Check(stream.good(), "write pack source file");
+}
+
+void CheckPackArchive(const std::filesystem::path &root, const std::vector<uint8_t> &payload) {
+  const auto source = root / "pack-source";
+  const auto unicode = Common::PathFromUtf8(ArchiveTests::UnicodeFilename);
+  std::error_code error;
+  Check(std::filesystem::create_directories(source / "assets/subdir", error) &&
+            std::filesystem::create_directories(source / "empty", error),
+        "create pack source tree");
+  WriteHostFile(source / "eboot.bin", ArchiveTests::Eboot.data(), ArchiveTests::Eboot.size());
+  WriteHostFile(source / "assets/subdir/data.bin", payload.data(), payload.size());
+  WriteHostFile(source / unicode, nullptr, 0);
+  uint64_t expected = ArchiveTests::Eboot.size() + payload.size();
+#if !defined(_WIN32)
+  std::filesystem::create_directory_symlink("assets", source / "linked", error);
+  Check(!error, "link a pack source directory");
+  expected += payload.size();
+#endif
+
+  const auto archive = root / "packed.zar";
+  uint64_t done = 0;
+  uint64_t total = 0;
+  Check(Common::PackArchive(source, archive,
+                            [&](uint64_t packed, uint64_t size) {
+                              done = packed;
+                              total = size;
+                              return true;
+                            }),
+        "pack a host directory");
+  Check(done == expected && total == expected, "report every packed byte");
+
+  const auto packed = Common::MakeArchivePath(archive);
+  const auto check_payload = [&](const std::filesystem::path &member) {
+    Common::File file(packed / member, Common::File::Mode::Read);
+    const auto data = file.ReadWholeBuffer();
+    Check(data.size() == payload.size() &&
+              std::memcmp(data.data(), payload.data(), payload.size()) == 0,
+          "round-trip packed file contents");
+  };
+  check_payload("assets/subdir/data.bin");
+#if !defined(_WIN32)
+  check_payload("linked/subdir/data.bin");
+#endif
+  const auto empty_file = Common::File::GetInfo(packed / unicode);
+  Check(empty_file && empty_file->is_file && empty_file->size == 0 &&
+            Common::File::IsDirectoryExisting(packed / "empty"),
+        "keep empty files and directories");
+
+  const auto failed = root / "failed.zar";
+  const auto rejected = [&](const std::filesystem::path &directory, bool proceed) {
+    return !Common::PackArchive(directory, failed, [=](uint64_t, uint64_t) { return proceed; }) &&
+           !std::filesystem::exists(failed, error);
+  };
+  Check(rejected(source, false), "cancelling removes the partial archive");
+  Check(rejected(root / "missing", true), "reject a missing source directory");
+#if defined(__linux__)
+  // Archive names compare case-insensitively, so such a folder cannot be packed faithfully.
+  WriteHostFile(source / "EBOOT.BIN", nullptr, 0);
+  Check(rejected(source, true), "reject names that differ only in case");
+#endif
+}
+
 #if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
 void CheckCancellation(const std::filesystem::path &path,
                        const std::vector<uint8_t> &payload) {
@@ -395,6 +462,8 @@ int main() {
             std::memcmp(replaced_data.data(), replacement.data(), replacement.size()) == 0,
         "reopening after the final reader closes sees replacement contents");
 
+  CheckPackArchive(temporary.Path(), payload);
+
   std::printf("ArchiveFileTests: all cases passed\n");
   return 0;
 }

tests/AudioOut2PortTests.cpp

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <array>
 #include <atomic>
+#include <chrono>
 #include <cmath>
 #include <condition_variable>
 #include <cstdint>
@@ -32,6 +33,8 @@ size_t                            g_capture_bytes = 0;
 int                               g_next_device  = 1;
 int                               g_open_waiters = 0;
 bool                              g_block_opens  = false;
+bool                              g_block_outputs = false;
+std::vector<int>                  g_blocked_output;
 
 void Check(bool value, const char* text) {
 	if (!value) {
@@ -190,6 +193,22 @@ void ReleaseDeviceOpens() {
 	g_device_cv.notify_all();
 }
 
+void BlockOutputs() {
+	std::lock_guard lock(g_device_mutex);
+	g_block_outputs = true;
+}
+
+void WaitForBlockedOutput() {
+	std::unique_lock lock(g_device_mutex);
+	g_device_cv.wait(lock, []() { return !g_blocked_output.empty(); });
+}
+
+void ReleaseOutputs() {
+	std::lock_guard lock(g_device_mutex);
+	g_block_outputs = false;
+	g_device_cv.notify_all();
+}
+
 int LiveDeviceCount() {
 	std::lock_guard lock(g_device_mutex);
 	return static_cast<int>(g_live_devices.size());
@@ -605,6 +624,46 @@ void TestObjectBusRouting() {
 	Check(LiveDeviceCount() == 0, "object bus device leaked");
 }
 
+void TestBlockedPushOnlyHoldsItsContext() {
+	const auto blocked      = CreateContext(4, 1);
+	const auto other        = CreateContext();
+	const auto blocked_port = CreatePort(blocked, MakeParam());
+	const auto other_port   = CreatePort(other, MakeParam());
+	auto       object_param = MakeParam(0x100);
+	object_param.port_type  = 0x100;
+	std::vector<float> pcm(512 * 2, 0.25f);
+	SetPcm(blocked_port, pcm.data());
+	SetPcm(CreatePort(blocked, object_param), pcm.data());
+	CaptureOutputPcm(pcm.size() * sizeof(float));
+
+	BlockOutputs();
+	std::thread pusher([&]() { Check(AudioOut2::AudioOut2ContextPush(blocked, 1) == OK, "blocked push failed"); });
+	WaitForBlockedOutput();
+	std::fill(pcm.begin(), pcm.end(), 0.5f);
+	SetPcm(blocked_port, pcm.data());
+	SetAttribute(blocked_port, 1, std::array {0.5f, 0.5f});
+	SetPcm(other_port, pcm.data());
+	Check(AudioOut2::AudioOut2ContextPush(other, 1) == OK, "push stalled behind another context");
+	// Give each destroyer time to close a device of the blocked output; the backend fails if it does.
+	std::thread port_destroyer([&]() { AudioOut2::AudioOut2PortDestroy(blocked_port); });
+	std::this_thread::sleep_for(std::chrono::milliseconds(20));
+	std::thread context_destroyer([&]() { AudioOut2::AudioOut2ContextDestroy(blocked); });
+	std::this_thread::sleep_for(std::chrono::milliseconds(20));
+	ReleaseOutputs();
+	pusher.join();
+	port_destroyer.join();
+	context_destroyer.join();
+
+	const auto output = OutputPcm();
+	Check(output.size() == 3, "pushes did not reach the backend once each");
+	CheckSamples(output[0], {0.5f}, "other context missed its PCM");
+	CheckSamples(output[1], {0.25f}, "PCM changed while its push was in flight");
+	Check(g_output_gains[0] == std::array {1.0f, 1.0f}, "gains changed while their push was in flight");
+	Check(LiveDeviceCount() == 1, "destroyed port or context kept its device");
+	CaptureOutputPcm(0);
+	AudioOut2::AudioOut2ContextDestroy(other);
+}
+
 } // namespace
 
 namespace Libs::Audio::AudioInternal {
@@ -625,6 +684,8 @@ int AudioOutOpen(int type, uint32_t /*samples_num*/, uint32_t /*freq*/, Format /
 void AudioOutClose(int handle) {
 	std::lock_guard lock(g_device_mutex);
 	const auto      it = std::find(g_live_devices.begin(), g_live_devices.end(), handle);
+	Check(std::find(g_blocked_output.begin(), g_blocked_output.end(), handle) == g_blocked_output.end(),
+	      "device closed during its output");
 	if (it != g_live_devices.end()) {
 		g_live_devices.erase(it);
 	}
@@ -642,8 +703,16 @@ bool AudioOutHasDevice(int handle) {
 }
 
 uint32_t AudioOutOutputs(const OutputParam* params, uint32_t num, bool blocking) {
-	std::lock_guard lock(g_device_mutex);
+	std::unique_lock lock(g_device_mutex);
 	g_output_blocking.push_back(blocking);
+	if (g_block_outputs && g_blocked_output.empty()) {
+		for (uint32_t i = 0; i < num; i++) {
+			g_blocked_output.push_back(params[i].handle);
+		}
+		g_device_cv.notify_all();
+		g_device_cv.wait(lock, []() { return !g_block_outputs; });
+		g_blocked_output.clear();
+	}
 	g_output_gains.clear();
 	if (g_capture_bytes != 0) {
 		for (uint32_t i = 0; i < num; i++) {
@@ -683,6 +752,7 @@ int main() {
 	TestPortGainAndValidation();
 	TestObjectGainAndAmbisonics();
 	TestObjectBusRouting();
+	TestBlockedPushOnlyHoldsItsContext();
 	Check(AudioOut2::AudioOut2UserDestroy(g_user_handle) == OK, "test user destroy failed");
 	std::printf("AudioOut2PortTests: all cases passed\n");
 	return 0;

tests/Ngs2SamplerTests.cpp

@@ -124,7 +124,7 @@ void TestWaveformQueueCapacity() {
 			for (uint32_t i = 0; i < capacity; ++i) {
 				Check(f.voice.blocks[i].data == reinterpret_cast<const uint8_t*>(pcm) &&
 				          f.voice.blocks[i].info.user_data == i + 1 &&
-				          f.voice.blocks[i].cursor == (i == 0 ? 1u : 0u),
+				          f.voice.blocks[i].data_cursor == (i == 0 ? sizeof(int16_t) : 0u),
 				      "rejected waveform request replaced an outstanding block");
 			}
 			f.Render(3);
@@ -846,13 +846,57 @@ void TestStarvationAndPause() {
 	f.Queue(pcm, 2, 0);
 	f.voice.SetEvent(16);
 	f.Render(2);
-	Check(!f.voice.has_samples && f.voice.blocks.front().cursor == 0, "paused voice consumed data");
+	Check(!f.voice.has_samples && f.voice.blocks.front().data_cursor == 0,
+	      "paused voice consumed data");
 	f.voice.SetEvent(32);
 	f.Render(2);
 	Check(f.voice.samples[0] == 0.5f && f.voice.samples[1] == 0.25f,
 	      "stream failed to resume after starvation/pause");
 }
 
+void TestPcmByteUnitStreaming() {
+	Fixture                 f(48000);
+	const int16_t           pcm[] = {0, 4096, 8192, 12288, 16384, 20480, 24576, 28672};
+	const Ngs2WaveformBlock first {0, 4, 0, 1, 5, 0, 0}, rest {4, 12, 0, 0, 99, 0, 0};
+	Check(f.QueueBlocks(pcm, &first, 1, 1) == OK && f.QueueBlocks(pcm, &rest, 1, 0x12) == OK &&
+	          f.voice.WaveformData() == reinterpret_cast<const uint8_t*>(pcm + 1),
+	      "byte-unit stream was rejected or its read address ignored the skipped frame");
+	f.Render(8);
+	Check(f.voice.samples == std::vector<float> {0.125f, 0.25f, 0.375f, 0.5f, 0.625f, 0, 0, 0} &&
+	          f.voice.blocks.empty() && f.voice.decoded_samples == 5,
+	      "byte-unit stream did not play exactly its duration across data-only blocks");
+}
+
+void TestPcmSkipAcrossBlocks() {
+	Fixture                 f(48000);
+	const int16_t           pcm[] = {1, 2, 3, 8192, 16384};
+	const Ngs2WaveformBlock first {0, 4, 0, 3, 2, 0, 0}, rest {4, 6, 0, 0, 0, 0, 0};
+	Check(f.QueueBlocks(pcm, &first, 1, 1) == OK, "skipping stream block was rejected");
+	f.Render(1);
+	Check(!f.voice.has_samples && f.voice.blocks.empty(), "skip-only block produced samples");
+	Check(f.QueueBlocks(pcm, &rest, 1, 2) == OK &&
+	          f.voice.WaveformData() == reinterpret_cast<const uint8_t*>(pcm + 3),
+	      "read address ignored the skip remaining from the previous block");
+	f.Render(2);
+	Check(f.voice.samples == std::vector<float> {0.25f, 0.5f} && f.voice.decoded_samples == 2 &&
+	          f.voice.decoded_bytes == sizeof(pcm),
+	      "skip spanning blocks played skipped frames or miscounted decoded data");
+}
+
+void TestPcmBlocksEndWithTheirData() {
+	// The Last Oricru queues blocks holding fewer frames than they declare, even none.
+	Fixture                 f(48000, 2);
+	const int16_t           pcm[] = {4096, -4096, 8192, -8192, 12288, -12288};
+	const Ngs2WaveformBlock blocks[] = {
+	    {0, 0, 0, 0, 16385, 0, 0}, {0, 8, 0, 0, 4, 0, 0}, {8, 4, 0, 0, 3, 0, 0}};
+	Check(f.QueueBlocks(pcm, blocks, 3, 0) == OK, "partially filled blocks were rejected");
+	f.Render(5);
+	Check(f.voice.samples == std::vector<float> {0.125f, 0.25f, 0.375f, 0, 0, -0.125f, -0.25f,
+	                                             -0.375f, 0, 0} &&
+	          f.voice.blocks.empty() && f.voice.decoded_samples == 3,
+	      "partially filled blocks did not play their data back to back");
+}
+
 void TestMonoRateAndRouting() {
 	Fixture              f(22050);
 	std::vector<int16_t> pcm(441, 16384);
@@ -930,7 +974,7 @@ void TestSamplerReuseResetsRouting() {
 		const int16_t pcm[] = {8192, 16384, -8192, -16384};
 		f.Queue(pcm, 2, 0);
 		f.Render(1);
-		Check(!f.voice.has_samples && f.voice.blocks.front().cursor == 0,
+		Check(!f.voice.has_samples && f.voice.blocks.front().data_cursor == 0,
 		      "sampler setup started the new waveform without a play event");
 		f.voice.ports[0].dest = &mastering;
 		f.voice.SetEvent(1);
@@ -1216,7 +1260,7 @@ void TestLoopCallbacksAndExit(Ngs2RackType type) {
 	active.Queue(pcm + 2, 1, 0);
 	active.Render(6);
 	Check(log.events.size() == 1 && active.voice.decoded_samples == 2 &&
-	          active.voice.blocks.front().cursor == 0 && active.voice.samples[2] == 0,
+	          active.voice.blocks.front().data_cursor == 0 && active.voice.samples[2] == 0,
 	      "loop callback pause consumed another iteration");
 	Ngs2VoiceParamHeader exit_loop {sizeof(Ngs2VoiceParamHeader), 0,
 	                                type == Ngs2RackType::Sampler ? 0x10000004u : 0x40010004u};
@@ -1423,7 +1467,7 @@ void TestCallbackStopsConsumption() {
 			      "completion callback failed to interrupt playback");
 			Check(f.voice.samples[0] == 0.0f && f.voice.samples[1] == 0.0f &&
 			          f.voice.decoded_samples == 1 && f.voice.blocks.size() == 1 &&
-			          f.voice.blocks.front().cursor == 0,
+			          f.voice.blocks.front().data_cursor == 0,
 			      "callback pause/stop consumed the next queued block");
 			if (event == 16) {
 				f.voice.SetEvent(32);
@@ -1523,7 +1567,7 @@ void TestCallbackReplacesSource() {
 				f.Render(4);
 				Check(!f.voice.has_samples && f.voice.samples.size() == (setup ? 4u : 8u) &&
 				          std::ranges::all_of(f.voice.samples, [](float value) { return value == 0; }) &&
-				          f.voice.blocks.size() == 1 && f.voice.blocks.front().cursor == 0,
+				          f.voice.blocks.size() == 1 && f.voice.blocks.front().data_cursor == 0,
 				      "callback source replacement used an old frame or channel layout");
 				f.Render(4);
 				const std::vector<float> expected = setup
@@ -1684,6 +1728,9 @@ int main() {
 	TestPitchLoopAndSkip();
 	TestPitchClamps();
 	TestStarvationAndPause();
+	TestPcmByteUnitStreaming();
+	TestPcmSkipAcrossBlocks();
+	TestPcmBlocksEndWithTheirData();
 	TestMonoRateAndRouting();
 	TestMatrixOutputChannels();
 	TestCustomPcmStillPlays();