launcher: compress game folders to ZAR

src/common/archive.h

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <filesystem>
+#include <functional>
 #include <memory>
 #include <vector>
 
@@ -45,6 +46,12 @@ private:
 // Accepts either a host archive filename or a virtual path inside the archive.
 [[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
 
+// Packs a host directory into a new ZArchive outside it. progress gets 0 < done <= total and
+// returns false to cancel. A failed or cancelled pack removes the partial archive.
+[[nodiscard]] bool PackArchive(const std::filesystem::path& directory,
+                               const std::filesystem::path& archive,
+                               const std::function<bool(uint64_t done, uint64_t total)>& progress);
+
 } // namespace Common
 
 #endif /* KYTY_COMMON_ARCHIVE_H_ */

src/common/archivePack.cpp

@@ -0,0 +1,84 @@
+// Kept out of archive.cpp so the statically linked emulator does not pull in the writer and the
+// zstd compressor.
+#include "common/archive.h"
+
+#include "common/stringUtils.h"
+
+#include <algorithm>
+#include <zarchive/zarchivewriter.h>
+
+namespace Common {
+
+bool PackArchive(const std::filesystem::path& directory, const std::filesystem::path& archive,
+                 const std::function<bool(uint64_t done, uint64_t total)>& progress) {
+	namespace fs = std::filesystem;
+	struct Output {
+		File file;
+		bool ok = true;
+	} output;
+	// Create the archive before walking, so a re-run started while a cancelled pack is still
+	// walking sees it exists instead of losing its archive to that pack's cleanup.
+	if (!output.file.Create(archive)) {
+		return false;
+	}
+	struct Entry {
+		fs::path path;
+		uint64_t size;
+		bool     is_file;
+	};
+	std::vector<Entry> entries;
+	uint64_t           total = 0;
+	std::error_code    error;
+	// Follow directory links, as the emulator does when it runs the folder.
+	fs::recursive_directory_iterator it(directory, fs::directory_options::follow_directory_symlink,
+	                                    error);
+	for (; !error && it != fs::end(it); it.increment(error)) {
+		const bool is_file = !it->is_directory(error);
+		const auto size    = is_file && !error ? it->file_size(error) : 0;
+		if (error) {
+			break;
+		}
+		entries.push_back({it->path(), size, is_file});
+		total += size;
+	}
+	output.ok = !error;
+	ZArchiveWriter writer(
+	    [](int32_t, void*) {},
+	    [](const void* data, size_t size, void* opaque) {
+		    auto&    out     = *static_cast<Output*>(opaque);
+		    uint32_t written = 0;
+		    out.file.Write(data, static_cast<uint32_t>(size), &written);
+		    out.ok = out.ok && written == size;
+	    },
+	    &output);
+	std::vector<uint8_t> buffer(1u << 20u);
+	uint64_t             done = 0;
+	for (size_t index = 0; output.ok && index < entries.size(); index++) {
+		const auto& entry = entries[index];
+		const auto  name  = PathToGenericString(entry.path.lexically_relative(directory));
+		File        input;
+		output.ok = entry.is_file ? writer.StartNewFile(name.c_str()) &&
+		                                input.Open(entry.path, File::Mode::Read)
+		                          : writer.MakeDir(name.c_str());
+		for (uint64_t left = entry.size; output.ok && left != 0;) {
+			uint32_t read = 0;
+			input.Read(buffer.data(), static_cast<uint32_t>(std::min<uint64_t>(left, buffer.size())),
+			           &read);
+			writer.AppendData(buffer.data(), read);
+			done += read;
+			left -= read;
+			output.ok = output.ok && read != 0 && progress(done, total);
+		}
+	}
+	if (output.ok) {
+		writer.Finalize();
+	}
+	output.ok = output.ok && output.file.Flush();
+	output.file.Close();
+	if (!output.ok) {
+		File::DeleteFile(archive);
+	}
+	return output.ok;
+}
+
+} // namespace Common

src/launcher/include/configurationListWidget.h

@@ -68,6 +68,7 @@ protected slots:
 	void list_itemDoubleClicked(QTreeWidgetItem* witem, int column);
 	void open_game_folder();
 	void remove_save_data();
+	void compress_game();
 	void filter_configurations(const QString& text);
 
 private:

src/launcher/src/configurationListWidget.cpp

@@ -27,6 +27,7 @@
 #include <QFile>
 #include <QFileDialog>
 #include <QFileInfo>
+#include <QFutureWatcher>
 #include <QHeaderView>
 #include <QIcon>
 #include <QInputDialog>
@@ -42,6 +43,8 @@
 #include <QPalette>
 #include <QPointer>
 #include <QProcess>
+#include <QProgressDialog>
+#include <QPromise>
 #include <QPushButton>
 #include <QRegularExpression>
 #include <QSaveFile>
@@ -54,6 +57,7 @@
 #include <QToolButton>
 #include <QTreeWidget>
 #include <QUrl>
+#include <QtConcurrentRun>
 #include <QtCore>
 
 #ifdef __linux__
@@ -62,6 +66,7 @@
 #include <QDBusPendingCallWatcher>
 #endif
 
+#include <atomic>
 #include <memory>
 #include <vector>
 
@@ -1171,6 +1176,71 @@ void ConfigurationListWidget::remove_save_data() {
 	}
 }
 
+void ConfigurationListWidget::compress_game() {
+	auto* item = static_cast<ConfigurationItem*>(m_ui->cfgs_list->currentItem());
+	if (item == nullptr) {
+		return;
+	}
+
+	const auto base    = item->GetInfo().basedir;
+	const auto archive = base + QStringLiteral(".zar");
+	const auto title   = tr("Compress to ZAR");
+	if (QFileInfo::exists(archive)) {
+		QMessageBox::warning(this, title,
+		                     tr("%1 already exists.").arg(QDir::toNativeSeparators(archive)));
+		return;
+	}
+
+	// Cancel through a flag, not the future, so a pack that already succeeded keeps its result.
+	auto  canceled = std::make_shared<std::atomic_bool>(false);
+	auto* progress = new QProgressDialog(
+	    tr("Compressing %1...").arg(QDir::toNativeSeparators(base)), tr("Cancel"), 0, 1000, this);
+	auto* watcher  = new QFutureWatcher<bool>(this);
+	// Stay modal through finalization and fsync, which follow the last progress step.
+	progress->setAutoReset(false);
+	connect(progress, &QProgressDialog::canceled, this, [canceled] { *canceled = true; });
+	connect(watcher, &QFutureWatcher<bool>::progressValueChanged, progress,
+	        &QProgressDialog::setValue);
+	connect(watcher, &QFutureWatcher<bool>::finished, this,
+	        [this, progress, watcher, canceled, base, archive, title] {
+		        // finished can arrive inside the modal dialog's setValue event processing.
+		        progress->deleteLater();
+		        watcher->deleteLater();
+		        if (!watcher->result()) {
+			        if (!*canceled) {
+				        QMessageBox::warning(
+				            this, title,
+				            tr("Could not compress %1.").arg(QDir::toNativeSeparators(base)));
+			        }
+			        return;
+		        }
+		        if (QMessageBox::Yes ==
+		                QMessageBox::question(this, title,
+		                                      tr("Created %1.\n\nDelete the original folder %2?")
+		                                          .arg(QDir::toNativeSeparators(archive),
+		                                               QDir::toNativeSeparators(base)),
+		                                      QMessageBox::Yes | QMessageBox::No, QMessageBox::No) &&
+		            !QDir(base).removeRecursively()) {
+			        QMessageBox::warning(
+			            this, title, tr("Could not remove %1.").arg(QDir::toNativeSeparators(base)));
+		        }
+		        ScanGameDirectory();
+	        });
+	progress->open();
+	watcher->setFuture(QtConcurrent::run(
+	    [directory = GameContent::ToPath(base), path = GameContent::ToPath(archive),
+	     canceled](QPromise<bool>& promise) {
+		    promise.setProgressRange(0, 1000);
+		    // A pack cancelled while queued must not touch a path a later pack may already own.
+		    promise.addResult(
+		        !*canceled &&
+		        Common::PackArchive(directory, path, [&](uint64_t done, uint64_t total) {
+			        promise.setProgressValue(static_cast<int>(done * 1000 / total));
+			        return !*canceled;
+		        }));
+	    }));
+}
+
 void ConfigurationListWidget::filter_configurations(const QString& text) {
 	const auto query              = text.trimmed();
 	const bool has_query          = !query.isEmpty();
@@ -1268,6 +1338,8 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		        }
 	        });
 	action_addcont->setEnabled(item != nullptr);
+	QAction* action_compress =
+	    menu.addAction(tr("Compress to ZAR..."), this, SLOT(compress_game()));
 	QAction* action_remove_save_data =
 	    menu.addAction(style()->standardIcon(QStyle::SP_DialogDiscardButton),
 	                   tr("Remove save data..."), this, SLOT(remove_save_data()));
@@ -1291,6 +1363,7 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		const auto& base = item->GetInfo().basedir;
 		action_open_folder->setDisabled(!QDir(base).exists() && !GameContent::IsArchive(base));
 		action_view_trophies->setDisabled(!has_trophy_data);
+		action_compress->setDisabled(item->IsRunning() || !QDir(base).exists());
 		action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty());
 		action_edit->setDisabled(item->IsRunning());
 		action_delete->setDisabled(item->IsRunning() || !item->GetInfo().custom_settings);
@@ -1298,6 +1371,7 @@ void ConfigurationListWidget::ShowContextMenu(QTreeWidgetItem* witem) {
 		action_run->setDisabled(true);
 		action_open_folder->setDisabled(true);
 		action_view_trophies->setDisabled(true);
+		action_compress->setDisabled(true);
 		action_remove_save_data->setDisabled(true);
 		action_edit->setDisabled(true);
 		action_delete->setDisabled(true);

tests/ArchiveFileTests.cpp

@@ -12,6 +12,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
+#include <fstream>
 #include <limits>
 #include <string>
 #include <thread>
@@ -143,6 +144,72 @@ void CheckMalformedArchives(const std::filesystem::path &source,
         "recover by reading an unaffected compressed block");
 }
 
+void WriteHostFile(const std::filesystem::path &path, const void *data, size_t size) {
+  std::ofstream stream(path, std::ios::binary);
+  stream.write(static_cast<const char *>(data), static_cast<std::streamsize>(size));
+  Check(stream.good(), "write pack source file");
+}
+
+void CheckPackArchive(const std::filesystem::path &root, const std::vector<uint8_t> &payload) {
+  const auto source = root / "pack-source";
+  const auto unicode = Common::PathFromUtf8(ArchiveTests::UnicodeFilename);
+  std::error_code error;
+  Check(std::filesystem::create_directories(source / "assets/subdir", error) &&
+            std::filesystem::create_directories(source / "empty", error),
+        "create pack source tree");
+  WriteHostFile(source / "eboot.bin", ArchiveTests::Eboot.data(), ArchiveTests::Eboot.size());
+  WriteHostFile(source / "assets/subdir/data.bin", payload.data(), payload.size());
+  WriteHostFile(source / unicode, nullptr, 0);
+  uint64_t expected = ArchiveTests::Eboot.size() + payload.size();
+#if !defined(_WIN32)
+  std::filesystem::create_directory_symlink("assets", source / "linked", error);
+  Check(!error, "link a pack source directory");
+  expected += payload.size();
+#endif
+
+  const auto archive = root / "packed.zar";
+  uint64_t done = 0;
+  uint64_t total = 0;
+  Check(Common::PackArchive(source, archive,
+                            [&](uint64_t packed, uint64_t size) {
+                              done = packed;
+                              total = size;
+                              return true;
+                            }),
+        "pack a host directory");
+  Check(done == expected && total == expected, "report every packed byte");
+
+  const auto packed = Common::MakeArchivePath(archive);
+  const auto check_payload = [&](const std::filesystem::path &member) {
+    Common::File file(packed / member, Common::File::Mode::Read);
+    const auto data = file.ReadWholeBuffer();
+    Check(data.size() == payload.size() &&
+              std::memcmp(data.data(), payload.data(), payload.size()) == 0,
+          "round-trip packed file contents");
+  };
+  check_payload("assets/subdir/data.bin");
+#if !defined(_WIN32)
+  check_payload("linked/subdir/data.bin");
+#endif
+  const auto empty_file = Common::File::GetInfo(packed / unicode);
+  Check(empty_file && empty_file->is_file && empty_file->size == 0 &&
+            Common::File::IsDirectoryExisting(packed / "empty"),
+        "keep empty files and directories");
+
+  const auto failed = root / "failed.zar";
+  const auto rejected = [&](const std::filesystem::path &directory, bool proceed) {
+    return !Common::PackArchive(directory, failed, [=](uint64_t, uint64_t) { return proceed; }) &&
+           !std::filesystem::exists(failed, error);
+  };
+  Check(rejected(source, false), "cancelling removes the partial archive");
+  Check(rejected(root / "missing", true), "reject a missing source directory");
+#if defined(__linux__)
+  // Archive names compare case-insensitively, so such a folder cannot be packed faithfully.
+  WriteHostFile(source / "EBOOT.BIN", nullptr, 0);
+  Check(rejected(source, true), "reject names that differ only in case");
+#endif
+}
+
 #if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
 void CheckCancellation(const std::filesystem::path &path,
                        const std::vector<uint8_t> &payload) {
@@ -395,6 +462,8 @@ int main() {
             std::memcmp(replaced_data.data(), replacement.data(), replacement.size()) == 0,
         "reopening after the final reader closes sees replacement contents");
 
+  CheckPackArchive(temporary.Path(), payload);
+
   std::printf("ArchiveFileTests: all cases passed\n");
   return 0;
 }