Zarchive Loading (#724)

* Simplify archive loading and harden filesystem integration

* Restore directory entry name length assertion

* Keep graphics-dependent filesystem tests out of CI

---------

Co-authored-by: nmzik <Nmzik@mail.ru>
This commit is contained in:
Aspen
2026-09-28 16:06:44 -04:00
committed by GitHub
parent 8e61798bfd
commit 4b0de1b672
29 changed files with 1956 additions and 262 deletions
+1
View File
@@ -26,6 +26,7 @@ target_link_libraries(common
cpuinfo
kyty_git_version
spdlog::spdlog
ZArchive::zarchive
)
get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES)
+397
View File
@@ -0,0 +1,397 @@
#include "common/archive.h"
#include "common/stringUtils.h"
#include <algorithm>
#include <condition_variable>
#include <cstring>
#include <mutex>
#include <optional>
#include <string_view>
#include <thread>
#include <unordered_map>
#include <utility>
#include <zarchive/zarchivereader.h>
// POSIX hosts and MinGW's pthread-backed libstdc++ have cancellable condition waits.
// MSVC/ClangCL and MinGW's native Win32 threading backend do not use pthread waits.
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
#include <pthread.h>
#define KYTY_ARCHIVE_PTHREAD_CANCELLATION
#endif
namespace Common {
// Backends expose immutable metadata and offset-based reads. Cursor management, path routing,
// reader lifetime and the host-stack worker are shared by every archive format.
class ArchiveReader {
public:
struct Entry {
uint64_t id;
uint64_t size;
bool is_file;
};
virtual ~ArchiveReader() = default;
virtual std::optional<Entry> Find(std::string_view member) = 0;
virtual std::vector<File::DirEntry> List(std::string_view member) = 0;
virtual uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) = 0;
};
namespace {
class ZArchiveReaderBackend final: public ArchiveReader {
public:
explicit ZArchiveReaderBackend(std::unique_ptr<ZArchiveReader> reader)
: m_reader(std::move(reader)) {}
std::optional<Entry> Find(std::string_view member) override {
const auto node = m_reader->LookUp(member);
if (node == ZARCHIVE_INVALID_NODE) {
return {};
}
return Entry {node, m_reader->GetFileSize(node), m_reader->IsFile(node)};
}
std::vector<File::DirEntry> List(std::string_view member) override {
const auto node = m_reader->LookUp(member);
if (!m_reader->IsDirectory(node)) {
return {};
}
const auto count = m_reader->GetDirEntryCount(node);
if (count > (1u << 20u)) {
return {};
}
std::vector<File::DirEntry> result;
result.reserve(count);
for (uint32_t index = 0; index < count; ++index) {
ZArchiveReader::DirEntry entry {};
if (m_reader->GetDirEntry(node, index, entry)) {
result.push_back({std::string(entry.name), entry.isFile});
}
}
return result;
}
uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) override {
// ZArchiveReader already serializes access to its stream and decompression cache.
return m_reader->ReadFromFile(static_cast<ZArchiveNodeHandle>(id), offset, size, data);
}
private:
std::unique_ptr<ZArchiveReader> m_reader;
};
std::shared_ptr<ArchiveReader> OpenZArchive(const std::filesystem::path& path) {
auto reader = std::unique_ptr<ZArchiveReader>(ZArchiveReader::OpenFromFile(path));
return reader ? std::make_shared<ZArchiveReaderBackend>(std::move(reader)) : nullptr;
}
struct ArchiveFormat {
std::string_view extension;
std::shared_ptr<ArchiveReader> (*open)(const std::filesystem::path&);
};
constexpr ArchiveFormat Formats[] = {{".zar", OpenZArchive}};
using NativeView = std::basic_string_view<std::filesystem::path::value_type>;
const ArchiveFormat* FindFormat(NativeView path) {
for (const auto& format: Formats) {
if (path.size() < format.extension.size()) {
continue;
}
const auto extension = path.substr(path.size() - format.extension.size());
if (std::equal(
extension.begin(), extension.end(), format.extension.begin(),
[](auto a, char b) { return (a >= 'A' && a <= 'Z' ? a + ('a' - 'A') : a) == b; })) {
return &format;
}
}
return nullptr;
}
size_t FindMarker(NativeView path) {
// Native files use this check on every operation. The usual path has no marker and allocates
// nothing.
for (auto marker = path.find('!'); marker != NativeView::npos;
marker = path.find('!', marker + 1)) {
if ((marker + 1 == path.size() || path[marker + 1] == '/' || path[marker + 1] == '\\') &&
FindFormat(path.substr(0, marker)) != nullptr) {
return marker;
}
}
return NativeView::npos;
}
struct ParsedPath {
std::filesystem::path archive;
std::string member;
};
std::optional<ParsedPath> ParsePath(const std::filesystem::path& path) {
const auto& native = path.native();
const auto marker = FindMarker(native);
if (marker == NativeView::npos) {
return {};
}
ParsedPath parsed {.archive = std::filesystem::path(native.substr(0, marker))};
auto member = PathToGenericString(std::filesystem::path(native.substr(marker + 1)));
std::replace(member.begin(), member.end(), '\\', '/');
std::string_view remaining = member;
while (!remaining.empty()) {
const auto separator = remaining.find('/');
const auto component = remaining.substr(0, separator);
remaining.remove_prefix(separator == std::string_view::npos ? remaining.size() : separator + 1);
if (component.empty() || component == ".") {
continue;
}
if (component == "..") {
if (parsed.member.empty()) {
return {};
}
const auto separator = parsed.member.rfind('/');
parsed.member.resize(separator == std::string::npos ? 0 : separator);
} else {
if (!parsed.member.empty()) {
parsed.member += '/';
}
parsed.member += component;
}
}
return parsed;
}
std::mutex g_readers_mutex;
std::unordered_map<std::filesystem::path, std::weak_ptr<ArchiveReader>> g_readers;
// Guest stacks can be only 64 KiB. Decompress on an ordinary host stack, then copy into guest
// memory on the caller so GPU-tracked writes use its fault handler. A single worker bounds host
// threads and avoids duplicating the archive's own block cache. Requests live on waiting callers.
class ArchiveIoThread {
public:
static ArchiveIoThread& Instance() {
// Guest exit can run static destructors while other guest threads are still reading.
// Keep the worker alive until process termination, as with the guest threads themselves.
static auto* instance = new ArchiveIoThread;
return *instance;
}
uint64_t Read(ArchiveReader& reader, uint64_t id, uint64_t offset, uint32_t size, void* data) {
Request request {reader, id, offset, size, data};
std::unique_lock lock(m_mutex);
if (m_last != nullptr) {
m_last->next = &request;
} else {
m_first = &request;
}
m_last = &request;
m_wake.notify_one();
request.ready.wait(lock, [&] { return request.done; });
return request.result;
}
KYTY_CLASS_NO_COPY(ArchiveIoThread);
private:
struct Request {
ArchiveReader& reader;
uint64_t id;
uint64_t offset;
uint32_t size;
void* data;
Request* next = nullptr;
uint64_t result = 0;
bool done = false;
std::condition_variable ready {};
};
ArchiveIoThread() {
std::thread([this] { Loop(); }).detach();
}
void Loop() {
for (;;) {
Request* request;
{
std::unique_lock lock(m_mutex);
m_wake.wait(lock, [this] { return m_first != nullptr; });
request = m_first;
m_first = request->next;
if (m_first == nullptr) {
m_last = nullptr;
}
}
const auto result =
request->reader.Read(request->id, request->offset, request->size, request->data);
{
std::lock_guard lock(m_mutex);
request->result = result;
request->done = true;
// Notify before releasing the mutex: the waiting caller owns the condition
// variable.
request->ready.notify_one();
}
}
}
std::mutex m_mutex;
std::condition_variable m_wake;
Request* m_first = nullptr;
Request* m_last = nullptr;
};
} // namespace
struct ArchiveFile::Private {
std::shared_ptr<ArchiveReader> reader;
ArchiveReader::Entry entry;
uint64_t position = 0;
};
ArchiveFile::ArchiveFile(std::unique_ptr<Private> p): m_p(std::move(p)) {}
ArchiveFile::~ArchiveFile() = default;
uint64_t ArchiveFile::Size() const {
return m_p->entry.size;
}
uint64_t ArchiveFile::Tell() const {
return m_p->position;
}
bool ArchiveFile::Seek(uint64_t offset) {
m_p->position = offset;
return true;
}
void ArchiveFile::Read(void* data, uint32_t size, uint32_t* bytes_read) {
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
// A pending cancellation must not destroy the waiting request or its TLS buffer while the
// worker still uses them. Restore the caller's state only after the copy is complete.
int cancellation_state = PTHREAD_CANCEL_ENABLE;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancellation_state);
#endif
constexpr uint32_t chunk_size = 1u << 20u;
thread_local std::vector<uint8_t> chunk;
uint32_t read = 0;
if (data != nullptr && size != 0 && m_p->position < Size()) {
const auto requested =
static_cast<uint32_t>(std::min<uint64_t>(size, Size() - m_p->position));
if (chunk.size() < std::min(chunk_size, requested)) {
chunk.resize(std::min(chunk_size, requested));
}
while (read < requested) {
const auto step = std::min(chunk_size, requested - read);
const auto got = ArchiveIoThread::Instance().Read(*m_p->reader, m_p->entry.id,
m_p->position, step, chunk.data());
if (got == 0 || got > step) {
break;
}
std::memcpy(static_cast<uint8_t*>(data) + read, chunk.data(), got);
read += static_cast<uint32_t>(got);
m_p->position += got;
if (got < step) {
break;
}
}
}
if (bytes_read != nullptr) {
*bytes_read = read;
}
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
pthread_setcancelstate(cancellation_state, nullptr);
#endif
}
std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
const std::filesystem::path& member) {
auto root = archive;
root += "!";
return member.empty() ? root : root / member.relative_path();
}
bool IsSupportedArchive(const std::filesystem::path& path) {
return FindFormat(path.native()) != nullptr;
}
bool IsArchivePath(const std::filesystem::path& path) {
return FindMarker(path.native()) != NativeView::npos;
}
std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path) {
const auto marker = FindMarker(path.native());
return marker == NativeView::npos ? std::filesystem::path {}
: std::filesystem::path(path.native().substr(0, marker));
}
std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path) {
const auto archive = IsArchivePath(path) ? GetArchiveHostPath(path) : path;
const auto* format = FindFormat(archive.native());
if (format == nullptr) {
return {};
}
std::error_code error;
auto key = std::filesystem::absolute(archive, error);
if (error) {
return {};
}
{
std::lock_guard lock(g_readers_mutex);
if (auto it = g_readers.find(key); it != g_readers.end()) {
if (auto reader = it->second.lock()) {
return reader;
}
}
}
// Do not hold the cache mutex while opening an unrelated archive from disk.
auto reader = format->open(archive);
if (reader == nullptr) {
return {};
}
std::lock_guard lock(g_readers_mutex);
std::erase_if(g_readers, [](const auto& entry) { return entry.second.expired(); });
auto& cached = g_readers[key];
if (auto existing = cached.lock()) {
return existing;
}
cached = reader;
return reader;
}
std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path) {
const auto parsed = ParsePath(path);
if (parsed) {
if (auto reader = OpenArchive(parsed->archive)) {
if (auto entry = reader->Find(parsed->member)) {
return File::Info {entry->is_file, entry->size};
}
}
}
return {};
}
std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path) {
const auto parsed = ParsePath(path);
if (parsed) {
if (auto reader = OpenArchive(parsed->archive)) {
return reader->List(parsed->member);
}
}
return {};
}
std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path) {
auto parsed = ParsePath(path);
if (!parsed) {
return {};
}
auto reader = OpenArchive(parsed->archive);
const auto entry = reader ? reader->Find(parsed->member) : std::nullopt;
if (!entry || !entry->is_file) {
return {};
}
auto p = std::make_unique<ArchiveFile::Private>();
p->reader = std::move(reader);
p->entry = *entry;
return std::unique_ptr<ArchiveFile>(new ArchiveFile(std::move(p)));
}
} // namespace Common
+50
View File
@@ -0,0 +1,50 @@
#ifndef KYTY_COMMON_ARCHIVE_H_
#define KYTY_COMMON_ARCHIVE_H_
#include "common/file.h"
#include <cstdint>
#include <filesystem>
#include <memory>
#include <vector>
namespace Common {
class ArchiveReader;
class ArchiveFile {
public:
~ArchiveFile();
[[nodiscard]] uint64_t Size() const;
[[nodiscard]] uint64_t Tell() const;
bool Seek(uint64_t offset);
void Read(void* data, uint32_t size, uint32_t* bytes_read);
KYTY_CLASS_NO_COPY(ArchiveFile);
private:
struct Private;
explicit ArchiveFile(std::unique_ptr<Private> p);
std::unique_ptr<Private> m_p;
friend std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
};
// Virtual paths use "archive.ext!/member" and retain native host-path encoding.
[[nodiscard]] std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
const std::filesystem::path& member = {});
[[nodiscard]] bool IsSupportedArchive(const std::filesystem::path& path);
[[nodiscard]] bool IsArchivePath(const std::filesystem::path& path);
[[nodiscard]] std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path);
[[nodiscard]] std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path);
[[nodiscard]] std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path);
[[nodiscard]] std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
// Retain this handle while a mount or metadata scan is active to reuse the index and block cache.
// Accepts either a host archive filename or a virtual path inside the archive.
[[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
} // namespace Common
#endif /* KYTY_COMMON_ARCHIVE_H_ */
+140 -17
View File
@@ -1,5 +1,6 @@
#include "common/file.h"
#include "common/archive.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/platform/sysFileIO.h"
@@ -61,7 +62,8 @@ SysFileTimeStruct DateTimeToFileTimeUtc(const DateTime& date_time) {
} // namespace
struct File::FilePrivate {
sys_file_t* f;
sys_file_t* f;
std::unique_ptr<ArchiveFile> archive;
};
File::File(): m_p(std::make_unique<FilePrivate>()) {
@@ -85,11 +87,15 @@ File::File(const std::filesystem::path& name, Mode mode): m_p(std::make_unique<F
}
bool File::IsInvalid() const {
return m_p->f == nullptr;
return m_p->f == nullptr && m_p->archive == nullptr;
}
bool File::Create(const std::filesystem::path& name) {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
if (IsArchivePath(name)) {
return false;
}
m_file_name = name;
@@ -104,9 +110,16 @@ bool File::Create(const std::filesystem::path& name) {
}
bool File::Open(const std::filesystem::path& name, Mode mode) {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
m_file_name = name;
if (IsArchivePath(name)) {
if (mode != Mode::Read) {
return false;
}
m_p->archive = OpenArchiveFile(name);
return m_p->archive != nullptr;
}
switch (mode) {
case Mode::Read: m_p->f = SysFileOpenR(name); break;
@@ -124,7 +137,7 @@ bool File::Open(const std::filesystem::path& name, Mode mode) {
}
bool File::OpenInMem(void* buf, uint32_t buf_size) {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
m_p->f = SysFileOpen(static_cast<uint8_t*>(buf), buf_size);
@@ -137,7 +150,7 @@ bool File::OpenInMem(void* buf, uint32_t buf_size) {
}
bool File::CreateInMem() {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
m_p->f = SysFileCreate();
@@ -150,6 +163,7 @@ bool File::CreateInMem() {
}
void File::Close() {
m_p->archive.reset();
if (m_p->f != nullptr) {
SysFileClose(m_p->f);
m_p->f = nullptr;
@@ -157,7 +171,11 @@ void File::Close() {
}
uint64_t File::Size() const {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Size();
}
if (m_p->f == nullptr) {
return 0;
@@ -167,41 +185,76 @@ uint64_t File::Size() const {
}
uint64_t File::Remaining() const {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
return Size() - Tell();
const auto size = Size();
const auto position = Tell();
return position < size ? size - position : 0;
}
uint64_t File::Size(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
const auto info = GetArchiveInfo(name);
return info ? info->size : 0;
}
return SysFileSize(name);
}
std::optional<File::Info> File::GetInfo(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetArchiveInfo(name);
}
Info info {};
return SysFileGetInfo(name, &info.is_file, &info.size) ? std::optional(info) : std::nullopt;
}
bool File::Seek(uint64_t offset) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Seek(offset);
}
return SysFileSeek(*m_p->f, offset);
}
bool File::Truncate(uint64_t size) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return false;
}
return SysFileTruncate(*m_p->f, size);
}
bool File::Unlink() {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return false;
}
return SysFileUnlink(*m_p->f, m_file_name);
}
uint64_t File::Tell() const {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Tell();
}
return SysFileTell(*m_p->f);
}
void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
m_p->archive->Read(data, size, bytes_read);
return;
}
if (m_p->f != nullptr) {
SysFileRead(data, size, *m_p->f, bytes_read);
@@ -209,7 +262,14 @@ void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
}
void File::Write(const void* data, uint32_t size, uint32_t* bytes_written) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
if (bytes_written != nullptr) {
*bytes_written = 0;
}
return;
}
SysFileWrite(data, size, *m_p->f, bytes_written);
}
@@ -257,24 +317,41 @@ static std::filesystem::path WithoutTrailingSeparator(
bool File::IsDirectoryExisting(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
const auto info = GetArchiveInfo(path);
return info && !info->is_file;
}
return SysFileIsDirectoryExisting(WithoutTrailingSeparator(path));
}
bool File::IsFileExisting(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
const auto info = GetArchiveInfo(name);
return info && info->is_file;
}
return SysFileIsFileExisting(name);
}
bool File::CreateDirectory(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
return SysFileCreateDirectory(
WithoutTrailingSeparator(path));
}
bool File::DeleteDirectory(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
return SysFileDeleteDirectory(
WithoutTrailingSeparator(path));
}
bool File::CreateDirectories(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
const auto normalized_path =
WithoutTrailingSeparator(path);
@@ -297,6 +374,9 @@ bool File::CreateDirectories(const std::filesystem::path& path) {
}
bool File::DeleteDirectories(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
const auto normalized_path =
WithoutTrailingSeparator(path);
@@ -322,11 +402,18 @@ bool File::DeleteDirectories(const std::filesystem::path& path) {
bool File::DeleteFile(
const std::filesystem::path& name) // @suppress("Member declaration not found")
{
if (IsArchivePath(name)) {
return false;
}
return SysFileDeleteFile(name);
}
bool File::Flush() {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return true;
}
return SysFileFlush(*m_p->f);
}
@@ -348,6 +435,9 @@ std::vector<std::byte> File::ReadWholeBuffer() {
}
DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetLastAccessTimeUTC(GetArchiveHostPath(name));
}
SysTimeStruct t {};
SysFileToSystemTimeUtc(SysFileGetLastAccessTimeUtc(name), t);
@@ -359,6 +449,9 @@ DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
}
DateTime File::GetLastWriteTimeUTC(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetLastWriteTimeUTC(GetArchiveHostPath(name));
}
SysTimeStruct t {};
SysFileToSystemTimeUtc(SysFileGetLastWriteTimeUtc(name), t);
@@ -373,6 +466,10 @@ void File::GetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, DateT
DateTime* write) {
EXIT_IF(access == nullptr);
EXIT_IF(write == nullptr);
if (IsArchivePath(name)) {
GetLastAccessAndWriteTimeUTC(GetArchiveHostPath(name), access, write);
return;
}
SysTimeStruct at {};
SysTimeStruct wt {};
@@ -399,7 +496,12 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
EXIT_IF(access == nullptr);
EXIT_IF(write == nullptr);
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
GetLastAccessAndWriteTimeUTC(m_file_name, access, write);
return;
}
if (m_p->f == nullptr) {
*access = DateTime();
@@ -429,23 +531,35 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
}
bool File::SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
if (IsArchivePath(name)) {
return false;
}
auto f = DateTimeToFileTimeUtc(dt);
return SysFileSetLastAccessTimeUtc(name, f);
}
bool File::SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
if (IsArchivePath(name)) {
return false;
}
auto f = DateTimeToFileTimeUtc(dt);
return SysFileSetLastWriteTimeUtc(name, f);
}
bool File::SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, const DateTime& access,
const DateTime& write) {
if (IsArchivePath(name)) {
return false;
}
auto af = DateTimeToFileTimeUtc(access);
auto wf = DateTimeToFileTimeUtc(write);
return SysFileSetLastAccessAndWriteTimeUtc(name, af, wf);
}
std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return GetArchiveDirEntries(path);
}
std::vector<sys_dir_entry_t> files;
SysFileGetDents(path, files);
@@ -463,12 +577,18 @@ std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& pat
bool File::CopyFile(const std::filesystem::path& src,
const std::filesystem::path& dst) // @suppress("Member declaration not found")
{
if (IsArchivePath(src) || IsArchivePath(dst)) {
return false;
}
return SysFileCopyFile(src, dst);
}
bool File::RenameFile(const std::filesystem::path& src,
const std::filesystem::path& dst) // @suppress("Member declaration not found")
{
if (IsArchivePath(src) || IsArchivePath(dst)) {
return false;
}
if (IsFileExisting(dst)) {
DeleteFile(dst); // @suppress("Invalid arguments")
}
@@ -477,6 +597,9 @@ bool File::RenameFile(const std::filesystem::path& src,
}
void File::RemoveReadonly(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return;
}
SysFileRemoveReadonly(name);
}
+6
View File
@@ -7,6 +7,7 @@
#include <cstddef>
#include <filesystem>
#include <memory>
#include <optional>
#include <string>
#include <vector>
@@ -34,6 +35,10 @@ public:
std::string name;
bool is_file;
};
struct Info {
bool is_file;
uint64_t size;
};
File();
explicit File(const std::filesystem::path& name);
@@ -72,6 +77,7 @@ public:
std::vector<std::byte> ReadWholeBuffer();
static uint64_t Size(const std::filesystem::path& name);
static std::optional<Info> GetInfo(const std::filesystem::path& name);
static bool IsDirectoryExisting(const std::filesystem::path& path);
static bool IsFileExisting(const std::filesystem::path& name);
+1
View File
@@ -45,6 +45,7 @@ bool SysFileTruncate(sys_file_t& f, uint64_t size); // NOLINT(google-runt
bool SysFileUnlink(sys_file_t& f,
const std::filesystem::path& name); // NOLINT(google-runtime-references)
uint64_t SysFileSize(const std::filesystem::path& file_name);
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size);
bool SysFileIsError(sys_file_t& f); // NOLINT(google-runtime-references)
bool SysFileIsDirectoryExisting(const std::filesystem::path& path);
bool SysFileIsFileExisting(const std::filesystem::path& name);
+20 -23
View File
@@ -297,10 +297,21 @@ uint64_t SysFileSize(sys_file_t& f) {
return 0;
}
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
const auto path = get_internal_name(name);
struct stat info {};
if (stat(path.c_str(), &info) != 0) {
return false;
}
*is_file = !S_ISDIR(info.st_mode);
*size = *is_file ? static_cast<uint64_t>(info.st_size) : 0;
return true;
}
uint64_t SysFileSize(const std::filesystem::path& file_name) {
sys_file_t* f = SysFileOpenR(file_name);
uint64_t size = SysFileSize(*f);
SysFileClose(f);
bool is_file;
uint64_t size = 0;
SysFileGetInfo(file_name, &is_file, &size);
return size;
}
@@ -353,29 +364,15 @@ bool SysFileIsError(sys_file_t& f) {
}
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
auto real_name = get_internal_name(path);
auto real_name_str = real_name.string();
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return false;
}
return S_ISDIR(s.st_mode); // NOLINT
bool is_file;
uint64_t size;
return SysFileGetInfo(path, &is_file, &size) && !is_file;
}
bool SysFileIsFileExisting(const std::filesystem::path& name) {
auto real_name = get_internal_name(name);
auto real_name_str = real_name.string();
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return false;
}
return !S_ISDIR(s.st_mode); // NOLINT
bool is_file;
uint64_t size;
return SysFileGetInfo(name, &is_file, &size) && is_file;
}
bool SysFileCreateDirectory(const std::filesystem::path& path) {
+25 -19
View File
@@ -266,19 +266,27 @@ uint64_t SysFileSize(sys_file_t& f) {
return 0;
}
uint64_t SysFileSize(const std::filesystem::path& file_name) {
LARGE_INTEGER s;
WIN32_FILE_ATTRIBUTE_DATA a;
auto wide = file_name.wstring();
if (GetFileAttributesExW(wide.c_str(), GetFileExInfoStandard, &a) == 0) {
return 0;
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
const bool directory_path = !name.empty() && !name.has_filename() && name != name.root_path();
WIN32_FILE_ATTRIBUTE_DATA info {};
if (GetFileAttributesExW(directory_path ? name.parent_path().c_str() : name.c_str(),
GetFileExInfoStandard, &info) == 0) {
return false;
}
const bool file = (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
if (directory_path && file) {
return false;
}
*is_file = file;
*size = *is_file ? ((static_cast<uint64_t>(info.nFileSizeHigh) << 32u) | info.nFileSizeLow) : 0;
return true;
}
s.HighPart = static_cast<LONG>(a.nFileSizeHigh);
s.LowPart = a.nFileSizeLow;
return s.QuadPart;
uint64_t SysFileSize(const std::filesystem::path& file_name) {
bool is_file;
uint64_t size = 0;
SysFileGetInfo(file_name, &is_file, &size);
return size;
}
bool SysFileTruncate(sys_file_t& f, uint64_t size) {
@@ -346,17 +354,15 @@ bool SysFileIsError(sys_file_t& f) {
}
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
auto wide = path.wstring();
DWORD a = GetFileAttributesW(wide.c_str());
return a != INVALID_FILE_ATTRIBUTES &&
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) != 0u);
bool is_file;
uint64_t size;
return SysFileGetInfo(path, &is_file, &size) && !is_file;
}
bool SysFileIsFileExisting(const std::filesystem::path& name) {
auto wide = name.wstring();
DWORD a = GetFileAttributesW(wide.c_str());
return a != INVALID_FILE_ATTRIBUTES &&
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) == 0u);
bool is_file;
uint64_t size;
return SysFileGetInfo(name, &is_file, &size) && is_file;
}
bool SysFileCreateDirectory(const std::filesystem::path& path) {
+33 -12
View File
@@ -1,5 +1,6 @@
#include "kernel/fileSystem.h"
#include "common/archive.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/dateTime.h"
@@ -41,6 +42,7 @@ public:
struct MountPair {
std::filesystem::path dir;
std::string point;
std::shared_ptr<Common::ArchiveReader> archive;
};
MountPoints() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); }
@@ -260,6 +262,7 @@ void MountPoints::Mount(const std::filesystem::path& folder, const std::string&
MountPair p;
p.dir = folder;
p.point = point_str;
p.archive = Common::OpenArchive(folder);
m_mount_pairs.push_back(std::move(p));
}
@@ -346,11 +349,13 @@ std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name)
}
const auto native_rel_path = Common::PathFromUtf8(rel_path);
if (Common::IsArchivePath(p.dir)) {
return p.dir / native_rel_path;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (HasWindowsForbiddenFilenameCharacter(rel_path)) {
::printf("FileSystem: Windows-incompatible guest filename: %s\n",
mounted_name.c_str());
::printf("FileSystem: Windows-incompatible guest filename: %s\n", mounted_name.c_str());
}
return p.dir / native_rel_path;
#else
@@ -466,14 +471,20 @@ int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_ENOENT;
}
if (Common::IsArchivePath(file->real_name) &&
(rw_mode != Common::File::Mode::Read || trunc || creat)) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_EROFS;
}
if (trunc && rw_mode == Common::File::Mode::Read) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_EACCES;
}
bool dir_exist = Common::File::IsDirectoryExisting(file->real_name);
bool file_exist = Common::File::IsFileExisting(file->real_name);
const auto info = Common::File::GetInfo(file->real_name);
const bool dir_exist = info && !info->is_file;
const bool file_exist = info && info->is_file;
// A missing path opened without O_CREAT is ENOENT
if (!creat && !dir_exist && !file_exist) {
@@ -671,7 +682,7 @@ int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) {
auto* file = g_files->GetFile(d);
if (file == nullptr) {
if (file == nullptr || !file->opened || !file->writable) {
return KERNEL_ERROR_EBADF;
}
@@ -886,7 +897,7 @@ int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_
auto* file = g_files->GetFile(d);
if (file == nullptr) {
if (file == nullptr || !file->opened || !file->writable) {
return KERNEL_ERROR_EBADF;
}
@@ -1063,15 +1074,13 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
auto real_file_name = g_mount_points->ResolvePath(path);
bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
bool is_file = Common::File::IsFileExisting(real_file_name);
if (!is_dir && !is_file) {
const auto info = Common::File::GetInfo(real_file_name);
if (!info) {
LOGF("\t file not found\n");
return KERNEL_ERROR_ENOENT;
}
EXIT_NOT_IMPLEMENTED(is_dir && is_file);
const bool is_dir = !info->is_file;
FileStat stat {};
stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u);
@@ -1085,7 +1094,7 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
stat.st_blksize = 512;
stat.st_blocks = stat.st_size / 512;
} else {
stat.st_size = static_cast<int64_t>(Common::File::Size(real_file_name));
stat.st_size = static_cast<int64_t>(info->size);
stat.st_blksize = 512;
stat.st_blocks = (stat.st_size + 511) / 512;
@@ -1222,6 +1231,9 @@ int KYTY_SYSV_ABI KernelUnlink(const char* path) {
}
auto real_file_name = g_mount_points->ResolvePath(path);
if (Common::IsArchivePath(real_file_name)) {
return KERNEL_ERROR_EROFS;
}
bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
bool is_file = Common::File::IsFileExisting(real_file_name);
@@ -1259,6 +1271,9 @@ int KYTY_SYSV_ABI KernelRename(const char* from, const char* to) {
auto to_path = std::string(to);
auto real_from = g_mount_points->ResolvePath(from_path);
auto real_to = g_mount_points->ResolvePath(to_path);
if (Common::IsArchivePath(real_from) || Common::IsArchivePath(real_to)) {
return KERNEL_ERROR_EROFS;
}
if (real_to.empty() || !Common::File::IsFileExisting(real_from)) {
return KERNEL_ERROR_ENOENT;
@@ -1343,6 +1358,9 @@ int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode) {
if (real_name.empty()) {
return KERNEL_ERROR_ENOENT;
}
if (Common::IsArchivePath(real_name)) {
return KERNEL_ERROR_EROFS;
}
if (Common::File::IsDirectoryExisting(real_name)) {
return KERNEL_ERROR_EEXIST;
@@ -1369,6 +1387,9 @@ int KYTY_SYSV_ABI KernelRmdir(const char* path) {
LOGF("\t path = %s\n", path);
auto real_name = g_mount_points->ResolvePath(std::string(path));
if (Common::IsArchivePath(real_name)) {
return KERNEL_ERROR_EROFS;
}
if (!Common::File::IsDirectoryExisting(real_name)) {
return KERNEL_ERROR_ENOENT;
+1 -1
View File
@@ -20,7 +20,7 @@ add_executable(launcher WIN32
)
target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version)
target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version common)
if(WIN32)
target_link_libraries(launcher setupapi)
endif()
+3 -1
View File
@@ -1,6 +1,7 @@
#ifndef CONFIGURATION_ITEM_H
#define CONFIGURATION_ITEM_H
#include <QIcon>
#include <QObject>
#include <QTreeWidgetItem>
@@ -18,7 +19,7 @@ public:
explicit ConfigurationItem(std::unique_ptr<Configuration> info, QTreeWidget* parent);
~ConfigurationItem() override;
void Update();
void Update(bool reload_icon = false);
bool operator<(const QTreeWidgetItem& other) const override;
Configuration& GetInfo() { return *m_info; }
@@ -35,6 +36,7 @@ private:
void UpdateStatusIndicator();
std::unique_ptr<Configuration> m_info;
QIcon m_icon;
bool m_running = false;
QComboBox* m_status_combo = nullptr;
QLabel* m_status_indicator = nullptr;
+28
View File
@@ -0,0 +1,28 @@
#ifndef GAME_CONTENT_H
#define GAME_CONTENT_H
#include <QByteArray>
#include <QString>
#include <QStringList>
#include <cstdint>
#include <filesystem>
namespace GameContent {
inline constexpr uint64_t MaxMetadataSize = uint64_t {1} << 20u;
inline constexpr uint64_t MaxImageSize = uint64_t {32} << 20u;
inline constexpr uint64_t MaxTrophyPackageSize = uint64_t {128} << 20u;
[[nodiscard]] std::filesystem::path ToPath(const QString& path);
[[nodiscard]] QString FromPath(const std::filesystem::path& path);
[[nodiscard]] bool IsArchive(const QString& base);
[[nodiscard]] std::filesystem::path Resolve(const QString& base, const QString& relative);
[[nodiscard]] bool FileExists(const QString& base, const QString& relative);
[[nodiscard]] QByteArray ReadPath(const std::filesystem::path& path, uint64_t max_size);
[[nodiscard]] QByteArray ReadFile(const QString& base, const QString& relative, uint64_t max_size);
[[nodiscard]] QStringList ListFiles(const QString& base, const QString& relative);
} // namespace GameContent
#endif // GAME_CONTENT_H
+8 -5
View File
@@ -1,6 +1,8 @@
#ifndef GAME_LIST_TREE_WIDGET_H
#define GAME_LIST_TREE_WIDGET_H
#include "gameContent.h"
#include <QPaintEvent>
#include <QPainter>
#include <QPalette>
@@ -17,14 +19,15 @@ public:
viewport()->setAutoFillBackground(false);
}
void SetBackgroundImage(const QString& path) {
if (m_path == path) {
void SetBackgroundImage(const QString& key) {
if (m_path == key) {
return;
}
m_path = path;
m_path = key;
m_source = QPixmap();
if (!path.isEmpty()) {
m_source.load(path);
if (!key.isEmpty()) {
m_source.loadFromData(GameContent::ReadFile(key, QStringLiteral("sce_sys/pic0.png"),
GameContent::MaxImageSize));
}
UpdateScaledBackground();
viewport()->update();
+20 -6
View File
@@ -1,6 +1,7 @@
#include "configurationItem.h"
#include "configuration.h"
#include "gameContent.h"
#include <QApplication>
#include <QComboBox>
@@ -14,6 +15,7 @@
#include <QLabel>
#include <QLineEdit>
#include <QLocale>
#include <QPixmap>
#include <QSize>
#include <QStringList>
#include <QStyle>
@@ -154,7 +156,7 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
m_comment_edit->setFrame(false);
parent->setItemWidget(this, CommentsColumn, m_comment_edit);
Update();
Update(true);
SetRunning(false);
setData(SizeColumn, Qt::UserRole, qint64(-1));
@@ -170,7 +172,14 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
watcher->deleteLater();
});
watcher->setFuture(QtConcurrent::run([path = m_info->basedir]() -> qint64 {
if (path.isEmpty() || !QDir(path).exists()) {
if (path.isEmpty()) {
return -1;
}
const QFileInfo info(path);
if (GameContent::IsArchive(path)) {
return info.size();
}
if (!QDir(path).exists()) {
return -1;
}
qint64 bytes = 0;
@@ -188,7 +197,7 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
ConfigurationItem::~ConfigurationItem() = default;
void ConfigurationItem::Update() {
void ConfigurationItem::Update(bool reload_icon) {
const auto display_text = GetDisplayText(*m_info);
const auto path = GetPathText(*m_info);
@@ -209,6 +218,12 @@ void ConfigurationItem::Update() {
m_comment_edit->setText(m_info->game_comment);
}
if (reload_icon) {
const auto icon_data = GameContent::ReadFile(
m_info->basedir, QStringLiteral("sce_sys/icon0.png"), GameContent::MaxImageSize);
QPixmap icon;
m_icon = !icon_data.isEmpty() && icon.loadFromData(icon_data) ? QIcon(icon) : QIcon {};
}
UpdateIcon();
UpdateStatusIndicator();
}
@@ -267,9 +282,8 @@ void ConfigurationItem::SetCompatibilityEditable(bool editable) {
}
void ConfigurationItem::UpdateIcon() {
const QString icon_file = QDir(m_info->basedir).filePath(QStringLiteral("sce_sys/icon0.png"));
if (QFileInfo::exists(icon_file)) {
setIcon(NameColumn, QIcon(icon_file));
if (!m_icon.isNull()) {
setIcon(NameColumn, m_icon);
return;
}
+106 -84
View File
@@ -1,9 +1,11 @@
#include "configurationListWidget.h"
#include "common/archive.h"
#include "compatibilityDatabase.h"
#include "configuration.h"
#include "configurationEditDialog.h"
#include "configurationItem.h"
#include "gameContent.h"
#include "gameListTreeWidget.h"
#include "inputMappingDialog.h"
#include "mainDialog.h"
@@ -127,15 +129,6 @@ static QStringList SettingsStringList(const QVariant& value) {
return text.isEmpty() ? QStringList() : QStringList({text});
}
static QString GetPic0Path(const Configuration& info) {
if (info.basedir.isEmpty()) {
return {};
}
const auto path = QDir(info.basedir).filePath(QStringLiteral("sce_sys/pic0.png"));
return QFileInfo::exists(path) ? QFileInfo(path).absoluteFilePath() : QString();
}
static void ConfigureGameList(Ui::ConfigurationListWidget* ui) {
ui->cfgs_list->setAlternatingRowColors(false);
ui->cfgs_list->setAllColumnsShowFocus(true);
@@ -490,17 +483,16 @@ static QString GetFirmwareVersion(const QJsonObject& root) {
return version;
}
static GameMetadata GetGameMetadata(const QString& param_file, const QString& fallback) {
static GameMetadata GetGameMetadata(const QByteArray& param_data, const QString& fallback) {
GameMetadata ret;
ret.title_name = fallback;
QFile param(param_file);
if (!param.open(QIODevice::ReadOnly)) {
if (param_data.isEmpty()) {
return ret;
}
QJsonParseError parse_error;
const auto doc = QJsonDocument::fromJson(param.readAll(), &parse_error);
const auto doc = QJsonDocument::fromJson(param_data, &parse_error);
if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) {
return ret;
}
@@ -522,18 +514,18 @@ static GameMetadata GetGameMetadata(const QString& param_file, const QString& fa
}
static void SetGameFiles(Configuration& info, const QString& game_dir, const QString& game_path,
const GameMetadata& metadata) {
QDir game(game_dir);
const GameMetadata& metadata, bool archive) {
const QFileInfo game(game_dir);
info.game_path = game_path;
info.basedir = game.absolutePath();
info.basedir = archive ? game.absoluteFilePath() : QDir(game_dir).absolutePath();
info.name = metadata.title_name;
info.title_id = metadata.title_id;
info.gameVersion = metadata.gameVersion;
info.firmwareVer = metadata.firmwareVer;
if (info.name.isEmpty()) {
info.name = game.dirName();
info.name = archive ? game.completeBaseName() : QDir(game_dir).dirName();
}
}
@@ -609,6 +601,94 @@ void ConfigurationListWidget::ScanGameDirectory() {
const QString eboot_name = QStringLiteral("eboot.bin");
QSet<QString> found_games;
const auto add_game = [this, &found_games, &eboot_name,
&running_items](const QString& base, const QString& game_path,
const QString& legacy_game_path, const QString& fallback,
bool archive) {
const QString game_key = PathKey(game_path);
if (game_key.isEmpty() || found_games.contains(game_key)) {
return;
}
// Keep the index and decompression cache alive through validation, metadata and icon
// reads, after rejecting duplicate candidates from overlapping game folders.
const auto reader = archive ? Common::OpenArchive(GameContent::ToPath(base)) : nullptr;
if (archive && (reader == nullptr || !GameContent::FileExists(base, eboot_name))) {
return;
}
found_games.insert(game_key);
const auto metadata =
GetGameMetadata(GameContent::ReadFile(base, QStringLiteral("sce_sys/param.json"),
GameContent::MaxMetadataSize),
fallback);
auto info = std::make_unique<Configuration>();
info->custom_settings =
FindCustomInfo(&m_custom_infos, game_path, legacy_game_path) != nullptr;
SetGameFiles(*info, base, game_path, metadata, archive);
const auto* compatibility = m_compatibility->Find(info->title_id);
if (compatibility != nullptr) {
info->game_status = compatibility->status;
info->game_comment = compatibility->comment;
}
if (auto* item = running_items.value(game_key); item != nullptr) {
// Refresh metadata without losing the running row's identity.
const QSignalBlocker status_blocker(item->GetStatusCombo());
item->GetInfo().CopyGameInfoFrom(*info);
item->Update(true);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
return;
}
auto* item = new ConfigurationItem(std::move(info), m_ui->cfgs_list);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
connect(item->GetStatusCombo(), &QComboBox::currentIndexChanged, item,
[this, item](int /*index*/) {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_status = static_cast<Configuration::GameStatus>(
item->GetStatusCombo()->currentData().toInt());
item->Update();
m_compatibility->SetStatus(title_id, item->GetInfo().game_status);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
connect(item->GetCommentEdit(), &QLineEdit::editingFinished, item, [this, item]() {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_comment = item->GetCommentEdit()->text();
item->Update();
m_compatibility->SetComment(title_id, item->GetInfo().game_comment);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
};
const auto scan_archives = [&add_game](const QDir& directory, const QDir& root) {
const auto entries = directory.entryInfoList(QDir::Files | QDir::NoSymLinks, QDir::Name);
for (const auto& archive: entries) {
const auto base = archive.absoluteFilePath();
if (!Common::IsSupportedArchive(GameContent::ToPath(base))) {
continue;
}
add_game(base, QDir::cleanPath(base), root.relativeFilePath(base),
archive.completeBaseName(), true);
}
};
for (const auto& root_path: m_game_dirs) {
QDir root(root_path);
@@ -616,6 +696,7 @@ void ConfigurationListWidget::ScanGameDirectory() {
continue;
}
scan_archives(root, root);
QList<QDir> pending_dirs;
const auto root_subdirs =
root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks);
@@ -625,74 +706,13 @@ void ConfigurationListWidget::ScanGameDirectory() {
while (!pending_dirs.isEmpty()) {
QDir game_dir = pending_dirs.takeFirst();
scan_archives(game_dir, root);
if (game_dir.exists(eboot_name)) {
const QString game_path = NormalizeGameDirectory(game_dir.absolutePath());
const QString legacy_game_path = root.relativeFilePath(game_dir.absolutePath());
const QString game_key = PathKey(game_path);
if (game_key.isEmpty() || found_games.contains(game_key)) {
continue;
}
found_games.insert(game_key);
auto metadata = GetGameMetadata(
game_dir.filePath(QStringLiteral("sce_sys/param.json")), game_dir.dirName());
auto info = std::make_unique<Configuration>();
info->custom_settings =
FindCustomInfo(&m_custom_infos, game_path, legacy_game_path) != nullptr;
SetGameFiles(*info, game_dir.absolutePath(), game_path, metadata);
const auto* compatibility = m_compatibility->Find(info->title_id);
if (compatibility != nullptr) {
info->game_status = compatibility->status;
info->game_comment = compatibility->comment;
}
if (auto* item = running_items.value(game_key); item != nullptr) {
// Refresh metadata without losing the running row's identity.
const QSignalBlocker status_blocker(item->GetStatusCombo());
item->GetInfo().CopyGameInfoFrom(*info);
item->Update();
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
continue;
}
auto* item = new ConfigurationItem(std::move(info), m_ui->cfgs_list);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
connect(item->GetStatusCombo(), &QComboBox::currentIndexChanged, item,
[this, item](int /*index*/) {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_status = static_cast<Configuration::GameStatus>(
item->GetStatusCombo()->currentData().toInt());
item->Update();
m_compatibility->SetStatus(title_id, item->GetInfo().game_status);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
connect(item->GetCommentEdit(), &QLineEdit::editingFinished, item, [this, item]() {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_comment = item->GetCommentEdit()->text();
item->Update();
m_compatibility->SetComment(title_id, item->GetInfo().game_comment);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
add_game(game_dir.absolutePath(), game_path, legacy_game_path, game_dir.dirName(),
false);
continue;
}
@@ -805,7 +825,8 @@ void ConfigurationListWidget::open_game_folder() {
return;
}
const QDir game_dir(item->GetInfo().basedir);
const auto base = item->GetInfo().basedir;
const QDir game_dir(GameContent::IsArchive(base) ? QFileInfo(base).absolutePath() : base);
if (!game_dir.exists()) {
QMessageBox::warning(this, tr("Open game folder"), tr("Game folder does not exist."));
return;
@@ -896,7 +917,7 @@ void ConfigurationListWidget::SelectItem(QTreeWidgetItem* witem) {
m_ui->edit_button->setEnabled(!item->IsRunning());
m_selected_item = item;
m_ui->cfgs_list->SetBackgroundImage(GetPic0Path(item->GetInfo()));
m_ui->cfgs_list->SetBackgroundImage(item->GetInfo().basedir);
emit Select();
}
@@ -959,7 +980,8 @@ void ConfigurationListWidget::show_context_menu(const QPoint& pos) {
if (item != nullptr) {
action_run->setDisabled(item->IsRunning());
action_open_folder->setDisabled(!QDir(item->GetInfo().basedir).exists());
const auto& base = item->GetInfo().basedir;
action_open_folder->setDisabled(!QDir(base).exists() && !GameContent::IsArchive(base));
action_view_trophies->setDisabled(!has_trophy_data);
action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty());
action_edit->setDisabled(item->IsRunning());
+80
View File
@@ -0,0 +1,80 @@
#include "gameContent.h"
#include "common/archive.h"
#include "common/file.h"
#include "common/stringUtils.h"
#include <QFileInfo>
#include <limits>
namespace GameContent {
std::filesystem::path ToPath(const QString& path) {
#if defined(_WIN32)
return std::filesystem::path(path.toStdWString());
#else
return std::filesystem::path(path.toStdString());
#endif
}
QString FromPath(const std::filesystem::path& path) {
#if defined(_WIN32)
return QString::fromStdWString(path.wstring());
#else
return QString::fromStdString(Common::PathToString(path));
#endif
}
bool IsArchive(const QString& base) {
const QFileInfo info(base);
return info.isFile() && Common::IsSupportedArchive(ToPath(base));
}
std::filesystem::path Resolve(const QString& base, const QString& relative) {
const auto root = ToPath(base);
return IsArchive(base) ? Common::MakeArchivePath(root, ToPath(relative))
: root / ToPath(relative);
}
bool FileExists(const QString& base, const QString& relative) {
return Common::File::IsFileExisting(Resolve(base, relative));
}
QByteArray ReadFile(const QString& base, const QString& relative, uint64_t max_size) {
return ReadPath(Resolve(base, relative), max_size);
}
QByteArray ReadPath(const std::filesystem::path& path, uint64_t max_size) {
Common::File file(path, Common::File::Mode::Read);
if (file.IsInvalid()) {
return {};
}
const auto size = file.Size();
if (size > max_size || size > static_cast<uint64_t>(std::numeric_limits<int>::max())) {
return {};
}
QByteArray data(static_cast<int>(size), Qt::Uninitialized);
uint32_t bytes_read = 0;
file.Read(data.data(), static_cast<uint32_t>(size), &bytes_read);
return bytes_read == size ? data : QByteArray {};
}
QStringList ListFiles(const QString& base, const QString& relative) {
const auto directory = Resolve(base, relative);
const bool archive = Common::IsArchivePath(directory);
QStringList result;
for (const auto& entry: Common::File::GetDirEntries(directory)) {
if (entry.is_file) {
const auto path = FromPath(directory / Common::PathFromUtf8(entry.name));
if (archive || !QFileInfo(path).isSymLink()) {
result.append(path);
}
}
}
return result;
}
} // namespace GameContent
+3 -2
View File
@@ -3,6 +3,7 @@
#include "configuration.h"
#include "configurationItem.h"
#include "configurationListWidget.h"
#include "gameContent.h"
#include "patchesDialog.h"
#include "updateChecker.h"
@@ -260,7 +261,7 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
}
QString game = info.basedir;
if (!info.elf.isEmpty()) {
if (!info.elf.isEmpty() && !GameContent::IsArchive(info.basedir)) {
game = QDir(info.basedir).filePath(info.elf);
}
args << "--game" << game;
@@ -507,7 +508,7 @@ void MainDialogPrivate::Update() {
if (run_enabled) {
const auto& info = item->GetInfo();
auto dir = info.basedir;
run_enabled = !dir.isEmpty() && QDir(dir).exists();
run_enabled = !dir.isEmpty() && (QDir(dir).exists() || GameContent::IsArchive(dir));
}
m_ui->widget->SetRunEnabled(run_enabled);
+54 -23
View File
@@ -1,13 +1,14 @@
#include "trophyViewerDialog.h"
#include "common/archive.h"
#include "configuration.h"
#include "gameContent.h"
#include <QAbstractItemView>
#include <QBrush>
#include <QByteArray>
#include <QDialogButtonBox>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QFont>
#include <QHeaderView>
@@ -39,12 +40,14 @@
namespace {
constexpr quint32 UCP_MAGIC = 0xb228c60a;
constexpr quint32 UCP_VERSION = 1;
constexpr int UCP_HEADER_LEN = 0x40;
constexpr int UCP_TOC_SKIP = 0x20;
constexpr int UCP_ENTRY_LEN = 0x40;
constexpr int UCP_NAME_LEN = 0x20;
constexpr quint32 UCP_MAGIC = 0xb228c60a;
constexpr quint32 UCP_VERSION = 1;
constexpr int UCP_HEADER_LEN = 0x40;
constexpr int UCP_TOC_SKIP = 0x20;
constexpr int UCP_ENTRY_LEN = 0x40;
constexpr int UCP_NAME_LEN = 0x20;
constexpr quint32 UCP_MAX_FILES = 4096;
constexpr quint64 UCP_MAX_EXTRACTED_SIZE = quint64 {64} << 20u;
struct UcpEntry {
QString name;
@@ -98,14 +101,22 @@ static QString ReadFixedString(const QByteArray& data, qsizetype offset, qsizety
return QString::fromLatin1(data.constData() + offset, len);
}
static bool IsUsedUcpEntry(const QString& name) {
const auto lower = name.toCaseFolded();
return lower == QStringLiteral("tropconf.json") || lower == QStringLiteral("tropmeta.json") ||
(lower.startsWith(QStringLiteral("tropmeta_")) &&
lower.endsWith(QStringLiteral(".json"))) ||
(lower.startsWith(QStringLiteral("trop")) && lower.endsWith(QStringLiteral(".png")));
}
static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files, QString& error) {
QFile file(file_name);
if (!file.open(QIODevice::ReadOnly)) {
const QByteArray data =
GameContent::ReadPath(GameContent::ToPath(file_name), GameContent::MaxTrophyPackageSize);
if (data.isEmpty()) {
error = QObject::tr("Could not open %1").arg(QDir::toNativeSeparators(file_name));
return false;
}
const QByteArray data = file.readAll();
if (data.size() < UCP_HEADER_LEN) {
error = QObject::tr("%1 is too small to be a trophy package.")
.arg(QFileInfo(file_name).fileName());
@@ -127,14 +138,18 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
}
const auto declared_size = qFromBigEndian<quint64>(data.constData() + 0x08);
if (declared_size > static_cast<quint64>(data.size())) {
if (declared_size < UCP_HEADER_LEN || declared_size > static_cast<quint64>(data.size())) {
error = QObject::tr("%1 is truncated.").arg(QFileInfo(file_name).fileName());
return false;
}
const auto file_count = qFromBigEndian<quint32>(data.constData() + 0x10);
if (file_count > UCP_MAX_FILES) {
error = QObject::tr("%1 contains too many files.").arg(QFileInfo(file_name).fileName());
return false;
}
const auto toc_offset = static_cast<quint64>(qFromBigEndian<quint32>(data.constData() + 0x14));
const auto data_size = static_cast<quint64>(data.size());
const auto data_size = declared_size;
const quint64 table_size = UCP_TOC_SKIP + static_cast<quint64>(file_count) * UCP_ENTRY_LEN;
if (toc_offset > data_size || table_size > data_size - toc_offset ||
@@ -144,6 +159,7 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
return false;
}
quint64 extracted_size = 0;
for (quint32 i = 0; i < file_count; i++) {
const auto entry_offset = static_cast<qsizetype>(toc_offset + UCP_TOC_SKIP +
static_cast<quint64>(i) * UCP_ENTRY_LEN);
@@ -161,9 +177,22 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
.arg(QFileInfo(file_name).fileName(), entry.name);
return false;
}
if (!IsUsedUcpEntry(entry.name)) {
continue;
}
const auto entry_limit = entry.name.endsWith(QStringLiteral(".png"), Qt::CaseInsensitive)
? GameContent::MaxImageSize
: GameContent::MaxMetadataSize;
if (entry.size > entry_limit || entry.size > UCP_MAX_EXTRACTED_SIZE - extracted_size) {
error = QObject::tr("%1 contains an oversized entry for %2.")
.arg(QFileInfo(file_name).fileName(), entry.name);
return false;
}
files.insert(entry.name.toCaseFolded(), data.mid(static_cast<qsizetype>(entry.offset),
static_cast<qsizetype>(entry.size)));
extracted_size += entry.size;
}
return true;
@@ -392,30 +421,31 @@ static QStringList FindTrophyFiles(const Configuration* info) {
return {};
}
const QDir trophy_dir(QDir(info->basedir).filePath(QStringLiteral("sce_sys/trophy2")));
if (!trophy_dir.exists()) {
return {};
QStringList files;
for (const auto& file:
GameContent::ListFiles(info->basedir, QStringLiteral("sce_sys/trophy2"))) {
const auto name = QFileInfo(file).fileName();
if (name.startsWith(QStringLiteral("trophy"), Qt::CaseInsensitive) &&
name.endsWith(QStringLiteral(".ucp"), Qt::CaseInsensitive)) {
files.append(file);
}
}
const auto files =
trophy_dir.entryInfoList({QStringLiteral("Trophy*.ucp"), QStringLiteral("trophy*.ucp")},
QDir::Files | QDir::NoSymLinks, QDir::Name | QDir::IgnoreCase);
files.sort(Qt::CaseInsensitive);
QStringList trophy_files;
QSet<QString> seen;
for (const auto& file: files) {
auto key = file.canonicalFilePath();
auto key = QFileInfo(file).canonicalFilePath();
if (key.isEmpty()) {
key = file.absoluteFilePath();
key = file;
}
key = QDir::cleanPath(key).toCaseFolded();
if (!seen.contains(key)) {
seen.insert(key);
trophy_files.append(file.absoluteFilePath());
trophy_files.append(file);
}
}
return trophy_files;
}
@@ -506,6 +536,7 @@ void TrophyViewerDialog::ShowForGame(const Configuration* info, QWidget* parent)
}
bool TrophyViewerDialog::LoadGame(const Configuration& info, QString& error) {
const auto reader = Common::OpenArchive(GameContent::ToPath(info.basedir));
const auto trophy_files = FindTrophyFiles(&info);
if (trophy_files.isEmpty()) {
error = tr("No trophy package found in sce_sys/trophy2.");
+72 -51
View File
@@ -1,6 +1,6 @@
#include "common/common.h"
#include "common/file.h"
#include "common/logging/log.h"
#include "common/stringUtils.h"
#include "kernel/fileSystem.h"
#include "kernel/pthread.h"
#include "libs/audio.h"
@@ -9,12 +9,14 @@
#include <algorithm>
#include <atomic>
#include <cctype>
#include <cerrno>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <deque>
#include <limits>
#include <magic_enum.hpp>
#include <memory>
#include <mutex>
@@ -577,30 +579,40 @@ public:
explicit FileStreamer(AvPlayerFileReplacement f): file(f) {}
~FileStreamer() {
if (ctx != nullptr) {
av_freep(&ctx->buffer);
avio_context_free(&ctx);
}
if (opened && file.close != nullptr) {
if (opened) {
file.close(file.object_pointer);
}
}
bool Init(const std::string& path) {
if (file.open == nullptr || file.close == nullptr || file.read_offset == nullptr ||
file.size == nullptr) {
if (file.open != nullptr) {
if (file.close == nullptr || file.read_offset == nullptr || file.size == nullptr ||
file.open(file.object_pointer, path.c_str()) < 0) {
return false;
}
opened = true;
size = file.size(file.object_pointer);
} else {
if (!local_file.Open(LibKernel::FileSystem::GetRealFilename(path),
Common::File::Mode::Read)) {
return false;
}
size = local_file.Size();
}
if (size == 0 || size > static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) {
return false;
}
if (file.open(file.object_pointer, path.c_str()) < 0) {
return false;
}
opened = true;
size = file.size(file.object_pointer);
if (size == 0) {
return false;
}
auto* buf = static_cast<uint8_t*>(av_malloc(4096));
constexpr int buffer_size = 64 * 1024;
auto* buf = static_cast<uint8_t*>(av_malloc(buffer_size));
if (buf == nullptr) {
return false;
}
ctx = avio_alloc_context(buf, 4096, 0, this, Read, nullptr, Seek);
ctx = avio_alloc_context(buf, buffer_size, 0, this, Read, nullptr, Seek);
if (ctx == nullptr) {
av_free(buf);
}
return ctx != nullptr;
}
AVIOContext* Context() const { return ctx; }
@@ -611,33 +623,50 @@ private:
if (s->pos >= s->size) {
return AVERROR_EOF;
}
len = static_cast<int>(std::min<uint64_t>(len, s->size - s->pos));
auto r =
s->file.read_offset(s->file.object_pointer, buf, s->pos, static_cast<uint32_t>(len));
if (r <= 0) {
return r == 0 ? AVERROR_EOF : r;
len = static_cast<int>(std::min<uint64_t>(len, s->size - s->pos));
int read = 0;
if (s->opened) {
read = s->file.read_offset(s->file.object_pointer, buf, s->pos,
static_cast<uint32_t>(len));
} else {
uint32_t bytes = 0;
s->local_file.Read(buf, static_cast<uint32_t>(len), &bytes);
read = static_cast<int>(bytes);
}
s->pos += static_cast<uint64_t>(r);
return r;
if (read <= 0 || read > len) {
return read < 0 ? read : AVERROR(EIO);
}
s->pos += static_cast<uint64_t>(read);
return read;
}
static int64_t Seek(void* opaque, int64_t off, int whence) {
static int64_t Seek(void* opaque, int64_t offset, int whence) {
auto* s = static_cast<FileStreamer*>(opaque);
if ((whence & AVSEEK_SIZE) != 0) {
return static_cast<int64_t>(s->size);
}
int64_t p = whence == SEEK_SET
? off
: (whence == SEEK_CUR
? static_cast<int64_t>(s->pos) + off
: (whence == SEEK_END ? static_cast<int64_t>(s->size) + off : -1));
if (p < 0) {
return -1;
whence &= ~AVSEEK_FORCE;
uint64_t base = 0;
switch (whence) {
case SEEK_SET: break;
case SEEK_CUR: base = s->pos; break;
case SEEK_END: base = s->size; break;
default: return AVERROR(EINVAL);
}
p = std::min<int64_t>(p, static_cast<int64_t>(s->size));
s->pos = static_cast<uint64_t>(p);
return p;
const auto distance = offset < 0 ? uint64_t {0} - static_cast<uint64_t>(offset)
: static_cast<uint64_t>(offset);
if ((offset < 0 && distance > base) ||
(offset >= 0 && distance > std::numeric_limits<int64_t>::max() - base)) {
return AVERROR(EINVAL);
}
const auto position = offset < 0 ? base - distance : base + distance;
if (!s->opened && !s->local_file.Seek(position)) {
return AVERROR(EIO);
}
s->pos = position;
return static_cast<int64_t>(position);
}
AvPlayerFileReplacement file;
Common::File local_file;
bool opened = false;
uint64_t pos = 0;
uint64_t size = 0;
@@ -675,25 +704,17 @@ public:
return static_cast<Source*>(opaque)->interrupt_io.load() ? 1 : 0;
};
raw->interrupt_callback.opaque = this;
if (file.open != nullptr) {
streamer = std::make_unique<FileStreamer>(file);
if (!streamer->Init(path)) {
avformat_free_context(raw);
return AVPLAYER_ERROR_OPERATION_FAILED;
}
raw->pb = streamer->Context();
if (auto rc = avformat_open_input(&raw, nullptr, nullptr, nullptr); rc < 0) {
LOGF("\t avformat_open_input callback failed: %s\n", fferr(rc).c_str());
return AVPLAYER_ERROR_OPERATION_FAILED;
}
} else {
auto real = LibKernel::FileSystem::GetRealFilename(std::string(path.c_str()));
auto real_str = Common::PathToString(real);
if (auto rc = avformat_open_input(&raw, real_str.c_str(), nullptr, nullptr); rc < 0) {
LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(),
real_str.c_str());
return AVPLAYER_ERROR_OPERATION_FAILED;
}
streamer = std::make_unique<FileStreamer>(file);
if (!streamer->Init(path)) {
avformat_free_context(raw);
return AVPLAYER_ERROR_OPERATION_FAILED;
}
raw->pb = streamer->Context();
if (const auto rc = avformat_open_input(&raw, file.open == nullptr ? path.c_str() : nullptr,
nullptr, nullptr);
rc < 0) {
LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(), path.c_str());
return AVPLAYER_ERROR_OPERATION_FAILED;
}
fmt = raw;
if (auto rc = avformat_find_stream_info(fmt, nullptr); rc < 0) {
+15 -4
View File
@@ -1,3 +1,4 @@
#include "common/archive.h"
#include "common/common.h"
#include "common/dateTime.h"
#include "common/debug.h"
@@ -10,6 +11,7 @@
#include <charconv>
#include <cstdio>
#include <filesystem>
#include <string_view>
#include <vector>
#include <fmt/format.h>
@@ -40,9 +42,9 @@ static std::string GetBuildString() {
static void PrintUsage() {
::printf("%s\n", GetBuildString().c_str());
::printf("kyty_emulator --game <dir|elf> [options]\n\n");
::printf("kyty_emulator --game <dir|elf|zar> [options]\n\n");
::printf("Options:\n");
::printf(" --game <dir|elf> Game directory or ELF to load.\n");
::printf(" --game <dir|elf|zar> Game directory, ELF, or ZArchive to load.\n");
::printf(" --game-patch <json> ETAHen cheat file.\n");
::printf(" --screen-width <num> Window width. Default: 1280.\n");
::printf(" --screen-height <num> Window height. Default: 720.\n");
@@ -234,12 +236,20 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
return false;
}
value = Common::FixFilenameSlash(value);
value = Common::FixFilenameSlash(value);
const auto path = Common::PathFromUtf8(value);
if (Common::File::IsDirectoryExisting(path)) {
options.app0_dir = path;
options.elf = "/app0/eboot.bin";
} else if (Common::IsSupportedArchive(path) && Common::File::IsFileExisting(path)) {
const auto root = Common::MakeArchivePath(path);
if (!Common::File::IsFileExisting(root / "eboot.bin")) {
::printf("Archive does not contain eboot.bin: %s\n", value.c_str());
return false;
}
options.app0_dir = root;
options.elf = "/app0/eboot.bin";
} else if (Common::File::IsFileExisting(path)) {
options.app0_dir = path.parent_path();
@@ -249,7 +259,8 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
options.elf = std::filesystem::path("/app0") / path.filename();
} else {
::printf("--game must point to an existing directory or ELF: %s\n", value.c_str());
::printf("--game must point to an existing directory, ELF, or archive: %s\n",
value.c_str());
return false;
}
} else if (arg == "--game-patch") {