raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-19 18:36:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Zarchive Loading (#724)

Browse Source 
* Simplify archive loading and harden filesystem integration

* Restore directory entry name length assertion

* Keep graphics-dependent filesystem tests out of CI

---------

Co-authored-by: nmzik <Nmzik@mail.ru>
This commit is contained in:
Aspen
2026-09-28 16:06:44 -04:00
committed by GitHub
parent 8e61798bfd
commit 4b0de1b672
29 changed files with 1956 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+6 -6
View File
@@ -115,12 +115,12 @@ jobs:
cmake --build _Build/windows --target launcher ^
audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
resource_tracking_tests scalar_provenance_tests ^
resource_materialization_tests shader_vertex_metadata_tests --parallel
resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests --parallel
- name: Test
shell: cmd
run: |
ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|virtual_memory_allocation)$"
ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
- name: Install
shell: cmd
@@ -203,14 +203,14 @@ jobs:
cmake --build _Build/macos \
--target launcher audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
resource_tracking_tests scalar_provenance_tests \
resource_materialization_tests shader_vertex_metadata_tests \
resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
--parallel
- name: Test
shell: bash
run: |
ctest --test-dir _Build/macos --output-on-failure \
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|virtual_memory_allocation)$'
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
- name: Install
shell: bash
@@ -386,14 +386,14 @@ jobs:
--target launcher page_manager_tests memory_tracker_tests \
audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
resource_tracking_tests scalar_provenance_tests \
resource_materialization_tests shader_vertex_metadata_tests \
resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
--parallel
- name: Test
shell: bash
run: |
ctest --test-dir _Build/linux --output-on-failure \
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|virtual_memory_allocation)$'
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
- name: Install
shell: bash
3rdparty/CMakeLists.txt
Vendored
+43
View File
@@ -28,6 +28,49 @@ FetchContent_Declare(
FetchContent_MakeAvailable(xbyak zydis)
# ZArchive provides random-access reads for compressed game dumps. Build only
# the reader/writer library and its zstd dependency; the command-line tool is
# not part of KytyPS5's runtime distribution.
set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE)
set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE)
set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE)
set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(ZSTD_LEGACY_SUPPORT OFF CACHE BOOL "" FORCE)
set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
zstd
GIT_REPOSITORY https://github.com/facebook/zstd.git
GIT_TAG f8745da6ff1ad1e7bab384bd1f9d742439278e99
GIT_SHALLOW FALSE
SOURCE_SUBDIR build/cmake
)
FetchContent_Declare(
zarchive_source
GIT_REPOSITORY https://github.com/Exzap/ZArchive.git
GIT_TAG d2c717730092c7bf8cbb033b12fd4001b7c4d932
GIT_SHALLOW FALSE
PATCH_COMMAND "${GIT_EXECUTABLE}" apply --ignore-whitespace
"${CMAKE_CURRENT_SOURCE_DIR}/patches/zarchive-reader.patch"
SOURCE_SUBDIR kyty-library-only
)
FetchContent_MakeAvailable(zstd zarchive_source)
add_library(zarchive STATIC
"${zarchive_source_SOURCE_DIR}/src/zarchivereader.cpp"
"${zarchive_source_SOURCE_DIR}/src/zarchivewriter.cpp"
"${zarchive_source_SOURCE_DIR}/src/sha_256.c"
)
add_library(ZArchive::zarchive ALIAS zarchive)
target_include_directories(zarchive SYSTEM PUBLIC "${zarchive_source_SOURCE_DIR}/include")
target_include_directories(zarchive PRIVATE "${zstd_SOURCE_DIR}/lib")
target_link_libraries(zarchive PRIVATE libzstd_static)
target_compile_features(zarchive PRIVATE cxx_std_20)
if(MSVC)
target_compile_options(zarchive PRIVATE /W0)
else()
target_compile_options(zarchive PRIVATE -w)
endif()
set(SDL_SHARED OFF CACHE BOOL "" FORCE)
if(APPLE)
# SDL's built-in iconv fallback does not support CP932.
3rdparty/patches/zarchive-reader.patch
Vendored
+100
View File
@@ -0,0 +1,100 @@
--- a/include/zarchive/zarchivecommon.h
+++ b/include/zarchive/zarchivecommon.h
@@ -256,7 +256,7 @@
bool IsWithinValidRange(uint64_t fileSize) const
{
- return (offset + size) <= fileSize;
+ return offset <= fileSize && size <= fileSize - offset;
}
};
--- a/src/zarchivereader.cpp
+++ b/src/zarchivereader.cpp
@@ -14,6 +14,7 @@
static bool _ifstream_readBytes(std::ifstream& file, uint64_t offset, void* buffer, uint32_t size)
{
+ file.clear();
file.seekg(offset, std::ios_base::beg);
file.read((char*)buffer, size);
return file.gcount() == size;
@@ -80,9 +81,27 @@
// verify file tree
if (fileTree[0].IsFile())
return nullptr; // first entry must be root directory
- auto rootName = GetName(nameTable, fileTree[0].GetNameOffset());
- if (!rootName.empty())
+ if (fileTree[0].GetNameOffset() != 0x7FFFFFFF)
return nullptr; // root node must not have a name
+ const uint64_t dataCapacity = (uint64_t)offsetRecords.size() * _ZARCHIVE::ENTRIES_PER_OFFSETRECORD * _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
+ for (size_t i = 0; i < fileTree.size(); ++i)
+ {
+ auto& entry = fileTree[i];
+ if (i != 0)
+ {
+ auto name = GetName(nameTable, entry.GetNameOffset());
+ if (name.empty() || name == "." || name == ".." || name.find_first_of("/\\") != std::string_view::npos || name.find('\0') != std::string_view::npos)
+ return nullptr;
+ }
+ if (entry.IsFile())
+ {
+ if (entry.GetFileOffset() > dataCapacity || entry.GetFileSize() > dataCapacity - entry.GetFileOffset())
+ return nullptr;
+ }
+ else if (entry.directoryRecord.nodeStartIndex > fileTree.size() ||
+ entry.directoryRecord.count > fileTree.size() - entry.directoryRecord.nodeStartIndex)
+ return nullptr;
+ }
// read meta data
// todo
@@ -238,7 +257,7 @@
uint32_t stepSize = std::min<uint32_t>(remainingBytes, _ZARCHIVE::COMPRESSED_BLOCK_SIZE - blockOffset);
CacheBlock* block = GetCachedBlock(blockIdx);
if (!block)
- return 0;
+ return bytesToRead - remainingBytes;
std::memcpy(bufferU8, block->data + blockOffset, stepSize);
rawReadOffset += stepSize;
remainingBytes -= stepSize;
@@ -325,14 +344,18 @@
// determine offset and size of compressed block
auto& record = m_offsetRecords[recordIndex];
uint64_t offset = record.baseOffset;
+ if (offset > m_compressedDataSize)
+ return false;
for (uint32_t i = 0; i < recordSubIndex; i++)
{
- offset += (uint64_t)record.size[i];
- offset++;
+ const uint64_t size = (uint64_t)record.size[i] + 1;
+ if (size > m_compressedDataSize - offset)
+ return false;
+ offset += size;
}
uint32_t compressedSize = (uint32_t)record.size[recordSubIndex] + 1;
// load file data
- if ((offset + compressedSize) > m_compressedDataSize)
+ if (compressedSize > m_compressedDataSize - offset)
return false;
offset += m_compressedDataOffset;
if (compressedSize == _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
@@ -350,7 +373,7 @@
// returns empty view on failure
std::string_view ZArchiveReader::GetName(const std::vector<uint8_t>& nameTable, uint32_t nameOffset)
{
- if (nameOffset == 0x7FFFFFFF || nameOffset > nameTable.size())
+ if (nameOffset == 0x7FFFFFFF || nameOffset >= nameTable.size())
return "";
// parse header
uint16_t nameLength = nameTable[nameOffset] & 0x7F;
@@ -359,7 +382,7 @@
// extended 2-byte length
if (nameOffset + 1 >= nameTable.size())
return "";
- nameLength |= ((uint16_t)nameTable[nameOffset] << 7);
+ nameLength |= ((uint16_t)nameTable[nameOffset + 1] << 7);
nameOffset += 2;
}
else
CMakeLists.txt
+16 -1
View File
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.16...4.3)
cmake_minimum_required(VERSION 3.18...4.3)
if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies")
@@ -407,6 +407,16 @@ add_executable(page_manager_tests EXCLUDE_FROM_ALL
target_link_libraries(page_manager_tests common)
target_include_directories(page_manager_tests PRIVATE ${inc_headers})
add_executable(archive_file_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/ArchiveFileTests.cpp"
)
target_link_libraries(archive_file_tests common ZArchive::zarchive)
target_include_directories(archive_file_tests PRIVATE ${inc_headers})
add_executable(avplayer_file_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp")
target_link_libraries(avplayer_file_tests common FFmpeg::ffmpeg ZArchive::zarchive)
target_include_directories(avplayer_file_tests PRIVATE ${inc_headers})
add_executable(bit_array_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/BitArrayTests.cpp"
)
@@ -514,6 +524,7 @@ target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
add_kyty_full_emulator_test(kernel_file_system_tests
"${KYTY_TESTS_DIR}/KernelFileSystemTests.cpp")
target_link_libraries(kernel_file_system_tests ZArchive::zarchive)
add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
@@ -580,6 +591,8 @@ if(BUILD_TESTING)
add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
add_test(NAME lru_cache COMMAND $<TARGET_FILE:lru_cache_tests>)
add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
@@ -649,6 +662,8 @@ if(BUILD_TESTING)
image_page_table_tests
memory_tracker_tests
page_manager_tests
archive_file_tests
avplayer_file_tests
bit_array_tests
lru_cache_tests
shader_vertex_metadata_tests
README.md
+11 -6
View File
@@ -236,9 +236,10 @@ cmake --build _Build/linux --target launcher --parallel
cmake --install _Build/linux --prefix _Build/linux/install
```
The configure step downloads the FFmpeg prebuilts and the `xbyak`/`zydis` sources, so it needs
network access; a fully sandboxed `nix build` would require vendoring those inputs. A Vulkan 1.3
driver must be available at runtime (on NixOS, `hardware.graphics.enable = true`).
The configure step downloads the FFmpeg prebuilts and the `xbyak`, `zydis`, `zstd`, and ZArchive
sources, so it needs network access; a fully sandboxed `nix build` would require vendoring those
inputs. A Vulkan 1.3 driver must be available at runtime (on NixOS,
`hardware.graphics.enable = true`).
### Building on macOS
@@ -332,17 +333,21 @@ open _Build/macos/install/KytyPS5.app # or double-click in Finder
```
On first launch, add one or more game folders in the global settings. The launcher searches those
folders recursively for game directories containing `eboot.bin`. Select a detected game and run it
from the game list.
folders recursively for game directories containing `eboot.bin` and ZArchive (`.zar`) game dumps
whose archive root contains `eboot.bin`. Select a detected game and run it from the game list.
ZArchive dumps are mounted read-only and streamed directly; they do not need to be extracted first.
The emulator can also be started directly with a legally obtained game directory or ELF file:
The emulator can also be started directly with a legally obtained game directory, ELF file, or
ZArchive dump:
```powershell
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame"
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame.zar"
```
```bash
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame.zar"
```
On macOS, the adjacent flat or app-bundled `libMoltenVK.dylib` is found automatically; no
src/common/CMakeLists.txt
+1
View File
@@ -26,6 +26,7 @@ target_link_libraries(common
cpuinfo
kyty_git_version
spdlog::spdlog
ZArchive::zarchive
)
get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES)
src/common/archive.cpp
+397
View File
@@ -0,0 +1,397 @@
#include "common/archive.h"
#include "common/stringUtils.h"
#include <algorithm>
#include <condition_variable>
#include <cstring>
#include <mutex>
#include <optional>
#include <string_view>
#include <thread>
#include <unordered_map>
#include <utility>
#include <zarchive/zarchivereader.h>
// POSIX hosts and MinGW's pthread-backed libstdc++ have cancellable condition waits.
// MSVC/ClangCL and MinGW's native Win32 threading backend do not use pthread waits.
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
#include <pthread.h>
#define KYTY_ARCHIVE_PTHREAD_CANCELLATION
#endif
namespace Common {
// Backends expose immutable metadata and offset-based reads. Cursor management, path routing,
// reader lifetime and the host-stack worker are shared by every archive format.
class ArchiveReader {
public:
struct Entry {
uint64_t id;
uint64_t size;
bool is_file;
};
virtual ~ArchiveReader() = default;
virtual std::optional<Entry> Find(std::string_view member) = 0;
virtual std::vector<File::DirEntry> List(std::string_view member) = 0;
virtual uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) = 0;
};
namespace {
class ZArchiveReaderBackend final: public ArchiveReader {
public:
explicit ZArchiveReaderBackend(std::unique_ptr<ZArchiveReader> reader)
: m_reader(std::move(reader)) {}
std::optional<Entry> Find(std::string_view member) override {
const auto node = m_reader->LookUp(member);
if (node == ZARCHIVE_INVALID_NODE) {
return {};
}
return Entry {node, m_reader->GetFileSize(node), m_reader->IsFile(node)};
}
std::vector<File::DirEntry> List(std::string_view member) override {
const auto node = m_reader->LookUp(member);
if (!m_reader->IsDirectory(node)) {
return {};
}
const auto count = m_reader->GetDirEntryCount(node);
if (count > (1u << 20u)) {
return {};
}
std::vector<File::DirEntry> result;
result.reserve(count);
for (uint32_t index = 0; index < count; ++index) {
ZArchiveReader::DirEntry entry {};
if (m_reader->GetDirEntry(node, index, entry)) {
result.push_back({std::string(entry.name), entry.isFile});
}
}
return result;
}
uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) override {
// ZArchiveReader already serializes access to its stream and decompression cache.
return m_reader->ReadFromFile(static_cast<ZArchiveNodeHandle>(id), offset, size, data);
}
private:
std::unique_ptr<ZArchiveReader> m_reader;
};
std::shared_ptr<ArchiveReader> OpenZArchive(const std::filesystem::path& path) {
auto reader = std::unique_ptr<ZArchiveReader>(ZArchiveReader::OpenFromFile(path));
return reader ? std::make_shared<ZArchiveReaderBackend>(std::move(reader)) : nullptr;
}
struct ArchiveFormat {
std::string_view extension;
std::shared_ptr<ArchiveReader> (*open)(const std::filesystem::path&);
};
constexpr ArchiveFormat Formats[] = {{".zar", OpenZArchive}};
using NativeView = std::basic_string_view<std::filesystem::path::value_type>;
const ArchiveFormat* FindFormat(NativeView path) {
for (const auto& format: Formats) {
if (path.size() < format.extension.size()) {
continue;
}
const auto extension = path.substr(path.size() - format.extension.size());
if (std::equal(
extension.begin(), extension.end(), format.extension.begin(),
[](auto a, char b) { return (a >= 'A' && a <= 'Z' ? a + ('a' - 'A') : a) == b; })) {
return &format;
}
}
return nullptr;
}
size_t FindMarker(NativeView path) {
// Native files use this check on every operation. The usual path has no marker and allocates
// nothing.
for (auto marker = path.find('!'); marker != NativeView::npos;
marker = path.find('!', marker + 1)) {
if ((marker + 1 == path.size() || path[marker + 1] == '/' || path[marker + 1] == '\\') &&
FindFormat(path.substr(0, marker)) != nullptr) {
return marker;
}
}
return NativeView::npos;
}
struct ParsedPath {
std::filesystem::path archive;
std::string member;
};
std::optional<ParsedPath> ParsePath(const std::filesystem::path& path) {
const auto& native = path.native();
const auto marker = FindMarker(native);
if (marker == NativeView::npos) {
return {};
}
ParsedPath parsed {.archive = std::filesystem::path(native.substr(0, marker))};
auto member = PathToGenericString(std::filesystem::path(native.substr(marker + 1)));
std::replace(member.begin(), member.end(), '\\', '/');
std::string_view remaining = member;
while (!remaining.empty()) {
const auto separator = remaining.find('/');
const auto component = remaining.substr(0, separator);
remaining.remove_prefix(separator == std::string_view::npos ? remaining.size() : separator + 1);
if (component.empty() || component == ".") {
continue;
}
if (component == "..") {
if (parsed.member.empty()) {
return {};
}
const auto separator = parsed.member.rfind('/');
parsed.member.resize(separator == std::string::npos ? 0 : separator);
} else {
if (!parsed.member.empty()) {
parsed.member += '/';
}
parsed.member += component;
}
}
return parsed;
}
std::mutex g_readers_mutex;
std::unordered_map<std::filesystem::path, std::weak_ptr<ArchiveReader>> g_readers;
// Guest stacks can be only 64 KiB. Decompress on an ordinary host stack, then copy into guest
// memory on the caller so GPU-tracked writes use its fault handler. A single worker bounds host
// threads and avoids duplicating the archive's own block cache. Requests live on waiting callers.
class ArchiveIoThread {
public:
static ArchiveIoThread& Instance() {
// Guest exit can run static destructors while other guest threads are still reading.
// Keep the worker alive until process termination, as with the guest threads themselves.
static auto* instance = new ArchiveIoThread;
return *instance;
}
uint64_t Read(ArchiveReader& reader, uint64_t id, uint64_t offset, uint32_t size, void* data) {
Request request {reader, id, offset, size, data};
std::unique_lock lock(m_mutex);
if (m_last != nullptr) {
m_last->next = &request;
} else {
m_first = &request;
}
m_last = &request;
m_wake.notify_one();
request.ready.wait(lock, [&] { return request.done; });
return request.result;
}
KYTY_CLASS_NO_COPY(ArchiveIoThread);
private:
struct Request {
ArchiveReader& reader;
uint64_t id;
uint64_t offset;
uint32_t size;
void* data;
Request* next = nullptr;
uint64_t result = 0;
bool done = false;
std::condition_variable ready {};
};
ArchiveIoThread() {
std::thread([this] { Loop(); }).detach();
}
void Loop() {
for (;;) {
Request* request;
{
std::unique_lock lock(m_mutex);
m_wake.wait(lock, [this] { return m_first != nullptr; });
request = m_first;
m_first = request->next;
if (m_first == nullptr) {
m_last = nullptr;
}
}
const auto result =
request->reader.Read(request->id, request->offset, request->size, request->data);
{
std::lock_guard lock(m_mutex);
request->result = result;
request->done = true;
// Notify before releasing the mutex: the waiting caller owns the condition
// variable.
request->ready.notify_one();
}
}
}
std::mutex m_mutex;
std::condition_variable m_wake;
Request* m_first = nullptr;
Request* m_last = nullptr;
};
} // namespace
struct ArchiveFile::Private {
std::shared_ptr<ArchiveReader> reader;
ArchiveReader::Entry entry;
uint64_t position = 0;
};
ArchiveFile::ArchiveFile(std::unique_ptr<Private> p): m_p(std::move(p)) {}
ArchiveFile::~ArchiveFile() = default;
uint64_t ArchiveFile::Size() const {
return m_p->entry.size;
}
uint64_t ArchiveFile::Tell() const {
return m_p->position;
}
bool ArchiveFile::Seek(uint64_t offset) {
m_p->position = offset;
return true;
}
void ArchiveFile::Read(void* data, uint32_t size, uint32_t* bytes_read) {
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
// A pending cancellation must not destroy the waiting request or its TLS buffer while the
// worker still uses them. Restore the caller's state only after the copy is complete.
int cancellation_state = PTHREAD_CANCEL_ENABLE;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancellation_state);
#endif
constexpr uint32_t chunk_size = 1u << 20u;
thread_local std::vector<uint8_t> chunk;
uint32_t read = 0;
if (data != nullptr && size != 0 && m_p->position < Size()) {
const auto requested =
static_cast<uint32_t>(std::min<uint64_t>(size, Size() - m_p->position));
if (chunk.size() < std::min(chunk_size, requested)) {
chunk.resize(std::min(chunk_size, requested));
}
while (read < requested) {
const auto step = std::min(chunk_size, requested - read);
const auto got = ArchiveIoThread::Instance().Read(*m_p->reader, m_p->entry.id,
m_p->position, step, chunk.data());
if (got == 0 || got > step) {
break;
}
std::memcpy(static_cast<uint8_t*>(data) + read, chunk.data(), got);
read += static_cast<uint32_t>(got);
m_p->position += got;
if (got < step) {
break;
}
}
}
if (bytes_read != nullptr) {
*bytes_read = read;
}
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
pthread_setcancelstate(cancellation_state, nullptr);
#endif
}
std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
const std::filesystem::path& member) {
auto root = archive;
root += "!";
return member.empty() ? root : root / member.relative_path();
}
bool IsSupportedArchive(const std::filesystem::path& path) {
return FindFormat(path.native()) != nullptr;
}
bool IsArchivePath(const std::filesystem::path& path) {
return FindMarker(path.native()) != NativeView::npos;
}
std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path) {
const auto marker = FindMarker(path.native());
return marker == NativeView::npos ? std::filesystem::path {}
: std::filesystem::path(path.native().substr(0, marker));
}
std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path) {
const auto archive = IsArchivePath(path) ? GetArchiveHostPath(path) : path;
const auto* format = FindFormat(archive.native());
if (format == nullptr) {
return {};
}
std::error_code error;
auto key = std::filesystem::absolute(archive, error);
if (error) {
return {};
}
{
std::lock_guard lock(g_readers_mutex);
if (auto it = g_readers.find(key); it != g_readers.end()) {
if (auto reader = it->second.lock()) {
return reader;
}
}
}
// Do not hold the cache mutex while opening an unrelated archive from disk.
auto reader = format->open(archive);
if (reader == nullptr) {
return {};
}
std::lock_guard lock(g_readers_mutex);
std::erase_if(g_readers, [](const auto& entry) { return entry.second.expired(); });
auto& cached = g_readers[key];
if (auto existing = cached.lock()) {
return existing;
}
cached = reader;
return reader;
}
std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path) {
const auto parsed = ParsePath(path);
if (parsed) {
if (auto reader = OpenArchive(parsed->archive)) {
if (auto entry = reader->Find(parsed->member)) {
return File::Info {entry->is_file, entry->size};
}
}
}
return {};
}
std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path) {
const auto parsed = ParsePath(path);
if (parsed) {
if (auto reader = OpenArchive(parsed->archive)) {
return reader->List(parsed->member);
}
}
return {};
}
std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path) {
auto parsed = ParsePath(path);
if (!parsed) {
return {};
}
auto reader = OpenArchive(parsed->archive);
const auto entry = reader ? reader->Find(parsed->member) : std::nullopt;
if (!entry || !entry->is_file) {
return {};
}
auto p = std::make_unique<ArchiveFile::Private>();
p->reader = std::move(reader);
p->entry = *entry;
return std::unique_ptr<ArchiveFile>(new ArchiveFile(std::move(p)));
}
} // namespace Common
src/common/archive.h
+50
View File
@@ -0,0 +1,50 @@
#ifndef KYTY_COMMON_ARCHIVE_H_
#define KYTY_COMMON_ARCHIVE_H_
#include "common/file.h"
#include <cstdint>
#include <filesystem>
#include <memory>
#include <vector>
namespace Common {
class ArchiveReader;
class ArchiveFile {
public:
~ArchiveFile();
[[nodiscard]] uint64_t Size() const;
[[nodiscard]] uint64_t Tell() const;
bool Seek(uint64_t offset);
void Read(void* data, uint32_t size, uint32_t* bytes_read);
KYTY_CLASS_NO_COPY(ArchiveFile);
private:
struct Private;
explicit ArchiveFile(std::unique_ptr<Private> p);
std::unique_ptr<Private> m_p;
friend std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
};
// Virtual paths use "archive.ext!/member" and retain native host-path encoding.
[[nodiscard]] std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
const std::filesystem::path& member = {});
[[nodiscard]] bool IsSupportedArchive(const std::filesystem::path& path);
[[nodiscard]] bool IsArchivePath(const std::filesystem::path& path);
[[nodiscard]] std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path);
[[nodiscard]] std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path);
[[nodiscard]] std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path);
[[nodiscard]] std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
// Retain this handle while a mount or metadata scan is active to reuse the index and block cache.
// Accepts either a host archive filename or a virtual path inside the archive.
[[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
} // namespace Common
#endif /* KYTY_COMMON_ARCHIVE_H_ */
src/common/file.cpp
+139 -16
View File
@@ -1,5 +1,6 @@
#include "common/file.h"
#include "common/archive.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/platform/sysFileIO.h"
@@ -62,6 +63,7 @@ SysFileTimeStruct DateTimeToFileTimeUtc(const DateTime& date_time) {
struct File::FilePrivate {
sys_file_t* f;
std::unique_ptr<ArchiveFile> archive;
};
File::File(): m_p(std::make_unique<FilePrivate>()) {
@@ -85,11 +87,15 @@ File::File(const std::filesystem::path& name, Mode mode): m_p(std::make_unique<F
}
bool File::IsInvalid() const {
return m_p->f == nullptr;
return m_p->f == nullptr && m_p->archive == nullptr;
}
bool File::Create(const std::filesystem::path& name) {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
if (IsArchivePath(name)) {
return false;
}
m_file_name = name;
@@ -104,9 +110,16 @@ bool File::Create(const std::filesystem::path& name) {
}
bool File::Open(const std::filesystem::path& name, Mode mode) {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
m_file_name = name;
if (IsArchivePath(name)) {
if (mode != Mode::Read) {
return false;
}
m_p->archive = OpenArchiveFile(name);
return m_p->archive != nullptr;
}
switch (mode) {
case Mode::Read: m_p->f = SysFileOpenR(name); break;
@@ -124,7 +137,7 @@ bool File::Open(const std::filesystem::path& name, Mode mode) {
}
bool File::OpenInMem(void* buf, uint32_t buf_size) {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
m_p->f = SysFileOpen(static_cast<uint8_t*>(buf), buf_size);
@@ -137,7 +150,7 @@ bool File::OpenInMem(void* buf, uint32_t buf_size) {
}
bool File::CreateInMem() {
EXIT_IF(m_p->f != nullptr);
EXIT_IF(!IsInvalid());
m_p->f = SysFileCreate();
@@ -150,6 +163,7 @@ bool File::CreateInMem() {
}
void File::Close() {
m_p->archive.reset();
if (m_p->f != nullptr) {
SysFileClose(m_p->f);
m_p->f = nullptr;
@@ -157,7 +171,11 @@ void File::Close() {
}
uint64_t File::Size() const {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Size();
}
if (m_p->f == nullptr) {
return 0;
@@ -167,41 +185,76 @@ uint64_t File::Size() const {
}
uint64_t File::Remaining() const {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
return Size() - Tell();
const auto size = Size();
const auto position = Tell();
return position < size ? size - position : 0;
}
uint64_t File::Size(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
const auto info = GetArchiveInfo(name);
return info ? info->size : 0;
}
return SysFileSize(name);
}
std::optional<File::Info> File::GetInfo(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetArchiveInfo(name);
}
Info info {};
return SysFileGetInfo(name, &info.is_file, &info.size) ? std::optional(info) : std::nullopt;
}
bool File::Seek(uint64_t offset) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Seek(offset);
}
return SysFileSeek(*m_p->f, offset);
}
bool File::Truncate(uint64_t size) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return false;
}
return SysFileTruncate(*m_p->f, size);
}
bool File::Unlink() {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return false;
}
return SysFileUnlink(*m_p->f, m_file_name);
}
uint64_t File::Tell() const {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Tell();
}
return SysFileTell(*m_p->f);
}
void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
m_p->archive->Read(data, size, bytes_read);
return;
}
if (m_p->f != nullptr) {
SysFileRead(data, size, *m_p->f, bytes_read);
@@ -209,7 +262,14 @@ void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
}
void File::Write(const void* data, uint32_t size, uint32_t* bytes_written) {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
if (bytes_written != nullptr) {
*bytes_written = 0;
}
return;
}
SysFileWrite(data, size, *m_p->f, bytes_written);
}
@@ -257,24 +317,41 @@ static std::filesystem::path WithoutTrailingSeparator(
bool File::IsDirectoryExisting(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
const auto info = GetArchiveInfo(path);
return info && !info->is_file;
}
return SysFileIsDirectoryExisting(WithoutTrailingSeparator(path));
}
bool File::IsFileExisting(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
const auto info = GetArchiveInfo(name);
return info && info->is_file;
}
return SysFileIsFileExisting(name);
}
bool File::CreateDirectory(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
return SysFileCreateDirectory(
WithoutTrailingSeparator(path));
}
bool File::DeleteDirectory(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
return SysFileDeleteDirectory(
WithoutTrailingSeparator(path));
}
bool File::CreateDirectories(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
const auto normalized_path =
WithoutTrailingSeparator(path);
@@ -297,6 +374,9 @@ bool File::CreateDirectories(const std::filesystem::path& path) {
}
bool File::DeleteDirectories(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
const auto normalized_path =
WithoutTrailingSeparator(path);
@@ -322,11 +402,18 @@ bool File::DeleteDirectories(const std::filesystem::path& path) {
bool File::DeleteFile(
const std::filesystem::path& name) // @suppress("Member declaration not found")
{
if (IsArchivePath(name)) {
return false;
}
return SysFileDeleteFile(name);
}
bool File::Flush() {
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return true;
}
return SysFileFlush(*m_p->f);
}
@@ -348,6 +435,9 @@ std::vector<std::byte> File::ReadWholeBuffer() {
}
DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetLastAccessTimeUTC(GetArchiveHostPath(name));
}
SysTimeStruct t {};
SysFileToSystemTimeUtc(SysFileGetLastAccessTimeUtc(name), t);
@@ -359,6 +449,9 @@ DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
}
DateTime File::GetLastWriteTimeUTC(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetLastWriteTimeUTC(GetArchiveHostPath(name));
}
SysTimeStruct t {};
SysFileToSystemTimeUtc(SysFileGetLastWriteTimeUtc(name), t);
@@ -373,6 +466,10 @@ void File::GetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, DateT
DateTime* write) {
EXIT_IF(access == nullptr);
EXIT_IF(write == nullptr);
if (IsArchivePath(name)) {
GetLastAccessAndWriteTimeUTC(GetArchiveHostPath(name), access, write);
return;
}
SysTimeStruct at {};
SysTimeStruct wt {};
@@ -399,7 +496,12 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
EXIT_IF(access == nullptr);
EXIT_IF(write == nullptr);
EXIT_IF(m_p->f == nullptr);
EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
GetLastAccessAndWriteTimeUTC(m_file_name, access, write);
return;
}
if (m_p->f == nullptr) {
*access = DateTime();
@@ -429,23 +531,35 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
}
bool File::SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
if (IsArchivePath(name)) {
return false;
}
auto f = DateTimeToFileTimeUtc(dt);
return SysFileSetLastAccessTimeUtc(name, f);
}
bool File::SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
if (IsArchivePath(name)) {
return false;
}
auto f = DateTimeToFileTimeUtc(dt);
return SysFileSetLastWriteTimeUtc(name, f);
}
bool File::SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, const DateTime& access,
const DateTime& write) {
if (IsArchivePath(name)) {
return false;
}
auto af = DateTimeToFileTimeUtc(access);
auto wf = DateTimeToFileTimeUtc(write);
return SysFileSetLastAccessAndWriteTimeUtc(name, af, wf);
}
std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return GetArchiveDirEntries(path);
}
std::vector<sys_dir_entry_t> files;
SysFileGetDents(path, files);
@@ -463,12 +577,18 @@ std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& pat
bool File::CopyFile(const std::filesystem::path& src,
const std::filesystem::path& dst) // @suppress("Member declaration not found")
{
if (IsArchivePath(src) || IsArchivePath(dst)) {
return false;
}
return SysFileCopyFile(src, dst);
}
bool File::RenameFile(const std::filesystem::path& src,
const std::filesystem::path& dst) // @suppress("Member declaration not found")
{
if (IsArchivePath(src) || IsArchivePath(dst)) {
return false;
}
if (IsFileExisting(dst)) {
DeleteFile(dst); // @suppress("Invalid arguments")
}
@@ -477,6 +597,9 @@ bool File::RenameFile(const std::filesystem::path& src,
}
void File::RemoveReadonly(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return;
}
SysFileRemoveReadonly(name);
}
src/common/file.h
+6
View File
@@ -7,6 +7,7 @@
#include <cstddef>
#include <filesystem>
#include <memory>
#include <optional>
#include <string>
#include <vector>
@@ -34,6 +35,10 @@ public:
std::string name;
bool is_file;
};
struct Info {
bool is_file;
uint64_t size;
};
File();
explicit File(const std::filesystem::path& name);
@@ -72,6 +77,7 @@ public:
std::vector<std::byte> ReadWholeBuffer();
static uint64_t Size(const std::filesystem::path& name);
static std::optional<Info> GetInfo(const std::filesystem::path& name);
static bool IsDirectoryExisting(const std::filesystem::path& path);
static bool IsFileExisting(const std::filesystem::path& name);
src/common/platform/sysFileIO.h
+1
View File
@@ -45,6 +45,7 @@ bool SysFileTruncate(sys_file_t& f, uint64_t size); // NOLINT(google-runt
bool SysFileUnlink(sys_file_t& f,
const std::filesystem::path& name); // NOLINT(google-runtime-references)
uint64_t SysFileSize(const std::filesystem::path& file_name);
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size);
bool SysFileIsError(sys_file_t& f); // NOLINT(google-runtime-references)
bool SysFileIsDirectoryExisting(const std::filesystem::path& path);
bool SysFileIsFileExisting(const std::filesystem::path& name);
src/common/platform/sysLinuxFileIO.cpp
+20 -23
View File
@@ -297,10 +297,21 @@ uint64_t SysFileSize(sys_file_t& f) {
return 0;
}
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
const auto path = get_internal_name(name);
struct stat info {};
if (stat(path.c_str(), &info) != 0) {
return false;
}
*is_file = !S_ISDIR(info.st_mode);
*size = *is_file ? static_cast<uint64_t>(info.st_size) : 0;
return true;
}
uint64_t SysFileSize(const std::filesystem::path& file_name) {
sys_file_t* f = SysFileOpenR(file_name);
uint64_t size = SysFileSize(*f);
SysFileClose(f);
bool is_file;
uint64_t size = 0;
SysFileGetInfo(file_name, &is_file, &size);
return size;
}
@@ -353,29 +364,15 @@ bool SysFileIsError(sys_file_t& f) {
}
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
auto real_name = get_internal_name(path);
auto real_name_str = real_name.string();
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return false;
}
return S_ISDIR(s.st_mode); // NOLINT
bool is_file;
uint64_t size;
return SysFileGetInfo(path, &is_file, &size) && !is_file;
}
bool SysFileIsFileExisting(const std::filesystem::path& name) {
auto real_name = get_internal_name(name);
auto real_name_str = real_name.string();
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return false;
}
return !S_ISDIR(s.st_mode); // NOLINT
bool is_file;
uint64_t size;
return SysFileGetInfo(name, &is_file, &size) && is_file;
}
bool SysFileCreateDirectory(const std::filesystem::path& path) {
src/common/platform/sysWindowsFileIO.cpp
+25 -19
View File
@@ -266,19 +266,27 @@ uint64_t SysFileSize(sys_file_t& f) {
return 0;
}
uint64_t SysFileSize(const std::filesystem::path& file_name) {
LARGE_INTEGER s;
WIN32_FILE_ATTRIBUTE_DATA a;
auto wide = file_name.wstring();
if (GetFileAttributesExW(wide.c_str(), GetFileExInfoStandard, &a) == 0) {
return 0;
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
const bool directory_path = !name.empty() && !name.has_filename() && name != name.root_path();
WIN32_FILE_ATTRIBUTE_DATA info {};
if (GetFileAttributesExW(directory_path ? name.parent_path().c_str() : name.c_str(),
GetFileExInfoStandard, &info) == 0) {
return false;
}
const bool file = (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
if (directory_path && file) {
return false;
}
*is_file = file;
*size = *is_file ? ((static_cast<uint64_t>(info.nFileSizeHigh) << 32u) | info.nFileSizeLow) : 0;
return true;
}
s.HighPart = static_cast<LONG>(a.nFileSizeHigh);
s.LowPart = a.nFileSizeLow;
return s.QuadPart;
uint64_t SysFileSize(const std::filesystem::path& file_name) {
bool is_file;
uint64_t size = 0;
SysFileGetInfo(file_name, &is_file, &size);
return size;
}
bool SysFileTruncate(sys_file_t& f, uint64_t size) {
@@ -346,17 +354,15 @@ bool SysFileIsError(sys_file_t& f) {
}
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
auto wide = path.wstring();
DWORD a = GetFileAttributesW(wide.c_str());
return a != INVALID_FILE_ATTRIBUTES &&
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) != 0u);
bool is_file;
uint64_t size;
return SysFileGetInfo(path, &is_file, &size) && !is_file;
}
bool SysFileIsFileExisting(const std::filesystem::path& name) {
auto wide = name.wstring();
DWORD a = GetFileAttributesW(wide.c_str());
return a != INVALID_FILE_ATTRIBUTES &&
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) == 0u);
bool is_file;
uint64_t size;
return SysFileGetInfo(name, &is_file, &size) && is_file;
}
bool SysFileCreateDirectory(const std::filesystem::path& path) {
src/kernel/fileSystem.cpp
+33 -12
View File
@@ -1,5 +1,6 @@
#include "kernel/fileSystem.h"
#include "common/archive.h"
#include "common/assert.h"
#include "common/common.h"
#include "common/dateTime.h"
@@ -41,6 +42,7 @@ public:
struct MountPair {
std::filesystem::path dir;
std::string point;
std::shared_ptr<Common::ArchiveReader> archive;
};
MountPoints() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); }
@@ -260,6 +262,7 @@ void MountPoints::Mount(const std::filesystem::path& folder, const std::string&
MountPair p;
p.dir = folder;
p.point = point_str;
p.archive = Common::OpenArchive(folder);
m_mount_pairs.push_back(std::move(p));
}
@@ -346,11 +349,13 @@ std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name)
}
const auto native_rel_path = Common::PathFromUtf8(rel_path);
if (Common::IsArchivePath(p.dir)) {
return p.dir / native_rel_path;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (HasWindowsForbiddenFilenameCharacter(rel_path)) {
::printf("FileSystem: Windows-incompatible guest filename: %s\n",
mounted_name.c_str());
::printf("FileSystem: Windows-incompatible guest filename: %s\n", mounted_name.c_str());
}
return p.dir / native_rel_path;
#else
@@ -466,14 +471,20 @@ int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_ENOENT;
}
if (Common::IsArchivePath(file->real_name) &&
(rw_mode != Common::File::Mode::Read || trunc || creat)) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_EROFS;
}
if (trunc && rw_mode == Common::File::Mode::Read) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_EACCES;
}
bool dir_exist = Common::File::IsDirectoryExisting(file->real_name);
bool file_exist = Common::File::IsFileExisting(file->real_name);
const auto info = Common::File::GetInfo(file->real_name);
const bool dir_exist = info && !info->is_file;
const bool file_exist = info && info->is_file;
// A missing path opened without O_CREAT is ENOENT
if (!creat && !dir_exist && !file_exist) {
@@ -671,7 +682,7 @@ int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) {
auto* file = g_files->GetFile(d);
if (file == nullptr) {
if (file == nullptr || !file->opened || !file->writable) {
return KERNEL_ERROR_EBADF;
}
@@ -886,7 +897,7 @@ int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_
auto* file = g_files->GetFile(d);
if (file == nullptr) {
if (file == nullptr || !file->opened || !file->writable) {
return KERNEL_ERROR_EBADF;
}
@@ -1063,15 +1074,13 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
auto real_file_name = g_mount_points->ResolvePath(path);
bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
bool is_file = Common::File::IsFileExisting(real_file_name);
if (!is_dir && !is_file) {
const auto info = Common::File::GetInfo(real_file_name);
if (!info) {
LOGF("\t file not found\n");
return KERNEL_ERROR_ENOENT;
}
EXIT_NOT_IMPLEMENTED(is_dir && is_file);
const bool is_dir = !info->is_file;
FileStat stat {};
stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u);
@@ -1085,7 +1094,7 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
stat.st_blksize = 512;
stat.st_blocks = stat.st_size / 512;
} else {
stat.st_size = static_cast<int64_t>(Common::File::Size(real_file_name));
stat.st_size = static_cast<int64_t>(info->size);
stat.st_blksize = 512;
stat.st_blocks = (stat.st_size + 511) / 512;
@@ -1222,6 +1231,9 @@ int KYTY_SYSV_ABI KernelUnlink(const char* path) {
}
auto real_file_name = g_mount_points->ResolvePath(path);
if (Common::IsArchivePath(real_file_name)) {
return KERNEL_ERROR_EROFS;
}
bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
bool is_file = Common::File::IsFileExisting(real_file_name);
@@ -1259,6 +1271,9 @@ int KYTY_SYSV_ABI KernelRename(const char* from, const char* to) {
auto to_path = std::string(to);
auto real_from = g_mount_points->ResolvePath(from_path);
auto real_to = g_mount_points->ResolvePath(to_path);
if (Common::IsArchivePath(real_from) || Common::IsArchivePath(real_to)) {
return KERNEL_ERROR_EROFS;
}
if (real_to.empty() || !Common::File::IsFileExisting(real_from)) {
return KERNEL_ERROR_ENOENT;
@@ -1343,6 +1358,9 @@ int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode) {
if (real_name.empty()) {
return KERNEL_ERROR_ENOENT;
}
if (Common::IsArchivePath(real_name)) {
return KERNEL_ERROR_EROFS;
}
if (Common::File::IsDirectoryExisting(real_name)) {
return KERNEL_ERROR_EEXIST;
@@ -1369,6 +1387,9 @@ int KYTY_SYSV_ABI KernelRmdir(const char* path) {
LOGF("\t path = %s\n", path);
auto real_name = g_mount_points->ResolvePath(std::string(path));
if (Common::IsArchivePath(real_name)) {
return KERNEL_ERROR_EROFS;
}
if (!Common::File::IsDirectoryExisting(real_name)) {
return KERNEL_ERROR_ENOENT;
src/launcher/CMakeLists.txt
+1 -1
View File
@@ -20,7 +20,7 @@ add_executable(launcher WIN32
)
target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version)
target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version common)
if(WIN32)
target_link_libraries(launcher setupapi)
endif()
src/launcher/include/configurationItem.h
+3 -1
View File
@@ -1,6 +1,7 @@
#ifndef CONFIGURATION_ITEM_H
#define CONFIGURATION_ITEM_H
#include <QIcon>
#include <QObject>
#include <QTreeWidgetItem>
@@ -18,7 +19,7 @@ public:
explicit ConfigurationItem(std::unique_ptr<Configuration> info, QTreeWidget* parent);
~ConfigurationItem() override;
void Update();
void Update(bool reload_icon = false);
bool operator<(const QTreeWidgetItem& other) const override;
Configuration& GetInfo() { return *m_info; }
@@ -35,6 +36,7 @@ private:
void UpdateStatusIndicator();
std::unique_ptr<Configuration> m_info;
QIcon m_icon;
bool m_running = false;
QComboBox* m_status_combo = nullptr;
QLabel* m_status_indicator = nullptr;
src/launcher/include/gameContent.h
+28
View File
@@ -0,0 +1,28 @@
#ifndef GAME_CONTENT_H
#define GAME_CONTENT_H
#include <QByteArray>
#include <QString>
#include <QStringList>
#include <cstdint>
#include <filesystem>
namespace GameContent {
inline constexpr uint64_t MaxMetadataSize = uint64_t {1} << 20u;
inline constexpr uint64_t MaxImageSize = uint64_t {32} << 20u;
inline constexpr uint64_t MaxTrophyPackageSize = uint64_t {128} << 20u;
[[nodiscard]] std::filesystem::path ToPath(const QString& path);
[[nodiscard]] QString FromPath(const std::filesystem::path& path);
[[nodiscard]] bool IsArchive(const QString& base);
[[nodiscard]] std::filesystem::path Resolve(const QString& base, const QString& relative);
[[nodiscard]] bool FileExists(const QString& base, const QString& relative);
[[nodiscard]] QByteArray ReadPath(const std::filesystem::path& path, uint64_t max_size);
[[nodiscard]] QByteArray ReadFile(const QString& base, const QString& relative, uint64_t max_size);
[[nodiscard]] QStringList ListFiles(const QString& base, const QString& relative);
} // namespace GameContent
#endif // GAME_CONTENT_H
src/launcher/include/gameListTreeWidget.h
+8 -5
View File
@@ -1,6 +1,8 @@
#ifndef GAME_LIST_TREE_WIDGET_H
#define GAME_LIST_TREE_WIDGET_H
#include "gameContent.h"
#include <QPaintEvent>
#include <QPainter>
#include <QPalette>
@@ -17,14 +19,15 @@ public:
viewport()->setAutoFillBackground(false);
}
void SetBackgroundImage(const QString& path) {
if (m_path == path) {
void SetBackgroundImage(const QString& key) {
if (m_path == key) {
return;
}
m_path = path;
m_path = key;
m_source = QPixmap();
if (!path.isEmpty()) {
m_source.load(path);
if (!key.isEmpty()) {
m_source.loadFromData(GameContent::ReadFile(key, QStringLiteral("sce_sys/pic0.png"),
GameContent::MaxImageSize));
}
UpdateScaledBackground();
viewport()->update();
src/launcher/src/configurationItem.cpp
+20 -6
View File
@@ -1,6 +1,7 @@
#include "configurationItem.h"
#include "configuration.h"
#include "gameContent.h"
#include <QApplication>
#include <QComboBox>
@@ -14,6 +15,7 @@
#include <QLabel>
#include <QLineEdit>
#include <QLocale>
#include <QPixmap>
#include <QSize>
#include <QStringList>
#include <QStyle>
@@ -154,7 +156,7 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
m_comment_edit->setFrame(false);
parent->setItemWidget(this, CommentsColumn, m_comment_edit);
Update();
Update(true);
SetRunning(false);
setData(SizeColumn, Qt::UserRole, qint64(-1));
@@ -170,7 +172,14 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
watcher->deleteLater();
});
watcher->setFuture(QtConcurrent::run([path = m_info->basedir]() -> qint64 {
if (path.isEmpty() || !QDir(path).exists()) {
if (path.isEmpty()) {
return -1;
}
const QFileInfo info(path);
if (GameContent::IsArchive(path)) {
return info.size();
}
if (!QDir(path).exists()) {
return -1;
}
qint64 bytes = 0;
@@ -188,7 +197,7 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
ConfigurationItem::~ConfigurationItem() = default;
void ConfigurationItem::Update() {
void ConfigurationItem::Update(bool reload_icon) {
const auto display_text = GetDisplayText(*m_info);
const auto path = GetPathText(*m_info);
@@ -209,6 +218,12 @@ void ConfigurationItem::Update() {
m_comment_edit->setText(m_info->game_comment);
}
if (reload_icon) {
const auto icon_data = GameContent::ReadFile(
m_info->basedir, QStringLiteral("sce_sys/icon0.png"), GameContent::MaxImageSize);
QPixmap icon;
m_icon = !icon_data.isEmpty() && icon.loadFromData(icon_data) ? QIcon(icon) : QIcon {};
}
UpdateIcon();
UpdateStatusIndicator();
}
@@ -267,9 +282,8 @@ void ConfigurationItem::SetCompatibilityEditable(bool editable) {
}
void ConfigurationItem::UpdateIcon() {
const QString icon_file = QDir(m_info->basedir).filePath(QStringLiteral("sce_sys/icon0.png"));
if (QFileInfo::exists(icon_file)) {
setIcon(NameColumn, QIcon(icon_file));
if (!m_icon.isNull()) {
setIcon(NameColumn, m_icon);
return;
}
src/launcher/src/configurationListWidget.cpp
+69 -47
View File
@@ -1,9 +1,11 @@
#include "configurationListWidget.h"
#include "common/archive.h"
#include "compatibilityDatabase.h"
#include "configuration.h"
#include "configurationEditDialog.h"
#include "configurationItem.h"
#include "gameContent.h"
#include "gameListTreeWidget.h"
#include "inputMappingDialog.h"
#include "mainDialog.h"
@@ -127,15 +129,6 @@ static QStringList SettingsStringList(const QVariant& value) {
return text.isEmpty() ? QStringList() : QStringList({text});
}
static QString GetPic0Path(const Configuration& info) {
if (info.basedir.isEmpty()) {
return {};
}
const auto path = QDir(info.basedir).filePath(QStringLiteral("sce_sys/pic0.png"));
return QFileInfo::exists(path) ? QFileInfo(path).absoluteFilePath() : QString();
}
static void ConfigureGameList(Ui::ConfigurationListWidget* ui) {
ui->cfgs_list->setAlternatingRowColors(false);
ui->cfgs_list->setAllColumnsShowFocus(true);
@@ -490,17 +483,16 @@ static QString GetFirmwareVersion(const QJsonObject& root) {
return version;
}
static GameMetadata GetGameMetadata(const QString& param_file, const QString& fallback) {
static GameMetadata GetGameMetadata(const QByteArray& param_data, const QString& fallback) {
GameMetadata ret;
ret.title_name = fallback;
QFile param(param_file);
if (!param.open(QIODevice::ReadOnly)) {
if (param_data.isEmpty()) {
return ret;
}
QJsonParseError parse_error;
const auto doc = QJsonDocument::fromJson(param.readAll(), &parse_error);
const auto doc = QJsonDocument::fromJson(param_data, &parse_error);
if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) {
return ret;
}
@@ -522,18 +514,18 @@ static GameMetadata GetGameMetadata(const QString& param_file, const QString& fa
}
static void SetGameFiles(Configuration& info, const QString& game_dir, const QString& game_path,
const GameMetadata& metadata) {
QDir game(game_dir);
const GameMetadata& metadata, bool archive) {
const QFileInfo game(game_dir);
info.game_path = game_path;
info.basedir = game.absolutePath();
info.basedir = archive ? game.absoluteFilePath() : QDir(game_dir).absolutePath();
info.name = metadata.title_name;
info.title_id = metadata.title_id;
info.gameVersion = metadata.gameVersion;
info.firmwareVer = metadata.firmwareVer;
if (info.name.isEmpty()) {
info.name = game.dirName();
info.name = archive ? game.completeBaseName() : QDir(game_dir).dirName();
}
}
@@ -609,39 +601,31 @@ void ConfigurationListWidget::ScanGameDirectory() {
const QString eboot_name = QStringLiteral("eboot.bin");
QSet<QString> found_games;
for (const auto& root_path: m_game_dirs) {
QDir root(root_path);
if (root_path.isEmpty() || !root.exists()) {
continue;
}
QList<QDir> pending_dirs;
const auto root_subdirs =
root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks);
for (const auto& subdir: root_subdirs) {
pending_dirs.append(QDir(subdir.absoluteFilePath()));
}
while (!pending_dirs.isEmpty()) {
QDir game_dir = pending_dirs.takeFirst();
if (game_dir.exists(eboot_name)) {
const QString game_path = NormalizeGameDirectory(game_dir.absolutePath());
const QString legacy_game_path = root.relativeFilePath(game_dir.absolutePath());
const auto add_game = [this, &found_games, &eboot_name,
&running_items](const QString& base, const QString& game_path,
const QString& legacy_game_path, const QString& fallback,
bool archive) {
const QString game_key = PathKey(game_path);
if (game_key.isEmpty() || found_games.contains(game_key)) {
continue;
return;
}
// Keep the index and decompression cache alive through validation, metadata and icon
// reads, after rejecting duplicate candidates from overlapping game folders.
const auto reader = archive ? Common::OpenArchive(GameContent::ToPath(base)) : nullptr;
if (archive && (reader == nullptr || !GameContent::FileExists(base, eboot_name))) {
return;
}
found_games.insert(game_key);
auto metadata = GetGameMetadata(
game_dir.filePath(QStringLiteral("sce_sys/param.json")), game_dir.dirName());
const auto metadata =
GetGameMetadata(GameContent::ReadFile(base, QStringLiteral("sce_sys/param.json"),
GameContent::MaxMetadataSize),
fallback);
auto info = std::make_unique<Configuration>();
info->custom_settings =
FindCustomInfo(&m_custom_infos, game_path, legacy_game_path) != nullptr;
SetGameFiles(*info, game_dir.absolutePath(), game_path, metadata);
SetGameFiles(*info, base, game_path, metadata, archive);
const auto* compatibility = m_compatibility->Find(info->title_id);
if (compatibility != nullptr) {
info->game_status = compatibility->status;
@@ -652,10 +636,10 @@ void ConfigurationListWidget::ScanGameDirectory() {
// Refresh metadata without losing the running row's identity.
const QSignalBlocker status_blocker(item->GetStatusCombo());
item->GetInfo().CopyGameInfoFrom(*info);
item->Update();
item->Update(true);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
continue;
return;
}
auto* item = new ConfigurationItem(std::move(info), m_ui->cfgs_list);
@@ -693,6 +677,42 @@ void ConfigurationListWidget::ScanGameDirectory() {
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
};
const auto scan_archives = [&add_game](const QDir& directory, const QDir& root) {
const auto entries = directory.entryInfoList(QDir::Files | QDir::NoSymLinks, QDir::Name);
for (const auto& archive: entries) {
const auto base = archive.absoluteFilePath();
if (!Common::IsSupportedArchive(GameContent::ToPath(base))) {
continue;
}
add_game(base, QDir::cleanPath(base), root.relativeFilePath(base),
archive.completeBaseName(), true);
}
};
for (const auto& root_path: m_game_dirs) {
QDir root(root_path);
if (root_path.isEmpty() || !root.exists()) {
continue;
}
scan_archives(root, root);
QList<QDir> pending_dirs;
const auto root_subdirs =
root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks);
for (const auto& subdir: root_subdirs) {
pending_dirs.append(QDir(subdir.absoluteFilePath()));
}
while (!pending_dirs.isEmpty()) {
QDir game_dir = pending_dirs.takeFirst();
scan_archives(game_dir, root);
if (game_dir.exists(eboot_name)) {
const QString game_path = NormalizeGameDirectory(game_dir.absolutePath());
const QString legacy_game_path = root.relativeFilePath(game_dir.absolutePath());
add_game(game_dir.absolutePath(), game_path, legacy_game_path, game_dir.dirName(),
false);
continue;
}
@@ -805,7 +825,8 @@ void ConfigurationListWidget::open_game_folder() {
return;
}
const QDir game_dir(item->GetInfo().basedir);
const auto base = item->GetInfo().basedir;
const QDir game_dir(GameContent::IsArchive(base) ? QFileInfo(base).absolutePath() : base);
if (!game_dir.exists()) {
QMessageBox::warning(this, tr("Open game folder"), tr("Game folder does not exist."));
return;
@@ -896,7 +917,7 @@ void ConfigurationListWidget::SelectItem(QTreeWidgetItem* witem) {
m_ui->edit_button->setEnabled(!item->IsRunning());
m_selected_item = item;
m_ui->cfgs_list->SetBackgroundImage(GetPic0Path(item->GetInfo()));
m_ui->cfgs_list->SetBackgroundImage(item->GetInfo().basedir);
emit Select();
}
@@ -959,7 +980,8 @@ void ConfigurationListWidget::show_context_menu(const QPoint& pos) {
if (item != nullptr) {
action_run->setDisabled(item->IsRunning());
action_open_folder->setDisabled(!QDir(item->GetInfo().basedir).exists());
const auto& base = item->GetInfo().basedir;
action_open_folder->setDisabled(!QDir(base).exists() && !GameContent::IsArchive(base));
action_view_trophies->setDisabled(!has_trophy_data);
action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty());
action_edit->setDisabled(item->IsRunning());
src/launcher/src/gameContent.cpp
+80
View File
@@ -0,0 +1,80 @@
#include "gameContent.h"
#include "common/archive.h"
#include "common/file.h"
#include "common/stringUtils.h"
#include <QFileInfo>
#include <limits>
namespace GameContent {
std::filesystem::path ToPath(const QString& path) {
#if defined(_WIN32)
return std::filesystem::path(path.toStdWString());
#else
return std::filesystem::path(path.toStdString());
#endif
}
QString FromPath(const std::filesystem::path& path) {
#if defined(_WIN32)
return QString::fromStdWString(path.wstring());
#else
return QString::fromStdString(Common::PathToString(path));
#endif
}
bool IsArchive(const QString& base) {
const QFileInfo info(base);
return info.isFile() && Common::IsSupportedArchive(ToPath(base));
}
std::filesystem::path Resolve(const QString& base, const QString& relative) {
const auto root = ToPath(base);
return IsArchive(base) ? Common::MakeArchivePath(root, ToPath(relative))
: root / ToPath(relative);
}
bool FileExists(const QString& base, const QString& relative) {
return Common::File::IsFileExisting(Resolve(base, relative));
}
QByteArray ReadFile(const QString& base, const QString& relative, uint64_t max_size) {
return ReadPath(Resolve(base, relative), max_size);
}
QByteArray ReadPath(const std::filesystem::path& path, uint64_t max_size) {
Common::File file(path, Common::File::Mode::Read);
if (file.IsInvalid()) {
return {};
}
const auto size = file.Size();
if (size > max_size || size > static_cast<uint64_t>(std::numeric_limits<int>::max())) {
return {};
}
QByteArray data(static_cast<int>(size), Qt::Uninitialized);
uint32_t bytes_read = 0;
file.Read(data.data(), static_cast<uint32_t>(size), &bytes_read);
return bytes_read == size ? data : QByteArray {};
}
QStringList ListFiles(const QString& base, const QString& relative) {
const auto directory = Resolve(base, relative);
const bool archive = Common::IsArchivePath(directory);
QStringList result;
for (const auto& entry: Common::File::GetDirEntries(directory)) {
if (entry.is_file) {
const auto path = FromPath(directory / Common::PathFromUtf8(entry.name));
if (archive || !QFileInfo(path).isSymLink()) {
result.append(path);
}
}
}
return result;
}
} // namespace GameContent
src/launcher/src/mainDialog.cpp
+3 -2
View File
@@ -3,6 +3,7 @@
#include "configuration.h"
#include "configurationItem.h"
#include "configurationListWidget.h"
#include "gameContent.h"
#include "patchesDialog.h"
#include "updateChecker.h"
@@ -260,7 +261,7 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
}
QString game = info.basedir;
if (!info.elf.isEmpty()) {
if (!info.elf.isEmpty() && !GameContent::IsArchive(info.basedir)) {
game = QDir(info.basedir).filePath(info.elf);
}
args << "--game" << game;
@@ -507,7 +508,7 @@ void MainDialogPrivate::Update() {
if (run_enabled) {
const auto& info = item->GetInfo();
auto dir = info.basedir;
run_enabled = !dir.isEmpty() && QDir(dir).exists();
run_enabled = !dir.isEmpty() && (QDir(dir).exists() || GameContent::IsArchive(dir));
}
m_ui->widget->SetRunEnabled(run_enabled);
src/launcher/src/trophyViewerDialog.cpp
+48 -17
View File
@@ -1,13 +1,14 @@
#include "trophyViewerDialog.h"
#include "common/archive.h"
#include "configuration.h"
#include "gameContent.h"
#include <QAbstractItemView>
#include <QBrush>
#include <QByteArray>
#include <QDialogButtonBox>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QFont>
#include <QHeaderView>
@@ -45,6 +46,8 @@ constexpr int UCP_HEADER_LEN = 0x40;
constexpr int UCP_TOC_SKIP = 0x20;
constexpr int UCP_ENTRY_LEN = 0x40;
constexpr int UCP_NAME_LEN = 0x20;
constexpr quint32 UCP_MAX_FILES = 4096;
constexpr quint64 UCP_MAX_EXTRACTED_SIZE = quint64 {64} << 20u;
struct UcpEntry {
QString name;
@@ -98,14 +101,22 @@ static QString ReadFixedString(const QByteArray& data, qsizetype offset, qsizety
return QString::fromLatin1(data.constData() + offset, len);
}
static bool IsUsedUcpEntry(const QString& name) {
const auto lower = name.toCaseFolded();
return lower == QStringLiteral("tropconf.json") || lower == QStringLiteral("tropmeta.json") ||
(lower.startsWith(QStringLiteral("tropmeta_")) &&
lower.endsWith(QStringLiteral(".json"))) ||
(lower.startsWith(QStringLiteral("trop")) && lower.endsWith(QStringLiteral(".png")));
}
static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files, QString& error) {
QFile file(file_name);
if (!file.open(QIODevice::ReadOnly)) {
const QByteArray data =
GameContent::ReadPath(GameContent::ToPath(file_name), GameContent::MaxTrophyPackageSize);
if (data.isEmpty()) {
error = QObject::tr("Could not open %1").arg(QDir::toNativeSeparators(file_name));
return false;
}
const QByteArray data = file.readAll();
if (data.size() < UCP_HEADER_LEN) {
error = QObject::tr("%1 is too small to be a trophy package.")
.arg(QFileInfo(file_name).fileName());
@@ -127,14 +138,18 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
}
const auto declared_size = qFromBigEndian<quint64>(data.constData() + 0x08);
if (declared_size > static_cast<quint64>(data.size())) {
if (declared_size < UCP_HEADER_LEN || declared_size > static_cast<quint64>(data.size())) {
error = QObject::tr("%1 is truncated.").arg(QFileInfo(file_name).fileName());
return false;
}
const auto file_count = qFromBigEndian<quint32>(data.constData() + 0x10);
if (file_count > UCP_MAX_FILES) {
error = QObject::tr("%1 contains too many files.").arg(QFileInfo(file_name).fileName());
return false;
}
const auto toc_offset = static_cast<quint64>(qFromBigEndian<quint32>(data.constData() + 0x14));
const auto data_size = static_cast<quint64>(data.size());
const auto data_size = declared_size;
const quint64 table_size = UCP_TOC_SKIP + static_cast<quint64>(file_count) * UCP_ENTRY_LEN;
if (toc_offset > data_size || table_size > data_size - toc_offset ||
@@ -144,6 +159,7 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
return false;
}
quint64 extracted_size = 0;
for (quint32 i = 0; i < file_count; i++) {
const auto entry_offset = static_cast<qsizetype>(toc_offset + UCP_TOC_SKIP +
static_cast<quint64>(i) * UCP_ENTRY_LEN);
@@ -161,9 +177,22 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
.arg(QFileInfo(file_name).fileName(), entry.name);
return false;
}
if (!IsUsedUcpEntry(entry.name)) {
continue;
}
const auto entry_limit = entry.name.endsWith(QStringLiteral(".png"), Qt::CaseInsensitive)
? GameContent::MaxImageSize
: GameContent::MaxMetadataSize;
if (entry.size > entry_limit || entry.size > UCP_MAX_EXTRACTED_SIZE - extracted_size) {
error = QObject::tr("%1 contains an oversized entry for %2.")
.arg(QFileInfo(file_name).fileName(), entry.name);
return false;
}
files.insert(entry.name.toCaseFolded(), data.mid(static_cast<qsizetype>(entry.offset),
static_cast<qsizetype>(entry.size)));
extracted_size += entry.size;
}
return true;
@@ -392,30 +421,31 @@ static QStringList FindTrophyFiles(const Configuration* info) {
return {};
}
const QDir trophy_dir(QDir(info->basedir).filePath(QStringLiteral("sce_sys/trophy2")));
if (!trophy_dir.exists()) {
return {};
QStringList files;
for (const auto& file:
GameContent::ListFiles(info->basedir, QStringLiteral("sce_sys/trophy2"))) {
const auto name = QFileInfo(file).fileName();
if (name.startsWith(QStringLiteral("trophy"), Qt::CaseInsensitive) &&
name.endsWith(QStringLiteral(".ucp"), Qt::CaseInsensitive)) {
files.append(file);
}
const auto files =
trophy_dir.entryInfoList({QStringLiteral("Trophy*.ucp"), QStringLiteral("trophy*.ucp")},
QDir::Files | QDir::NoSymLinks, QDir::Name | QDir::IgnoreCase);
}
files.sort(Qt::CaseInsensitive);
QStringList trophy_files;
QSet<QString> seen;
for (const auto& file: files) {
auto key = file.canonicalFilePath();
auto key = QFileInfo(file).canonicalFilePath();
if (key.isEmpty()) {
key = file.absoluteFilePath();
key = file;
}
key = QDir::cleanPath(key).toCaseFolded();
if (!seen.contains(key)) {
seen.insert(key);
trophy_files.append(file.absoluteFilePath());
trophy_files.append(file);
}
}
return trophy_files;
}
@@ -506,6 +536,7 @@ void TrophyViewerDialog::ShowForGame(const Configuration* info, QWidget* parent)
}
bool TrophyViewerDialog::LoadGame(const Configuration& info, QString& error) {
const auto reader = Common::OpenArchive(GameContent::ToPath(info.basedir));
const auto trophy_files = FindTrophyFiles(&info);
if (trophy_files.isEmpty()) {
error = tr("No trophy package found in sce_sys/trophy2.");
src/libs/avPlayer.cpp
+60 -39
View File
@@ -1,6 +1,6 @@
#include "common/common.h"
#include "common/file.h"
#include "common/logging/log.h"
#include "common/stringUtils.h"
#include "kernel/fileSystem.h"
#include "kernel/pthread.h"
#include "libs/audio.h"
@@ -9,12 +9,14 @@
#include <algorithm>
#include <atomic>
#include <cctype>
#include <cerrno>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <deque>
#include <limits>
#include <magic_enum.hpp>
#include <memory>
#include <mutex>
@@ -577,30 +579,40 @@ public:
explicit FileStreamer(AvPlayerFileReplacement f): file(f) {}
~FileStreamer() {
if (ctx != nullptr) {
av_freep(&ctx->buffer);
avio_context_free(&ctx);
}
if (opened && file.close != nullptr) {
if (opened) {
file.close(file.object_pointer);
}
}
bool Init(const std::string& path) {
if (file.open == nullptr || file.close == nullptr || file.read_offset == nullptr ||
file.size == nullptr) {
return false;
}
if (file.open(file.object_pointer, path.c_str()) < 0) {
if (file.open != nullptr) {
if (file.close == nullptr || file.read_offset == nullptr || file.size == nullptr ||
file.open(file.object_pointer, path.c_str()) < 0) {
return false;
}
opened = true;
size = file.size(file.object_pointer);
if (size == 0) {
} else {
if (!local_file.Open(LibKernel::FileSystem::GetRealFilename(path),
Common::File::Mode::Read)) {
return false;
}
auto* buf = static_cast<uint8_t*>(av_malloc(4096));
size = local_file.Size();
}
if (size == 0 || size > static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) {
return false;
}
constexpr int buffer_size = 64 * 1024;
auto* buf = static_cast<uint8_t*>(av_malloc(buffer_size));
if (buf == nullptr) {
return false;
}
ctx = avio_alloc_context(buf, 4096, 0, this, Read, nullptr, Seek);
ctx = avio_alloc_context(buf, buffer_size, 0, this, Read, nullptr, Seek);
if (ctx == nullptr) {
av_free(buf);
}
return ctx != nullptr;
}
AVIOContext* Context() const { return ctx; }
@@ -612,32 +624,49 @@ private:
return AVERROR_EOF;
}
len = static_cast<int>(std::min<uint64_t>(len, s->size - s->pos));
auto r =
s->file.read_offset(s->file.object_pointer, buf, s->pos, static_cast<uint32_t>(len));
if (r <= 0) {
return r == 0 ? AVERROR_EOF : r;
int read = 0;
if (s->opened) {
read = s->file.read_offset(s->file.object_pointer, buf, s->pos,
static_cast<uint32_t>(len));
} else {
uint32_t bytes = 0;
s->local_file.Read(buf, static_cast<uint32_t>(len), &bytes);
read = static_cast<int>(bytes);
}
s->pos += static_cast<uint64_t>(r);
return r;
if (read <= 0 || read > len) {
return read < 0 ? read : AVERROR(EIO);
}
static int64_t Seek(void* opaque, int64_t off, int whence) {
s->pos += static_cast<uint64_t>(read);
return read;
}
static int64_t Seek(void* opaque, int64_t offset, int whence) {
auto* s = static_cast<FileStreamer*>(opaque);
if ((whence & AVSEEK_SIZE) != 0) {
return static_cast<int64_t>(s->size);
}
int64_t p = whence == SEEK_SET
? off
: (whence == SEEK_CUR
? static_cast<int64_t>(s->pos) + off
: (whence == SEEK_END ? static_cast<int64_t>(s->size) + off : -1));
if (p < 0) {
return -1;
whence &= ~AVSEEK_FORCE;
uint64_t base = 0;
switch (whence) {
case SEEK_SET: break;
case SEEK_CUR: base = s->pos; break;
case SEEK_END: base = s->size; break;
default: return AVERROR(EINVAL);
}
p = std::min<int64_t>(p, static_cast<int64_t>(s->size));
s->pos = static_cast<uint64_t>(p);
return p;
const auto distance = offset < 0 ? uint64_t {0} - static_cast<uint64_t>(offset)
: static_cast<uint64_t>(offset);
if ((offset < 0 && distance > base) ||
(offset >= 0 && distance > std::numeric_limits<int64_t>::max() - base)) {
return AVERROR(EINVAL);
}
const auto position = offset < 0 ? base - distance : base + distance;
if (!s->opened && !s->local_file.Seek(position)) {
return AVERROR(EIO);
}
s->pos = position;
return static_cast<int64_t>(position);
}
AvPlayerFileReplacement file;
Common::File local_file;
bool opened = false;
uint64_t pos = 0;
uint64_t size = 0;
@@ -675,26 +704,18 @@ public:
return static_cast<Source*>(opaque)->interrupt_io.load() ? 1 : 0;
};
raw->interrupt_callback.opaque = this;
if (file.open != nullptr) {
streamer = std::make_unique<FileStreamer>(file);
if (!streamer->Init(path)) {
avformat_free_context(raw);
return AVPLAYER_ERROR_OPERATION_FAILED;
}
raw->pb = streamer->Context();
if (auto rc = avformat_open_input(&raw, nullptr, nullptr, nullptr); rc < 0) {
LOGF("\t avformat_open_input callback failed: %s\n", fferr(rc).c_str());
if (const auto rc = avformat_open_input(&raw, file.open == nullptr ? path.c_str() : nullptr,
nullptr, nullptr);
rc < 0) {
LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(), path.c_str());
return AVPLAYER_ERROR_OPERATION_FAILED;
}
} else {
auto real = LibKernel::FileSystem::GetRealFilename(std::string(path.c_str()));
auto real_str = Common::PathToString(real);
if (auto rc = avformat_open_input(&raw, real_str.c_str(), nullptr, nullptr); rc < 0) {
LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(),
real_str.c_str());
return AVPLAYER_ERROR_OPERATION_FAILED;
}
}
fmt = raw;
if (auto rc = avformat_find_stream_info(fmt, nullptr); rc < 0) {
LOGF("\t avformat_find_stream_info failed: %s\n", fferr(rc).c_str());
src/main.cpp
+14 -3
View File
@@ -1,3 +1,4 @@
#include "common/archive.h"
#include "common/common.h"
#include "common/dateTime.h"
#include "common/debug.h"
@@ -10,6 +11,7 @@
#include <charconv>
#include <cstdio>
#include <filesystem>
#include <string_view>
#include <vector>
#include <fmt/format.h>
@@ -40,9 +42,9 @@ static std::string GetBuildString() {
static void PrintUsage() {
::printf("%s\n", GetBuildString().c_str());
::printf("kyty_emulator --game <dir|elf> [options]\n\n");
::printf("kyty_emulator --game <dir|elf|zar> [options]\n\n");
::printf("Options:\n");
::printf(" --game <dir|elf> Game directory or ELF to load.\n");
::printf(" --game <dir|elf|zar> Game directory, ELF, or ZArchive to load.\n");
::printf(" --game-patch <json> ETAHen cheat file.\n");
::printf(" --screen-width <num> Window width. Default: 1280.\n");
::printf(" --screen-height <num> Window height. Default: 720.\n");
@@ -240,6 +242,14 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
if (Common::File::IsDirectoryExisting(path)) {
options.app0_dir = path;
options.elf = "/app0/eboot.bin";
} else if (Common::IsSupportedArchive(path) && Common::File::IsFileExisting(path)) {
const auto root = Common::MakeArchivePath(path);
if (!Common::File::IsFileExisting(root / "eboot.bin")) {
::printf("Archive does not contain eboot.bin: %s\n", value.c_str());
return false;
}
options.app0_dir = root;
options.elf = "/app0/eboot.bin";
} else if (Common::File::IsFileExisting(path)) {
options.app0_dir = path.parent_path();
@@ -249,7 +259,8 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
options.elf = std::filesystem::path("/app0") / path.filename();
} else {
::printf("--game must point to an existing directory or ELF: %s\n", value.c_str());
::printf("--game must point to an existing directory, ELF, or archive: %s\n",
value.c_str());
return false;
}
} else if (arg == "--game-patch") {
tests/ArchiveFileTests.cpp
+400
View File
@@ -0,0 +1,400 @@
#include "common/file.h"
#include "common/archive.h"
#include "common/stringUtils.h"
#include "ArchiveTestFixture.h"
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <limits>
#include <string>
#include <thread>
#include <vector>
#include <zarchive/zarchivecommon.h>
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
#include <pthread.h>
#endif
namespace {
void Check(bool value, const char *text) {
if (!value) {
std::fprintf(stderr, "ArchiveFileTests: failed: %s\n", text);
std::abort();
}
}
class TempDirectory {
public:
TempDirectory() {
const auto unique =
std::chrono::steady_clock::now().time_since_epoch().count();
m_path = std::filesystem::temp_directory_path() /
("kyty_zarchive_test_!_" + std::to_string(unique));
Check(std::filesystem::create_directories(m_path),
"create temporary directory");
}
~TempDirectory() {
std::error_code error;
std::filesystem::remove_all(m_path, error);
}
[[nodiscard]] const std::filesystem::path &Path() const { return m_path; }
KYTY_CLASS_NO_COPY(TempDirectory);
private:
std::filesystem::path m_path;
};
template <typename T>
void SetValue(std::vector<std::byte> &bytes, size_t offset, T value) {
value = _ZARCHIVE::_store(value);
Check(offset <= bytes.size() && sizeof(value) <= bytes.size() - offset,
"patch lies inside fixture");
std::memcpy(bytes.data() + offset, &value, sizeof(value));
}
void SetMemberNameOffset(std::vector<std::byte> &bytes,
const _ZARCHIVE::Footer &footer, uint32_t name_offset) {
const auto entry_offset = footer.sectionFileTree.offset + sizeof(_ZARCHIVE::FileDirectoryEntry);
uint32_t original = 0;
std::memcpy(&original, bytes.data() + entry_offset, sizeof(original));
SetValue(bytes, entry_offset, (_ZARCHIVE::_load(original) & 0x80000000u) | name_offset);
}
template <typename Edit>
std::filesystem::path MutateArchive(const std::filesystem::path &source,
const char *name, Edit edit) {
Common::File input(source, Common::File::Mode::Read);
auto bytes = input.ReadWholeBuffer();
_ZARCHIVE::Footer footer{};
Check(bytes.size() >= sizeof(footer), "fixture has a footer");
std::memcpy(&footer, bytes.data() + bytes.size() - sizeof(footer), sizeof(footer));
_ZARCHIVE::Footer::Deserialize(&footer, &footer);
edit(bytes, footer);
_ZARCHIVE::Footer::Serialize(&footer, &footer);
std::memcpy(bytes.data() + bytes.size() - sizeof(footer), &footer, sizeof(footer));
const auto destination = source.parent_path() / name;
Common::File output;
Check(output.Create(destination), "create malformed archive fixture");
uint32_t written = 0;
output.Write(bytes.data(), static_cast<uint32_t>(bytes.size()), &written);
Check(written == bytes.size(), "write malformed archive fixture");
return destination;
}
void CheckMalformedArchives(const std::filesystem::path &source,
const std::vector<uint8_t> &payload) {
const auto bad_name = MutateArchive(source, "bad-name.zar", [](auto &bytes, auto &footer) {
SetMemberNameOffset(bytes, footer, static_cast<uint32_t>(footer.sectionNames.size));
});
Check(!Common::OpenArchive(bad_name), "reject name offset at the end of name table");
const auto bad_length = MutateArchive(source, "bad-length.zar", [](auto &bytes, auto &footer) {
SetMemberNameOffset(bytes, footer, static_cast<uint32_t>(footer.sectionNames.size - 1));
bytes[footer.sectionNames.offset + footer.sectionNames.size - 1] = std::byte{0x80};
});
Check(!Common::OpenArchive(bad_length), "reject truncated extended name length");
const auto bad_children = MutateArchive(source, "bad-children.zar", [](auto &bytes, auto &footer) {
SetValue(bytes, footer.sectionFileTree.offset + sizeof(uint32_t), UINT32_MAX);
});
Check(!Common::OpenArchive(bad_children), "reject invalid directory children without throwing");
const auto bad_range = MutateArchive(source, "bad-range.zar", [](auto &, auto &footer) {
footer.sectionCompressedData.offset = std::numeric_limits<uint64_t>::max();
});
Check(!Common::OpenArchive(bad_range), "reject overflowing footer section range");
const auto truncated = source.parent_path() / "truncated.zar";
Check(std::filesystem::copy_file(source, truncated), "copy truncated archive fixture");
std::filesystem::resize_file(truncated, std::filesystem::file_size(truncated) - 1);
Check(!Common::OpenArchive(truncated), "reject missing footer byte");
const auto corrupted = MutateArchive(source, "corrupt-data.zar", [](auto &bytes, auto &footer) {
_ZARCHIVE::CompressionOffsetRecord record{};
std::memcpy(&record, bytes.data() + footer.sectionOffsetRecords.offset, sizeof(record));
_ZARCHIVE::CompressionOffsetRecord::Deserialize(&record, 1, &record);
const auto second_block = footer.sectionCompressedData.offset + record.baseOffset + record.size[0] + 1;
std::fill_n(bytes.data() + second_block, record.size[1] + 1, std::byte{});
});
Common::File damaged(Common::MakeArchivePath(corrupted, "assets/subdir/data.bin"),
Common::File::Mode::Read);
Check(!damaged.IsInvalid(), "open archive with damaged compressed data");
std::vector<uint8_t> bytes(payload.size());
uint32_t read = 0;
damaged.Read(bytes.data(), static_cast<uint32_t>(bytes.size()), &read);
Check(read > 0 && read < 64 * 1024 && damaged.Tell() == read &&
std::equal(bytes.begin(), bytes.begin() + read, payload.begin()),
"report the valid prefix before a corrupt compressed block");
Check(damaged.Seek(2 * 64 * 1024), "seek past corrupt compressed block");
damaged.Read(bytes.data(), 97, &read);
Check(read == 97 && std::equal(bytes.begin(), bytes.begin() + read,
payload.begin() + 2 * 64 * 1024),
"recover by reading an unaffected compressed block");
}
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
void CheckCancellation(const std::filesystem::path &path,
const std::vector<uint8_t> &payload) {
// Keep the file and destination on the parent thread: cancellation must only release the
// reader thread's request and staging buffer after the worker has finished using them.
struct CancelledRead {
Common::File file;
std::vector<uint8_t> data;
std::atomic_bool ready = false;
std::atomic_bool start = false;
uint32_t bytes = 0;
} read;
Check(read.file.Open(path, Common::File::Mode::Read), "open cancellation fixture");
read.data.resize(payload.size());
pthread_t thread;
Check(pthread_create(&thread, nullptr, [](void *opaque) -> void * {
auto &read = *static_cast<CancelledRead *>(opaque);
read.ready = true;
while (!read.start.load()) {}
read.file.Read(read.data.data(), static_cast<uint32_t>(read.data.size()), &read.bytes);
pthread_testcancel();
return nullptr;
}, &read) == 0, "start cancellable archive reader");
while (!read.ready.load()) {}
Check(pthread_cancel(thread) == 0, "queue cancellation before the archive worker wait");
read.start = true;
void *result = nullptr;
Check(pthread_join(thread, &result) == 0 && result == PTHREAD_CANCELED,
"restore cancellation after the archive read");
Check(read.bytes == payload.size() && read.data == payload,
"finish worker reads and copies before releasing a cancelled caller");
int previous = PTHREAD_CANCEL_ENABLE;
int observed = PTHREAD_CANCEL_ENABLE;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &previous);
read.file.Seek(0);
read.file.Read(read.data.data(), 97, &read.bytes);
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &observed);
pthread_setcancelstate(previous, nullptr);
Check(observed == PTHREAD_CANCEL_DISABLE && read.bytes == 97,
"preserve a caller's disabled cancellation state");
}
#endif
} // namespace
int main() {
TempDirectory temporary;
const auto archive_path =
temporary.Path() / std::filesystem::path(u8"game-テスト.zar");
std::vector<uint8_t> payload(3 * 1024 * 1024 + 37);
for (size_t index = 0; index < payload.size(); index++) {
payload[index] = static_cast<uint8_t>((index * 37 + 11) & 0xff);
}
Check(ArchiveTests::CreateArchive(archive_path, payload), "create archive fixture");
const auto native_file = Common::File::GetInfo(archive_path);
const auto native_directory = Common::File::GetInfo(temporary.Path() / "");
Check(native_file && native_file->is_file &&
native_file->size == std::filesystem::file_size(archive_path),
"query native file type and size together");
Check(native_directory && !native_directory->is_file && native_directory->size == 0,
"query native directory with trailing separator");
Check(!Common::File::GetInfo(archive_path / "") && Common::File::Size(archive_path / "") == 0 &&
!Common::File::GetInfo(temporary.Path() / "missing.bin"),
"reject missing native paths and files with a trailing separator");
const auto root = Common::MakeArchivePath(archive_path);
auto reader_pin = Common::OpenArchive(archive_path);
Check(reader_pin && Common::OpenArchive(root) == reader_pin,
"mounted archive pin reuses the same reader and index");
const auto archive_file = Common::File::GetInfo(root / "assets/subdir/data.bin");
const auto archive_directory = Common::File::GetInfo(root / "assets/");
Check(archive_file && archive_file->is_file && archive_file->size == payload.size(),
"query archive member type and size together");
Check(archive_directory && !archive_directory->is_file && archive_directory->size == 0 &&
!Common::File::GetInfo(root / "missing.bin"),
"distinguish archive directories from missing members");
Check(Common::IsSupportedArchive(archive_path) &&
Common::IsSupportedArchive("GAME.ZAR") &&
!Common::IsSupportedArchive("game.zip"),
"select an archive backend by supported extension");
Check(!Common::IsArchivePath(temporary.Path() / "ordinary!" / "asset.bin"),
"ordinary host paths with exclamation marks stay host paths");
Check(Common::IsArchivePath(root), "recognize virtual archive root");
Check(Common::GetArchiveHostPath(root) == archive_path,
"recover host archive path");
Check(Common::File::IsDirectoryExisting(root), "find archive root directory");
Check(Common::File::IsFileExisting(root / "EBOOT.BIN"),
"lookup is case insensitive");
Check(!Common::File::IsFileExisting(root / "missing.bin"),
"reject missing member");
Check(Common::IsArchivePath(root / "../outside.bin"),
"retain archive classification for invalid traversal");
Check(!Common::File::IsFileExisting(root / "../outside.bin"),
"reject leading parent traversal");
Check(Common::File::IsFileExisting(root / "assets/../eboot.bin"),
"normalize internal parent traversal");
Check(Common::File::IsFileExisting(root / ArchiveTests::LongFilename),
"decode extended-length archive names");
Check(Common::File::IsFileExisting(root / Common::PathFromUtf8(ArchiveTests::UnicodeFilename)),
"lookup Unicode archive member");
const auto entries = Common::File::GetDirEntries(root);
Check(std::ranges::any_of(entries,
[](const auto &entry) {
return entry.is_file && entry.name == "eboot.bin";
}),
"enumerate root file");
Check(std::ranges::any_of(entries,
[](const auto &entry) {
return !entry.is_file && entry.name == "assets";
}),
"enumerate root directory");
Check(std::ranges::any_of(entries, [](const auto &entry) {
return entry.is_file && entry.name == ArchiveTests::UnicodeFilename;
}), "enumerate Unicode member names as UTF-8");
const auto data_path = root / "assets/subdir/data.bin";
Check(Common::File::Size(data_path) == payload.size(), "report member size");
Common::File file(data_path, Common::File::Mode::Read);
Check(!file.IsInvalid(), "open archive member");
Check(file.Remaining() == payload.size(), "report initial remaining bytes");
Check(file.Seek(64 * 1024 - 19), "seek before compression block boundary");
std::vector<uint8_t> slice(97);
uint32_t bytes_read = 0;
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == slice.size(), "read across compression block boundary");
Check(
std::equal(slice.begin(), slice.end(), payload.begin() + 64 * 1024 - 19),
"preserve data across compression block boundary");
Check(file.Seek(payload.size() - 9), "seek near member end");
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == 9, "truncate read at member end");
Check(file.IsEOF(), "report member end of file");
Check(file.Seek(payload.size() + 100), "allow seek past member end");
Check(file.Remaining() == 0, "report no bytes remaining past member end");
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == 0, "read past member end returns no data");
Check(file.Seek(0), "seek to member start");
std::vector<uint8_t> full(payload.size());
file.Read(full.data(), static_cast<uint32_t>(full.size()), &bytes_read);
Check(bytes_read == full.size(), "read a multi-chunk member");
Check(full == payload, "preserve a multi-chunk member");
std::atomic_bool concurrent_reads_ok = true;
std::vector<std::thread> readers;
for (uint32_t worker = 0; worker < 8; worker++) {
readers.emplace_back([&, worker] {
Common::File concurrent_file(data_path, Common::File::Mode::Read);
std::array<uint8_t, 4096> concurrent_data{};
for (uint32_t iteration = 0; iteration < 16; iteration++) {
const auto offset =
static_cast<size_t>(worker * 7919 + iteration * 12347) %
(payload.size() - concurrent_data.size());
uint32_t concurrent_bytes_read = 0;
if (concurrent_file.IsInvalid() || !concurrent_file.Seek(offset)) {
concurrent_reads_ok = false;
return;
}
concurrent_file.Read(concurrent_data.data(), concurrent_data.size(),
&concurrent_bytes_read);
if (concurrent_bytes_read != concurrent_data.size() ||
!std::equal(concurrent_data.begin(), concurrent_data.end(),
payload.begin() +
static_cast<std::ptrdiff_t>(offset))) {
concurrent_reads_ok = false;
return;
}
}
});
}
for (auto &reader : readers) {
reader.join();
}
Check(concurrent_reads_ok, "read concurrently through independent file handles");
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
CheckCancellation(data_path, payload);
#endif
#if !defined(_WIN32)
// Guest threads have much smaller stacks than std::thread's default stack.
pthread_attr_t attributes;
Check(pthread_attr_init(&attributes) == 0 &&
pthread_attr_setstacksize(&attributes, 64 * 1024) == 0,
"configure a guest-sized stack");
struct SmallStackRead {
const std::filesystem::path &path;
const std::vector<uint8_t> &expected;
bool passed = false;
} small_stack{data_path, payload};
pthread_t thread;
Check(pthread_create(&thread, &attributes, [](void *opaque) -> void * {
auto &test = *static_cast<SmallStackRead *>(opaque);
Common::File input(test.path, Common::File::Mode::Read);
auto data = input.ReadWholeBuffer();
test.passed = data.size() == test.expected.size() &&
std::memcmp(data.data(), test.expected.data(), data.size()) == 0;
return nullptr;
}, &small_stack) == 0, "start guest-sized-stack reader");
pthread_attr_destroy(&attributes);
Check(pthread_join(thread, nullptr) == 0 && small_stack.passed,
"decompress a multi-block file on a guest-sized stack");
#endif
Common::File metadata(root / "sce_sys/param.json", Common::File::Mode::Read);
const auto metadata_data = metadata.ReadWholeBuffer();
const std::string metadata_text(
reinterpret_cast<const char *>(metadata_data.data()),
metadata_data.size());
Check(metadata_text.find("Archive Test") != std::string::npos,
"read metadata member");
Common::File writable;
Check(!writable.Open(data_path, Common::File::Mode::ReadWrite),
"reject writable member open");
Check(!Common::File::DeleteFile(data_path), "reject member deletion");
Check(!Common::File::CreateDirectory(root / "new-dir"),
"reject member directory creation");
Check(!file.Truncate(0) && !file.Unlink(), "reject descriptor mutations");
const auto copied = temporary.Path() / "copied.bin";
Common::File host_fixture;
Check(host_fixture.Create(copied), "create host mutation fixture");
host_fixture.Close();
Check(!Common::File::CopyFile(data_path, copied) &&
!Common::File::CopyFile(copied, data_path) &&
!Common::File::RenameFile(data_path, copied) &&
!Common::File::RenameFile(copied, data_path),
"reject unsupported copy and rename operations involving archives");
CheckMalformedArchives(archive_path, payload);
file.Close();
metadata.Close();
Check(Common::OpenArchive(archive_path) == reader_pin,
"mount pin retains the archive reader after the last member closes");
reader_pin.reset();
std::error_code remove_error;
Check(std::filesystem::remove(archive_path, remove_error) && !remove_error,
"release archive handle after the final reader closes");
const std::array<uint8_t, 3> replacement{42, 7, 19};
Check(ArchiveTests::CreateArchive(archive_path, replacement), "replace a closed archive");
Common::File replaced(data_path, Common::File::Mode::Read);
const auto replaced_data = replaced.ReadWholeBuffer();
Check(replaced_data.size() == replacement.size() &&
std::memcmp(replaced_data.data(), replacement.data(), replacement.size()) == 0,
"reopening after the final reader closes sees replacement contents");
std::printf("ArchiveFileTests: all cases passed\n");
return 0;
}
tests/ArchiveTestFixture.h
+66
View File
@@ -0,0 +1,66 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <fstream>
#include <span>
#include <string>
#include <string_view>
#include <zarchive/zarchivewriter.h>
namespace ArchiveTests {
inline constexpr std::string_view Eboot = "ELF fixture";
inline constexpr std::string_view Param =
R"({"titleId":"TEST00001","titleName":"Archive Test"})";
inline constexpr std::string_view UnicodeFilename =
"asset-\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e.bin";
inline const std::string LongFilename = std::string(160, 'a') + ".bin";
// Both Common::File and guest filesystem tests read the same archive layout.
inline bool CreateArchive(const std::filesystem::path &path,
std::span<const uint8_t> payload) {
struct Output {
std::filesystem::path path;
std::ofstream stream;
bool good = true;
} output{.path = path};
{
ZArchiveWriter writer(
[](int32_t, void *opaque) {
auto &out = *static_cast<Output *>(opaque);
out.stream.open(out.path, std::ios::binary | std::ios::trunc);
out.good = out.stream.is_open();
},
[](const void *data, size_t size, void *opaque) {
auto &out = *static_cast<Output *>(opaque);
out.stream.write(static_cast<const char *>(data),
static_cast<std::streamsize>(size));
out.good = out.good && out.stream.good();
},
&output);
if (!output.good || !writer.MakeDir("sce_sys", true) ||
!writer.MakeDir("assets/subdir", true)) {
return false;
}
const auto add = [&](std::string_view name, const void *data, size_t size) {
if (!writer.StartNewFile(name.data())) {
return false;
}
writer.AppendData(data, size);
return true;
};
if (!add("eboot.bin", Eboot.data(), Eboot.size()) ||
!add("sce_sys/param.json", Param.data(), Param.size()) ||
!add("assets/subdir/data.bin", payload.data(), payload.size()) ||
!add(UnicodeFilename, Eboot.data(), Eboot.size()) ||
!add(LongFilename, Eboot.data(), Eboot.size())) {
return false;
}
writer.Finalize();
}
output.stream.close();
return output.good && !output.stream.fail();
}
} // namespace ArchiveTests
tests/AvPlayerFileTests.cpp
+134
View File
@@ -0,0 +1,134 @@
#include "libs/avPlayer.cpp"
#include "common/archive.h"
#include "ArchiveTestFixture.h"
#include <array>
#include <chrono>
namespace {
std::filesystem::path content_root;
}
namespace Libs::LibKernel {
int KYTY_SYSV_ABI PthreadCreate(Pthread*, const PthreadAttr*, pthread_entry_func_t, void*, const char*) {
std::abort();
}
int KYTY_SYSV_ABI PthreadJoin(Pthread, void**) { std::abort(); }
namespace FileSystem {
std::filesystem::path GetRealFilename(const std::string& path) {
return path.starts_with("/app0/") ? content_root / path.substr(6) : std::filesystem::path {};
}
}
}
namespace {
using namespace Libs::Audio::AvPlayer;
void Check(bool condition, const char* message) {
if (!condition) {
std::fprintf(stderr, "AvPlayerFileTests: %s\n", message);
std::abort();
}
}
void CheckStream(FileStreamer& stream, const std::vector<uint8_t>& data) {
auto* io = stream.Context();
Check(io->seek(io->opaque, 0, AVSEEK_SIZE) == static_cast<int64_t>(data.size()), "query size");
Check(io->seek(io->opaque, 65531, SEEK_SET | AVSEEK_FORCE) == 65531, "seek across block boundary");
std::array<uint8_t, 32> bytes {};
Check(io->read_packet(io->opaque, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), data.begin() + 65531), "read across block boundary");
Check(io->seek(io->opaque, -16, SEEK_CUR) == 65547, "seek relative to cursor");
Check(io->seek(io->opaque, INT64_MIN, SEEK_CUR) < 0 &&
io->seek(io->opaque, INT64_MAX, SEEK_END) < 0 &&
io->seek(io->opaque, 0, 12345) < 0, "reject invalid and overflowing seeks");
Check(io->seek(io->opaque, 0, SEEK_CUR) == 65547, "rejected seeks preserve the cursor");
Check(io->seek(io->opaque, -7, SEEK_END) == static_cast<int64_t>(data.size() - 7), "seek near end");
Check(io->read_packet(io->opaque, bytes.data(), bytes.size()) == 7 &&
std::equal(bytes.begin(), bytes.begin() + 7, data.end() - 7), "short read at end");
Check(io->read_packet(io->opaque, bytes.data(), bytes.size()) == AVERROR_EOF, "report EOF");
Check(io->seek(io->opaque, 11, SEEK_END) == static_cast<int64_t>(data.size() + 11) &&
io->read_packet(io->opaque, bytes.data(), bytes.size()) == AVERROR_EOF, "read beyond end");
Check(io->seek(io->opaque, 0, SEEK_SET) == 0, "rewind");
std::vector<uint8_t> buffered(data.size());
Check(avio_read(io, buffered.data(), static_cast<int>(buffered.size())) == buffered.size() &&
buffered == data, "FFmpeg buffered reads preserve content");
Check(avio_seek(io, 123, SEEK_SET) == 123 &&
avio_read(io, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), data.begin() + 123), "FFmpeg seeks reset buffered reads");
}
struct Callbacks {
const std::vector<uint8_t>& data;
unsigned opens = 0;
unsigned closes = 0;
bool premature_eof = false;
};
int KYTY_SYSV_ABI Open(void* object, const char*) {
++static_cast<Callbacks*>(object)->opens;
return 0;
}
int KYTY_SYSV_ABI Close(void* object) {
++static_cast<Callbacks*>(object)->closes;
return 0;
}
int KYTY_SYSV_ABI Read(void* object, uint8_t* buffer, uint64_t offset, uint32_t size) {
auto& state = *static_cast<Callbacks*>(object);
if (state.premature_eof) {
return 0;
}
const auto bytes = std::min<uint64_t>(size, state.data.size() - offset);
std::memcpy(buffer, state.data.data() + offset, bytes);
return static_cast<int>(bytes);
}
uint64_t KYTY_SYSV_ABI Size(void* object) { return static_cast<Callbacks*>(object)->data.size(); }
}
int main() {
const auto root = std::filesystem::temp_directory_path() /
("kyty_avio_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
Check(std::filesystem::create_directories(root), "create fixture directory");
std::vector<uint8_t> payload(128 * 1024 + 37);
for (size_t i = 0; i < payload.size(); ++i) {
payload[i] = static_cast<uint8_t>(i * 37);
}
{
Common::File output;
Check(output.Create(root / "movie.bin"), "create native fixture");
output.Write(payload.data(), static_cast<uint32_t>(payload.size()));
}
Check(ArchiveTests::CreateArchive(root / "game.zar", payload), "create archive fixture");
content_root = root;
{
FileStreamer native({});
Check(native.Init("/app0/movie.bin"), "open native stream");
CheckStream(native, payload);
}
content_root = Common::MakeArchivePath(root / "game.zar");
{
FileStreamer archive({});
Check(archive.Init("/app0/assets/subdir/data.bin"), "open archived stream");
CheckStream(archive, payload);
}
Callbacks callbacks {payload};
const AvPlayerFileReplacement replacement {&callbacks, Open, Close, Read, Size};
{
FileStreamer callback(replacement);
Check(callback.Init("callback"), "open guest callback stream");
CheckStream(callback, payload);
callbacks.premature_eof = true;
auto* io = callback.Context();
Check(io->seek(io->opaque, 0, SEEK_SET) == 0, "rewind callback");
uint8_t byte = 0;
Check(io->read_packet(io->opaque, &byte, 1) == AVERROR(EIO), "reject premature EOF");
}
Check(callbacks.opens == 1 && callbacks.closes == 1, "close callback exactly once");
{
FileStreamer missing({});
Check(!missing.Init("/unmounted/movie.bin"), "reject unmapped media");
}
std::error_code error;
std::filesystem::remove_all(root, error);
Check(!error, "release all archive handles");
std::puts("AvPlayerFileTests: all cases passed");
}
tests/KernelFileSystemTests.cpp
+117 -1
View File
@@ -2,7 +2,9 @@
#include <SDL3/SDL_main.h>
#include "common/emulatorConfig.h"
#include "common/archive.h"
#include "common/file.h"
#include "ArchiveTestFixture.h"
#include "common/logging/log.h"
#include "common/subsystems.h"
#include "common/threads.h"
@@ -214,6 +216,119 @@ void CheckUnmappedPaths(const std::filesystem::path &root) {
FileSystem::Umount("/app0");
}
void CheckArchiveMount(const std::filesystem::path &root) {
const auto archive = root / u8"game-日本語.zar";
std::vector<uint8_t> payload(2 * 64 * 1024 + 33);
for (size_t i = 0; i < payload.size(); ++i) {
payload[i] = static_cast<uint8_t>(i * 37 + 11);
}
Check(ArchiveTests::CreateArchive(archive, payload), "create mounted archive fixture");
const auto archive_root = Common::MakeArchivePath(archive);
const auto host_member = archive_root / "assets/subdir/data.bin";
constexpr char GuestMember[] = "/app0/assets/subdir/data.bin";
FileSystem::Mount(archive_root, "/app0");
Check(FileSystem::GetRealFilename(GuestMember) == host_member &&
FileSystem::GetRealFilename("/app01/eboot.bin").empty(),
"resolve archive mounts at path-component boundaries");
Check(FileSystem::KernelOpen("/app0/../outside.bin", 0, 0) ==
Libs::LibKernel::KERNEL_ERROR_ENOENT &&
FileSystem::KernelOpen("/app0/missing.bin", 0, 0) ==
Libs::LibKernel::KERNEL_ERROR_ENOENT,
"reject traversal outside an archive and missing members");
FileSystem::FileStat path_stat{}, descriptor_stat{};
Check(FileSystem::KernelStat(GuestMember, &path_stat) == OK &&
path_stat.st_size == payload.size() &&
path_stat.st_size == Common::File::Size(host_member) &&
FileSystem::KernelCheckReachability(GuestMember) == OK,
"archive path stat and reachability agree with Common::File");
const int fd = FileSystem::KernelOpen("/app0/ASSETS/subdir/DATA.BIN", 0, 0);
Check(fd >= 3 && FileSystem::KernelFstat(fd, &descriptor_stat) == OK &&
descriptor_stat.st_size == path_stat.st_size &&
descriptor_stat.st_mode == path_stat.st_mode,
"case-insensitive archive open and descriptor stat agree");
std::array<uint8_t, 97> bytes{};
constexpr int64_t Offset = 64 * 1024 - 19;
Check(FileSystem::KernelPread(fd, bytes.data(), bytes.size(), Offset) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), payload.begin() + Offset) &&
FileSystem::KernelLseek(fd, 0, 1) == 0,
"archive pread crosses a compression block without moving position");
Check(FileSystem::KernelLseek(fd, Offset, 0) == Offset &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), payload.begin() + Offset) &&
FileSystem::KernelLseek(fd, 0, 1) == Offset + bytes.size(),
"archive seek and sequential read share descriptor position");
Check(FileSystem::KernelLseek(fd, -9, 2) == payload.size() - 9 &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == 9 &&
std::equal(bytes.begin(), bytes.begin() + 9, payload.end() - 9) &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == 0,
"archive reads stop at the member boundary");
Check(FileSystem::KernelWrite(fd, bytes.data(), 1) ==
Libs::LibKernel::KERNEL_ERROR_EBADF &&
FileSystem::KernelPwrite(fd, bytes.data(), 1, 0) ==
Libs::LibKernel::KERNEL_ERROR_EBADF &&
FileSystem::KernelFtruncate(fd, 0) ==
Libs::LibKernel::KERNEL_ERROR_EBADF,
"archive read-only descriptors reject write and truncate");
const auto unicode_guest = std::string("/app0/") + std::string(ArchiveTests::UnicodeFilename);
const int unicode = FileSystem::KernelOpen(unicode_guest.c_str(), 0, 0);
Check(unicode >= 3 && FileSystem::KernelRead(unicode, bytes.data(), bytes.size()) ==
ArchiveTests::Eboot.size() &&
std::memcmp(bytes.data(), ArchiveTests::Eboot.data(), ArchiveTests::Eboot.size()) == 0,
"open and read a Unicode archive member");
Check(FileSystem::KernelClose(unicode) == OK, "close Unicode archive member");
const int directory = FileSystem::KernelOpen("/app0/", 0x00020000, 0);
Check(directory >= 3, "open mounted archive directory");
std::array<char, 512> block{};
const auto expected_entries = Common::File::GetDirEntries(archive_root);
size_t entries_seen = 0;
for (;;) {
const int length = FileSystem::KernelGetdents(directory, block.data(), block.size());
Check(length >= 0 && length <= block.size(), "read archive directory records");
if (length == 0) {
break;
}
for (size_t offset = 0; offset < static_cast<size_t>(length);) {
uint16_t record_length = 0;
Check(length - offset >= 8, "archive directory record header fits");
std::memcpy(&record_length, block.data() + offset + 4, sizeof(record_length));
const auto name_length = static_cast<uint8_t>(block[offset + 7]);
Check(record_length >= 9 + name_length && record_length <= length - offset,
"archive directory record fits");
const std::string_view name(block.data() + offset + 8, name_length);
Check(std::any_of(expected_entries.begin(), expected_entries.end(), [&](const auto &entry) {
return entry.name == name && block[offset + 6] == (entry.is_file ? 8 : 4);
}), "guest directory entry matches Common::File name and type");
++entries_seen;
offset += record_length;
}
}
Check(entries_seen == expected_entries.size(), "guest enumerates every archive directory entry");
Check(FileSystem::KernelClose(directory) == OK, "close archive directory");
for (const int flags : {1, 2, 0x0200, 0x0400}) {
Check(FileSystem::KernelOpen(GuestMember, flags, 0777) ==
Libs::LibKernel::KERNEL_ERROR_EROFS,
"archive rejects write, create and truncate open flags");
}
Check(FileSystem::KernelUnlink(GuestMember) == Libs::LibKernel::KERNEL_ERROR_EROFS &&
FileSystem::KernelMkdir("/app0/new-dir", 0777) == Libs::LibKernel::KERNEL_ERROR_EROFS &&
FileSystem::KernelRmdir("/app0/assets") == Libs::LibKernel::KERNEL_ERROR_EROFS &&
FileSystem::KernelRename(GuestMember, "/app0/renamed.bin") ==
Libs::LibKernel::KERNEL_ERROR_EROFS,
"archive mount rejects path mutations");
FileSystem::Umount("/app0");
Check(FileSystem::GetRealFilename(GuestMember).empty() &&
FileSystem::KernelOpen(GuestMember, 0, 0) == Libs::LibKernel::KERNEL_ERROR_ENOENT &&
FileSystem::KernelLseek(fd, 0, 0) == 0 &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), payload.begin()),
"unmount hides archive paths while open descriptors retain the reader");
Check(FileSystem::KernelClose(fd) == OK, "close last archive descriptor");
}
void CheckUnicodePaths(const std::filesystem::path &root) {
constexpr std::string_view HostDirectory =
"Test\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e";
@@ -567,7 +682,7 @@ void CheckSocketWakeup() {
} // namespace
int main() {
int main(int, char**) {
Common::InitializeThreads();
Common::Subsystems subsystems;
subsystems.Initialize<Config::Lifecycle>();
@@ -589,6 +704,7 @@ int main() {
FileSystem::Initialize();
CheckMountRoot(temporary.Path());
CheckUnmappedPaths(temporary.Path());
CheckArchiveMount(temporary.Path());
CheckUnicodePaths(temporary.Path());
CheckUnicodeLogPath(temporary.Path());
CheckDirectoryStream(temporary.Path());