Zarchive Loading (#724)

* Simplify archive loading and harden filesystem integration

* Restore directory entry name length assertion

* Keep graphics-dependent filesystem tests out of CI

---------

Co-authored-by: nmzik <Nmzik@mail.ru>
This commit is contained in:
Aspen
2026-09-28 16:06:44 -04:00
committed by GitHub
parent 8e61798bfd
commit 4b0de1b672
29 changed files with 1956 additions and 262 deletions
+6 -6
View File
@@ -115,12 +115,12 @@ jobs:
cmake --build _Build/windows --target launcher ^ cmake --build _Build/windows --target launcher ^
audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^ audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
resource_tracking_tests scalar_provenance_tests ^ resource_tracking_tests scalar_provenance_tests ^
resource_materialization_tests shader_vertex_metadata_tests --parallel resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests --parallel
- name: Test - name: Test
shell: cmd shell: cmd
run: | run: |
ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|virtual_memory_allocation)$" ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
- name: Install - name: Install
shell: cmd shell: cmd
@@ -203,14 +203,14 @@ jobs:
cmake --build _Build/macos \ cmake --build _Build/macos \
--target launcher audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \ --target launcher audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
resource_tracking_tests scalar_provenance_tests \ resource_tracking_tests scalar_provenance_tests \
resource_materialization_tests shader_vertex_metadata_tests \ resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
--parallel --parallel
- name: Test - name: Test
shell: bash shell: bash
run: | run: |
ctest --test-dir _Build/macos --output-on-failure \ ctest --test-dir _Build/macos --output-on-failure \
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|virtual_memory_allocation)$' -R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
- name: Install - name: Install
shell: bash shell: bash
@@ -386,14 +386,14 @@ jobs:
--target launcher page_manager_tests memory_tracker_tests \ --target launcher page_manager_tests memory_tracker_tests \
audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \ audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
resource_tracking_tests scalar_provenance_tests \ resource_tracking_tests scalar_provenance_tests \
resource_materialization_tests shader_vertex_metadata_tests \ resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
--parallel --parallel
- name: Test - name: Test
shell: bash shell: bash
run: | run: |
ctest --test-dir _Build/linux --output-on-failure \ ctest --test-dir _Build/linux --output-on-failure \
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|virtual_memory_allocation)$' -R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
- name: Install - name: Install
shell: bash shell: bash
+43
View File
@@ -28,6 +28,49 @@ FetchContent_Declare(
FetchContent_MakeAvailable(xbyak zydis) FetchContent_MakeAvailable(xbyak zydis)
# ZArchive provides random-access reads for compressed game dumps. Build only
# the reader/writer library and its zstd dependency; the command-line tool is
# not part of KytyPS5's runtime distribution.
set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE)
set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE)
set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE)
set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(ZSTD_LEGACY_SUPPORT OFF CACHE BOOL "" FORCE)
set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
zstd
GIT_REPOSITORY https://github.com/facebook/zstd.git
GIT_TAG f8745da6ff1ad1e7bab384bd1f9d742439278e99
GIT_SHALLOW FALSE
SOURCE_SUBDIR build/cmake
)
FetchContent_Declare(
zarchive_source
GIT_REPOSITORY https://github.com/Exzap/ZArchive.git
GIT_TAG d2c717730092c7bf8cbb033b12fd4001b7c4d932
GIT_SHALLOW FALSE
PATCH_COMMAND "${GIT_EXECUTABLE}" apply --ignore-whitespace
"${CMAKE_CURRENT_SOURCE_DIR}/patches/zarchive-reader.patch"
SOURCE_SUBDIR kyty-library-only
)
FetchContent_MakeAvailable(zstd zarchive_source)
add_library(zarchive STATIC
"${zarchive_source_SOURCE_DIR}/src/zarchivereader.cpp"
"${zarchive_source_SOURCE_DIR}/src/zarchivewriter.cpp"
"${zarchive_source_SOURCE_DIR}/src/sha_256.c"
)
add_library(ZArchive::zarchive ALIAS zarchive)
target_include_directories(zarchive SYSTEM PUBLIC "${zarchive_source_SOURCE_DIR}/include")
target_include_directories(zarchive PRIVATE "${zstd_SOURCE_DIR}/lib")
target_link_libraries(zarchive PRIVATE libzstd_static)
target_compile_features(zarchive PRIVATE cxx_std_20)
if(MSVC)
target_compile_options(zarchive PRIVATE /W0)
else()
target_compile_options(zarchive PRIVATE -w)
endif()
set(SDL_SHARED OFF CACHE BOOL "" FORCE) set(SDL_SHARED OFF CACHE BOOL "" FORCE)
if(APPLE) if(APPLE)
# SDL's built-in iconv fallback does not support CP932. # SDL's built-in iconv fallback does not support CP932.
+100
View File
@@ -0,0 +1,100 @@
--- a/include/zarchive/zarchivecommon.h
+++ b/include/zarchive/zarchivecommon.h
@@ -256,7 +256,7 @@
bool IsWithinValidRange(uint64_t fileSize) const
{
- return (offset + size) <= fileSize;
+ return offset <= fileSize && size <= fileSize - offset;
}
};
--- a/src/zarchivereader.cpp
+++ b/src/zarchivereader.cpp
@@ -14,6 +14,7 @@
static bool _ifstream_readBytes(std::ifstream& file, uint64_t offset, void* buffer, uint32_t size)
{
+ file.clear();
file.seekg(offset, std::ios_base::beg);
file.read((char*)buffer, size);
return file.gcount() == size;
@@ -80,9 +81,27 @@
// verify file tree
if (fileTree[0].IsFile())
return nullptr; // first entry must be root directory
- auto rootName = GetName(nameTable, fileTree[0].GetNameOffset());
- if (!rootName.empty())
+ if (fileTree[0].GetNameOffset() != 0x7FFFFFFF)
return nullptr; // root node must not have a name
+ const uint64_t dataCapacity = (uint64_t)offsetRecords.size() * _ZARCHIVE::ENTRIES_PER_OFFSETRECORD * _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
+ for (size_t i = 0; i < fileTree.size(); ++i)
+ {
+ auto& entry = fileTree[i];
+ if (i != 0)
+ {
+ auto name = GetName(nameTable, entry.GetNameOffset());
+ if (name.empty() || name == "." || name == ".." || name.find_first_of("/\\") != std::string_view::npos || name.find('\0') != std::string_view::npos)
+ return nullptr;
+ }
+ if (entry.IsFile())
+ {
+ if (entry.GetFileOffset() > dataCapacity || entry.GetFileSize() > dataCapacity - entry.GetFileOffset())
+ return nullptr;
+ }
+ else if (entry.directoryRecord.nodeStartIndex > fileTree.size() ||
+ entry.directoryRecord.count > fileTree.size() - entry.directoryRecord.nodeStartIndex)
+ return nullptr;
+ }
// read meta data
// todo
@@ -238,7 +257,7 @@
uint32_t stepSize = std::min<uint32_t>(remainingBytes, _ZARCHIVE::COMPRESSED_BLOCK_SIZE - blockOffset);
CacheBlock* block = GetCachedBlock(blockIdx);
if (!block)
- return 0;
+ return bytesToRead - remainingBytes;
std::memcpy(bufferU8, block->data + blockOffset, stepSize);
rawReadOffset += stepSize;
remainingBytes -= stepSize;
@@ -325,14 +344,18 @@
// determine offset and size of compressed block
auto& record = m_offsetRecords[recordIndex];
uint64_t offset = record.baseOffset;
+ if (offset > m_compressedDataSize)
+ return false;
for (uint32_t i = 0; i < recordSubIndex; i++)
{
- offset += (uint64_t)record.size[i];
- offset++;
+ const uint64_t size = (uint64_t)record.size[i] + 1;
+ if (size > m_compressedDataSize - offset)
+ return false;
+ offset += size;
}
uint32_t compressedSize = (uint32_t)record.size[recordSubIndex] + 1;
// load file data
- if ((offset + compressedSize) > m_compressedDataSize)
+ if (compressedSize > m_compressedDataSize - offset)
return false;
offset += m_compressedDataOffset;
if (compressedSize == _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
@@ -350,7 +373,7 @@
// returns empty view on failure
std::string_view ZArchiveReader::GetName(const std::vector<uint8_t>& nameTable, uint32_t nameOffset)
{
- if (nameOffset == 0x7FFFFFFF || nameOffset > nameTable.size())
+ if (nameOffset == 0x7FFFFFFF || nameOffset >= nameTable.size())
return "";
// parse header
uint16_t nameLength = nameTable[nameOffset] & 0x7F;
@@ -359,7 +382,7 @@
// extended 2-byte length
if (nameOffset + 1 >= nameTable.size())
return "";
- nameLength |= ((uint16_t)nameTable[nameOffset] << 7);
+ nameLength |= ((uint16_t)nameTable[nameOffset + 1] << 7);
nameOffset += 2;
}
else
+16 -1
View File
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.16...4.3) cmake_minimum_required(VERSION 3.18...4.3)
if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0) if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies") set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies")
@@ -407,6 +407,16 @@ add_executable(page_manager_tests EXCLUDE_FROM_ALL
target_link_libraries(page_manager_tests common) target_link_libraries(page_manager_tests common)
target_include_directories(page_manager_tests PRIVATE ${inc_headers}) target_include_directories(page_manager_tests PRIVATE ${inc_headers})
add_executable(archive_file_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/ArchiveFileTests.cpp"
)
target_link_libraries(archive_file_tests common ZArchive::zarchive)
target_include_directories(archive_file_tests PRIVATE ${inc_headers})
add_executable(avplayer_file_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp")
target_link_libraries(avplayer_file_tests common FFmpeg::ffmpeg ZArchive::zarchive)
target_include_directories(avplayer_file_tests PRIVATE ${inc_headers})
add_executable(bit_array_tests EXCLUDE_FROM_ALL add_executable(bit_array_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/BitArrayTests.cpp" "${KYTY_TESTS_DIR}/BitArrayTests.cpp"
) )
@@ -514,6 +524,7 @@ target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
add_kyty_full_emulator_test(kernel_file_system_tests add_kyty_full_emulator_test(kernel_file_system_tests
"${KYTY_TESTS_DIR}/KernelFileSystemTests.cpp") "${KYTY_TESTS_DIR}/KernelFileSystemTests.cpp")
target_link_libraries(kernel_file_system_tests ZArchive::zarchive)
add_executable(sync_on_address_tests EXCLUDE_FROM_ALL add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp" "${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
@@ -580,6 +591,8 @@ if(BUILD_TESTING)
add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>) add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>) add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>) add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>) add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
add_test(NAME lru_cache COMMAND $<TARGET_FILE:lru_cache_tests>) add_test(NAME lru_cache COMMAND $<TARGET_FILE:lru_cache_tests>)
add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>) add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
@@ -649,6 +662,8 @@ if(BUILD_TESTING)
image_page_table_tests image_page_table_tests
memory_tracker_tests memory_tracker_tests
page_manager_tests page_manager_tests
archive_file_tests
avplayer_file_tests
bit_array_tests bit_array_tests
lru_cache_tests lru_cache_tests
shader_vertex_metadata_tests shader_vertex_metadata_tests
+11 -6
View File
@@ -236,9 +236,10 @@ cmake --build _Build/linux --target launcher --parallel
cmake --install _Build/linux --prefix _Build/linux/install cmake --install _Build/linux --prefix _Build/linux/install
``` ```
The configure step downloads the FFmpeg prebuilts and the `xbyak`/`zydis` sources, so it needs The configure step downloads the FFmpeg prebuilts and the `xbyak`, `zydis`, `zstd`, and ZArchive
network access; a fully sandboxed `nix build` would require vendoring those inputs. A Vulkan 1.3 sources, so it needs network access; a fully sandboxed `nix build` would require vendoring those
driver must be available at runtime (on NixOS, `hardware.graphics.enable = true`). inputs. A Vulkan 1.3 driver must be available at runtime (on NixOS,
`hardware.graphics.enable = true`).
### Building on macOS ### Building on macOS
@@ -332,17 +333,21 @@ open _Build/macos/install/KytyPS5.app # or double-click in Finder
``` ```
On first launch, add one or more game folders in the global settings. The launcher searches those On first launch, add one or more game folders in the global settings. The launcher searches those
folders recursively for game directories containing `eboot.bin`. Select a detected game and run it folders recursively for game directories containing `eboot.bin` and ZArchive (`.zar`) game dumps
from the game list. whose archive root contains `eboot.bin`. Select a detected game and run it from the game list.
ZArchive dumps are mounted read-only and streamed directly; they do not need to be extracted first.
The emulator can also be started directly with a legally obtained game directory or ELF file: The emulator can also be started directly with a legally obtained game directory, ELF file, or
ZArchive dump:
```powershell ```powershell
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame" .\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame"
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame.zar"
``` ```
```bash ```bash
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame" ./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame.zar"
``` ```
On macOS, the adjacent flat or app-bundled `libMoltenVK.dylib` is found automatically; no On macOS, the adjacent flat or app-bundled `libMoltenVK.dylib` is found automatically; no
+1
View File
@@ -26,6 +26,7 @@ target_link_libraries(common
cpuinfo cpuinfo
kyty_git_version kyty_git_version
spdlog::spdlog spdlog::spdlog
ZArchive::zarchive
) )
get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES) get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES)
+397
View File
@@ -0,0 +1,397 @@
#include "common/archive.h"
#include "common/stringUtils.h"
#include <algorithm>
#include <condition_variable>
#include <cstring>
#include <mutex>
#include <optional>
#include <string_view>
#include <thread>
#include <unordered_map>
#include <utility>
#include <zarchive/zarchivereader.h>
// POSIX hosts and MinGW's pthread-backed libstdc++ have cancellable condition waits.
// MSVC/ClangCL and MinGW's native Win32 threading backend do not use pthread waits.
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
#include <pthread.h>
#define KYTY_ARCHIVE_PTHREAD_CANCELLATION
#endif
namespace Common {
// Backends expose immutable metadata and offset-based reads. Cursor management, path routing,
// reader lifetime and the host-stack worker are shared by every archive format.
class ArchiveReader {
public:
struct Entry {
uint64_t id;
uint64_t size;
bool is_file;
};
virtual ~ArchiveReader() = default;
virtual std::optional<Entry> Find(std::string_view member) = 0;
virtual std::vector<File::DirEntry> List(std::string_view member) = 0;
virtual uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) = 0;
};
namespace {
class ZArchiveReaderBackend final: public ArchiveReader {
public:
explicit ZArchiveReaderBackend(std::unique_ptr<ZArchiveReader> reader)
: m_reader(std::move(reader)) {}
std::optional<Entry> Find(std::string_view member) override {
const auto node = m_reader->LookUp(member);
if (node == ZARCHIVE_INVALID_NODE) {
return {};
}
return Entry {node, m_reader->GetFileSize(node), m_reader->IsFile(node)};
}
std::vector<File::DirEntry> List(std::string_view member) override {
const auto node = m_reader->LookUp(member);
if (!m_reader->IsDirectory(node)) {
return {};
}
const auto count = m_reader->GetDirEntryCount(node);
if (count > (1u << 20u)) {
return {};
}
std::vector<File::DirEntry> result;
result.reserve(count);
for (uint32_t index = 0; index < count; ++index) {
ZArchiveReader::DirEntry entry {};
if (m_reader->GetDirEntry(node, index, entry)) {
result.push_back({std::string(entry.name), entry.isFile});
}
}
return result;
}
uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) override {
// ZArchiveReader already serializes access to its stream and decompression cache.
return m_reader->ReadFromFile(static_cast<ZArchiveNodeHandle>(id), offset, size, data);
}
private:
std::unique_ptr<ZArchiveReader> m_reader;
};
std::shared_ptr<ArchiveReader> OpenZArchive(const std::filesystem::path& path) {
auto reader = std::unique_ptr<ZArchiveReader>(ZArchiveReader::OpenFromFile(path));
return reader ? std::make_shared<ZArchiveReaderBackend>(std::move(reader)) : nullptr;
}
struct ArchiveFormat {
std::string_view extension;
std::shared_ptr<ArchiveReader> (*open)(const std::filesystem::path&);
};
constexpr ArchiveFormat Formats[] = {{".zar", OpenZArchive}};
using NativeView = std::basic_string_view<std::filesystem::path::value_type>;
const ArchiveFormat* FindFormat(NativeView path) {
for (const auto& format: Formats) {
if (path.size() < format.extension.size()) {
continue;
}
const auto extension = path.substr(path.size() - format.extension.size());
if (std::equal(
extension.begin(), extension.end(), format.extension.begin(),
[](auto a, char b) { return (a >= 'A' && a <= 'Z' ? a + ('a' - 'A') : a) == b; })) {
return &format;
}
}
return nullptr;
}
size_t FindMarker(NativeView path) {
// Native files use this check on every operation. The usual path has no marker and allocates
// nothing.
for (auto marker = path.find('!'); marker != NativeView::npos;
marker = path.find('!', marker + 1)) {
if ((marker + 1 == path.size() || path[marker + 1] == '/' || path[marker + 1] == '\\') &&
FindFormat(path.substr(0, marker)) != nullptr) {
return marker;
}
}
return NativeView::npos;
}
struct ParsedPath {
std::filesystem::path archive;
std::string member;
};
std::optional<ParsedPath> ParsePath(const std::filesystem::path& path) {
const auto& native = path.native();
const auto marker = FindMarker(native);
if (marker == NativeView::npos) {
return {};
}
ParsedPath parsed {.archive = std::filesystem::path(native.substr(0, marker))};
auto member = PathToGenericString(std::filesystem::path(native.substr(marker + 1)));
std::replace(member.begin(), member.end(), '\\', '/');
std::string_view remaining = member;
while (!remaining.empty()) {
const auto separator = remaining.find('/');
const auto component = remaining.substr(0, separator);
remaining.remove_prefix(separator == std::string_view::npos ? remaining.size() : separator + 1);
if (component.empty() || component == ".") {
continue;
}
if (component == "..") {
if (parsed.member.empty()) {
return {};
}
const auto separator = parsed.member.rfind('/');
parsed.member.resize(separator == std::string::npos ? 0 : separator);
} else {
if (!parsed.member.empty()) {
parsed.member += '/';
}
parsed.member += component;
}
}
return parsed;
}
std::mutex g_readers_mutex;
std::unordered_map<std::filesystem::path, std::weak_ptr<ArchiveReader>> g_readers;
// Guest stacks can be only 64 KiB. Decompress on an ordinary host stack, then copy into guest
// memory on the caller so GPU-tracked writes use its fault handler. A single worker bounds host
// threads and avoids duplicating the archive's own block cache. Requests live on waiting callers.
class ArchiveIoThread {
public:
static ArchiveIoThread& Instance() {
// Guest exit can run static destructors while other guest threads are still reading.
// Keep the worker alive until process termination, as with the guest threads themselves.
static auto* instance = new ArchiveIoThread;
return *instance;
}
uint64_t Read(ArchiveReader& reader, uint64_t id, uint64_t offset, uint32_t size, void* data) {
Request request {reader, id, offset, size, data};
std::unique_lock lock(m_mutex);
if (m_last != nullptr) {
m_last->next = &request;
} else {
m_first = &request;
}
m_last = &request;
m_wake.notify_one();
request.ready.wait(lock, [&] { return request.done; });
return request.result;
}
KYTY_CLASS_NO_COPY(ArchiveIoThread);
private:
struct Request {
ArchiveReader& reader;
uint64_t id;
uint64_t offset;
uint32_t size;
void* data;
Request* next = nullptr;
uint64_t result = 0;
bool done = false;
std::condition_variable ready {};
};
ArchiveIoThread() {
std::thread([this] { Loop(); }).detach();
}
void Loop() {
for (;;) {
Request* request;
{
std::unique_lock lock(m_mutex);
m_wake.wait(lock, [this] { return m_first != nullptr; });
request = m_first;
m_first = request->next;
if (m_first == nullptr) {
m_last = nullptr;
}
}
const auto result =
request->reader.Read(request->id, request->offset, request->size, request->data);
{
std::lock_guard lock(m_mutex);
request->result = result;
request->done = true;
// Notify before releasing the mutex: the waiting caller owns the condition
// variable.
request->ready.notify_one();
}
}
}
std::mutex m_mutex;
std::condition_variable m_wake;
Request* m_first = nullptr;
Request* m_last = nullptr;
};
} // namespace
struct ArchiveFile::Private {
std::shared_ptr<ArchiveReader> reader;
ArchiveReader::Entry entry;
uint64_t position = 0;
};
ArchiveFile::ArchiveFile(std::unique_ptr<Private> p): m_p(std::move(p)) {}
ArchiveFile::~ArchiveFile() = default;
uint64_t ArchiveFile::Size() const {
return m_p->entry.size;
}
uint64_t ArchiveFile::Tell() const {
return m_p->position;
}
bool ArchiveFile::Seek(uint64_t offset) {
m_p->position = offset;
return true;
}
void ArchiveFile::Read(void* data, uint32_t size, uint32_t* bytes_read) {
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
// A pending cancellation must not destroy the waiting request or its TLS buffer while the
// worker still uses them. Restore the caller's state only after the copy is complete.
int cancellation_state = PTHREAD_CANCEL_ENABLE;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancellation_state);
#endif
constexpr uint32_t chunk_size = 1u << 20u;
thread_local std::vector<uint8_t> chunk;
uint32_t read = 0;
if (data != nullptr && size != 0 && m_p->position < Size()) {
const auto requested =
static_cast<uint32_t>(std::min<uint64_t>(size, Size() - m_p->position));
if (chunk.size() < std::min(chunk_size, requested)) {
chunk.resize(std::min(chunk_size, requested));
}
while (read < requested) {
const auto step = std::min(chunk_size, requested - read);
const auto got = ArchiveIoThread::Instance().Read(*m_p->reader, m_p->entry.id,
m_p->position, step, chunk.data());
if (got == 0 || got > step) {
break;
}
std::memcpy(static_cast<uint8_t*>(data) + read, chunk.data(), got);
read += static_cast<uint32_t>(got);
m_p->position += got;
if (got < step) {
break;
}
}
}
if (bytes_read != nullptr) {
*bytes_read = read;
}
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
pthread_setcancelstate(cancellation_state, nullptr);
#endif
}
std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
const std::filesystem::path& member) {
auto root = archive;
root += "!";
return member.empty() ? root : root / member.relative_path();
}
bool IsSupportedArchive(const std::filesystem::path& path) {
return FindFormat(path.native()) != nullptr;
}
bool IsArchivePath(const std::filesystem::path& path) {
return FindMarker(path.native()) != NativeView::npos;
}
std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path) {
const auto marker = FindMarker(path.native());
return marker == NativeView::npos ? std::filesystem::path {}
: std::filesystem::path(path.native().substr(0, marker));
}
std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path) {
const auto archive = IsArchivePath(path) ? GetArchiveHostPath(path) : path;
const auto* format = FindFormat(archive.native());
if (format == nullptr) {
return {};
}
std::error_code error;
auto key = std::filesystem::absolute(archive, error);
if (error) {
return {};
}
{
std::lock_guard lock(g_readers_mutex);
if (auto it = g_readers.find(key); it != g_readers.end()) {
if (auto reader = it->second.lock()) {
return reader;
}
}
}
// Do not hold the cache mutex while opening an unrelated archive from disk.
auto reader = format->open(archive);
if (reader == nullptr) {
return {};
}
std::lock_guard lock(g_readers_mutex);
std::erase_if(g_readers, [](const auto& entry) { return entry.second.expired(); });
auto& cached = g_readers[key];
if (auto existing = cached.lock()) {
return existing;
}
cached = reader;
return reader;
}
std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path) {
const auto parsed = ParsePath(path);
if (parsed) {
if (auto reader = OpenArchive(parsed->archive)) {
if (auto entry = reader->Find(parsed->member)) {
return File::Info {entry->is_file, entry->size};
}
}
}
return {};
}
std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path) {
const auto parsed = ParsePath(path);
if (parsed) {
if (auto reader = OpenArchive(parsed->archive)) {
return reader->List(parsed->member);
}
}
return {};
}
std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path) {
auto parsed = ParsePath(path);
if (!parsed) {
return {};
}
auto reader = OpenArchive(parsed->archive);
const auto entry = reader ? reader->Find(parsed->member) : std::nullopt;
if (!entry || !entry->is_file) {
return {};
}
auto p = std::make_unique<ArchiveFile::Private>();
p->reader = std::move(reader);
p->entry = *entry;
return std::unique_ptr<ArchiveFile>(new ArchiveFile(std::move(p)));
}
} // namespace Common
+50
View File
@@ -0,0 +1,50 @@
#ifndef KYTY_COMMON_ARCHIVE_H_
#define KYTY_COMMON_ARCHIVE_H_
#include "common/file.h"
#include <cstdint>
#include <filesystem>
#include <memory>
#include <vector>
namespace Common {
class ArchiveReader;
class ArchiveFile {
public:
~ArchiveFile();
[[nodiscard]] uint64_t Size() const;
[[nodiscard]] uint64_t Tell() const;
bool Seek(uint64_t offset);
void Read(void* data, uint32_t size, uint32_t* bytes_read);
KYTY_CLASS_NO_COPY(ArchiveFile);
private:
struct Private;
explicit ArchiveFile(std::unique_ptr<Private> p);
std::unique_ptr<Private> m_p;
friend std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
};
// Virtual paths use "archive.ext!/member" and retain native host-path encoding.
[[nodiscard]] std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
const std::filesystem::path& member = {});
[[nodiscard]] bool IsSupportedArchive(const std::filesystem::path& path);
[[nodiscard]] bool IsArchivePath(const std::filesystem::path& path);
[[nodiscard]] std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path);
[[nodiscard]] std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path);
[[nodiscard]] std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path);
[[nodiscard]] std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
// Retain this handle while a mount or metadata scan is active to reuse the index and block cache.
// Accepts either a host archive filename or a virtual path inside the archive.
[[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
} // namespace Common
#endif /* KYTY_COMMON_ARCHIVE_H_ */
+140 -17
View File
@@ -1,5 +1,6 @@
#include "common/file.h" #include "common/file.h"
#include "common/archive.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/common.h" #include "common/common.h"
#include "common/platform/sysFileIO.h" #include "common/platform/sysFileIO.h"
@@ -61,7 +62,8 @@ SysFileTimeStruct DateTimeToFileTimeUtc(const DateTime& date_time) {
} // namespace } // namespace
struct File::FilePrivate { struct File::FilePrivate {
sys_file_t* f; sys_file_t* f;
std::unique_ptr<ArchiveFile> archive;
}; };
File::File(): m_p(std::make_unique<FilePrivate>()) { File::File(): m_p(std::make_unique<FilePrivate>()) {
@@ -85,11 +87,15 @@ File::File(const std::filesystem::path& name, Mode mode): m_p(std::make_unique<F
} }
bool File::IsInvalid() const { bool File::IsInvalid() const {
return m_p->f == nullptr; return m_p->f == nullptr && m_p->archive == nullptr;
} }
bool File::Create(const std::filesystem::path& name) { bool File::Create(const std::filesystem::path& name) {
EXIT_IF(m_p->f != nullptr); EXIT_IF(!IsInvalid());
if (IsArchivePath(name)) {
return false;
}
m_file_name = name; m_file_name = name;
@@ -104,9 +110,16 @@ bool File::Create(const std::filesystem::path& name) {
} }
bool File::Open(const std::filesystem::path& name, Mode mode) { bool File::Open(const std::filesystem::path& name, Mode mode) {
EXIT_IF(m_p->f != nullptr); EXIT_IF(!IsInvalid());
m_file_name = name; m_file_name = name;
if (IsArchivePath(name)) {
if (mode != Mode::Read) {
return false;
}
m_p->archive = OpenArchiveFile(name);
return m_p->archive != nullptr;
}
switch (mode) { switch (mode) {
case Mode::Read: m_p->f = SysFileOpenR(name); break; case Mode::Read: m_p->f = SysFileOpenR(name); break;
@@ -124,7 +137,7 @@ bool File::Open(const std::filesystem::path& name, Mode mode) {
} }
bool File::OpenInMem(void* buf, uint32_t buf_size) { bool File::OpenInMem(void* buf, uint32_t buf_size) {
EXIT_IF(m_p->f != nullptr); EXIT_IF(!IsInvalid());
m_p->f = SysFileOpen(static_cast<uint8_t*>(buf), buf_size); m_p->f = SysFileOpen(static_cast<uint8_t*>(buf), buf_size);
@@ -137,7 +150,7 @@ bool File::OpenInMem(void* buf, uint32_t buf_size) {
} }
bool File::CreateInMem() { bool File::CreateInMem() {
EXIT_IF(m_p->f != nullptr); EXIT_IF(!IsInvalid());
m_p->f = SysFileCreate(); m_p->f = SysFileCreate();
@@ -150,6 +163,7 @@ bool File::CreateInMem() {
} }
void File::Close() { void File::Close() {
m_p->archive.reset();
if (m_p->f != nullptr) { if (m_p->f != nullptr) {
SysFileClose(m_p->f); SysFileClose(m_p->f);
m_p->f = nullptr; m_p->f = nullptr;
@@ -157,7 +171,11 @@ void File::Close() {
} }
uint64_t File::Size() const { uint64_t File::Size() const {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Size();
}
if (m_p->f == nullptr) { if (m_p->f == nullptr) {
return 0; return 0;
@@ -167,41 +185,76 @@ uint64_t File::Size() const {
} }
uint64_t File::Remaining() const { uint64_t File::Remaining() const {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
return Size() - Tell(); const auto size = Size();
const auto position = Tell();
return position < size ? size - position : 0;
} }
uint64_t File::Size(const std::filesystem::path& name) { uint64_t File::Size(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
const auto info = GetArchiveInfo(name);
return info ? info->size : 0;
}
return SysFileSize(name); return SysFileSize(name);
} }
std::optional<File::Info> File::GetInfo(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetArchiveInfo(name);
}
Info info {};
return SysFileGetInfo(name, &info.is_file, &info.size) ? std::optional(info) : std::nullopt;
}
bool File::Seek(uint64_t offset) { bool File::Seek(uint64_t offset) {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Seek(offset);
}
return SysFileSeek(*m_p->f, offset); return SysFileSeek(*m_p->f, offset);
} }
bool File::Truncate(uint64_t size) { bool File::Truncate(uint64_t size) {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return false;
}
return SysFileTruncate(*m_p->f, size); return SysFileTruncate(*m_p->f, size);
} }
bool File::Unlink() { bool File::Unlink() {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return false;
}
return SysFileUnlink(*m_p->f, m_file_name); return SysFileUnlink(*m_p->f, m_file_name);
} }
uint64_t File::Tell() const { uint64_t File::Tell() const {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return m_p->archive->Tell();
}
return SysFileTell(*m_p->f); return SysFileTell(*m_p->f);
} }
void File::Read(void* data, uint32_t size, uint32_t* bytes_read) { void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
m_p->archive->Read(data, size, bytes_read);
return;
}
if (m_p->f != nullptr) { if (m_p->f != nullptr) {
SysFileRead(data, size, *m_p->f, bytes_read); SysFileRead(data, size, *m_p->f, bytes_read);
@@ -209,7 +262,14 @@ void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
} }
void File::Write(const void* data, uint32_t size, uint32_t* bytes_written) { void File::Write(const void* data, uint32_t size, uint32_t* bytes_written) {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
if (bytes_written != nullptr) {
*bytes_written = 0;
}
return;
}
SysFileWrite(data, size, *m_p->f, bytes_written); SysFileWrite(data, size, *m_p->f, bytes_written);
} }
@@ -257,24 +317,41 @@ static std::filesystem::path WithoutTrailingSeparator(
bool File::IsDirectoryExisting(const std::filesystem::path& path) { bool File::IsDirectoryExisting(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
const auto info = GetArchiveInfo(path);
return info && !info->is_file;
}
return SysFileIsDirectoryExisting(WithoutTrailingSeparator(path)); return SysFileIsDirectoryExisting(WithoutTrailingSeparator(path));
} }
bool File::IsFileExisting(const std::filesystem::path& name) { bool File::IsFileExisting(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
const auto info = GetArchiveInfo(name);
return info && info->is_file;
}
return SysFileIsFileExisting(name); return SysFileIsFileExisting(name);
} }
bool File::CreateDirectory(const std::filesystem::path& path) { bool File::CreateDirectory(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
return SysFileCreateDirectory( return SysFileCreateDirectory(
WithoutTrailingSeparator(path)); WithoutTrailingSeparator(path));
} }
bool File::DeleteDirectory(const std::filesystem::path& path) { bool File::DeleteDirectory(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
return SysFileDeleteDirectory( return SysFileDeleteDirectory(
WithoutTrailingSeparator(path)); WithoutTrailingSeparator(path));
} }
bool File::CreateDirectories(const std::filesystem::path& path) { bool File::CreateDirectories(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
const auto normalized_path = const auto normalized_path =
WithoutTrailingSeparator(path); WithoutTrailingSeparator(path);
@@ -297,6 +374,9 @@ bool File::CreateDirectories(const std::filesystem::path& path) {
} }
bool File::DeleteDirectories(const std::filesystem::path& path) { bool File::DeleteDirectories(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return false;
}
const auto normalized_path = const auto normalized_path =
WithoutTrailingSeparator(path); WithoutTrailingSeparator(path);
@@ -322,11 +402,18 @@ bool File::DeleteDirectories(const std::filesystem::path& path) {
bool File::DeleteFile( bool File::DeleteFile(
const std::filesystem::path& name) // @suppress("Member declaration not found") const std::filesystem::path& name) // @suppress("Member declaration not found")
{ {
if (IsArchivePath(name)) {
return false;
}
return SysFileDeleteFile(name); return SysFileDeleteFile(name);
} }
bool File::Flush() { bool File::Flush() {
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
return true;
}
return SysFileFlush(*m_p->f); return SysFileFlush(*m_p->f);
} }
@@ -348,6 +435,9 @@ std::vector<std::byte> File::ReadWholeBuffer() {
} }
DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) { DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetLastAccessTimeUTC(GetArchiveHostPath(name));
}
SysTimeStruct t {}; SysTimeStruct t {};
SysFileToSystemTimeUtc(SysFileGetLastAccessTimeUtc(name), t); SysFileToSystemTimeUtc(SysFileGetLastAccessTimeUtc(name), t);
@@ -359,6 +449,9 @@ DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
} }
DateTime File::GetLastWriteTimeUTC(const std::filesystem::path& name) { DateTime File::GetLastWriteTimeUTC(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return GetLastWriteTimeUTC(GetArchiveHostPath(name));
}
SysTimeStruct t {}; SysTimeStruct t {};
SysFileToSystemTimeUtc(SysFileGetLastWriteTimeUtc(name), t); SysFileToSystemTimeUtc(SysFileGetLastWriteTimeUtc(name), t);
@@ -373,6 +466,10 @@ void File::GetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, DateT
DateTime* write) { DateTime* write) {
EXIT_IF(access == nullptr); EXIT_IF(access == nullptr);
EXIT_IF(write == nullptr); EXIT_IF(write == nullptr);
if (IsArchivePath(name)) {
GetLastAccessAndWriteTimeUTC(GetArchiveHostPath(name), access, write);
return;
}
SysTimeStruct at {}; SysTimeStruct at {};
SysTimeStruct wt {}; SysTimeStruct wt {};
@@ -399,7 +496,12 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
EXIT_IF(access == nullptr); EXIT_IF(access == nullptr);
EXIT_IF(write == nullptr); EXIT_IF(write == nullptr);
EXIT_IF(m_p->f == nullptr); EXIT_IF(IsInvalid());
if (m_p->archive != nullptr) {
GetLastAccessAndWriteTimeUTC(m_file_name, access, write);
return;
}
if (m_p->f == nullptr) { if (m_p->f == nullptr) {
*access = DateTime(); *access = DateTime();
@@ -429,23 +531,35 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
} }
bool File::SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt) { bool File::SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
if (IsArchivePath(name)) {
return false;
}
auto f = DateTimeToFileTimeUtc(dt); auto f = DateTimeToFileTimeUtc(dt);
return SysFileSetLastAccessTimeUtc(name, f); return SysFileSetLastAccessTimeUtc(name, f);
} }
bool File::SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt) { bool File::SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
if (IsArchivePath(name)) {
return false;
}
auto f = DateTimeToFileTimeUtc(dt); auto f = DateTimeToFileTimeUtc(dt);
return SysFileSetLastWriteTimeUtc(name, f); return SysFileSetLastWriteTimeUtc(name, f);
} }
bool File::SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, const DateTime& access, bool File::SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, const DateTime& access,
const DateTime& write) { const DateTime& write) {
if (IsArchivePath(name)) {
return false;
}
auto af = DateTimeToFileTimeUtc(access); auto af = DateTimeToFileTimeUtc(access);
auto wf = DateTimeToFileTimeUtc(write); auto wf = DateTimeToFileTimeUtc(write);
return SysFileSetLastAccessAndWriteTimeUtc(name, af, wf); return SysFileSetLastAccessAndWriteTimeUtc(name, af, wf);
} }
std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& path) { std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& path) {
if (IsArchivePath(path)) {
return GetArchiveDirEntries(path);
}
std::vector<sys_dir_entry_t> files; std::vector<sys_dir_entry_t> files;
SysFileGetDents(path, files); SysFileGetDents(path, files);
@@ -463,12 +577,18 @@ std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& pat
bool File::CopyFile(const std::filesystem::path& src, bool File::CopyFile(const std::filesystem::path& src,
const std::filesystem::path& dst) // @suppress("Member declaration not found") const std::filesystem::path& dst) // @suppress("Member declaration not found")
{ {
if (IsArchivePath(src) || IsArchivePath(dst)) {
return false;
}
return SysFileCopyFile(src, dst); return SysFileCopyFile(src, dst);
} }
bool File::RenameFile(const std::filesystem::path& src, bool File::RenameFile(const std::filesystem::path& src,
const std::filesystem::path& dst) // @suppress("Member declaration not found") const std::filesystem::path& dst) // @suppress("Member declaration not found")
{ {
if (IsArchivePath(src) || IsArchivePath(dst)) {
return false;
}
if (IsFileExisting(dst)) { if (IsFileExisting(dst)) {
DeleteFile(dst); // @suppress("Invalid arguments") DeleteFile(dst); // @suppress("Invalid arguments")
} }
@@ -477,6 +597,9 @@ bool File::RenameFile(const std::filesystem::path& src,
} }
void File::RemoveReadonly(const std::filesystem::path& name) { void File::RemoveReadonly(const std::filesystem::path& name) {
if (IsArchivePath(name)) {
return;
}
SysFileRemoveReadonly(name); SysFileRemoveReadonly(name);
} }
+6
View File
@@ -7,6 +7,7 @@
#include <cstddef> #include <cstddef>
#include <filesystem> #include <filesystem>
#include <memory> #include <memory>
#include <optional>
#include <string> #include <string>
#include <vector> #include <vector>
@@ -34,6 +35,10 @@ public:
std::string name; std::string name;
bool is_file; bool is_file;
}; };
struct Info {
bool is_file;
uint64_t size;
};
File(); File();
explicit File(const std::filesystem::path& name); explicit File(const std::filesystem::path& name);
@@ -72,6 +77,7 @@ public:
std::vector<std::byte> ReadWholeBuffer(); std::vector<std::byte> ReadWholeBuffer();
static uint64_t Size(const std::filesystem::path& name); static uint64_t Size(const std::filesystem::path& name);
static std::optional<Info> GetInfo(const std::filesystem::path& name);
static bool IsDirectoryExisting(const std::filesystem::path& path); static bool IsDirectoryExisting(const std::filesystem::path& path);
static bool IsFileExisting(const std::filesystem::path& name); static bool IsFileExisting(const std::filesystem::path& name);
+1
View File
@@ -45,6 +45,7 @@ bool SysFileTruncate(sys_file_t& f, uint64_t size); // NOLINT(google-runt
bool SysFileUnlink(sys_file_t& f, bool SysFileUnlink(sys_file_t& f,
const std::filesystem::path& name); // NOLINT(google-runtime-references) const std::filesystem::path& name); // NOLINT(google-runtime-references)
uint64_t SysFileSize(const std::filesystem::path& file_name); uint64_t SysFileSize(const std::filesystem::path& file_name);
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size);
bool SysFileIsError(sys_file_t& f); // NOLINT(google-runtime-references) bool SysFileIsError(sys_file_t& f); // NOLINT(google-runtime-references)
bool SysFileIsDirectoryExisting(const std::filesystem::path& path); bool SysFileIsDirectoryExisting(const std::filesystem::path& path);
bool SysFileIsFileExisting(const std::filesystem::path& name); bool SysFileIsFileExisting(const std::filesystem::path& name);
+20 -23
View File
@@ -297,10 +297,21 @@ uint64_t SysFileSize(sys_file_t& f) {
return 0; return 0;
} }
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
const auto path = get_internal_name(name);
struct stat info {};
if (stat(path.c_str(), &info) != 0) {
return false;
}
*is_file = !S_ISDIR(info.st_mode);
*size = *is_file ? static_cast<uint64_t>(info.st_size) : 0;
return true;
}
uint64_t SysFileSize(const std::filesystem::path& file_name) { uint64_t SysFileSize(const std::filesystem::path& file_name) {
sys_file_t* f = SysFileOpenR(file_name); bool is_file;
uint64_t size = SysFileSize(*f); uint64_t size = 0;
SysFileClose(f); SysFileGetInfo(file_name, &is_file, &size);
return size; return size;
} }
@@ -353,29 +364,15 @@ bool SysFileIsError(sys_file_t& f) {
} }
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) { bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
auto real_name = get_internal_name(path); bool is_file;
auto real_name_str = real_name.string(); uint64_t size;
return SysFileGetInfo(path, &is_file, &size) && !is_file;
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return false;
}
return S_ISDIR(s.st_mode); // NOLINT
} }
bool SysFileIsFileExisting(const std::filesystem::path& name) { bool SysFileIsFileExisting(const std::filesystem::path& name) {
auto real_name = get_internal_name(name); bool is_file;
auto real_name_str = real_name.string(); uint64_t size;
return SysFileGetInfo(name, &is_file, &size) && is_file;
struct stat s {};
if (0 != stat(real_name_str.c_str(), &s)) {
return false;
}
return !S_ISDIR(s.st_mode); // NOLINT
} }
bool SysFileCreateDirectory(const std::filesystem::path& path) { bool SysFileCreateDirectory(const std::filesystem::path& path) {
+25 -19
View File
@@ -266,19 +266,27 @@ uint64_t SysFileSize(sys_file_t& f) {
return 0; return 0;
} }
uint64_t SysFileSize(const std::filesystem::path& file_name) { bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
LARGE_INTEGER s; const bool directory_path = !name.empty() && !name.has_filename() && name != name.root_path();
WIN32_FILE_ATTRIBUTE_DATA a; WIN32_FILE_ATTRIBUTE_DATA info {};
if (GetFileAttributesExW(directory_path ? name.parent_path().c_str() : name.c_str(),
auto wide = file_name.wstring(); GetFileExInfoStandard, &info) == 0) {
if (GetFileAttributesExW(wide.c_str(), GetFileExInfoStandard, &a) == 0) { return false;
return 0;
} }
const bool file = (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
if (directory_path && file) {
return false;
}
*is_file = file;
*size = *is_file ? ((static_cast<uint64_t>(info.nFileSizeHigh) << 32u) | info.nFileSizeLow) : 0;
return true;
}
s.HighPart = static_cast<LONG>(a.nFileSizeHigh); uint64_t SysFileSize(const std::filesystem::path& file_name) {
s.LowPart = a.nFileSizeLow; bool is_file;
uint64_t size = 0;
return s.QuadPart; SysFileGetInfo(file_name, &is_file, &size);
return size;
} }
bool SysFileTruncate(sys_file_t& f, uint64_t size) { bool SysFileTruncate(sys_file_t& f, uint64_t size) {
@@ -346,17 +354,15 @@ bool SysFileIsError(sys_file_t& f) {
} }
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) { bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
auto wide = path.wstring(); bool is_file;
DWORD a = GetFileAttributesW(wide.c_str()); uint64_t size;
return a != INVALID_FILE_ATTRIBUTES && return SysFileGetInfo(path, &is_file, &size) && !is_file;
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) != 0u);
} }
bool SysFileIsFileExisting(const std::filesystem::path& name) { bool SysFileIsFileExisting(const std::filesystem::path& name) {
auto wide = name.wstring(); bool is_file;
DWORD a = GetFileAttributesW(wide.c_str()); uint64_t size;
return a != INVALID_FILE_ATTRIBUTES && return SysFileGetInfo(name, &is_file, &size) && is_file;
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) == 0u);
} }
bool SysFileCreateDirectory(const std::filesystem::path& path) { bool SysFileCreateDirectory(const std::filesystem::path& path) {
+33 -12
View File
@@ -1,5 +1,6 @@
#include "kernel/fileSystem.h" #include "kernel/fileSystem.h"
#include "common/archive.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/common.h" #include "common/common.h"
#include "common/dateTime.h" #include "common/dateTime.h"
@@ -41,6 +42,7 @@ public:
struct MountPair { struct MountPair {
std::filesystem::path dir; std::filesystem::path dir;
std::string point; std::string point;
std::shared_ptr<Common::ArchiveReader> archive;
}; };
MountPoints() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } MountPoints() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); }
@@ -260,6 +262,7 @@ void MountPoints::Mount(const std::filesystem::path& folder, const std::string&
MountPair p; MountPair p;
p.dir = folder; p.dir = folder;
p.point = point_str; p.point = point_str;
p.archive = Common::OpenArchive(folder);
m_mount_pairs.push_back(std::move(p)); m_mount_pairs.push_back(std::move(p));
} }
@@ -346,11 +349,13 @@ std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name)
} }
const auto native_rel_path = Common::PathFromUtf8(rel_path); const auto native_rel_path = Common::PathFromUtf8(rel_path);
if (Common::IsArchivePath(p.dir)) {
return p.dir / native_rel_path;
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
if (HasWindowsForbiddenFilenameCharacter(rel_path)) { if (HasWindowsForbiddenFilenameCharacter(rel_path)) {
::printf("FileSystem: Windows-incompatible guest filename: %s\n", ::printf("FileSystem: Windows-incompatible guest filename: %s\n", mounted_name.c_str());
mounted_name.c_str());
} }
return p.dir / native_rel_path; return p.dir / native_rel_path;
#else #else
@@ -466,14 +471,20 @@ int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
g_files->DeleteDescriptor(descriptor); g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_ENOENT; return KERNEL_ERROR_ENOENT;
} }
if (Common::IsArchivePath(file->real_name) &&
(rw_mode != Common::File::Mode::Read || trunc || creat)) {
g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_EROFS;
}
if (trunc && rw_mode == Common::File::Mode::Read) { if (trunc && rw_mode == Common::File::Mode::Read) {
g_files->DeleteDescriptor(descriptor); g_files->DeleteDescriptor(descriptor);
return KERNEL_ERROR_EACCES; return KERNEL_ERROR_EACCES;
} }
bool dir_exist = Common::File::IsDirectoryExisting(file->real_name); const auto info = Common::File::GetInfo(file->real_name);
bool file_exist = Common::File::IsFileExisting(file->real_name); const bool dir_exist = info && !info->is_file;
const bool file_exist = info && info->is_file;
// A missing path opened without O_CREAT is ENOENT // A missing path opened without O_CREAT is ENOENT
if (!creat && !dir_exist && !file_exist) { if (!creat && !dir_exist && !file_exist) {
@@ -671,7 +682,7 @@ int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) {
auto* file = g_files->GetFile(d); auto* file = g_files->GetFile(d);
if (file == nullptr) { if (file == nullptr || !file->opened || !file->writable) {
return KERNEL_ERROR_EBADF; return KERNEL_ERROR_EBADF;
} }
@@ -886,7 +897,7 @@ int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_
auto* file = g_files->GetFile(d); auto* file = g_files->GetFile(d);
if (file == nullptr) { if (file == nullptr || !file->opened || !file->writable) {
return KERNEL_ERROR_EBADF; return KERNEL_ERROR_EBADF;
} }
@@ -1063,15 +1074,13 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
auto real_file_name = g_mount_points->ResolvePath(path); auto real_file_name = g_mount_points->ResolvePath(path);
bool is_dir = Common::File::IsDirectoryExisting(real_file_name); const auto info = Common::File::GetInfo(real_file_name);
bool is_file = Common::File::IsFileExisting(real_file_name); if (!info) {
if (!is_dir && !is_file) {
LOGF("\t file not found\n"); LOGF("\t file not found\n");
return KERNEL_ERROR_ENOENT; return KERNEL_ERROR_ENOENT;
} }
EXIT_NOT_IMPLEMENTED(is_dir && is_file); const bool is_dir = !info->is_file;
FileStat stat {}; FileStat stat {};
stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u); stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u);
@@ -1085,7 +1094,7 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
stat.st_blksize = 512; stat.st_blksize = 512;
stat.st_blocks = stat.st_size / 512; stat.st_blocks = stat.st_size / 512;
} else { } else {
stat.st_size = static_cast<int64_t>(Common::File::Size(real_file_name)); stat.st_size = static_cast<int64_t>(info->size);
stat.st_blksize = 512; stat.st_blksize = 512;
stat.st_blocks = (stat.st_size + 511) / 512; stat.st_blocks = (stat.st_size + 511) / 512;
@@ -1222,6 +1231,9 @@ int KYTY_SYSV_ABI KernelUnlink(const char* path) {
} }
auto real_file_name = g_mount_points->ResolvePath(path); auto real_file_name = g_mount_points->ResolvePath(path);
if (Common::IsArchivePath(real_file_name)) {
return KERNEL_ERROR_EROFS;
}
bool is_dir = Common::File::IsDirectoryExisting(real_file_name); bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
bool is_file = Common::File::IsFileExisting(real_file_name); bool is_file = Common::File::IsFileExisting(real_file_name);
@@ -1259,6 +1271,9 @@ int KYTY_SYSV_ABI KernelRename(const char* from, const char* to) {
auto to_path = std::string(to); auto to_path = std::string(to);
auto real_from = g_mount_points->ResolvePath(from_path); auto real_from = g_mount_points->ResolvePath(from_path);
auto real_to = g_mount_points->ResolvePath(to_path); auto real_to = g_mount_points->ResolvePath(to_path);
if (Common::IsArchivePath(real_from) || Common::IsArchivePath(real_to)) {
return KERNEL_ERROR_EROFS;
}
if (real_to.empty() || !Common::File::IsFileExisting(real_from)) { if (real_to.empty() || !Common::File::IsFileExisting(real_from)) {
return KERNEL_ERROR_ENOENT; return KERNEL_ERROR_ENOENT;
@@ -1343,6 +1358,9 @@ int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode) {
if (real_name.empty()) { if (real_name.empty()) {
return KERNEL_ERROR_ENOENT; return KERNEL_ERROR_ENOENT;
} }
if (Common::IsArchivePath(real_name)) {
return KERNEL_ERROR_EROFS;
}
if (Common::File::IsDirectoryExisting(real_name)) { if (Common::File::IsDirectoryExisting(real_name)) {
return KERNEL_ERROR_EEXIST; return KERNEL_ERROR_EEXIST;
@@ -1369,6 +1387,9 @@ int KYTY_SYSV_ABI KernelRmdir(const char* path) {
LOGF("\t path = %s\n", path); LOGF("\t path = %s\n", path);
auto real_name = g_mount_points->ResolvePath(std::string(path)); auto real_name = g_mount_points->ResolvePath(std::string(path));
if (Common::IsArchivePath(real_name)) {
return KERNEL_ERROR_EROFS;
}
if (!Common::File::IsDirectoryExisting(real_name)) { if (!Common::File::IsDirectoryExisting(real_name)) {
return KERNEL_ERROR_ENOENT; return KERNEL_ERROR_ENOENT;
+1 -1
View File
@@ -20,7 +20,7 @@ add_executable(launcher WIN32
) )
target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version) target_link_libraries(launcher Qt6::Concurrent Qt6::Network Qt6::Widgets Vulkan::Headers SDL3::SDL3-static kyty_git_version common)
if(WIN32) if(WIN32)
target_link_libraries(launcher setupapi) target_link_libraries(launcher setupapi)
endif() endif()
+3 -1
View File
@@ -1,6 +1,7 @@
#ifndef CONFIGURATION_ITEM_H #ifndef CONFIGURATION_ITEM_H
#define CONFIGURATION_ITEM_H #define CONFIGURATION_ITEM_H
#include <QIcon>
#include <QObject> #include <QObject>
#include <QTreeWidgetItem> #include <QTreeWidgetItem>
@@ -18,7 +19,7 @@ public:
explicit ConfigurationItem(std::unique_ptr<Configuration> info, QTreeWidget* parent); explicit ConfigurationItem(std::unique_ptr<Configuration> info, QTreeWidget* parent);
~ConfigurationItem() override; ~ConfigurationItem() override;
void Update(); void Update(bool reload_icon = false);
bool operator<(const QTreeWidgetItem& other) const override; bool operator<(const QTreeWidgetItem& other) const override;
Configuration& GetInfo() { return *m_info; } Configuration& GetInfo() { return *m_info; }
@@ -35,6 +36,7 @@ private:
void UpdateStatusIndicator(); void UpdateStatusIndicator();
std::unique_ptr<Configuration> m_info; std::unique_ptr<Configuration> m_info;
QIcon m_icon;
bool m_running = false; bool m_running = false;
QComboBox* m_status_combo = nullptr; QComboBox* m_status_combo = nullptr;
QLabel* m_status_indicator = nullptr; QLabel* m_status_indicator = nullptr;
+28
View File
@@ -0,0 +1,28 @@
#ifndef GAME_CONTENT_H
#define GAME_CONTENT_H
#include <QByteArray>
#include <QString>
#include <QStringList>
#include <cstdint>
#include <filesystem>
namespace GameContent {
inline constexpr uint64_t MaxMetadataSize = uint64_t {1} << 20u;
inline constexpr uint64_t MaxImageSize = uint64_t {32} << 20u;
inline constexpr uint64_t MaxTrophyPackageSize = uint64_t {128} << 20u;
[[nodiscard]] std::filesystem::path ToPath(const QString& path);
[[nodiscard]] QString FromPath(const std::filesystem::path& path);
[[nodiscard]] bool IsArchive(const QString& base);
[[nodiscard]] std::filesystem::path Resolve(const QString& base, const QString& relative);
[[nodiscard]] bool FileExists(const QString& base, const QString& relative);
[[nodiscard]] QByteArray ReadPath(const std::filesystem::path& path, uint64_t max_size);
[[nodiscard]] QByteArray ReadFile(const QString& base, const QString& relative, uint64_t max_size);
[[nodiscard]] QStringList ListFiles(const QString& base, const QString& relative);
} // namespace GameContent
#endif // GAME_CONTENT_H
+8 -5
View File
@@ -1,6 +1,8 @@
#ifndef GAME_LIST_TREE_WIDGET_H #ifndef GAME_LIST_TREE_WIDGET_H
#define GAME_LIST_TREE_WIDGET_H #define GAME_LIST_TREE_WIDGET_H
#include "gameContent.h"
#include <QPaintEvent> #include <QPaintEvent>
#include <QPainter> #include <QPainter>
#include <QPalette> #include <QPalette>
@@ -17,14 +19,15 @@ public:
viewport()->setAutoFillBackground(false); viewport()->setAutoFillBackground(false);
} }
void SetBackgroundImage(const QString& path) { void SetBackgroundImage(const QString& key) {
if (m_path == path) { if (m_path == key) {
return; return;
} }
m_path = path; m_path = key;
m_source = QPixmap(); m_source = QPixmap();
if (!path.isEmpty()) { if (!key.isEmpty()) {
m_source.load(path); m_source.loadFromData(GameContent::ReadFile(key, QStringLiteral("sce_sys/pic0.png"),
GameContent::MaxImageSize));
} }
UpdateScaledBackground(); UpdateScaledBackground();
viewport()->update(); viewport()->update();
+20 -6
View File
@@ -1,6 +1,7 @@
#include "configurationItem.h" #include "configurationItem.h"
#include "configuration.h" #include "configuration.h"
#include "gameContent.h"
#include <QApplication> #include <QApplication>
#include <QComboBox> #include <QComboBox>
@@ -14,6 +15,7 @@
#include <QLabel> #include <QLabel>
#include <QLineEdit> #include <QLineEdit>
#include <QLocale> #include <QLocale>
#include <QPixmap>
#include <QSize> #include <QSize>
#include <QStringList> #include <QStringList>
#include <QStyle> #include <QStyle>
@@ -154,7 +156,7 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
m_comment_edit->setFrame(false); m_comment_edit->setFrame(false);
parent->setItemWidget(this, CommentsColumn, m_comment_edit); parent->setItemWidget(this, CommentsColumn, m_comment_edit);
Update(); Update(true);
SetRunning(false); SetRunning(false);
setData(SizeColumn, Qt::UserRole, qint64(-1)); setData(SizeColumn, Qt::UserRole, qint64(-1));
@@ -170,7 +172,14 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
watcher->deleteLater(); watcher->deleteLater();
}); });
watcher->setFuture(QtConcurrent::run([path = m_info->basedir]() -> qint64 { watcher->setFuture(QtConcurrent::run([path = m_info->basedir]() -> qint64 {
if (path.isEmpty() || !QDir(path).exists()) { if (path.isEmpty()) {
return -1;
}
const QFileInfo info(path);
if (GameContent::IsArchive(path)) {
return info.size();
}
if (!QDir(path).exists()) {
return -1; return -1;
} }
qint64 bytes = 0; qint64 bytes = 0;
@@ -188,7 +197,7 @@ ConfigurationItem::ConfigurationItem(std::unique_ptr<Configuration> info, QTreeW
ConfigurationItem::~ConfigurationItem() = default; ConfigurationItem::~ConfigurationItem() = default;
void ConfigurationItem::Update() { void ConfigurationItem::Update(bool reload_icon) {
const auto display_text = GetDisplayText(*m_info); const auto display_text = GetDisplayText(*m_info);
const auto path = GetPathText(*m_info); const auto path = GetPathText(*m_info);
@@ -209,6 +218,12 @@ void ConfigurationItem::Update() {
m_comment_edit->setText(m_info->game_comment); m_comment_edit->setText(m_info->game_comment);
} }
if (reload_icon) {
const auto icon_data = GameContent::ReadFile(
m_info->basedir, QStringLiteral("sce_sys/icon0.png"), GameContent::MaxImageSize);
QPixmap icon;
m_icon = !icon_data.isEmpty() && icon.loadFromData(icon_data) ? QIcon(icon) : QIcon {};
}
UpdateIcon(); UpdateIcon();
UpdateStatusIndicator(); UpdateStatusIndicator();
} }
@@ -267,9 +282,8 @@ void ConfigurationItem::SetCompatibilityEditable(bool editable) {
} }
void ConfigurationItem::UpdateIcon() { void ConfigurationItem::UpdateIcon() {
const QString icon_file = QDir(m_info->basedir).filePath(QStringLiteral("sce_sys/icon0.png")); if (!m_icon.isNull()) {
if (QFileInfo::exists(icon_file)) { setIcon(NameColumn, m_icon);
setIcon(NameColumn, QIcon(icon_file));
return; return;
} }
+106 -84
View File
@@ -1,9 +1,11 @@
#include "configurationListWidget.h" #include "configurationListWidget.h"
#include "common/archive.h"
#include "compatibilityDatabase.h" #include "compatibilityDatabase.h"
#include "configuration.h" #include "configuration.h"
#include "configurationEditDialog.h" #include "configurationEditDialog.h"
#include "configurationItem.h" #include "configurationItem.h"
#include "gameContent.h"
#include "gameListTreeWidget.h" #include "gameListTreeWidget.h"
#include "inputMappingDialog.h" #include "inputMappingDialog.h"
#include "mainDialog.h" #include "mainDialog.h"
@@ -127,15 +129,6 @@ static QStringList SettingsStringList(const QVariant& value) {
return text.isEmpty() ? QStringList() : QStringList({text}); return text.isEmpty() ? QStringList() : QStringList({text});
} }
static QString GetPic0Path(const Configuration& info) {
if (info.basedir.isEmpty()) {
return {};
}
const auto path = QDir(info.basedir).filePath(QStringLiteral("sce_sys/pic0.png"));
return QFileInfo::exists(path) ? QFileInfo(path).absoluteFilePath() : QString();
}
static void ConfigureGameList(Ui::ConfigurationListWidget* ui) { static void ConfigureGameList(Ui::ConfigurationListWidget* ui) {
ui->cfgs_list->setAlternatingRowColors(false); ui->cfgs_list->setAlternatingRowColors(false);
ui->cfgs_list->setAllColumnsShowFocus(true); ui->cfgs_list->setAllColumnsShowFocus(true);
@@ -490,17 +483,16 @@ static QString GetFirmwareVersion(const QJsonObject& root) {
return version; return version;
} }
static GameMetadata GetGameMetadata(const QString& param_file, const QString& fallback) { static GameMetadata GetGameMetadata(const QByteArray& param_data, const QString& fallback) {
GameMetadata ret; GameMetadata ret;
ret.title_name = fallback; ret.title_name = fallback;
QFile param(param_file); if (param_data.isEmpty()) {
if (!param.open(QIODevice::ReadOnly)) {
return ret; return ret;
} }
QJsonParseError parse_error; QJsonParseError parse_error;
const auto doc = QJsonDocument::fromJson(param.readAll(), &parse_error); const auto doc = QJsonDocument::fromJson(param_data, &parse_error);
if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) { if (parse_error.error != QJsonParseError::NoError || !doc.isObject()) {
return ret; return ret;
} }
@@ -522,18 +514,18 @@ static GameMetadata GetGameMetadata(const QString& param_file, const QString& fa
} }
static void SetGameFiles(Configuration& info, const QString& game_dir, const QString& game_path, static void SetGameFiles(Configuration& info, const QString& game_dir, const QString& game_path,
const GameMetadata& metadata) { const GameMetadata& metadata, bool archive) {
QDir game(game_dir); const QFileInfo game(game_dir);
info.game_path = game_path; info.game_path = game_path;
info.basedir = game.absolutePath(); info.basedir = archive ? game.absoluteFilePath() : QDir(game_dir).absolutePath();
info.name = metadata.title_name; info.name = metadata.title_name;
info.title_id = metadata.title_id; info.title_id = metadata.title_id;
info.gameVersion = metadata.gameVersion; info.gameVersion = metadata.gameVersion;
info.firmwareVer = metadata.firmwareVer; info.firmwareVer = metadata.firmwareVer;
if (info.name.isEmpty()) { if (info.name.isEmpty()) {
info.name = game.dirName(); info.name = archive ? game.completeBaseName() : QDir(game_dir).dirName();
} }
} }
@@ -609,6 +601,94 @@ void ConfigurationListWidget::ScanGameDirectory() {
const QString eboot_name = QStringLiteral("eboot.bin"); const QString eboot_name = QStringLiteral("eboot.bin");
QSet<QString> found_games; QSet<QString> found_games;
const auto add_game = [this, &found_games, &eboot_name,
&running_items](const QString& base, const QString& game_path,
const QString& legacy_game_path, const QString& fallback,
bool archive) {
const QString game_key = PathKey(game_path);
if (game_key.isEmpty() || found_games.contains(game_key)) {
return;
}
// Keep the index and decompression cache alive through validation, metadata and icon
// reads, after rejecting duplicate candidates from overlapping game folders.
const auto reader = archive ? Common::OpenArchive(GameContent::ToPath(base)) : nullptr;
if (archive && (reader == nullptr || !GameContent::FileExists(base, eboot_name))) {
return;
}
found_games.insert(game_key);
const auto metadata =
GetGameMetadata(GameContent::ReadFile(base, QStringLiteral("sce_sys/param.json"),
GameContent::MaxMetadataSize),
fallback);
auto info = std::make_unique<Configuration>();
info->custom_settings =
FindCustomInfo(&m_custom_infos, game_path, legacy_game_path) != nullptr;
SetGameFiles(*info, base, game_path, metadata, archive);
const auto* compatibility = m_compatibility->Find(info->title_id);
if (compatibility != nullptr) {
info->game_status = compatibility->status;
info->game_comment = compatibility->comment;
}
if (auto* item = running_items.value(game_key); item != nullptr) {
// Refresh metadata without losing the running row's identity.
const QSignalBlocker status_blocker(item->GetStatusCombo());
item->GetInfo().CopyGameInfoFrom(*info);
item->Update(true);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
return;
}
auto* item = new ConfigurationItem(std::move(info), m_ui->cfgs_list);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
connect(item->GetStatusCombo(), &QComboBox::currentIndexChanged, item,
[this, item](int /*index*/) {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_status = static_cast<Configuration::GameStatus>(
item->GetStatusCombo()->currentData().toInt());
item->Update();
m_compatibility->SetStatus(title_id, item->GetInfo().game_status);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
connect(item->GetCommentEdit(), &QLineEdit::editingFinished, item, [this, item]() {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_comment = item->GetCommentEdit()->text();
item->Update();
m_compatibility->SetComment(title_id, item->GetInfo().game_comment);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
};
const auto scan_archives = [&add_game](const QDir& directory, const QDir& root) {
const auto entries = directory.entryInfoList(QDir::Files | QDir::NoSymLinks, QDir::Name);
for (const auto& archive: entries) {
const auto base = archive.absoluteFilePath();
if (!Common::IsSupportedArchive(GameContent::ToPath(base))) {
continue;
}
add_game(base, QDir::cleanPath(base), root.relativeFilePath(base),
archive.completeBaseName(), true);
}
};
for (const auto& root_path: m_game_dirs) { for (const auto& root_path: m_game_dirs) {
QDir root(root_path); QDir root(root_path);
@@ -616,6 +696,7 @@ void ConfigurationListWidget::ScanGameDirectory() {
continue; continue;
} }
scan_archives(root, root);
QList<QDir> pending_dirs; QList<QDir> pending_dirs;
const auto root_subdirs = const auto root_subdirs =
root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks); root.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks);
@@ -625,74 +706,13 @@ void ConfigurationListWidget::ScanGameDirectory() {
while (!pending_dirs.isEmpty()) { while (!pending_dirs.isEmpty()) {
QDir game_dir = pending_dirs.takeFirst(); QDir game_dir = pending_dirs.takeFirst();
scan_archives(game_dir, root);
if (game_dir.exists(eboot_name)) { if (game_dir.exists(eboot_name)) {
const QString game_path = NormalizeGameDirectory(game_dir.absolutePath()); const QString game_path = NormalizeGameDirectory(game_dir.absolutePath());
const QString legacy_game_path = root.relativeFilePath(game_dir.absolutePath()); const QString legacy_game_path = root.relativeFilePath(game_dir.absolutePath());
const QString game_key = PathKey(game_path); add_game(game_dir.absolutePath(), game_path, legacy_game_path, game_dir.dirName(),
if (game_key.isEmpty() || found_games.contains(game_key)) { false);
continue;
}
found_games.insert(game_key);
auto metadata = GetGameMetadata(
game_dir.filePath(QStringLiteral("sce_sys/param.json")), game_dir.dirName());
auto info = std::make_unique<Configuration>();
info->custom_settings =
FindCustomInfo(&m_custom_infos, game_path, legacy_game_path) != nullptr;
SetGameFiles(*info, game_dir.absolutePath(), game_path, metadata);
const auto* compatibility = m_compatibility->Find(info->title_id);
if (compatibility != nullptr) {
info->game_status = compatibility->status;
info->game_comment = compatibility->comment;
}
if (auto* item = running_items.value(game_key); item != nullptr) {
// Refresh metadata without losing the running row's identity.
const QSignalBlocker status_blocker(item->GetStatusCombo());
item->GetInfo().CopyGameInfoFrom(*info);
item->Update();
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
continue;
}
auto* item = new ConfigurationItem(std::move(info), m_ui->cfgs_list);
item->SetCompatibilityEditable(m_compatibility->IsLocal() &&
!item->GetInfo().title_id.trimmed().isEmpty());
connect(item->GetStatusCombo(), &QComboBox::currentIndexChanged, item,
[this, item](int /*index*/) {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_status = static_cast<Configuration::GameStatus>(
item->GetStatusCombo()->currentData().toInt());
item->Update();
m_compatibility->SetStatus(title_id, item->GetInfo().game_status);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
connect(item->GetCommentEdit(), &QLineEdit::editingFinished, item, [this, item]() {
if (!m_compatibility->IsLocal()) {
return;
}
const auto& title_id = item->GetInfo().title_id;
if (title_id.trimmed().isEmpty()) {
return;
}
item->GetInfo().game_comment = item->GetCommentEdit()->text();
item->Update();
m_compatibility->SetComment(title_id, item->GetInfo().game_comment);
m_ui->cfgs_list->setCurrentItem(item);
SelectItem(item);
});
continue; continue;
} }
@@ -805,7 +825,8 @@ void ConfigurationListWidget::open_game_folder() {
return; return;
} }
const QDir game_dir(item->GetInfo().basedir); const auto base = item->GetInfo().basedir;
const QDir game_dir(GameContent::IsArchive(base) ? QFileInfo(base).absolutePath() : base);
if (!game_dir.exists()) { if (!game_dir.exists()) {
QMessageBox::warning(this, tr("Open game folder"), tr("Game folder does not exist.")); QMessageBox::warning(this, tr("Open game folder"), tr("Game folder does not exist."));
return; return;
@@ -896,7 +917,7 @@ void ConfigurationListWidget::SelectItem(QTreeWidgetItem* witem) {
m_ui->edit_button->setEnabled(!item->IsRunning()); m_ui->edit_button->setEnabled(!item->IsRunning());
m_selected_item = item; m_selected_item = item;
m_ui->cfgs_list->SetBackgroundImage(GetPic0Path(item->GetInfo())); m_ui->cfgs_list->SetBackgroundImage(item->GetInfo().basedir);
emit Select(); emit Select();
} }
@@ -959,7 +980,8 @@ void ConfigurationListWidget::show_context_menu(const QPoint& pos) {
if (item != nullptr) { if (item != nullptr) {
action_run->setDisabled(item->IsRunning()); action_run->setDisabled(item->IsRunning());
action_open_folder->setDisabled(!QDir(item->GetInfo().basedir).exists()); const auto& base = item->GetInfo().basedir;
action_open_folder->setDisabled(!QDir(base).exists() && !GameContent::IsArchive(base));
action_view_trophies->setDisabled(!has_trophy_data); action_view_trophies->setDisabled(!has_trophy_data);
action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty()); action_remove_save_data->setDisabled(item->IsRunning() || save_data_dirs.isEmpty());
action_edit->setDisabled(item->IsRunning()); action_edit->setDisabled(item->IsRunning());
+80
View File
@@ -0,0 +1,80 @@
#include "gameContent.h"
#include "common/archive.h"
#include "common/file.h"
#include "common/stringUtils.h"
#include <QFileInfo>
#include <limits>
namespace GameContent {
std::filesystem::path ToPath(const QString& path) {
#if defined(_WIN32)
return std::filesystem::path(path.toStdWString());
#else
return std::filesystem::path(path.toStdString());
#endif
}
QString FromPath(const std::filesystem::path& path) {
#if defined(_WIN32)
return QString::fromStdWString(path.wstring());
#else
return QString::fromStdString(Common::PathToString(path));
#endif
}
bool IsArchive(const QString& base) {
const QFileInfo info(base);
return info.isFile() && Common::IsSupportedArchive(ToPath(base));
}
std::filesystem::path Resolve(const QString& base, const QString& relative) {
const auto root = ToPath(base);
return IsArchive(base) ? Common::MakeArchivePath(root, ToPath(relative))
: root / ToPath(relative);
}
bool FileExists(const QString& base, const QString& relative) {
return Common::File::IsFileExisting(Resolve(base, relative));
}
QByteArray ReadFile(const QString& base, const QString& relative, uint64_t max_size) {
return ReadPath(Resolve(base, relative), max_size);
}
QByteArray ReadPath(const std::filesystem::path& path, uint64_t max_size) {
Common::File file(path, Common::File::Mode::Read);
if (file.IsInvalid()) {
return {};
}
const auto size = file.Size();
if (size > max_size || size > static_cast<uint64_t>(std::numeric_limits<int>::max())) {
return {};
}
QByteArray data(static_cast<int>(size), Qt::Uninitialized);
uint32_t bytes_read = 0;
file.Read(data.data(), static_cast<uint32_t>(size), &bytes_read);
return bytes_read == size ? data : QByteArray {};
}
QStringList ListFiles(const QString& base, const QString& relative) {
const auto directory = Resolve(base, relative);
const bool archive = Common::IsArchivePath(directory);
QStringList result;
for (const auto& entry: Common::File::GetDirEntries(directory)) {
if (entry.is_file) {
const auto path = FromPath(directory / Common::PathFromUtf8(entry.name));
if (archive || !QFileInfo(path).isSymLink()) {
result.append(path);
}
}
}
return result;
}
} // namespace GameContent
+3 -2
View File
@@ -3,6 +3,7 @@
#include "configuration.h" #include "configuration.h"
#include "configurationItem.h" #include "configurationItem.h"
#include "configurationListWidget.h" #include "configurationListWidget.h"
#include "gameContent.h"
#include "patchesDialog.h" #include "patchesDialog.h"
#include "updateChecker.h" #include "updateChecker.h"
@@ -260,7 +261,7 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
} }
QString game = info.basedir; QString game = info.basedir;
if (!info.elf.isEmpty()) { if (!info.elf.isEmpty() && !GameContent::IsArchive(info.basedir)) {
game = QDir(info.basedir).filePath(info.elf); game = QDir(info.basedir).filePath(info.elf);
} }
args << "--game" << game; args << "--game" << game;
@@ -507,7 +508,7 @@ void MainDialogPrivate::Update() {
if (run_enabled) { if (run_enabled) {
const auto& info = item->GetInfo(); const auto& info = item->GetInfo();
auto dir = info.basedir; auto dir = info.basedir;
run_enabled = !dir.isEmpty() && QDir(dir).exists(); run_enabled = !dir.isEmpty() && (QDir(dir).exists() || GameContent::IsArchive(dir));
} }
m_ui->widget->SetRunEnabled(run_enabled); m_ui->widget->SetRunEnabled(run_enabled);
+54 -23
View File
@@ -1,13 +1,14 @@
#include "trophyViewerDialog.h" #include "trophyViewerDialog.h"
#include "common/archive.h"
#include "configuration.h" #include "configuration.h"
#include "gameContent.h"
#include <QAbstractItemView> #include <QAbstractItemView>
#include <QBrush> #include <QBrush>
#include <QByteArray> #include <QByteArray>
#include <QDialogButtonBox> #include <QDialogButtonBox>
#include <QDir> #include <QDir>
#include <QFile>
#include <QFileInfo> #include <QFileInfo>
#include <QFont> #include <QFont>
#include <QHeaderView> #include <QHeaderView>
@@ -39,12 +40,14 @@
namespace { namespace {
constexpr quint32 UCP_MAGIC = 0xb228c60a; constexpr quint32 UCP_MAGIC = 0xb228c60a;
constexpr quint32 UCP_VERSION = 1; constexpr quint32 UCP_VERSION = 1;
constexpr int UCP_HEADER_LEN = 0x40; constexpr int UCP_HEADER_LEN = 0x40;
constexpr int UCP_TOC_SKIP = 0x20; constexpr int UCP_TOC_SKIP = 0x20;
constexpr int UCP_ENTRY_LEN = 0x40; constexpr int UCP_ENTRY_LEN = 0x40;
constexpr int UCP_NAME_LEN = 0x20; constexpr int UCP_NAME_LEN = 0x20;
constexpr quint32 UCP_MAX_FILES = 4096;
constexpr quint64 UCP_MAX_EXTRACTED_SIZE = quint64 {64} << 20u;
struct UcpEntry { struct UcpEntry {
QString name; QString name;
@@ -98,14 +101,22 @@ static QString ReadFixedString(const QByteArray& data, qsizetype offset, qsizety
return QString::fromLatin1(data.constData() + offset, len); return QString::fromLatin1(data.constData() + offset, len);
} }
static bool IsUsedUcpEntry(const QString& name) {
const auto lower = name.toCaseFolded();
return lower == QStringLiteral("tropconf.json") || lower == QStringLiteral("tropmeta.json") ||
(lower.startsWith(QStringLiteral("tropmeta_")) &&
lower.endsWith(QStringLiteral(".json"))) ||
(lower.startsWith(QStringLiteral("trop")) && lower.endsWith(QStringLiteral(".png")));
}
static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files, QString& error) { static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files, QString& error) {
QFile file(file_name); const QByteArray data =
if (!file.open(QIODevice::ReadOnly)) { GameContent::ReadPath(GameContent::ToPath(file_name), GameContent::MaxTrophyPackageSize);
if (data.isEmpty()) {
error = QObject::tr("Could not open %1").arg(QDir::toNativeSeparators(file_name)); error = QObject::tr("Could not open %1").arg(QDir::toNativeSeparators(file_name));
return false; return false;
} }
const QByteArray data = file.readAll();
if (data.size() < UCP_HEADER_LEN) { if (data.size() < UCP_HEADER_LEN) {
error = QObject::tr("%1 is too small to be a trophy package.") error = QObject::tr("%1 is too small to be a trophy package.")
.arg(QFileInfo(file_name).fileName()); .arg(QFileInfo(file_name).fileName());
@@ -127,14 +138,18 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
} }
const auto declared_size = qFromBigEndian<quint64>(data.constData() + 0x08); const auto declared_size = qFromBigEndian<quint64>(data.constData() + 0x08);
if (declared_size > static_cast<quint64>(data.size())) { if (declared_size < UCP_HEADER_LEN || declared_size > static_cast<quint64>(data.size())) {
error = QObject::tr("%1 is truncated.").arg(QFileInfo(file_name).fileName()); error = QObject::tr("%1 is truncated.").arg(QFileInfo(file_name).fileName());
return false; return false;
} }
const auto file_count = qFromBigEndian<quint32>(data.constData() + 0x10); const auto file_count = qFromBigEndian<quint32>(data.constData() + 0x10);
if (file_count > UCP_MAX_FILES) {
error = QObject::tr("%1 contains too many files.").arg(QFileInfo(file_name).fileName());
return false;
}
const auto toc_offset = static_cast<quint64>(qFromBigEndian<quint32>(data.constData() + 0x14)); const auto toc_offset = static_cast<quint64>(qFromBigEndian<quint32>(data.constData() + 0x14));
const auto data_size = static_cast<quint64>(data.size()); const auto data_size = declared_size;
const quint64 table_size = UCP_TOC_SKIP + static_cast<quint64>(file_count) * UCP_ENTRY_LEN; const quint64 table_size = UCP_TOC_SKIP + static_cast<quint64>(file_count) * UCP_ENTRY_LEN;
if (toc_offset > data_size || table_size > data_size - toc_offset || if (toc_offset > data_size || table_size > data_size - toc_offset ||
@@ -144,6 +159,7 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
return false; return false;
} }
quint64 extracted_size = 0;
for (quint32 i = 0; i < file_count; i++) { for (quint32 i = 0; i < file_count; i++) {
const auto entry_offset = static_cast<qsizetype>(toc_offset + UCP_TOC_SKIP + const auto entry_offset = static_cast<qsizetype>(toc_offset + UCP_TOC_SKIP +
static_cast<quint64>(i) * UCP_ENTRY_LEN); static_cast<quint64>(i) * UCP_ENTRY_LEN);
@@ -161,9 +177,22 @@ static bool ReadUcp(const QString& file_name, QMap<QString, QByteArray>& files,
.arg(QFileInfo(file_name).fileName(), entry.name); .arg(QFileInfo(file_name).fileName(), entry.name);
return false; return false;
} }
if (!IsUsedUcpEntry(entry.name)) {
continue;
}
const auto entry_limit = entry.name.endsWith(QStringLiteral(".png"), Qt::CaseInsensitive)
? GameContent::MaxImageSize
: GameContent::MaxMetadataSize;
if (entry.size > entry_limit || entry.size > UCP_MAX_EXTRACTED_SIZE - extracted_size) {
error = QObject::tr("%1 contains an oversized entry for %2.")
.arg(QFileInfo(file_name).fileName(), entry.name);
return false;
}
files.insert(entry.name.toCaseFolded(), data.mid(static_cast<qsizetype>(entry.offset), files.insert(entry.name.toCaseFolded(), data.mid(static_cast<qsizetype>(entry.offset),
static_cast<qsizetype>(entry.size))); static_cast<qsizetype>(entry.size)));
extracted_size += entry.size;
} }
return true; return true;
@@ -392,30 +421,31 @@ static QStringList FindTrophyFiles(const Configuration* info) {
return {}; return {};
} }
const QDir trophy_dir(QDir(info->basedir).filePath(QStringLiteral("sce_sys/trophy2"))); QStringList files;
if (!trophy_dir.exists()) { for (const auto& file:
return {}; GameContent::ListFiles(info->basedir, QStringLiteral("sce_sys/trophy2"))) {
const auto name = QFileInfo(file).fileName();
if (name.startsWith(QStringLiteral("trophy"), Qt::CaseInsensitive) &&
name.endsWith(QStringLiteral(".ucp"), Qt::CaseInsensitive)) {
files.append(file);
}
} }
files.sort(Qt::CaseInsensitive);
const auto files =
trophy_dir.entryInfoList({QStringLiteral("Trophy*.ucp"), QStringLiteral("trophy*.ucp")},
QDir::Files | QDir::NoSymLinks, QDir::Name | QDir::IgnoreCase);
QStringList trophy_files; QStringList trophy_files;
QSet<QString> seen; QSet<QString> seen;
for (const auto& file: files) { for (const auto& file: files) {
auto key = file.canonicalFilePath(); auto key = QFileInfo(file).canonicalFilePath();
if (key.isEmpty()) { if (key.isEmpty()) {
key = file.absoluteFilePath(); key = file;
} }
key = QDir::cleanPath(key).toCaseFolded(); key = QDir::cleanPath(key).toCaseFolded();
if (!seen.contains(key)) { if (!seen.contains(key)) {
seen.insert(key); seen.insert(key);
trophy_files.append(file.absoluteFilePath()); trophy_files.append(file);
} }
} }
return trophy_files; return trophy_files;
} }
@@ -506,6 +536,7 @@ void TrophyViewerDialog::ShowForGame(const Configuration* info, QWidget* parent)
} }
bool TrophyViewerDialog::LoadGame(const Configuration& info, QString& error) { bool TrophyViewerDialog::LoadGame(const Configuration& info, QString& error) {
const auto reader = Common::OpenArchive(GameContent::ToPath(info.basedir));
const auto trophy_files = FindTrophyFiles(&info); const auto trophy_files = FindTrophyFiles(&info);
if (trophy_files.isEmpty()) { if (trophy_files.isEmpty()) {
error = tr("No trophy package found in sce_sys/trophy2."); error = tr("No trophy package found in sce_sys/trophy2.");
+72 -51
View File
@@ -1,6 +1,6 @@
#include "common/common.h" #include "common/common.h"
#include "common/file.h"
#include "common/logging/log.h" #include "common/logging/log.h"
#include "common/stringUtils.h"
#include "kernel/fileSystem.h" #include "kernel/fileSystem.h"
#include "kernel/pthread.h" #include "kernel/pthread.h"
#include "libs/audio.h" #include "libs/audio.h"
@@ -9,12 +9,14 @@
#include <algorithm> #include <algorithm>
#include <atomic> #include <atomic>
#include <cctype> #include <cctype>
#include <cerrno>
#include <condition_variable> #include <condition_variable>
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <cstring> #include <cstring>
#include <deque> #include <deque>
#include <limits>
#include <magic_enum.hpp> #include <magic_enum.hpp>
#include <memory> #include <memory>
#include <mutex> #include <mutex>
@@ -577,30 +579,40 @@ public:
explicit FileStreamer(AvPlayerFileReplacement f): file(f) {} explicit FileStreamer(AvPlayerFileReplacement f): file(f) {}
~FileStreamer() { ~FileStreamer() {
if (ctx != nullptr) { if (ctx != nullptr) {
av_freep(&ctx->buffer);
avio_context_free(&ctx); avio_context_free(&ctx);
} }
if (opened && file.close != nullptr) { if (opened) {
file.close(file.object_pointer); file.close(file.object_pointer);
} }
} }
bool Init(const std::string& path) { bool Init(const std::string& path) {
if (file.open == nullptr || file.close == nullptr || file.read_offset == nullptr || if (file.open != nullptr) {
file.size == nullptr) { if (file.close == nullptr || file.read_offset == nullptr || file.size == nullptr ||
file.open(file.object_pointer, path.c_str()) < 0) {
return false;
}
opened = true;
size = file.size(file.object_pointer);
} else {
if (!local_file.Open(LibKernel::FileSystem::GetRealFilename(path),
Common::File::Mode::Read)) {
return false;
}
size = local_file.Size();
}
if (size == 0 || size > static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) {
return false; return false;
} }
if (file.open(file.object_pointer, path.c_str()) < 0) { constexpr int buffer_size = 64 * 1024;
return false; auto* buf = static_cast<uint8_t*>(av_malloc(buffer_size));
}
opened = true;
size = file.size(file.object_pointer);
if (size == 0) {
return false;
}
auto* buf = static_cast<uint8_t*>(av_malloc(4096));
if (buf == nullptr) { if (buf == nullptr) {
return false; return false;
} }
ctx = avio_alloc_context(buf, 4096, 0, this, Read, nullptr, Seek); ctx = avio_alloc_context(buf, buffer_size, 0, this, Read, nullptr, Seek);
if (ctx == nullptr) {
av_free(buf);
}
return ctx != nullptr; return ctx != nullptr;
} }
AVIOContext* Context() const { return ctx; } AVIOContext* Context() const { return ctx; }
@@ -611,33 +623,50 @@ private:
if (s->pos >= s->size) { if (s->pos >= s->size) {
return AVERROR_EOF; return AVERROR_EOF;
} }
len = static_cast<int>(std::min<uint64_t>(len, s->size - s->pos)); len = static_cast<int>(std::min<uint64_t>(len, s->size - s->pos));
auto r = int read = 0;
s->file.read_offset(s->file.object_pointer, buf, s->pos, static_cast<uint32_t>(len)); if (s->opened) {
if (r <= 0) { read = s->file.read_offset(s->file.object_pointer, buf, s->pos,
return r == 0 ? AVERROR_EOF : r; static_cast<uint32_t>(len));
} else {
uint32_t bytes = 0;
s->local_file.Read(buf, static_cast<uint32_t>(len), &bytes);
read = static_cast<int>(bytes);
} }
s->pos += static_cast<uint64_t>(r); if (read <= 0 || read > len) {
return r; return read < 0 ? read : AVERROR(EIO);
}
s->pos += static_cast<uint64_t>(read);
return read;
} }
static int64_t Seek(void* opaque, int64_t off, int whence) { static int64_t Seek(void* opaque, int64_t offset, int whence) {
auto* s = static_cast<FileStreamer*>(opaque); auto* s = static_cast<FileStreamer*>(opaque);
if ((whence & AVSEEK_SIZE) != 0) { if ((whence & AVSEEK_SIZE) != 0) {
return static_cast<int64_t>(s->size); return static_cast<int64_t>(s->size);
} }
int64_t p = whence == SEEK_SET whence &= ~AVSEEK_FORCE;
? off uint64_t base = 0;
: (whence == SEEK_CUR switch (whence) {
? static_cast<int64_t>(s->pos) + off case SEEK_SET: break;
: (whence == SEEK_END ? static_cast<int64_t>(s->size) + off : -1)); case SEEK_CUR: base = s->pos; break;
if (p < 0) { case SEEK_END: base = s->size; break;
return -1; default: return AVERROR(EINVAL);
} }
p = std::min<int64_t>(p, static_cast<int64_t>(s->size)); const auto distance = offset < 0 ? uint64_t {0} - static_cast<uint64_t>(offset)
s->pos = static_cast<uint64_t>(p); : static_cast<uint64_t>(offset);
return p; if ((offset < 0 && distance > base) ||
(offset >= 0 && distance > std::numeric_limits<int64_t>::max() - base)) {
return AVERROR(EINVAL);
}
const auto position = offset < 0 ? base - distance : base + distance;
if (!s->opened && !s->local_file.Seek(position)) {
return AVERROR(EIO);
}
s->pos = position;
return static_cast<int64_t>(position);
} }
AvPlayerFileReplacement file; AvPlayerFileReplacement file;
Common::File local_file;
bool opened = false; bool opened = false;
uint64_t pos = 0; uint64_t pos = 0;
uint64_t size = 0; uint64_t size = 0;
@@ -675,25 +704,17 @@ public:
return static_cast<Source*>(opaque)->interrupt_io.load() ? 1 : 0; return static_cast<Source*>(opaque)->interrupt_io.load() ? 1 : 0;
}; };
raw->interrupt_callback.opaque = this; raw->interrupt_callback.opaque = this;
if (file.open != nullptr) { streamer = std::make_unique<FileStreamer>(file);
streamer = std::make_unique<FileStreamer>(file); if (!streamer->Init(path)) {
if (!streamer->Init(path)) { avformat_free_context(raw);
avformat_free_context(raw); return AVPLAYER_ERROR_OPERATION_FAILED;
return AVPLAYER_ERROR_OPERATION_FAILED; }
} raw->pb = streamer->Context();
raw->pb = streamer->Context(); if (const auto rc = avformat_open_input(&raw, file.open == nullptr ? path.c_str() : nullptr,
if (auto rc = avformat_open_input(&raw, nullptr, nullptr, nullptr); rc < 0) { nullptr, nullptr);
LOGF("\t avformat_open_input callback failed: %s\n", fferr(rc).c_str()); rc < 0) {
return AVPLAYER_ERROR_OPERATION_FAILED; LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(), path.c_str());
} return AVPLAYER_ERROR_OPERATION_FAILED;
} else {
auto real = LibKernel::FileSystem::GetRealFilename(std::string(path.c_str()));
auto real_str = Common::PathToString(real);
if (auto rc = avformat_open_input(&raw, real_str.c_str(), nullptr, nullptr); rc < 0) {
LOGF("\t avformat_open_input failed: %s path=%s\n", fferr(rc).c_str(),
real_str.c_str());
return AVPLAYER_ERROR_OPERATION_FAILED;
}
} }
fmt = raw; fmt = raw;
if (auto rc = avformat_find_stream_info(fmt, nullptr); rc < 0) { if (auto rc = avformat_find_stream_info(fmt, nullptr); rc < 0) {
+15 -4
View File
@@ -1,3 +1,4 @@
#include "common/archive.h"
#include "common/common.h" #include "common/common.h"
#include "common/dateTime.h" #include "common/dateTime.h"
#include "common/debug.h" #include "common/debug.h"
@@ -10,6 +11,7 @@
#include <charconv> #include <charconv>
#include <cstdio> #include <cstdio>
#include <filesystem>
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <fmt/format.h> #include <fmt/format.h>
@@ -40,9 +42,9 @@ static std::string GetBuildString() {
static void PrintUsage() { static void PrintUsage() {
::printf("%s\n", GetBuildString().c_str()); ::printf("%s\n", GetBuildString().c_str());
::printf("kyty_emulator --game <dir|elf> [options]\n\n"); ::printf("kyty_emulator --game <dir|elf|zar> [options]\n\n");
::printf("Options:\n"); ::printf("Options:\n");
::printf(" --game <dir|elf> Game directory or ELF to load.\n"); ::printf(" --game <dir|elf|zar> Game directory, ELF, or ZArchive to load.\n");
::printf(" --game-patch <json> ETAHen cheat file.\n"); ::printf(" --game-patch <json> ETAHen cheat file.\n");
::printf(" --screen-width <num> Window width. Default: 1280.\n"); ::printf(" --screen-width <num> Window width. Default: 1280.\n");
::printf(" --screen-height <num> Window height. Default: 720.\n"); ::printf(" --screen-height <num> Window height. Default: 720.\n");
@@ -234,12 +236,20 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
return false; return false;
} }
value = Common::FixFilenameSlash(value); value = Common::FixFilenameSlash(value);
const auto path = Common::PathFromUtf8(value); const auto path = Common::PathFromUtf8(value);
if (Common::File::IsDirectoryExisting(path)) { if (Common::File::IsDirectoryExisting(path)) {
options.app0_dir = path; options.app0_dir = path;
options.elf = "/app0/eboot.bin"; options.elf = "/app0/eboot.bin";
} else if (Common::IsSupportedArchive(path) && Common::File::IsFileExisting(path)) {
const auto root = Common::MakeArchivePath(path);
if (!Common::File::IsFileExisting(root / "eboot.bin")) {
::printf("Archive does not contain eboot.bin: %s\n", value.c_str());
return false;
}
options.app0_dir = root;
options.elf = "/app0/eboot.bin";
} else if (Common::File::IsFileExisting(path)) { } else if (Common::File::IsFileExisting(path)) {
options.app0_dir = path.parent_path(); options.app0_dir = path.parent_path();
@@ -249,7 +259,8 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
options.elf = std::filesystem::path("/app0") / path.filename(); options.elf = std::filesystem::path("/app0") / path.filename();
} else { } else {
::printf("--game must point to an existing directory or ELF: %s\n", value.c_str()); ::printf("--game must point to an existing directory, ELF, or archive: %s\n",
value.c_str());
return false; return false;
} }
} else if (arg == "--game-patch") { } else if (arg == "--game-patch") {
+400
View File
@@ -0,0 +1,400 @@
#include "common/file.h"
#include "common/archive.h"
#include "common/stringUtils.h"
#include "ArchiveTestFixture.h"
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <limits>
#include <string>
#include <thread>
#include <vector>
#include <zarchive/zarchivecommon.h>
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
#include <pthread.h>
#endif
namespace {
void Check(bool value, const char *text) {
if (!value) {
std::fprintf(stderr, "ArchiveFileTests: failed: %s\n", text);
std::abort();
}
}
class TempDirectory {
public:
TempDirectory() {
const auto unique =
std::chrono::steady_clock::now().time_since_epoch().count();
m_path = std::filesystem::temp_directory_path() /
("kyty_zarchive_test_!_" + std::to_string(unique));
Check(std::filesystem::create_directories(m_path),
"create temporary directory");
}
~TempDirectory() {
std::error_code error;
std::filesystem::remove_all(m_path, error);
}
[[nodiscard]] const std::filesystem::path &Path() const { return m_path; }
KYTY_CLASS_NO_COPY(TempDirectory);
private:
std::filesystem::path m_path;
};
template <typename T>
void SetValue(std::vector<std::byte> &bytes, size_t offset, T value) {
value = _ZARCHIVE::_store(value);
Check(offset <= bytes.size() && sizeof(value) <= bytes.size() - offset,
"patch lies inside fixture");
std::memcpy(bytes.data() + offset, &value, sizeof(value));
}
void SetMemberNameOffset(std::vector<std::byte> &bytes,
const _ZARCHIVE::Footer &footer, uint32_t name_offset) {
const auto entry_offset = footer.sectionFileTree.offset + sizeof(_ZARCHIVE::FileDirectoryEntry);
uint32_t original = 0;
std::memcpy(&original, bytes.data() + entry_offset, sizeof(original));
SetValue(bytes, entry_offset, (_ZARCHIVE::_load(original) & 0x80000000u) | name_offset);
}
template <typename Edit>
std::filesystem::path MutateArchive(const std::filesystem::path &source,
const char *name, Edit edit) {
Common::File input(source, Common::File::Mode::Read);
auto bytes = input.ReadWholeBuffer();
_ZARCHIVE::Footer footer{};
Check(bytes.size() >= sizeof(footer), "fixture has a footer");
std::memcpy(&footer, bytes.data() + bytes.size() - sizeof(footer), sizeof(footer));
_ZARCHIVE::Footer::Deserialize(&footer, &footer);
edit(bytes, footer);
_ZARCHIVE::Footer::Serialize(&footer, &footer);
std::memcpy(bytes.data() + bytes.size() - sizeof(footer), &footer, sizeof(footer));
const auto destination = source.parent_path() / name;
Common::File output;
Check(output.Create(destination), "create malformed archive fixture");
uint32_t written = 0;
output.Write(bytes.data(), static_cast<uint32_t>(bytes.size()), &written);
Check(written == bytes.size(), "write malformed archive fixture");
return destination;
}
void CheckMalformedArchives(const std::filesystem::path &source,
const std::vector<uint8_t> &payload) {
const auto bad_name = MutateArchive(source, "bad-name.zar", [](auto &bytes, auto &footer) {
SetMemberNameOffset(bytes, footer, static_cast<uint32_t>(footer.sectionNames.size));
});
Check(!Common::OpenArchive(bad_name), "reject name offset at the end of name table");
const auto bad_length = MutateArchive(source, "bad-length.zar", [](auto &bytes, auto &footer) {
SetMemberNameOffset(bytes, footer, static_cast<uint32_t>(footer.sectionNames.size - 1));
bytes[footer.sectionNames.offset + footer.sectionNames.size - 1] = std::byte{0x80};
});
Check(!Common::OpenArchive(bad_length), "reject truncated extended name length");
const auto bad_children = MutateArchive(source, "bad-children.zar", [](auto &bytes, auto &footer) {
SetValue(bytes, footer.sectionFileTree.offset + sizeof(uint32_t), UINT32_MAX);
});
Check(!Common::OpenArchive(bad_children), "reject invalid directory children without throwing");
const auto bad_range = MutateArchive(source, "bad-range.zar", [](auto &, auto &footer) {
footer.sectionCompressedData.offset = std::numeric_limits<uint64_t>::max();
});
Check(!Common::OpenArchive(bad_range), "reject overflowing footer section range");
const auto truncated = source.parent_path() / "truncated.zar";
Check(std::filesystem::copy_file(source, truncated), "copy truncated archive fixture");
std::filesystem::resize_file(truncated, std::filesystem::file_size(truncated) - 1);
Check(!Common::OpenArchive(truncated), "reject missing footer byte");
const auto corrupted = MutateArchive(source, "corrupt-data.zar", [](auto &bytes, auto &footer) {
_ZARCHIVE::CompressionOffsetRecord record{};
std::memcpy(&record, bytes.data() + footer.sectionOffsetRecords.offset, sizeof(record));
_ZARCHIVE::CompressionOffsetRecord::Deserialize(&record, 1, &record);
const auto second_block = footer.sectionCompressedData.offset + record.baseOffset + record.size[0] + 1;
std::fill_n(bytes.data() + second_block, record.size[1] + 1, std::byte{});
});
Common::File damaged(Common::MakeArchivePath(corrupted, "assets/subdir/data.bin"),
Common::File::Mode::Read);
Check(!damaged.IsInvalid(), "open archive with damaged compressed data");
std::vector<uint8_t> bytes(payload.size());
uint32_t read = 0;
damaged.Read(bytes.data(), static_cast<uint32_t>(bytes.size()), &read);
Check(read > 0 && read < 64 * 1024 && damaged.Tell() == read &&
std::equal(bytes.begin(), bytes.begin() + read, payload.begin()),
"report the valid prefix before a corrupt compressed block");
Check(damaged.Seek(2 * 64 * 1024), "seek past corrupt compressed block");
damaged.Read(bytes.data(), 97, &read);
Check(read == 97 && std::equal(bytes.begin(), bytes.begin() + read,
payload.begin() + 2 * 64 * 1024),
"recover by reading an unaffected compressed block");
}
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
void CheckCancellation(const std::filesystem::path &path,
const std::vector<uint8_t> &payload) {
// Keep the file and destination on the parent thread: cancellation must only release the
// reader thread's request and staging buffer after the worker has finished using them.
struct CancelledRead {
Common::File file;
std::vector<uint8_t> data;
std::atomic_bool ready = false;
std::atomic_bool start = false;
uint32_t bytes = 0;
} read;
Check(read.file.Open(path, Common::File::Mode::Read), "open cancellation fixture");
read.data.resize(payload.size());
pthread_t thread;
Check(pthread_create(&thread, nullptr, [](void *opaque) -> void * {
auto &read = *static_cast<CancelledRead *>(opaque);
read.ready = true;
while (!read.start.load()) {}
read.file.Read(read.data.data(), static_cast<uint32_t>(read.data.size()), &read.bytes);
pthread_testcancel();
return nullptr;
}, &read) == 0, "start cancellable archive reader");
while (!read.ready.load()) {}
Check(pthread_cancel(thread) == 0, "queue cancellation before the archive worker wait");
read.start = true;
void *result = nullptr;
Check(pthread_join(thread, &result) == 0 && result == PTHREAD_CANCELED,
"restore cancellation after the archive read");
Check(read.bytes == payload.size() && read.data == payload,
"finish worker reads and copies before releasing a cancelled caller");
int previous = PTHREAD_CANCEL_ENABLE;
int observed = PTHREAD_CANCEL_ENABLE;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &previous);
read.file.Seek(0);
read.file.Read(read.data.data(), 97, &read.bytes);
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &observed);
pthread_setcancelstate(previous, nullptr);
Check(observed == PTHREAD_CANCEL_DISABLE && read.bytes == 97,
"preserve a caller's disabled cancellation state");
}
#endif
} // namespace
int main() {
TempDirectory temporary;
const auto archive_path =
temporary.Path() / std::filesystem::path(u8"game-テスト.zar");
std::vector<uint8_t> payload(3 * 1024 * 1024 + 37);
for (size_t index = 0; index < payload.size(); index++) {
payload[index] = static_cast<uint8_t>((index * 37 + 11) & 0xff);
}
Check(ArchiveTests::CreateArchive(archive_path, payload), "create archive fixture");
const auto native_file = Common::File::GetInfo(archive_path);
const auto native_directory = Common::File::GetInfo(temporary.Path() / "");
Check(native_file && native_file->is_file &&
native_file->size == std::filesystem::file_size(archive_path),
"query native file type and size together");
Check(native_directory && !native_directory->is_file && native_directory->size == 0,
"query native directory with trailing separator");
Check(!Common::File::GetInfo(archive_path / "") && Common::File::Size(archive_path / "") == 0 &&
!Common::File::GetInfo(temporary.Path() / "missing.bin"),
"reject missing native paths and files with a trailing separator");
const auto root = Common::MakeArchivePath(archive_path);
auto reader_pin = Common::OpenArchive(archive_path);
Check(reader_pin && Common::OpenArchive(root) == reader_pin,
"mounted archive pin reuses the same reader and index");
const auto archive_file = Common::File::GetInfo(root / "assets/subdir/data.bin");
const auto archive_directory = Common::File::GetInfo(root / "assets/");
Check(archive_file && archive_file->is_file && archive_file->size == payload.size(),
"query archive member type and size together");
Check(archive_directory && !archive_directory->is_file && archive_directory->size == 0 &&
!Common::File::GetInfo(root / "missing.bin"),
"distinguish archive directories from missing members");
Check(Common::IsSupportedArchive(archive_path) &&
Common::IsSupportedArchive("GAME.ZAR") &&
!Common::IsSupportedArchive("game.zip"),
"select an archive backend by supported extension");
Check(!Common::IsArchivePath(temporary.Path() / "ordinary!" / "asset.bin"),
"ordinary host paths with exclamation marks stay host paths");
Check(Common::IsArchivePath(root), "recognize virtual archive root");
Check(Common::GetArchiveHostPath(root) == archive_path,
"recover host archive path");
Check(Common::File::IsDirectoryExisting(root), "find archive root directory");
Check(Common::File::IsFileExisting(root / "EBOOT.BIN"),
"lookup is case insensitive");
Check(!Common::File::IsFileExisting(root / "missing.bin"),
"reject missing member");
Check(Common::IsArchivePath(root / "../outside.bin"),
"retain archive classification for invalid traversal");
Check(!Common::File::IsFileExisting(root / "../outside.bin"),
"reject leading parent traversal");
Check(Common::File::IsFileExisting(root / "assets/../eboot.bin"),
"normalize internal parent traversal");
Check(Common::File::IsFileExisting(root / ArchiveTests::LongFilename),
"decode extended-length archive names");
Check(Common::File::IsFileExisting(root / Common::PathFromUtf8(ArchiveTests::UnicodeFilename)),
"lookup Unicode archive member");
const auto entries = Common::File::GetDirEntries(root);
Check(std::ranges::any_of(entries,
[](const auto &entry) {
return entry.is_file && entry.name == "eboot.bin";
}),
"enumerate root file");
Check(std::ranges::any_of(entries,
[](const auto &entry) {
return !entry.is_file && entry.name == "assets";
}),
"enumerate root directory");
Check(std::ranges::any_of(entries, [](const auto &entry) {
return entry.is_file && entry.name == ArchiveTests::UnicodeFilename;
}), "enumerate Unicode member names as UTF-8");
const auto data_path = root / "assets/subdir/data.bin";
Check(Common::File::Size(data_path) == payload.size(), "report member size");
Common::File file(data_path, Common::File::Mode::Read);
Check(!file.IsInvalid(), "open archive member");
Check(file.Remaining() == payload.size(), "report initial remaining bytes");
Check(file.Seek(64 * 1024 - 19), "seek before compression block boundary");
std::vector<uint8_t> slice(97);
uint32_t bytes_read = 0;
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == slice.size(), "read across compression block boundary");
Check(
std::equal(slice.begin(), slice.end(), payload.begin() + 64 * 1024 - 19),
"preserve data across compression block boundary");
Check(file.Seek(payload.size() - 9), "seek near member end");
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == 9, "truncate read at member end");
Check(file.IsEOF(), "report member end of file");
Check(file.Seek(payload.size() + 100), "allow seek past member end");
Check(file.Remaining() == 0, "report no bytes remaining past member end");
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == 0, "read past member end returns no data");
Check(file.Seek(0), "seek to member start");
std::vector<uint8_t> full(payload.size());
file.Read(full.data(), static_cast<uint32_t>(full.size()), &bytes_read);
Check(bytes_read == full.size(), "read a multi-chunk member");
Check(full == payload, "preserve a multi-chunk member");
std::atomic_bool concurrent_reads_ok = true;
std::vector<std::thread> readers;
for (uint32_t worker = 0; worker < 8; worker++) {
readers.emplace_back([&, worker] {
Common::File concurrent_file(data_path, Common::File::Mode::Read);
std::array<uint8_t, 4096> concurrent_data{};
for (uint32_t iteration = 0; iteration < 16; iteration++) {
const auto offset =
static_cast<size_t>(worker * 7919 + iteration * 12347) %
(payload.size() - concurrent_data.size());
uint32_t concurrent_bytes_read = 0;
if (concurrent_file.IsInvalid() || !concurrent_file.Seek(offset)) {
concurrent_reads_ok = false;
return;
}
concurrent_file.Read(concurrent_data.data(), concurrent_data.size(),
&concurrent_bytes_read);
if (concurrent_bytes_read != concurrent_data.size() ||
!std::equal(concurrent_data.begin(), concurrent_data.end(),
payload.begin() +
static_cast<std::ptrdiff_t>(offset))) {
concurrent_reads_ok = false;
return;
}
}
});
}
for (auto &reader : readers) {
reader.join();
}
Check(concurrent_reads_ok, "read concurrently through independent file handles");
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
CheckCancellation(data_path, payload);
#endif
#if !defined(_WIN32)
// Guest threads have much smaller stacks than std::thread's default stack.
pthread_attr_t attributes;
Check(pthread_attr_init(&attributes) == 0 &&
pthread_attr_setstacksize(&attributes, 64 * 1024) == 0,
"configure a guest-sized stack");
struct SmallStackRead {
const std::filesystem::path &path;
const std::vector<uint8_t> &expected;
bool passed = false;
} small_stack{data_path, payload};
pthread_t thread;
Check(pthread_create(&thread, &attributes, [](void *opaque) -> void * {
auto &test = *static_cast<SmallStackRead *>(opaque);
Common::File input(test.path, Common::File::Mode::Read);
auto data = input.ReadWholeBuffer();
test.passed = data.size() == test.expected.size() &&
std::memcmp(data.data(), test.expected.data(), data.size()) == 0;
return nullptr;
}, &small_stack) == 0, "start guest-sized-stack reader");
pthread_attr_destroy(&attributes);
Check(pthread_join(thread, nullptr) == 0 && small_stack.passed,
"decompress a multi-block file on a guest-sized stack");
#endif
Common::File metadata(root / "sce_sys/param.json", Common::File::Mode::Read);
const auto metadata_data = metadata.ReadWholeBuffer();
const std::string metadata_text(
reinterpret_cast<const char *>(metadata_data.data()),
metadata_data.size());
Check(metadata_text.find("Archive Test") != std::string::npos,
"read metadata member");
Common::File writable;
Check(!writable.Open(data_path, Common::File::Mode::ReadWrite),
"reject writable member open");
Check(!Common::File::DeleteFile(data_path), "reject member deletion");
Check(!Common::File::CreateDirectory(root / "new-dir"),
"reject member directory creation");
Check(!file.Truncate(0) && !file.Unlink(), "reject descriptor mutations");
const auto copied = temporary.Path() / "copied.bin";
Common::File host_fixture;
Check(host_fixture.Create(copied), "create host mutation fixture");
host_fixture.Close();
Check(!Common::File::CopyFile(data_path, copied) &&
!Common::File::CopyFile(copied, data_path) &&
!Common::File::RenameFile(data_path, copied) &&
!Common::File::RenameFile(copied, data_path),
"reject unsupported copy and rename operations involving archives");
CheckMalformedArchives(archive_path, payload);
file.Close();
metadata.Close();
Check(Common::OpenArchive(archive_path) == reader_pin,
"mount pin retains the archive reader after the last member closes");
reader_pin.reset();
std::error_code remove_error;
Check(std::filesystem::remove(archive_path, remove_error) && !remove_error,
"release archive handle after the final reader closes");
const std::array<uint8_t, 3> replacement{42, 7, 19};
Check(ArchiveTests::CreateArchive(archive_path, replacement), "replace a closed archive");
Common::File replaced(data_path, Common::File::Mode::Read);
const auto replaced_data = replaced.ReadWholeBuffer();
Check(replaced_data.size() == replacement.size() &&
std::memcmp(replaced_data.data(), replacement.data(), replacement.size()) == 0,
"reopening after the final reader closes sees replacement contents");
std::printf("ArchiveFileTests: all cases passed\n");
return 0;
}
+66
View File
@@ -0,0 +1,66 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <fstream>
#include <span>
#include <string>
#include <string_view>
#include <zarchive/zarchivewriter.h>
namespace ArchiveTests {
inline constexpr std::string_view Eboot = "ELF fixture";
inline constexpr std::string_view Param =
R"({"titleId":"TEST00001","titleName":"Archive Test"})";
inline constexpr std::string_view UnicodeFilename =
"asset-\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e.bin";
inline const std::string LongFilename = std::string(160, 'a') + ".bin";
// Both Common::File and guest filesystem tests read the same archive layout.
inline bool CreateArchive(const std::filesystem::path &path,
std::span<const uint8_t> payload) {
struct Output {
std::filesystem::path path;
std::ofstream stream;
bool good = true;
} output{.path = path};
{
ZArchiveWriter writer(
[](int32_t, void *opaque) {
auto &out = *static_cast<Output *>(opaque);
out.stream.open(out.path, std::ios::binary | std::ios::trunc);
out.good = out.stream.is_open();
},
[](const void *data, size_t size, void *opaque) {
auto &out = *static_cast<Output *>(opaque);
out.stream.write(static_cast<const char *>(data),
static_cast<std::streamsize>(size));
out.good = out.good && out.stream.good();
},
&output);
if (!output.good || !writer.MakeDir("sce_sys", true) ||
!writer.MakeDir("assets/subdir", true)) {
return false;
}
const auto add = [&](std::string_view name, const void *data, size_t size) {
if (!writer.StartNewFile(name.data())) {
return false;
}
writer.AppendData(data, size);
return true;
};
if (!add("eboot.bin", Eboot.data(), Eboot.size()) ||
!add("sce_sys/param.json", Param.data(), Param.size()) ||
!add("assets/subdir/data.bin", payload.data(), payload.size()) ||
!add(UnicodeFilename, Eboot.data(), Eboot.size()) ||
!add(LongFilename, Eboot.data(), Eboot.size())) {
return false;
}
writer.Finalize();
}
output.stream.close();
return output.good && !output.stream.fail();
}
} // namespace ArchiveTests
+134
View File
@@ -0,0 +1,134 @@
#include "libs/avPlayer.cpp"
#include "common/archive.h"
#include "ArchiveTestFixture.h"
#include <array>
#include <chrono>
namespace {
std::filesystem::path content_root;
}
namespace Libs::LibKernel {
int KYTY_SYSV_ABI PthreadCreate(Pthread*, const PthreadAttr*, pthread_entry_func_t, void*, const char*) {
std::abort();
}
int KYTY_SYSV_ABI PthreadJoin(Pthread, void**) { std::abort(); }
namespace FileSystem {
std::filesystem::path GetRealFilename(const std::string& path) {
return path.starts_with("/app0/") ? content_root / path.substr(6) : std::filesystem::path {};
}
}
}
namespace {
using namespace Libs::Audio::AvPlayer;
void Check(bool condition, const char* message) {
if (!condition) {
std::fprintf(stderr, "AvPlayerFileTests: %s\n", message);
std::abort();
}
}
void CheckStream(FileStreamer& stream, const std::vector<uint8_t>& data) {
auto* io = stream.Context();
Check(io->seek(io->opaque, 0, AVSEEK_SIZE) == static_cast<int64_t>(data.size()), "query size");
Check(io->seek(io->opaque, 65531, SEEK_SET | AVSEEK_FORCE) == 65531, "seek across block boundary");
std::array<uint8_t, 32> bytes {};
Check(io->read_packet(io->opaque, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), data.begin() + 65531), "read across block boundary");
Check(io->seek(io->opaque, -16, SEEK_CUR) == 65547, "seek relative to cursor");
Check(io->seek(io->opaque, INT64_MIN, SEEK_CUR) < 0 &&
io->seek(io->opaque, INT64_MAX, SEEK_END) < 0 &&
io->seek(io->opaque, 0, 12345) < 0, "reject invalid and overflowing seeks");
Check(io->seek(io->opaque, 0, SEEK_CUR) == 65547, "rejected seeks preserve the cursor");
Check(io->seek(io->opaque, -7, SEEK_END) == static_cast<int64_t>(data.size() - 7), "seek near end");
Check(io->read_packet(io->opaque, bytes.data(), bytes.size()) == 7 &&
std::equal(bytes.begin(), bytes.begin() + 7, data.end() - 7), "short read at end");
Check(io->read_packet(io->opaque, bytes.data(), bytes.size()) == AVERROR_EOF, "report EOF");
Check(io->seek(io->opaque, 11, SEEK_END) == static_cast<int64_t>(data.size() + 11) &&
io->read_packet(io->opaque, bytes.data(), bytes.size()) == AVERROR_EOF, "read beyond end");
Check(io->seek(io->opaque, 0, SEEK_SET) == 0, "rewind");
std::vector<uint8_t> buffered(data.size());
Check(avio_read(io, buffered.data(), static_cast<int>(buffered.size())) == buffered.size() &&
buffered == data, "FFmpeg buffered reads preserve content");
Check(avio_seek(io, 123, SEEK_SET) == 123 &&
avio_read(io, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), data.begin() + 123), "FFmpeg seeks reset buffered reads");
}
struct Callbacks {
const std::vector<uint8_t>& data;
unsigned opens = 0;
unsigned closes = 0;
bool premature_eof = false;
};
int KYTY_SYSV_ABI Open(void* object, const char*) {
++static_cast<Callbacks*>(object)->opens;
return 0;
}
int KYTY_SYSV_ABI Close(void* object) {
++static_cast<Callbacks*>(object)->closes;
return 0;
}
int KYTY_SYSV_ABI Read(void* object, uint8_t* buffer, uint64_t offset, uint32_t size) {
auto& state = *static_cast<Callbacks*>(object);
if (state.premature_eof) {
return 0;
}
const auto bytes = std::min<uint64_t>(size, state.data.size() - offset);
std::memcpy(buffer, state.data.data() + offset, bytes);
return static_cast<int>(bytes);
}
uint64_t KYTY_SYSV_ABI Size(void* object) { return static_cast<Callbacks*>(object)->data.size(); }
}
int main() {
const auto root = std::filesystem::temp_directory_path() /
("kyty_avio_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
Check(std::filesystem::create_directories(root), "create fixture directory");
std::vector<uint8_t> payload(128 * 1024 + 37);
for (size_t i = 0; i < payload.size(); ++i) {
payload[i] = static_cast<uint8_t>(i * 37);
}
{
Common::File output;
Check(output.Create(root / "movie.bin"), "create native fixture");
output.Write(payload.data(), static_cast<uint32_t>(payload.size()));
}
Check(ArchiveTests::CreateArchive(root / "game.zar", payload), "create archive fixture");
content_root = root;
{
FileStreamer native({});
Check(native.Init("/app0/movie.bin"), "open native stream");
CheckStream(native, payload);
}
content_root = Common::MakeArchivePath(root / "game.zar");
{
FileStreamer archive({});
Check(archive.Init("/app0/assets/subdir/data.bin"), "open archived stream");
CheckStream(archive, payload);
}
Callbacks callbacks {payload};
const AvPlayerFileReplacement replacement {&callbacks, Open, Close, Read, Size};
{
FileStreamer callback(replacement);
Check(callback.Init("callback"), "open guest callback stream");
CheckStream(callback, payload);
callbacks.premature_eof = true;
auto* io = callback.Context();
Check(io->seek(io->opaque, 0, SEEK_SET) == 0, "rewind callback");
uint8_t byte = 0;
Check(io->read_packet(io->opaque, &byte, 1) == AVERROR(EIO), "reject premature EOF");
}
Check(callbacks.opens == 1 && callbacks.closes == 1, "close callback exactly once");
{
FileStreamer missing({});
Check(!missing.Init("/unmounted/movie.bin"), "reject unmapped media");
}
std::error_code error;
std::filesystem::remove_all(root, error);
Check(!error, "release all archive handles");
std::puts("AvPlayerFileTests: all cases passed");
}
+117 -1
View File
@@ -2,7 +2,9 @@
#include <SDL3/SDL_main.h> #include <SDL3/SDL_main.h>
#include "common/emulatorConfig.h" #include "common/emulatorConfig.h"
#include "common/archive.h"
#include "common/file.h" #include "common/file.h"
#include "ArchiveTestFixture.h"
#include "common/logging/log.h" #include "common/logging/log.h"
#include "common/subsystems.h" #include "common/subsystems.h"
#include "common/threads.h" #include "common/threads.h"
@@ -214,6 +216,119 @@ void CheckUnmappedPaths(const std::filesystem::path &root) {
FileSystem::Umount("/app0"); FileSystem::Umount("/app0");
} }
void CheckArchiveMount(const std::filesystem::path &root) {
const auto archive = root / u8"game-日本語.zar";
std::vector<uint8_t> payload(2 * 64 * 1024 + 33);
for (size_t i = 0; i < payload.size(); ++i) {
payload[i] = static_cast<uint8_t>(i * 37 + 11);
}
Check(ArchiveTests::CreateArchive(archive, payload), "create mounted archive fixture");
const auto archive_root = Common::MakeArchivePath(archive);
const auto host_member = archive_root / "assets/subdir/data.bin";
constexpr char GuestMember[] = "/app0/assets/subdir/data.bin";
FileSystem::Mount(archive_root, "/app0");
Check(FileSystem::GetRealFilename(GuestMember) == host_member &&
FileSystem::GetRealFilename("/app01/eboot.bin").empty(),
"resolve archive mounts at path-component boundaries");
Check(FileSystem::KernelOpen("/app0/../outside.bin", 0, 0) ==
Libs::LibKernel::KERNEL_ERROR_ENOENT &&
FileSystem::KernelOpen("/app0/missing.bin", 0, 0) ==
Libs::LibKernel::KERNEL_ERROR_ENOENT,
"reject traversal outside an archive and missing members");
FileSystem::FileStat path_stat{}, descriptor_stat{};
Check(FileSystem::KernelStat(GuestMember, &path_stat) == OK &&
path_stat.st_size == payload.size() &&
path_stat.st_size == Common::File::Size(host_member) &&
FileSystem::KernelCheckReachability(GuestMember) == OK,
"archive path stat and reachability agree with Common::File");
const int fd = FileSystem::KernelOpen("/app0/ASSETS/subdir/DATA.BIN", 0, 0);
Check(fd >= 3 && FileSystem::KernelFstat(fd, &descriptor_stat) == OK &&
descriptor_stat.st_size == path_stat.st_size &&
descriptor_stat.st_mode == path_stat.st_mode,
"case-insensitive archive open and descriptor stat agree");
std::array<uint8_t, 97> bytes{};
constexpr int64_t Offset = 64 * 1024 - 19;
Check(FileSystem::KernelPread(fd, bytes.data(), bytes.size(), Offset) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), payload.begin() + Offset) &&
FileSystem::KernelLseek(fd, 0, 1) == 0,
"archive pread crosses a compression block without moving position");
Check(FileSystem::KernelLseek(fd, Offset, 0) == Offset &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), payload.begin() + Offset) &&
FileSystem::KernelLseek(fd, 0, 1) == Offset + bytes.size(),
"archive seek and sequential read share descriptor position");
Check(FileSystem::KernelLseek(fd, -9, 2) == payload.size() - 9 &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == 9 &&
std::equal(bytes.begin(), bytes.begin() + 9, payload.end() - 9) &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == 0,
"archive reads stop at the member boundary");
Check(FileSystem::KernelWrite(fd, bytes.data(), 1) ==
Libs::LibKernel::KERNEL_ERROR_EBADF &&
FileSystem::KernelPwrite(fd, bytes.data(), 1, 0) ==
Libs::LibKernel::KERNEL_ERROR_EBADF &&
FileSystem::KernelFtruncate(fd, 0) ==
Libs::LibKernel::KERNEL_ERROR_EBADF,
"archive read-only descriptors reject write and truncate");
const auto unicode_guest = std::string("/app0/") + std::string(ArchiveTests::UnicodeFilename);
const int unicode = FileSystem::KernelOpen(unicode_guest.c_str(), 0, 0);
Check(unicode >= 3 && FileSystem::KernelRead(unicode, bytes.data(), bytes.size()) ==
ArchiveTests::Eboot.size() &&
std::memcmp(bytes.data(), ArchiveTests::Eboot.data(), ArchiveTests::Eboot.size()) == 0,
"open and read a Unicode archive member");
Check(FileSystem::KernelClose(unicode) == OK, "close Unicode archive member");
const int directory = FileSystem::KernelOpen("/app0/", 0x00020000, 0);
Check(directory >= 3, "open mounted archive directory");
std::array<char, 512> block{};
const auto expected_entries = Common::File::GetDirEntries(archive_root);
size_t entries_seen = 0;
for (;;) {
const int length = FileSystem::KernelGetdents(directory, block.data(), block.size());
Check(length >= 0 && length <= block.size(), "read archive directory records");
if (length == 0) {
break;
}
for (size_t offset = 0; offset < static_cast<size_t>(length);) {
uint16_t record_length = 0;
Check(length - offset >= 8, "archive directory record header fits");
std::memcpy(&record_length, block.data() + offset + 4, sizeof(record_length));
const auto name_length = static_cast<uint8_t>(block[offset + 7]);
Check(record_length >= 9 + name_length && record_length <= length - offset,
"archive directory record fits");
const std::string_view name(block.data() + offset + 8, name_length);
Check(std::any_of(expected_entries.begin(), expected_entries.end(), [&](const auto &entry) {
return entry.name == name && block[offset + 6] == (entry.is_file ? 8 : 4);
}), "guest directory entry matches Common::File name and type");
++entries_seen;
offset += record_length;
}
}
Check(entries_seen == expected_entries.size(), "guest enumerates every archive directory entry");
Check(FileSystem::KernelClose(directory) == OK, "close archive directory");
for (const int flags : {1, 2, 0x0200, 0x0400}) {
Check(FileSystem::KernelOpen(GuestMember, flags, 0777) ==
Libs::LibKernel::KERNEL_ERROR_EROFS,
"archive rejects write, create and truncate open flags");
}
Check(FileSystem::KernelUnlink(GuestMember) == Libs::LibKernel::KERNEL_ERROR_EROFS &&
FileSystem::KernelMkdir("/app0/new-dir", 0777) == Libs::LibKernel::KERNEL_ERROR_EROFS &&
FileSystem::KernelRmdir("/app0/assets") == Libs::LibKernel::KERNEL_ERROR_EROFS &&
FileSystem::KernelRename(GuestMember, "/app0/renamed.bin") ==
Libs::LibKernel::KERNEL_ERROR_EROFS,
"archive mount rejects path mutations");
FileSystem::Umount("/app0");
Check(FileSystem::GetRealFilename(GuestMember).empty() &&
FileSystem::KernelOpen(GuestMember, 0, 0) == Libs::LibKernel::KERNEL_ERROR_ENOENT &&
FileSystem::KernelLseek(fd, 0, 0) == 0 &&
FileSystem::KernelRead(fd, bytes.data(), bytes.size()) == bytes.size() &&
std::equal(bytes.begin(), bytes.end(), payload.begin()),
"unmount hides archive paths while open descriptors retain the reader");
Check(FileSystem::KernelClose(fd) == OK, "close last archive descriptor");
}
void CheckUnicodePaths(const std::filesystem::path &root) { void CheckUnicodePaths(const std::filesystem::path &root) {
constexpr std::string_view HostDirectory = constexpr std::string_view HostDirectory =
"Test\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; "Test\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e";
@@ -567,7 +682,7 @@ void CheckSocketWakeup() {
} // namespace } // namespace
int main() { int main(int, char**) {
Common::InitializeThreads(); Common::InitializeThreads();
Common::Subsystems subsystems; Common::Subsystems subsystems;
subsystems.Initialize<Config::Lifecycle>(); subsystems.Initialize<Config::Lifecycle>();
@@ -589,6 +704,7 @@ int main() {
FileSystem::Initialize(); FileSystem::Initialize();
CheckMountRoot(temporary.Path()); CheckMountRoot(temporary.Path());
CheckUnmappedPaths(temporary.Path()); CheckUnmappedPaths(temporary.Path());
CheckArchiveMount(temporary.Path());
CheckUnicodePaths(temporary.Path()); CheckUnicodePaths(temporary.Path());
CheckUnicodeLogPath(temporary.Path()); CheckUnicodeLogPath(temporary.Path());
CheckDirectoryStream(temporary.Path()); CheckDirectoryStream(temporary.Path());