raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-26 12:56:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

pipeline: write the shader store on a background thread

Browse Source
This commit is contained in:
nmzik
2026-10-25 01:59:49 +02:00
parent 72903bfce6
commit dd5bc26c64
3 changed files with 108 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/pipeline/pipelineCache.h
+30 -2
View File
@@ -5,6 +5,7 @@
#include "common/assert.h"
#include "common/common.h"
#include "common/file.h"
#include "common/uniqueFunction.h"
#include "graphics/host_gpu/renderer/colorRenderTarget.h"
#include "graphics/host_gpu/renderer/depthRenderTarget.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
@@ -14,10 +15,14 @@
#include "graphics/shader/shader.h"
#include <chrono>
#include <condition_variable>
#include <cstddef>
#include <filesystem>
#include <memory>
#include <mutex>
#include <queue>
#include <span>
#include <thread>
#include <type_traits>
#include <unordered_map>
#include <vector>
@@ -141,6 +146,29 @@ struct DrawRenderState {
}
};
// Writes the shader store on its own thread, running jobs in the order they were posted.
class ShaderStoreWriter {
public:
// Keeps the first `kept` bytes of the store at `path`, or starts it over with `header`. The
// thread runs only if that succeeded.
void Open(const std::filesystem::path& path, std::span<const uint8_t> header, uint64_t kept);
explicit operator bool() const { return m_thread.joinable(); }
// Compresses `record` and appends it on the store thread, in call order.
void Append(std::vector<uint8_t>&& record);
private:
friend struct PipelineCacheTestAccess;
void Post(Common::UniqueFunction<void>&& job);
Common::File m_file;
std::mutex m_mutex;
std::condition_variable_any m_ready;
std::queue<Common::UniqueFunction<void>> m_jobs;
// Declared last, so it drains the queue and joins before the rest is destroyed.
std::jthread m_thread;
};
// The owning renderer serializes access, including saves while the GPU is running.
class PipelineCache {
public:
@@ -219,8 +247,8 @@ private:
std::unordered_map<GraphicsPipelineKey, Pipeline, GraphicsPipelineKeyHash> m_graphics_pipelines;
std::unordered_map<uint64_t, Pipeline> m_compute_pipelines;
std::unordered_map<uint64_t, ShaderTessellationInputInfo> m_tess_layouts;
// Appends records of everything first created; invalid without a shader store.
Common::File m_store;
// Records everything first created; closed without a shader store.
ShaderStoreWriter m_store;
template <typename InputInfo>
const ShaderModule* GetProgram(const ShaderParams& params, InputInfo& input_info,
src/graphics/host_gpu/renderer/pipeline/pipelineSerialization.cpp
+58 -29
View File
@@ -72,19 +72,6 @@ std::vector<uint8_t> StoreHeader(const GraphicContext& graphics) {
return std::move(header.bytes);
}
void Append(Common::File& store, const Writer& record) {
std::vector<uint8_t> bytes(sizeof(Frame) + ZSTD_compressBound(record.bytes.size()));
const auto size = ZSTD_compress(bytes.data() + sizeof(Frame), bytes.size() - sizeof(Frame),
record.bytes.data(), record.bytes.size(), 3);
EXIT_IF(ZSTD_isError(size));
const Frame frame {.compressed_size = static_cast<uint32_t>(size),
.raw_size = static_cast<uint32_t>(record.bytes.size()),
.checksum = XXH3_64bits(bytes.data() + sizeof(Frame), size)};
std::memcpy(bytes.data(), &frame, sizeof(frame));
// A failed write leaves a partial record; the next run drops it and everything after it.
store.Write(bytes.data(), static_cast<uint32_t>(sizeof(Frame) + size));
}
// Record fields, shared by writing and reading.
template <class S, class Key> void IoKey(S& io, Key& key) {
auto& [stage, hash, user_data_count, static_state] = key;
@@ -99,6 +86,56 @@ template <class S, class Module> void IoModule(S& io, Module& stage) {
} // namespace
void ShaderStoreWriter::Open(const std::filesystem::path& path, std::span<const uint8_t> header,
uint64_t kept) {
Common::File::CreateDirectories(path.parent_path());
if (kept == 0) {
if (m_file.Create(path)) {
m_file.Write(header.data(), static_cast<uint32_t>(header.size()));
}
} else if (m_file.Open(path, Common::File::Mode::ReadWrite)) {
m_file.Truncate(kept);
m_file.Seek(kept);
}
if (m_file.IsInvalid()) {
return;
}
m_thread = std::jthread([this](std::stop_token stop) {
std::unique_lock lock(m_mutex);
// Once stop is requested the wait returns false only when the queue is empty.
while (m_ready.wait(lock, stop, [this] { return !m_jobs.empty(); })) {
const auto job = std::move(m_jobs.front());
m_jobs.pop();
lock.unlock();
job();
lock.lock();
}
});
}
void ShaderStoreWriter::Append(std::vector<uint8_t>&& record) {
Post([this, record = std::move(record)] {
std::vector<uint8_t> bytes(sizeof(Frame) + ZSTD_compressBound(record.size()));
const auto size = ZSTD_compress(bytes.data() + sizeof(Frame), bytes.size() - sizeof(Frame),
record.data(), record.size(), 3);
EXIT_IF(ZSTD_isError(size));
const Frame frame {.compressed_size = static_cast<uint32_t>(size),
.raw_size = static_cast<uint32_t>(record.size()),
.checksum = XXH3_64bits(bytes.data() + sizeof(Frame), size)};
std::memcpy(bytes.data(), &frame, sizeof(frame));
// A failed write leaves a partial record; the next run drops it and everything after it.
m_file.Write(bytes.data(), static_cast<uint32_t>(sizeof(Frame) + size));
});
}
void ShaderStoreWriter::Post(Common::UniqueFunction<void>&& job) {
{
std::scoped_lock lock(m_mutex);
m_jobs.push(std::move(job));
}
m_ready.notify_one();
}
void PipelineCache::WarmUp(const std::filesystem::path& path) {
if (path.empty()) {
return;
@@ -126,16 +163,7 @@ void PipelineCache::WarmUp(const std::filesystem::path& path) {
}
}
// A damaged record ends the store; the records before it stay.
const auto kept = reusable ? bytes.size() - records.size() : 0;
Common::File::CreateDirectories(path.parent_path());
if (kept == 0) {
if (m_store.Create(path)) {
m_store.Write(header.data(), static_cast<uint32_t>(header.size()));
}
} else if (m_store.Open(path, Common::File::Mode::ReadWrite)) {
m_store.Truncate(kept);
m_store.Seek(kept);
}
m_store.Open(path, header, reusable ? bytes.size() - records.size() : 0);
const auto modules = MapModules();
for (const auto& key: pipelines) {
@@ -238,9 +266,10 @@ bool PipelineCache::LoadRecord(std::span<const uint8_t>& records,
return true;
}
// Records are encoded here, since programs and modules change later.
void PipelineCache::StoreModule(const ProgramKey& key, const Program& program,
const ShaderModule& module, const std::vector<uint32_t>& spirv) {
if (m_store.IsInvalid()) {
if (!m_store) {
return;
}
Writer record;
@@ -249,26 +278,26 @@ void PipelineCache::StoreModule(const ProgramKey& key, const Program& program,
IoAll(record, ShaderRecompiler::IR::SaveResourcePlan(program.resource_plan));
IoModule(record, module);
IoAll(record, spirv);
Append(m_store, record);
m_store.Append(std::move(record.bytes));
}
void PipelineCache::StorePipeline(const GraphicsPipelineKey& key) {
if (m_store.IsInvalid()) {
if (!m_store) {
return;
}
Writer record;
IoAll(record, Record::GraphicsPipeline);
record.Raw(&key, sizeof(key));
Append(m_store, record);
m_store.Append(std::move(record.bytes));
}
void PipelineCache::StoreLayout(uint64_t key, const ShaderTessellationInputInfo& layout) {
if (m_store.IsInvalid()) {
if (!m_store) {
return;
}
Writer record;
IoAll(record, Record::TessellationLayout, key, layout);
Append(m_store, record);
m_store.Append(std::move(record.bytes));
}
PipelineCache::ModuleMap PipelineCache::MapModules() const {