raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 10:56:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

ampr: schedule address waits without blocking other priorities

Browse Source 
Co-authored-by: Mac <me@itsmemac.com>
This commit is contained in:
nmzik
2026-09-29 21:10:44 +04:00
parent 5bf9218b32
commit c045f08904
2 changed files with 403 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/libAmpr.cpp
+278 -288
View File
@@ -12,11 +12,16 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <cstring>
#include <deque>
#include <limits>
#include <mutex>
#include <string>
#include <thread>
#include <unordered_map>
#include <variant>
#include <vector>
@@ -37,13 +42,6 @@ struct ResultBuffer {
uint32_t errorOffset = 0;
};
struct SubmissionState {
uint64_t command_buffer = 0;
uint64_t result = 0;
int32_t execution_result = 0;
uint32_t error_offset = 0;
};
struct ResolvedPathInfo {
int result = OK;
uint32_t file_id = 0xffffffffu;
@@ -54,7 +52,8 @@ struct ResolvedPathInfo {
static std::mutex g_mutex;
static uint32_t g_next_submission_id = 1;
static std::unordered_map<uint32_t, SubmissionState> g_submissions;
static std::unordered_map<uint32_t, bool> g_submissions;
static std::condition_variable g_submission_cv;
static std::unordered_map<uint32_t, std::string> g_files;
static std::unordered_map<uint32_t, uint64_t> g_file_sizes;
static std::unordered_map<std::string, ResolvedPathInfo> g_resolved_paths;
@@ -340,38 +339,35 @@ static int ResolvePathsCommon(const char* const* path_list, uint32_t count, uint
return results != nullptr ? static_cast<int>(success_count) : OK;
}
static uint32_t AllocateSubmissionId(uint64_t command_buffer, uint64_t result) {
static uint32_t AllocateSubmissionId() {
std::scoped_lock lock(g_mutex);
auto id = g_next_submission_id++;
if (id == 0) {
id = g_next_submission_id++;
}
g_submissions[id] = SubmissionState {command_buffer, result};
g_submissions.emplace(id, false);
return id;
}
static void SetSubmissionResult(uint32_t submission_id, int32_t execution_result,
uint32_t error_offset) {
std::scoped_lock lock(g_mutex);
auto it = g_submissions.find(submission_id);
if (it != g_submissions.end()) {
it->second.execution_result = execution_result;
it->second.error_offset = error_offset;
static void SetSubmissionComplete(uint32_t submission_id) {
if (submission_id == 0) {
return;
}
std::scoped_lock lock(g_mutex);
g_submissions.at(submission_id) = true;
g_submission_cv.notify_all();
}
static bool CompleteSubmission(uint32_t submission_id, SubmissionState* state) {
std::scoped_lock lock(g_mutex);
auto it = g_submissions.find(submission_id);
if (it == g_submissions.end()) {
static bool CompleteSubmission(uint32_t submission_id) {
std::unique_lock lock(g_mutex);
if (!g_submissions.contains(submission_id)) {
return false;
}
if (state != nullptr) {
*state = it->second;
}
g_submissions.erase(it);
return true;
g_submission_cv.wait(lock, [submission_id] {
const auto it = g_submissions.find(submission_id);
return it == g_submissions.end() || it->second;
});
return g_submissions.erase(submission_id) != 0;
}
static int WriteResult(void* result, int32_t execution_result = 0, uint32_t error_offset = 0) {
@@ -390,8 +386,9 @@ static int WriteResult(void* result, int32_t execution_result = 0, uint32_t erro
} // namespace AprShared
namespace LibAmpr::Ampr {
static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result,
uint32_t* error_offset);
enum class Engine { Apr, Amm };
static int QueueCommandBuffer(Engine engine, uint64_t command_buffer, uint32_t bytes,
uint32_t priority, void* result, uint32_t* out_submission_id);
}
namespace LibKernelApr {
@@ -537,89 +534,37 @@ static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIdsAndFileSizesForEach(
AprShared::ResolvePathsCommon(path_list, count, ids, sizes, nullptr, prefix, results));
}
static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint64_t,
static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint32_t priority,
void* result,
uint32_t* out_submission_id) {
PRINT_NAME();
if (command_buffer == nullptr) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL);
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer);
const auto id =
AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast<uint64_t>(result));
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr,
&execution_result, &error_offset);
if (submit_result != OK) {
return KernelSyscallResult(submit_result);
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (out_submission_id != nullptr) {
if (!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT);
}
}
return KernelSyscallResult(AprShared::WriteResult(result, execution_result, error_offset));
return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer(
LibAmpr::Ampr::Engine::Apr, reinterpret_cast<uint64_t>(command_buffer), 0, priority, result,
out_submission_id));
}
static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint64_t) {
static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint32_t priority) {
PRINT_NAME();
if (command_buffer == nullptr) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL);
}
int32_t execution_result = OK;
uint32_t error_offset = 0;
return KernelSyscallResult(LibAmpr::Ampr::ExecuteAprCommandBuffer(
reinterpret_cast<uint64_t>(command_buffer), &execution_result, &error_offset));
return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer(
LibAmpr::Ampr::Engine::Apr, reinterpret_cast<uint64_t>(command_buffer), 0, priority,
nullptr, nullptr));
}
static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint64_t,
static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint32_t priority,
uint32_t* out_submission_id) {
PRINT_NAME();
if (command_buffer == nullptr || out_submission_id == nullptr) {
if (out_submission_id == nullptr) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL);
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer);
const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0);
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr,
&execution_result, &error_offset);
if (submit_result != OK) {
return KernelSyscallResult(submit_result);
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT);
}
return OK;
return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer(
LibAmpr::Ampr::Engine::Apr, reinterpret_cast<uint64_t>(command_buffer), 0, priority,
nullptr, out_submission_id));
}
static int KYTY_SYSV_ABI WaitCommandBuffer(uint32_t submission_id) {
PRINT_NAME();
AprShared::SubmissionState state {};
if (!AprShared::CompleteSubmission(submission_id, &state)) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_ESRCH);
}
if (state.result != 0) {
return KernelSyscallResult(AprShared::WriteResult(
reinterpret_cast<void*>(state.result), state.execution_result, state.error_offset));
}
return OK;
return KernelSyscallResult(
AprShared::CompleteSubmission(submission_id) ? OK : LibKernel::KERNEL_ERROR_ESRCH);
}
} // namespace Apr
@@ -720,6 +665,11 @@ struct CommandBufferState {
uint64_t address = 0;
uint64_t value = 0;
};
struct WaitAddressCommand {
uint64_t address;
uint64_t value;
uint8_t compare;
};
struct AmmMapCommand {
AmmCommandKind kind = AmmCommandKind::MapAuto;
uint64_t va = 0;
@@ -729,9 +679,11 @@ struct CommandBufferState {
int32_t prot = 0;
uint8_t gpu_mask_id = 0;
};
using CommandData = std::variant<ReadFileCommand, KernelEventCommand, WriteAddressCommand,
AmmMapCommand, WaitAddressCommand>;
struct Command {
uint64_t record_offset;
std::variant<ReadFileCommand, KernelEventCommand, WriteAddressCommand, AmmMapCommand> data;
CommandData data;
};
std::vector<Command> commands;
bool gather_scatter_valid = false;
@@ -1141,27 +1093,6 @@ static bool AppendKernelEventRecord(uint64_t command_buffer, uint64_t eq, int32_
return CommitCommandBufferRecord(command_buffer, &state, KERNEL_EVENT_RECORD_SIZE);
}
static bool AppendWriteAddressRecord(uint64_t command_buffer, uint64_t address, uint64_t value,
uint64_t record_size = 0x20) {
std::scoped_lock lock(g_command_buffer_mutex);
CommandBufferIterator it;
if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) {
return false;
}
auto& state = it->second;
if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) {
return false;
}
const auto record_offset = state.write_offset;
std::memset(reinterpret_cast<void*>(state.buffer + record_offset), 0,
static_cast<size_t>(record_size));
state.commands.push_back(
{record_offset, CommandBufferState::WriteAddressCommand {address, value}});
return CommitCommandBufferRecord(command_buffer, &state, record_size);
}
static bool ValidateAmmMapArgs(uint64_t va, uint64_t size) {
return va != 0 && size != 0 && (va & (AMM_PAGE_SIZE - 1u)) == 0 &&
(size & (AMM_PAGE_SIZE - 1u)) == 0 && va + size >= va;
@@ -1219,8 +1150,8 @@ static int ExecuteAmmMapCommand(const CommandBufferState::AmmMapCommand& command
static_cast<int64_t>(command.dmem_offset), AMM_PAGE_SIZE);
}
static bool AppendAmmMapRecord(uint64_t command_buffer,
const CommandBufferState::AmmMapCommand& cmd, uint64_t record_size) {
static bool AppendCommandRecord(uint64_t command_buffer, CommandBufferState::CommandData command,
uint64_t record_size) {
std::scoped_lock lock(g_command_buffer_mutex);
CommandBufferIterator it;
if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) {
@@ -1234,101 +1165,202 @@ static bool AppendAmmMapRecord(uint64_t command_
auto* record_addr = reinterpret_cast<void*>(state.buffer + state.write_offset);
std::memset(record_addr, 0, static_cast<size_t>(record_size));
state.commands.push_back({state.write_offset, cmd});
state.commands.push_back({state.write_offset, std::move(command)});
return CommitCommandBufferRecord(command_buffer, &state, record_size);
}
static int ReadHostFileToGuest(const std::string& host_path, uint64_t file_offset,
uint64_t destination, uint64_t size, uint64_t* bytes_read);
static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result,
uint32_t* error_offset) {
if (execution_result == nullptr || error_offset == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL;
static int ExecuteCommand(const CommandBufferState::Command& entry) {
if (const auto* payload = std::get_if<CommandBufferState::ReadFileCommand>(&entry.data)) {
const auto& command = *payload;
std::string host_path;
if (!AprShared::TryGetHostPath(command.file_id, &host_path)) {
LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", command.file_id);
return LibKernel::KERNEL_ERROR_ENOENT;
}
uint64_t bytes_read = 0;
auto result = ReadHostFileToGuest(host_path, command.file_offset, command.destination,
command.size, &bytes_read);
if (result != OK) {
LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 ", path=%s\n",
command.file_id, static_cast<uint32_t>(result), host_path.c_str());
return result;
}
} else if (const auto* payload =
std::get_if<CommandBufferState::KernelEventCommand>(&entry.data)) {
const auto& command = *payload;
const auto eq = static_cast<LibKernel::EventQueue::KernelEqueue>(command.eq);
auto result = LibKernel::EventQueue::KernelTriggerUserEvent(
eq, command.id, reinterpret_cast<void*>(command.data));
if (result != OK) {
LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32
", result=0x%08" PRIx32 "\n",
command.eq, command.id, static_cast<uint32_t>(result));
return result;
}
} else if (const auto* payload =
std::get_if<CommandBufferState::WriteAddressCommand>(&entry.data)) {
const auto& command = *payload;
std::atomic_ref(*reinterpret_cast<uint64_t*>(command.address))
.store(command.value, std::memory_order_release);
} else if (const auto* payload = std::get_if<CommandBufferState::AmmMapCommand>(&entry.data)) {
const auto& command = *payload;
int result = OK;
if (command.kind == AmmCommandKind::Unmap) {
result = LibKernel::Memory::KernelMunmap(command.va, command.size);
} else {
result = ExecuteAmmMapCommand(command);
}
if (result != OK) {
LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 " dmem=0x%016" PRIx64
" size=0x%016" PRIx64 " type=%" PRId32 " prot=0x%08" PRIx32 " result=0x%08" PRIx32
"\n",
static_cast<uint32_t>(command.kind), command.va, command.dmem_offset, command.size,
command.type, static_cast<uint32_t>(command.prot), static_cast<uint32_t>(result));
return result;
}
}
return OK;
}
struct Submission {
std::vector<CommandBufferState::Command> commands;
size_t cursor = 0;
void* result = nullptr;
uint32_t id = 0;
};
static bool IsWaitSatisfied(const CommandBufferState::WaitAddressCommand& wait) {
const uint64_t value =
std::atomic_ref(*reinterpret_cast<uint64_t*>(wait.address)).load(std::memory_order_acquire);
switch (wait.compare) {
case 0: return value == wait.value;
case 1: return value > wait.value;
case 2: return value < wait.value;
case 3: return value != wait.value;
case 4: return static_cast<int64_t>(value - wait.value) >= 0;
case 5: return static_cast<int64_t>(value) > static_cast<int64_t>(wait.value);
default: return static_cast<int64_t>(value) < static_cast<int64_t>(wait.value);
}
}
class CommandEngine {
public:
explicit CommandEngine(size_t priorities)
: m_queues(priorities), m_worker([this](std::stop_token stop) { Run(stop); }) {}
~CommandEngine() {
std::scoped_lock lock(m_mutex);
m_worker.request_stop();
m_cv.notify_all();
}
*execution_result = OK;
*error_offset = 0;
void Submit(uint32_t priority, Submission submission) {
std::scoped_lock lock(m_mutex);
m_queues[priority].push_back(std::move(submission));
m_cv.notify_one();
}
private:
void Run(std::stop_token stop) {
std::unique_lock lock(m_mutex);
while (!stop.stop_requested()) {
std::deque<Submission>* ready = nullptr;
bool pending = false;
for (auto& queue: m_queues) {
if (queue.empty()) {
continue;
}
pending = true;
auto& submission = queue.front();
if (submission.cursor < submission.commands.size()) {
const auto* wait = std::get_if<CommandBufferState::WaitAddressCommand>(
&submission.commands[submission.cursor].data);
if (wait != nullptr && !IsWaitSatisfied(*wait)) {
continue;
}
}
ready = &queue;
break;
}
if (ready == nullptr) {
if (pending) {
// CPU and GPU fence writes do not notify the submission queue.
m_cv.wait_for(lock, std::chrono::microseconds(100));
} else {
m_cv.wait(lock);
}
continue;
}
auto& submission = ready->front();
lock.unlock();
int result = OK;
uint32_t error_offset = 0;
if (submission.cursor < submission.commands.size()) {
const auto& command = submission.commands[submission.cursor++];
result = ExecuteCommand(command);
if (result != OK) {
error_offset = static_cast<uint32_t>(command.record_offset);
}
}
const bool complete = result != OK || submission.cursor == submission.commands.size();
if (complete) {
AprShared::WriteResult(submission.result, result, error_offset);
AprShared::SetSubmissionComplete(submission.id);
}
lock.lock();
if (complete) {
ready->pop_front();
}
}
}
std::mutex m_mutex;
std::condition_variable m_cv;
std::vector<std::deque<Submission>> m_queues;
std::jthread m_worker;
};
static int QueueCommandBuffer(Engine engine, uint64_t command_buffer, uint32_t bytes,
uint32_t priority, void* result, uint32_t* out_submission_id) {
const bool amm = engine == Engine::Amm;
if (command_buffer == 0 || priority > (amm ? 2u : 6u)) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
if ((result != nullptr &&
!AprShared::IsValidGuestRange(reinterpret_cast<uint64_t>(result),
sizeof(AprShared::ResultBuffer), true)) ||
(out_submission_id != nullptr &&
!AprShared::IsValidGuestRange(reinterpret_cast<uint64_t>(out_submission_id),
sizeof(uint32_t), true))) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
CommandBufferState state {};
if (!TryGetCommandBufferState(command_buffer, &state)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
for (const auto& entry: state.commands) {
if (entry.record_offset >= state.write_offset) {
break;
}
if (const auto* payload = std::get_if<CommandBufferState::ReadFileCommand>(&entry.data)) {
const auto& command = *payload;
std::string host_path;
if (!AprShared::TryGetHostPath(command.file_id, &host_path)) {
LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", command.file_id);
*execution_result = LibKernel::KERNEL_ERROR_ENOENT;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
}
uint64_t bytes_read = 0;
auto result = ReadHostFileToGuest(host_path, command.file_offset, command.destination,
command.size, &bytes_read);
if (result != OK) {
LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32
", path=%s\n",
command.file_id, static_cast<uint32_t>(result), host_path.c_str());
*execution_result = result;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
}
} else if (const auto* payload =
std::get_if<CommandBufferState::KernelEventCommand>(&entry.data)) {
const auto& command = *payload;
const auto eq = static_cast<LibKernel::EventQueue::KernelEqueue>(command.eq);
auto result = LibKernel::EventQueue::KernelTriggerUserEvent(
eq, command.id, reinterpret_cast<void*>(command.data));
if (result != OK) {
LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32
", result=0x%08" PRIx32 "\n",
command.eq, command.id, static_cast<uint32_t>(result));
*execution_result = result;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
}
} else if (const auto* payload =
std::get_if<CommandBufferState::WriteAddressCommand>(&entry.data)) {
const auto& command = *payload;
if (!AprShared::WriteGuest(command.address, command.value)) {
LOGF("\tAMPR submit write-address failed: address=0x%016" PRIx64
" value=0x%016" PRIx64 "\n",
command.address, command.value);
*execution_result = LibKernel::KERNEL_ERROR_EFAULT;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
}
} else if (const auto* payload =
std::get_if<CommandBufferState::AmmMapCommand>(&entry.data)) {
const auto& command = *payload;
int result = OK;
if (command.kind == AmmCommandKind::Unmap) {
result = LibKernel::Memory::KernelMunmap(command.va, command.size);
} else {
result = ExecuteAmmMapCommand(command);
}
if (result != OK) {
LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 " dmem=0x%016" PRIx64
" size=0x%016" PRIx64 " type=%" PRId32 " prot=0x%08" PRIx32
" result=0x%08" PRIx32 "\n",
static_cast<uint32_t>(command.kind), command.va, command.dmem_offset,
command.size, command.type, static_cast<uint32_t>(command.prot),
static_cast<uint32_t>(result));
*execution_result = result;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
}
}
const uint64_t end = amm ? bytes : state.write_offset;
if (end == 0 || end > state.write_offset) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
state.commands.erase(std::lower_bound(state.commands.begin(), state.commands.end(), end,
[](const auto& command, uint64_t offset) {
return command.record_offset < offset;
}),
state.commands.end());
const uint32_t id = out_submission_id != nullptr ? AprShared::AllocateSubmissionId() : 0;
if (out_submission_id != nullptr) {
AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id);
}
static CommandEngine apr_engine(7);
static CommandEngine amm_engine(3);
(amm ? amm_engine : apr_engine)
.Submit(priority, {std::move(state.commands), 0, result, id});
return OK;
}
@@ -1882,11 +1914,20 @@ static int KYTY_SYSV_ABI CommandBufferPopMarker(void* command_buffer) {
return AppendNoOpCommand(command_buffer, sizeof(uint32_t));
}
static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer, volatile uint64_t*,
uint64_t, uint8_t, uint8_t) {
static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer,
volatile uint64_t* address, uint64_t value,
uint8_t compare, uint8_t) {
PRINT_NAME();
return AppendNoOpCommand(command_buffer, 0x20);
if (command_buffer == nullptr || address == nullptr ||
(reinterpret_cast<uint64_t>(address) & 7) != 0 || compare > 6) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer),
CommandBufferState::WaitAddressCommand {
reinterpret_cast<uint64_t>(address), value, compare},
0x20)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_t, uint8_t,
@@ -1898,11 +1939,14 @@ static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_
static int AppendWriteAddressCommand(void* command_buffer, volatile uint64_t* address,
uint64_t value) {
if (command_buffer == nullptr || address == nullptr) {
if (command_buffer == nullptr || address == nullptr ||
(reinterpret_cast<uint64_t>(address) & 7) != 0) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
return AppendWriteAddressRecord(reinterpret_cast<uint64_t>(command_buffer),
reinterpret_cast<uint64_t>(address), value)
return AppendCommandRecord(
reinterpret_cast<uint64_t>(command_buffer),
CommandBufferState::WriteAddressCommand {reinterpret_cast<uint64_t>(address), value},
0x20)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
@@ -2160,8 +2204,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMap(void* command_buffer, uint64_t va,
command.type = type;
command.prot = prot;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_RECORD_SIZE)
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_RECORD_SIZE)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
@@ -2183,8 +2227,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapWithGpuMaskId(void* command_buffer,
command.prot = prot;
command.gpu_mask_id = gpu_mask_id;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_RECORD_SIZE)
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_RECORD_SIZE)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
@@ -2207,8 +2251,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapDirect(void* command_buffer, uint64_
command.type = type;
command.prot = prot;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_DIRECT_RECORD_SIZE)
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_DIRECT_RECORD_SIZE)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
@@ -2233,8 +2277,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapDirectWithGpuMaskId(void* command_bu
command.prot = prot;
command.gpu_mask_id = gpu_mask_id;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_DIRECT_RECORD_SIZE)
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_DIRECT_RECORD_SIZE)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
@@ -2251,8 +2295,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferUnmap(void* command_buffer, uint64_t va
command.va = va;
command.size = size;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_UNMAP_RECORD_SIZE)
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_UNMAP_RECORD_SIZE)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
}
@@ -2316,90 +2360,36 @@ static int KYTY_SYSV_ABI AmmSetPageTablePoolOccupancyNotificationThreshold(uint3
return OK;
}
static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t, uint32_t) {
static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t bytes,
uint32_t priority) {
PRINT_NAME();
if (command_buffer_base == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
int32_t execution_result = OK;
uint32_t error_offset = 0;
return ExecuteAprCommandBuffer(reinterpret_cast<uint64_t>(command_buffer_base),
&execution_result, &error_offset);
return QueueCommandBuffer(Engine::Amm, reinterpret_cast<uint64_t>(command_buffer_base), bytes,
priority, nullptr, nullptr);
}
static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t,
uint32_t, uint32_t* out_submission_id) {
static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t bytes,
uint32_t priority,
uint32_t* out_submission_id) {
PRINT_NAME();
if (command_buffer_base == nullptr || out_submission_id == nullptr) {
if (out_submission_id == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer_base);
const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0);
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result =
ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset);
if (submit_result != OK) {
return submit_result;
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
return OK;
return QueueCommandBuffer(Engine::Amm, reinterpret_cast<uint64_t>(command_buffer_base), bytes,
priority, nullptr, out_submission_id);
}
static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base, uint32_t,
uint32_t, void* result,
static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base,
uint32_t bytes, uint32_t priority,
void* result,
uint32_t* out_submission_id) {
PRINT_NAME();
if (command_buffer_base == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer_base);
const auto id =
AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast<uint64_t>(result));
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result =
ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset);
if (submit_result != OK) {
return submit_result;
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (out_submission_id != nullptr &&
!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
return AprShared::WriteResult(result, execution_result, error_offset);
return QueueCommandBuffer(Engine::Amm, reinterpret_cast<uint64_t>(command_buffer_base), bytes,
priority, result, out_submission_id);
}
static int KYTY_SYSV_ABI AmmWaitCommandBufferCompletion(uint32_t submission_id) {
PRINT_NAME();
AprShared::SubmissionState state {};
if (!AprShared::CompleteSubmission(submission_id, &state)) {
return LibKernel::KERNEL_ERROR_ESRCH;
}
if (state.result != 0) {
return AprShared::WriteResult(reinterpret_cast<void*>(state.result), state.execution_result,
state.error_offset);
}
return OK;
return AprShared::CompleteSubmission(submission_id) ? OK : LibKernel::KERNEL_ERROR_ESRCH;
}
} // namespace Ampr
tests/KernelFileSystemTests.cpp
+125
View File
@@ -17,6 +17,7 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <csignal>
@@ -34,6 +35,10 @@ namespace Libs::LibKernelApr {
void InitLibKernel_1_Apr(Loader::SymbolDatabase *symbols);
}
namespace Libs::LibAmpr {
void InitAmpr_1(Loader::SymbolDatabase *symbols);
}
namespace Libs::LibNet {
void InitNet_1_Net(Loader::SymbolDatabase *symbols);
}
@@ -496,6 +501,125 @@ void CheckDirectoryStream(const std::filesystem::path &root) {
FileSystem::Umount("/app0");
}
void CheckAmprOrdering(Loader::SymbolDatabase &symbols, uint32_t file_id) {
Libs::LibAmpr::InitAmpr_1(&symbols);
const auto find = [&](const char *nid) {
const auto *symbol = symbols.FindByNid(nid, Loader::SymbolType::Func);
Check(symbol != nullptr, "AMPR command and submission exports resolve");
return symbol->vaddr;
};
using Unary = void (KYTY_SYSV_ABI *)(void *);
using AprConstructor = void (KYTY_SYSV_ABI *)(void *, void *, void *);
using SetBuffer = int (KYTY_SYSV_ABI *)(void *, void *, uint32_t);
using Offset = uint64_t (KYTY_SYSV_ABI *)(void *);
using WaitAddress = int (KYTY_SYSV_ABI *)(void *, volatile uint64_t *, uint64_t,
uint8_t, uint8_t);
using WriteAddress = int (KYTY_SYSV_ABI *)(void *, volatile uint64_t *, uint64_t);
using ReadFile = int (KYTY_SYSV_ABI *)(void *, uint64_t, uint64_t, uint32_t,
void *, uint64_t, uint64_t);
struct Result { int32_t result; uint32_t error_offset; };
using SubmitApr = int (KYTY_SYSV_ABI *)(void *, uint32_t, Result *, uint32_t *);
using SubmitAmm = int (KYTY_SYSV_ABI *)(void *, uint32_t, uint32_t, uint32_t *);
using WaitSubmission = int (KYTY_SYSV_ABI *)(uint32_t);
const auto construct = reinterpret_cast<Unary>(find("8aI7R7WaOlc"));
const auto construct_apr = reinterpret_cast<AprConstructor>(find("a8uLzYY--tM"));
const auto construct_amm = reinterpret_cast<Unary>(find("EDq5bqCqYpA"));
const auto destroy = reinterpret_cast<Unary>(find("GuchCTefuZw"));
const auto set_buffer = reinterpret_cast<SetBuffer>(find("N-FSPA4S3nI"));
const auto offset = reinterpret_cast<Offset>(find("GnxKOHEawhk"));
const auto wait_address = reinterpret_cast<WaitAddress>(find("DLfoNxTFNVk"));
const auto write_address = reinterpret_cast<WriteAddress>(find("sJXyWHjP-F8"));
const auto read_file = reinterpret_cast<ReadFile>(find("mQ16-QdKv7k"));
const auto submit_apr = reinterpret_cast<SubmitApr>(find("ASoW5WE-UPo"));
const auto submit_amm = reinterpret_cast<SubmitAmm>(find("NnKhlMJtIsI"));
const auto wait_apr = reinterpret_cast<WaitSubmission>(find("rqwFKI4PAiM"));
const auto wait_amm = reinterpret_cast<WaitSubmission>(find("HXymib4T8gc"));
struct Buffer {
std::array<uint64_t, 5> header {};
std::array<uint32_t, 256> data {};
};
std::array<Buffer, 4> buffers;
const auto reset = [&](size_t apr_count) {
for (size_t i = 0; i < buffers.size(); ++i) {
auto &buffer = buffers[i];
construct(buffer.header.data());
if (i < apr_count) {
construct_apr(buffer.header.data(), &buffer.header[3], &buffer.header[4]);
} else {
construct_amm(buffer.header.data());
}
Check(set_buffer(buffer.header.data(), buffer.data.data(), sizeof(buffer.data)) == OK,
"initialize AMPR command buffer");
}
};
// SDK WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<.
struct Comparison { uint8_t compare; uint64_t blocked, reference, released; };
constexpr std::array comparisons {
Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3,
0x40000000000019c4},
Comparison{1, 0, INT64_MAX, uint64_t{1} << 63},
Comparison{2, UINT64_MAX, uint64_t{1} << 63, INT64_MAX}, Comparison{3, 2, 2, 3},
Comparison{4, UINT64_MAX - 1, UINT64_MAX, 0},
Comparison{5, UINT64_MAX, 0, 1}, Comparison{6, 0, 0, UINT64_MAX}};
for (const auto &comparison : comparisons) {
reset(3);
uint64_t fence = comparison.blocked;
uint64_t blocked_done = 0, read_done = 0, lower_done = 0;
std::array<char, 3> output {};
std::array<Result, 3> results {{{1234, 5678}, {1234, 5678}, {1234, 5678}}};
std::array<uint32_t, 4> ids {};
auto *blocked = buffers[0].header.data();
auto *reader = buffers[1].header.data();
auto *lower = buffers[2].header.data();
auto *producer = buffers[3].header.data();
Check(wait_address(blocked, &fence, comparison.reference, comparison.compare, 0) == OK &&
write_address(blocked, &blocked_done, 1) == OK &&
read_file(reader, reinterpret_cast<uint64_t>(&buffers[1].header[3]),
reinterpret_cast<uint64_t>(&buffers[1].header[4]), file_id,
output.data(), output.size(), 0) == OK &&
write_address(reader, &read_done, 2) == OK &&
write_address(lower, &lower_done, 3) == OK &&
write_address(producer, &fence, comparison.released) == OK,
"build dependent APR read and later AMM fence producer");
Check(submit_apr(blocked, 3, &results[0], &ids[0]) == OK &&
submit_apr(reader, 3, &results[1], &ids[1]) == OK &&
submit_apr(lower, 4, &results[2], &ids[2]) == OK && wait_apr(ids[2]) == OK,
"APR submit returns while blocked and a lower priority completes");
Check(lower_done == 3 && blocked_done == 0 && read_done == 0 &&
output == std::array<char, 3>{} && results[0].result == 1234 &&
results[1].result == 1234,
"unsatisfied wait blocks later buffers at its priority without publishing completion");
Check(submit_amm(buffers[3].data.data(), static_cast<uint32_t>(offset(producer)), 1,
&ids[3]) == OK && wait_amm(ids[3]) == OK &&
wait_apr(ids[0]) == OK && wait_apr(ids[1]) == OK,
"later AMM submission releases the APR dependency");
Check(blocked_done == 1 && read_done == 2 && std::memcmp(output.data(), "APR", 3) == 0,
"APR reads real file bytes only after its dependency completes");
for (const auto &result : results) {
Check(result.result == OK,
"submission wait observes the completed result");
}
}
reset(0);
uint64_t cpu_fence = 0, amm_done = 0, lower_done = 0;
std::array<uint32_t, 2> ids {};
Check(wait_address(buffers[0].header.data(), &cpu_fence, 0, 1, 0) == OK &&
write_address(buffers[0].header.data(), &amm_done, 1) == OK &&
write_address(buffers[1].header.data(), &lower_done, 1) == OK &&
submit_amm(buffers[0].data.data(), static_cast<uint32_t>(offset(buffers[0].header.data())),
0, &ids[0]) == OK &&
submit_amm(buffers[1].data.data(), static_cast<uint32_t>(offset(buffers[1].header.data())),
1, &ids[1]) == OK && wait_amm(ids[1]) == OK,
"AMM lower priority progresses while its high priority waits");
Check(lower_done == 1 && amm_done == 0, "AMM wait preserves its dependency");
std::atomic_ref(cpu_fence).store(1, std::memory_order_release);
Check(wait_amm(ids[0]) == OK && amm_done == 1,
"AMM observes an external CPU fence store without a submission notification");
for (auto &buffer : buffers) {
destroy(buffer.header.data());
}
}
void CheckAprPaths(const std::filesystem::path &root) {
Loader::SymbolDatabase symbols;
Libs::LibKernelApr::InitLibKernel_1_Apr(&symbols);
@@ -562,6 +686,7 @@ void CheckAprPaths(const std::filesystem::path &root) {
Check(resolve(unterminated.data(), paths, 1, ids, sizes, &error_index) == -1 &&
*Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_ENAMETOOLONG,
"APR rejects an unterminated prefix");
CheckAmprOrdering(symbols, expected_id);
FileSystem::Umount("/app0");
}