diff --git a/src/libs/libAmpr.cpp b/src/libs/libAmpr.cpp index a38787129..107fd310d 100644 --- a/src/libs/libAmpr.cpp +++ b/src/libs/libAmpr.cpp @@ -12,11 +12,16 @@ #include #include +#include +#include +#include #include #include +#include #include #include #include +#include #include #include #include @@ -37,13 +42,6 @@ struct ResultBuffer { uint32_t errorOffset = 0; }; -struct SubmissionState { - uint64_t command_buffer = 0; - uint64_t result = 0; - int32_t execution_result = 0; - uint32_t error_offset = 0; -}; - struct ResolvedPathInfo { int result = OK; uint32_t file_id = 0xffffffffu; @@ -54,7 +52,8 @@ struct ResolvedPathInfo { static std::mutex g_mutex; static uint32_t g_next_submission_id = 1; -static std::unordered_map g_submissions; +static std::unordered_map g_submissions; +static std::condition_variable g_submission_cv; static std::unordered_map g_files; static std::unordered_map g_file_sizes; static std::unordered_map g_resolved_paths; @@ -340,38 +339,35 @@ static int ResolvePathsCommon(const char* const* path_list, uint32_t count, uint return results != nullptr ? static_cast(success_count) : OK; } -static uint32_t AllocateSubmissionId(uint64_t command_buffer, uint64_t result) { +static uint32_t AllocateSubmissionId() { std::scoped_lock lock(g_mutex); - auto id = g_next_submission_id++; if (id == 0) { id = g_next_submission_id++; } - g_submissions[id] = SubmissionState {command_buffer, result}; + g_submissions.emplace(id, false); return id; } -static void SetSubmissionResult(uint32_t submission_id, int32_t execution_result, - uint32_t error_offset) { - std::scoped_lock lock(g_mutex); - auto it = g_submissions.find(submission_id); - if (it != g_submissions.end()) { - it->second.execution_result = execution_result; - it->second.error_offset = error_offset; +static void SetSubmissionComplete(uint32_t submission_id) { + if (submission_id == 0) { + return; } + std::scoped_lock lock(g_mutex); + g_submissions.at(submission_id) = true; + g_submission_cv.notify_all(); } -static bool CompleteSubmission(uint32_t submission_id, SubmissionState* state) { - std::scoped_lock lock(g_mutex); - auto it = g_submissions.find(submission_id); - if (it == g_submissions.end()) { +static bool CompleteSubmission(uint32_t submission_id) { + std::unique_lock lock(g_mutex); + if (!g_submissions.contains(submission_id)) { return false; } - if (state != nullptr) { - *state = it->second; - } - g_submissions.erase(it); - return true; + g_submission_cv.wait(lock, [submission_id] { + const auto it = g_submissions.find(submission_id); + return it == g_submissions.end() || it->second; + }); + return g_submissions.erase(submission_id) != 0; } static int WriteResult(void* result, int32_t execution_result = 0, uint32_t error_offset = 0) { @@ -390,8 +386,9 @@ static int WriteResult(void* result, int32_t execution_result = 0, uint32_t erro } // namespace AprShared namespace LibAmpr::Ampr { -static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result, - uint32_t* error_offset); +enum class Engine { Apr, Amm }; +static int QueueCommandBuffer(Engine engine, uint64_t command_buffer, uint32_t bytes, + uint32_t priority, void* result, uint32_t* out_submission_id); } namespace LibKernelApr { @@ -537,89 +534,37 @@ static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIdsAndFileSizesForEach( AprShared::ResolvePathsCommon(path_list, count, ids, sizes, nullptr, prefix, results)); } -static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint64_t, +static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint32_t priority, void* result, uint32_t* out_submission_id) { PRINT_NAME(); - - if (command_buffer == nullptr) { - return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); - } - - const auto command_buffer_addr = reinterpret_cast(command_buffer); - const auto id = - AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast(result)); - - int32_t execution_result = OK; - uint32_t error_offset = 0; - auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr, - &execution_result, &error_offset); - if (submit_result != OK) { - return KernelSyscallResult(submit_result); - } - AprShared::SetSubmissionResult(id, execution_result, error_offset); - - if (out_submission_id != nullptr) { - if (!AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { - return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); - } - } - - return KernelSyscallResult(AprShared::WriteResult(result, execution_result, error_offset)); + return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer( + LibAmpr::Ampr::Engine::Apr, reinterpret_cast(command_buffer), 0, priority, result, + out_submission_id)); } -static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint64_t) { +static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint32_t priority) { PRINT_NAME(); - - if (command_buffer == nullptr) { - return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); - } - - int32_t execution_result = OK; - uint32_t error_offset = 0; - return KernelSyscallResult(LibAmpr::Ampr::ExecuteAprCommandBuffer( - reinterpret_cast(command_buffer), &execution_result, &error_offset)); + return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer( + LibAmpr::Ampr::Engine::Apr, reinterpret_cast(command_buffer), 0, priority, + nullptr, nullptr)); } -static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint64_t, +static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint32_t priority, uint32_t* out_submission_id) { PRINT_NAME(); - - if (command_buffer == nullptr || out_submission_id == nullptr) { + if (out_submission_id == nullptr) { return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); } - - const auto command_buffer_addr = reinterpret_cast(command_buffer); - const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0); - - int32_t execution_result = OK; - uint32_t error_offset = 0; - auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr, - &execution_result, &error_offset); - if (submit_result != OK) { - return KernelSyscallResult(submit_result); - } - AprShared::SetSubmissionResult(id, execution_result, error_offset); - - if (!AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { - return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT); - } - - return OK; + return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer( + LibAmpr::Ampr::Engine::Apr, reinterpret_cast(command_buffer), 0, priority, + nullptr, out_submission_id)); } static int KYTY_SYSV_ABI WaitCommandBuffer(uint32_t submission_id) { PRINT_NAME(); - - AprShared::SubmissionState state {}; - if (!AprShared::CompleteSubmission(submission_id, &state)) { - return KernelSyscallResult(LibKernel::KERNEL_ERROR_ESRCH); - } - if (state.result != 0) { - return KernelSyscallResult(AprShared::WriteResult( - reinterpret_cast(state.result), state.execution_result, state.error_offset)); - } - return OK; + return KernelSyscallResult( + AprShared::CompleteSubmission(submission_id) ? OK : LibKernel::KERNEL_ERROR_ESRCH); } } // namespace Apr @@ -720,6 +665,11 @@ struct CommandBufferState { uint64_t address = 0; uint64_t value = 0; }; + struct WaitAddressCommand { + uint64_t address; + uint64_t value; + uint8_t compare; + }; struct AmmMapCommand { AmmCommandKind kind = AmmCommandKind::MapAuto; uint64_t va = 0; @@ -729,9 +679,11 @@ struct CommandBufferState { int32_t prot = 0; uint8_t gpu_mask_id = 0; }; + using CommandData = std::variant; struct Command { uint64_t record_offset; - std::variant data; + CommandData data; }; std::vector commands; bool gather_scatter_valid = false; @@ -1141,27 +1093,6 @@ static bool AppendKernelEventRecord(uint64_t command_buffer, uint64_t eq, int32_ return CommitCommandBufferRecord(command_buffer, &state, KERNEL_EVENT_RECORD_SIZE); } -static bool AppendWriteAddressRecord(uint64_t command_buffer, uint64_t address, uint64_t value, - uint64_t record_size = 0x20) { - std::scoped_lock lock(g_command_buffer_mutex); - CommandBufferIterator it; - if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { - return false; - } - - auto& state = it->second; - if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) { - return false; - } - - const auto record_offset = state.write_offset; - std::memset(reinterpret_cast(state.buffer + record_offset), 0, - static_cast(record_size)); - state.commands.push_back( - {record_offset, CommandBufferState::WriteAddressCommand {address, value}}); - return CommitCommandBufferRecord(command_buffer, &state, record_size); -} - static bool ValidateAmmMapArgs(uint64_t va, uint64_t size) { return va != 0 && size != 0 && (va & (AMM_PAGE_SIZE - 1u)) == 0 && (size & (AMM_PAGE_SIZE - 1u)) == 0 && va + size >= va; @@ -1219,8 +1150,8 @@ static int ExecuteAmmMapCommand(const CommandBufferState::AmmMapCommand& command static_cast(command.dmem_offset), AMM_PAGE_SIZE); } -static bool AppendAmmMapRecord(uint64_t command_buffer, - const CommandBufferState::AmmMapCommand& cmd, uint64_t record_size) { +static bool AppendCommandRecord(uint64_t command_buffer, CommandBufferState::CommandData command, + uint64_t record_size) { std::scoped_lock lock(g_command_buffer_mutex); CommandBufferIterator it; if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { @@ -1234,101 +1165,202 @@ static bool AppendAmmMapRecord(uint64_t command_ auto* record_addr = reinterpret_cast(state.buffer + state.write_offset); std::memset(record_addr, 0, static_cast(record_size)); - state.commands.push_back({state.write_offset, cmd}); + state.commands.push_back({state.write_offset, std::move(command)}); return CommitCommandBufferRecord(command_buffer, &state, record_size); } static int ReadHostFileToGuest(const std::string& host_path, uint64_t file_offset, uint64_t destination, uint64_t size, uint64_t* bytes_read); -static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result, - uint32_t* error_offset) { - if (execution_result == nullptr || error_offset == nullptr) { - return LibKernel::KERNEL_ERROR_EINVAL; +static int ExecuteCommand(const CommandBufferState::Command& entry) { + if (const auto* payload = std::get_if(&entry.data)) { + const auto& command = *payload; + std::string host_path; + if (!AprShared::TryGetHostPath(command.file_id, &host_path)) { + LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", command.file_id); + return LibKernel::KERNEL_ERROR_ENOENT; + } + + uint64_t bytes_read = 0; + auto result = ReadHostFileToGuest(host_path, command.file_offset, command.destination, + command.size, &bytes_read); + if (result != OK) { + LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 ", path=%s\n", + command.file_id, static_cast(result), host_path.c_str()); + return result; + } + } else if (const auto* payload = + std::get_if(&entry.data)) { + const auto& command = *payload; + const auto eq = static_cast(command.eq); + auto result = LibKernel::EventQueue::KernelTriggerUserEvent( + eq, command.id, reinterpret_cast(command.data)); + if (result != OK) { + LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32 + ", result=0x%08" PRIx32 "\n", + command.eq, command.id, static_cast(result)); + return result; + } + } else if (const auto* payload = + std::get_if(&entry.data)) { + const auto& command = *payload; + std::atomic_ref(*reinterpret_cast(command.address)) + .store(command.value, std::memory_order_release); + } else if (const auto* payload = std::get_if(&entry.data)) { + const auto& command = *payload; + int result = OK; + if (command.kind == AmmCommandKind::Unmap) { + result = LibKernel::Memory::KernelMunmap(command.va, command.size); + } else { + result = ExecuteAmmMapCommand(command); + } + + if (result != OK) { + LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 " dmem=0x%016" PRIx64 + " size=0x%016" PRIx64 " type=%" PRId32 " prot=0x%08" PRIx32 " result=0x%08" PRIx32 + "\n", + static_cast(command.kind), command.va, command.dmem_offset, command.size, + command.type, static_cast(command.prot), static_cast(result)); + return result; + } + } + return OK; +} + +struct Submission { + std::vector commands; + size_t cursor = 0; + void* result = nullptr; + uint32_t id = 0; +}; + +static bool IsWaitSatisfied(const CommandBufferState::WaitAddressCommand& wait) { + const uint64_t value = + std::atomic_ref(*reinterpret_cast(wait.address)).load(std::memory_order_acquire); + switch (wait.compare) { + case 0: return value == wait.value; + case 1: return value > wait.value; + case 2: return value < wait.value; + case 3: return value != wait.value; + case 4: return static_cast(value - wait.value) >= 0; + case 5: return static_cast(value) > static_cast(wait.value); + default: return static_cast(value) < static_cast(wait.value); + } +} + +class CommandEngine { +public: + explicit CommandEngine(size_t priorities) + : m_queues(priorities), m_worker([this](std::stop_token stop) { Run(stop); }) {} + + ~CommandEngine() { + std::scoped_lock lock(m_mutex); + m_worker.request_stop(); + m_cv.notify_all(); } - *execution_result = OK; - *error_offset = 0; + void Submit(uint32_t priority, Submission submission) { + std::scoped_lock lock(m_mutex); + m_queues[priority].push_back(std::move(submission)); + m_cv.notify_one(); + } +private: + void Run(std::stop_token stop) { + std::unique_lock lock(m_mutex); + while (!stop.stop_requested()) { + std::deque* ready = nullptr; + bool pending = false; + for (auto& queue: m_queues) { + if (queue.empty()) { + continue; + } + pending = true; + auto& submission = queue.front(); + if (submission.cursor < submission.commands.size()) { + const auto* wait = std::get_if( + &submission.commands[submission.cursor].data); + if (wait != nullptr && !IsWaitSatisfied(*wait)) { + continue; + } + } + ready = &queue; + break; + } + if (ready == nullptr) { + if (pending) { + // CPU and GPU fence writes do not notify the submission queue. + m_cv.wait_for(lock, std::chrono::microseconds(100)); + } else { + m_cv.wait(lock); + } + continue; + } + + auto& submission = ready->front(); + lock.unlock(); + int result = OK; + uint32_t error_offset = 0; + if (submission.cursor < submission.commands.size()) { + const auto& command = submission.commands[submission.cursor++]; + result = ExecuteCommand(command); + if (result != OK) { + error_offset = static_cast(command.record_offset); + } + } + const bool complete = result != OK || submission.cursor == submission.commands.size(); + if (complete) { + AprShared::WriteResult(submission.result, result, error_offset); + AprShared::SetSubmissionComplete(submission.id); + } + lock.lock(); + if (complete) { + ready->pop_front(); + } + } + } + + std::mutex m_mutex; + std::condition_variable m_cv; + std::vector> m_queues; + std::jthread m_worker; +}; + +static int QueueCommandBuffer(Engine engine, uint64_t command_buffer, uint32_t bytes, + uint32_t priority, void* result, uint32_t* out_submission_id) { + const bool amm = engine == Engine::Amm; + if (command_buffer == 0 || priority > (amm ? 2u : 6u)) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + if ((result != nullptr && + !AprShared::IsValidGuestRange(reinterpret_cast(result), + sizeof(AprShared::ResultBuffer), true)) || + (out_submission_id != nullptr && + !AprShared::IsValidGuestRange(reinterpret_cast(out_submission_id), + sizeof(uint32_t), true))) { + return LibKernel::KERNEL_ERROR_EFAULT; + } CommandBufferState state {}; if (!TryGetCommandBufferState(command_buffer, &state)) { return LibKernel::KERNEL_ERROR_EFAULT; } - - for (const auto& entry: state.commands) { - if (entry.record_offset >= state.write_offset) { - break; - } - if (const auto* payload = std::get_if(&entry.data)) { - const auto& command = *payload; - std::string host_path; - if (!AprShared::TryGetHostPath(command.file_id, &host_path)) { - LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", command.file_id); - *execution_result = LibKernel::KERNEL_ERROR_ENOENT; - *error_offset = static_cast(entry.record_offset); - return OK; - } - - uint64_t bytes_read = 0; - auto result = ReadHostFileToGuest(host_path, command.file_offset, command.destination, - command.size, &bytes_read); - if (result != OK) { - LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 - ", path=%s\n", - command.file_id, static_cast(result), host_path.c_str()); - *execution_result = result; - *error_offset = static_cast(entry.record_offset); - return OK; - } - } else if (const auto* payload = - std::get_if(&entry.data)) { - const auto& command = *payload; - const auto eq = static_cast(command.eq); - auto result = LibKernel::EventQueue::KernelTriggerUserEvent( - eq, command.id, reinterpret_cast(command.data)); - if (result != OK) { - LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32 - ", result=0x%08" PRIx32 "\n", - command.eq, command.id, static_cast(result)); - *execution_result = result; - *error_offset = static_cast(entry.record_offset); - return OK; - } - } else if (const auto* payload = - std::get_if(&entry.data)) { - const auto& command = *payload; - if (!AprShared::WriteGuest(command.address, command.value)) { - LOGF("\tAMPR submit write-address failed: address=0x%016" PRIx64 - " value=0x%016" PRIx64 "\n", - command.address, command.value); - *execution_result = LibKernel::KERNEL_ERROR_EFAULT; - *error_offset = static_cast(entry.record_offset); - return OK; - } - } else if (const auto* payload = - std::get_if(&entry.data)) { - const auto& command = *payload; - int result = OK; - if (command.kind == AmmCommandKind::Unmap) { - result = LibKernel::Memory::KernelMunmap(command.va, command.size); - } else { - result = ExecuteAmmMapCommand(command); - } - - if (result != OK) { - LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 " dmem=0x%016" PRIx64 - " size=0x%016" PRIx64 " type=%" PRId32 " prot=0x%08" PRIx32 - " result=0x%08" PRIx32 "\n", - static_cast(command.kind), command.va, command.dmem_offset, - command.size, command.type, static_cast(command.prot), - static_cast(result)); - *execution_result = result; - *error_offset = static_cast(entry.record_offset); - return OK; - } - } + const uint64_t end = amm ? bytes : state.write_offset; + if (end == 0 || end > state.write_offset) { + return LibKernel::KERNEL_ERROR_EINVAL; } - + state.commands.erase(std::lower_bound(state.commands.begin(), state.commands.end(), end, + [](const auto& command, uint64_t offset) { + return command.record_offset < offset; + }), + state.commands.end()); + const uint32_t id = out_submission_id != nullptr ? AprShared::AllocateSubmissionId() : 0; + if (out_submission_id != nullptr) { + AprShared::WriteGuest(reinterpret_cast(out_submission_id), id); + } + static CommandEngine apr_engine(7); + static CommandEngine amm_engine(3); + (amm ? amm_engine : apr_engine) + .Submit(priority, {std::move(state.commands), 0, result, id}); return OK; } @@ -1882,11 +1914,20 @@ static int KYTY_SYSV_ABI CommandBufferPopMarker(void* command_buffer) { return AppendNoOpCommand(command_buffer, sizeof(uint32_t)); } -static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer, volatile uint64_t*, - uint64_t, uint8_t, uint8_t) { +static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer, + volatile uint64_t* address, uint64_t value, + uint8_t compare, uint8_t) { PRINT_NAME(); - - return AppendNoOpCommand(command_buffer, 0x20); + if (command_buffer == nullptr || address == nullptr || + (reinterpret_cast(address) & 7) != 0 || compare > 6) { + return LibKernel::KERNEL_ERROR_EINVAL; + } + return AppendCommandRecord(reinterpret_cast(command_buffer), + CommandBufferState::WaitAddressCommand { + reinterpret_cast(address), value, compare}, + 0x20) + ? OK + : LibKernel::KERNEL_ERROR_EBUSY; } static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_t, uint8_t, @@ -1898,11 +1939,14 @@ static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_ static int AppendWriteAddressCommand(void* command_buffer, volatile uint64_t* address, uint64_t value) { - if (command_buffer == nullptr || address == nullptr) { + if (command_buffer == nullptr || address == nullptr || + (reinterpret_cast(address) & 7) != 0) { return LibKernel::KERNEL_ERROR_EINVAL; } - return AppendWriteAddressRecord(reinterpret_cast(command_buffer), - reinterpret_cast(address), value) + return AppendCommandRecord( + reinterpret_cast(command_buffer), + CommandBufferState::WriteAddressCommand {reinterpret_cast(address), value}, + 0x20) ? OK : LibKernel::KERNEL_ERROR_EBUSY; } @@ -2160,8 +2204,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMap(void* command_buffer, uint64_t va, command.type = type; command.prot = prot; - return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, - AMM_MAP_RECORD_SIZE) + return AppendCommandRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_RECORD_SIZE) ? OK : LibKernel::KERNEL_ERROR_EBUSY; } @@ -2183,8 +2227,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapWithGpuMaskId(void* command_buffer, command.prot = prot; command.gpu_mask_id = gpu_mask_id; - return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, - AMM_MAP_RECORD_SIZE) + return AppendCommandRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_RECORD_SIZE) ? OK : LibKernel::KERNEL_ERROR_EBUSY; } @@ -2207,8 +2251,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapDirect(void* command_buffer, uint64_ command.type = type; command.prot = prot; - return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, - AMM_MAP_DIRECT_RECORD_SIZE) + return AppendCommandRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_DIRECT_RECORD_SIZE) ? OK : LibKernel::KERNEL_ERROR_EBUSY; } @@ -2233,8 +2277,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapDirectWithGpuMaskId(void* command_bu command.prot = prot; command.gpu_mask_id = gpu_mask_id; - return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, - AMM_MAP_DIRECT_RECORD_SIZE) + return AppendCommandRecord(reinterpret_cast(command_buffer), command, + AMM_MAP_DIRECT_RECORD_SIZE) ? OK : LibKernel::KERNEL_ERROR_EBUSY; } @@ -2251,8 +2295,8 @@ static int KYTY_SYSV_ABI AmmCommandBufferUnmap(void* command_buffer, uint64_t va command.va = va; command.size = size; - return AppendAmmMapRecord(reinterpret_cast(command_buffer), command, - AMM_UNMAP_RECORD_SIZE) + return AppendCommandRecord(reinterpret_cast(command_buffer), command, + AMM_UNMAP_RECORD_SIZE) ? OK : LibKernel::KERNEL_ERROR_EBUSY; } @@ -2316,90 +2360,36 @@ static int KYTY_SYSV_ABI AmmSetPageTablePoolOccupancyNotificationThreshold(uint3 return OK; } -static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t, uint32_t) { +static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t bytes, + uint32_t priority) { PRINT_NAME(); - - if (command_buffer_base == nullptr) { - return LibKernel::KERNEL_ERROR_EINVAL; - } - - int32_t execution_result = OK; - uint32_t error_offset = 0; - return ExecuteAprCommandBuffer(reinterpret_cast(command_buffer_base), - &execution_result, &error_offset); + return QueueCommandBuffer(Engine::Amm, reinterpret_cast(command_buffer_base), bytes, + priority, nullptr, nullptr); } -static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t, - uint32_t, uint32_t* out_submission_id) { +static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t bytes, + uint32_t priority, + uint32_t* out_submission_id) { PRINT_NAME(); - - if (command_buffer_base == nullptr || out_submission_id == nullptr) { + if (out_submission_id == nullptr) { return LibKernel::KERNEL_ERROR_EINVAL; } - - const auto command_buffer_addr = reinterpret_cast(command_buffer_base); - const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0); - - int32_t execution_result = OK; - uint32_t error_offset = 0; - auto submit_result = - ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset); - if (submit_result != OK) { - return submit_result; - } - AprShared::SetSubmissionResult(id, execution_result, error_offset); - - if (!AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { - return LibKernel::KERNEL_ERROR_EFAULT; - } - - return OK; + return QueueCommandBuffer(Engine::Amm, reinterpret_cast(command_buffer_base), bytes, + priority, nullptr, out_submission_id); } -static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base, uint32_t, - uint32_t, void* result, +static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base, + uint32_t bytes, uint32_t priority, + void* result, uint32_t* out_submission_id) { PRINT_NAME(); - - if (command_buffer_base == nullptr) { - return LibKernel::KERNEL_ERROR_EINVAL; - } - - const auto command_buffer_addr = reinterpret_cast(command_buffer_base); - const auto id = - AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast(result)); - - int32_t execution_result = OK; - uint32_t error_offset = 0; - auto submit_result = - ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset); - if (submit_result != OK) { - return submit_result; - } - AprShared::SetSubmissionResult(id, execution_result, error_offset); - - if (out_submission_id != nullptr && - !AprShared::WriteGuest(reinterpret_cast(out_submission_id), id)) { - return LibKernel::KERNEL_ERROR_EFAULT; - } - - return AprShared::WriteResult(result, execution_result, error_offset); + return QueueCommandBuffer(Engine::Amm, reinterpret_cast(command_buffer_base), bytes, + priority, result, out_submission_id); } static int KYTY_SYSV_ABI AmmWaitCommandBufferCompletion(uint32_t submission_id) { PRINT_NAME(); - - AprShared::SubmissionState state {}; - if (!AprShared::CompleteSubmission(submission_id, &state)) { - return LibKernel::KERNEL_ERROR_ESRCH; - } - - if (state.result != 0) { - return AprShared::WriteResult(reinterpret_cast(state.result), state.execution_result, - state.error_offset); - } - - return OK; + return AprShared::CompleteSubmission(submission_id) ? OK : LibKernel::KERNEL_ERROR_ESRCH; } } // namespace Ampr diff --git a/tests/KernelFileSystemTests.cpp b/tests/KernelFileSystemTests.cpp index 112a101a9..39b52e609 100644 --- a/tests/KernelFileSystemTests.cpp +++ b/tests/KernelFileSystemTests.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -34,6 +35,10 @@ namespace Libs::LibKernelApr { void InitLibKernel_1_Apr(Loader::SymbolDatabase *symbols); } +namespace Libs::LibAmpr { +void InitAmpr_1(Loader::SymbolDatabase *symbols); +} + namespace Libs::LibNet { void InitNet_1_Net(Loader::SymbolDatabase *symbols); } @@ -496,6 +501,125 @@ void CheckDirectoryStream(const std::filesystem::path &root) { FileSystem::Umount("/app0"); } +void CheckAmprOrdering(Loader::SymbolDatabase &symbols, uint32_t file_id) { + Libs::LibAmpr::InitAmpr_1(&symbols); + const auto find = [&](const char *nid) { + const auto *symbol = symbols.FindByNid(nid, Loader::SymbolType::Func); + Check(symbol != nullptr, "AMPR command and submission exports resolve"); + return symbol->vaddr; + }; + using Unary = void (KYTY_SYSV_ABI *)(void *); + using AprConstructor = void (KYTY_SYSV_ABI *)(void *, void *, void *); + using SetBuffer = int (KYTY_SYSV_ABI *)(void *, void *, uint32_t); + using Offset = uint64_t (KYTY_SYSV_ABI *)(void *); + using WaitAddress = int (KYTY_SYSV_ABI *)(void *, volatile uint64_t *, uint64_t, + uint8_t, uint8_t); + using WriteAddress = int (KYTY_SYSV_ABI *)(void *, volatile uint64_t *, uint64_t); + using ReadFile = int (KYTY_SYSV_ABI *)(void *, uint64_t, uint64_t, uint32_t, + void *, uint64_t, uint64_t); + struct Result { int32_t result; uint32_t error_offset; }; + using SubmitApr = int (KYTY_SYSV_ABI *)(void *, uint32_t, Result *, uint32_t *); + using SubmitAmm = int (KYTY_SYSV_ABI *)(void *, uint32_t, uint32_t, uint32_t *); + using WaitSubmission = int (KYTY_SYSV_ABI *)(uint32_t); + const auto construct = reinterpret_cast(find("8aI7R7WaOlc")); + const auto construct_apr = reinterpret_cast(find("a8uLzYY--tM")); + const auto construct_amm = reinterpret_cast(find("EDq5bqCqYpA")); + const auto destroy = reinterpret_cast(find("GuchCTefuZw")); + const auto set_buffer = reinterpret_cast(find("N-FSPA4S3nI")); + const auto offset = reinterpret_cast(find("GnxKOHEawhk")); + const auto wait_address = reinterpret_cast(find("DLfoNxTFNVk")); + const auto write_address = reinterpret_cast(find("sJXyWHjP-F8")); + const auto read_file = reinterpret_cast(find("mQ16-QdKv7k")); + const auto submit_apr = reinterpret_cast(find("ASoW5WE-UPo")); + const auto submit_amm = reinterpret_cast(find("NnKhlMJtIsI")); + const auto wait_apr = reinterpret_cast(find("rqwFKI4PAiM")); + const auto wait_amm = reinterpret_cast(find("HXymib4T8gc")); + struct Buffer { + std::array header {}; + std::array data {}; + }; + std::array buffers; + const auto reset = [&](size_t apr_count) { + for (size_t i = 0; i < buffers.size(); ++i) { + auto &buffer = buffers[i]; + construct(buffer.header.data()); + if (i < apr_count) { + construct_apr(buffer.header.data(), &buffer.header[3], &buffer.header[4]); + } else { + construct_amm(buffer.header.data()); + } + Check(set_buffer(buffer.header.data(), buffer.data.data(), sizeof(buffer.data)) == OK, + "initialize AMPR command buffer"); + } + }; + // SDK WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<. + struct Comparison { uint8_t compare; uint64_t blocked, reference, released; }; + constexpr std::array comparisons { + Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3, + 0x40000000000019c4}, + Comparison{1, 0, INT64_MAX, uint64_t{1} << 63}, + Comparison{2, UINT64_MAX, uint64_t{1} << 63, INT64_MAX}, Comparison{3, 2, 2, 3}, + Comparison{4, UINT64_MAX - 1, UINT64_MAX, 0}, + Comparison{5, UINT64_MAX, 0, 1}, Comparison{6, 0, 0, UINT64_MAX}}; + for (const auto &comparison : comparisons) { + reset(3); + uint64_t fence = comparison.blocked; + uint64_t blocked_done = 0, read_done = 0, lower_done = 0; + std::array output {}; + std::array results {{{1234, 5678}, {1234, 5678}, {1234, 5678}}}; + std::array ids {}; + auto *blocked = buffers[0].header.data(); + auto *reader = buffers[1].header.data(); + auto *lower = buffers[2].header.data(); + auto *producer = buffers[3].header.data(); + Check(wait_address(blocked, &fence, comparison.reference, comparison.compare, 0) == OK && + write_address(blocked, &blocked_done, 1) == OK && + read_file(reader, reinterpret_cast(&buffers[1].header[3]), + reinterpret_cast(&buffers[1].header[4]), file_id, + output.data(), output.size(), 0) == OK && + write_address(reader, &read_done, 2) == OK && + write_address(lower, &lower_done, 3) == OK && + write_address(producer, &fence, comparison.released) == OK, + "build dependent APR read and later AMM fence producer"); + Check(submit_apr(blocked, 3, &results[0], &ids[0]) == OK && + submit_apr(reader, 3, &results[1], &ids[1]) == OK && + submit_apr(lower, 4, &results[2], &ids[2]) == OK && wait_apr(ids[2]) == OK, + "APR submit returns while blocked and a lower priority completes"); + Check(lower_done == 3 && blocked_done == 0 && read_done == 0 && + output == std::array{} && results[0].result == 1234 && + results[1].result == 1234, + "unsatisfied wait blocks later buffers at its priority without publishing completion"); + Check(submit_amm(buffers[3].data.data(), static_cast(offset(producer)), 1, + &ids[3]) == OK && wait_amm(ids[3]) == OK && + wait_apr(ids[0]) == OK && wait_apr(ids[1]) == OK, + "later AMM submission releases the APR dependency"); + Check(blocked_done == 1 && read_done == 2 && std::memcmp(output.data(), "APR", 3) == 0, + "APR reads real file bytes only after its dependency completes"); + for (const auto &result : results) { + Check(result.result == OK, + "submission wait observes the completed result"); + } + } + reset(0); + uint64_t cpu_fence = 0, amm_done = 0, lower_done = 0; + std::array ids {}; + Check(wait_address(buffers[0].header.data(), &cpu_fence, 0, 1, 0) == OK && + write_address(buffers[0].header.data(), &amm_done, 1) == OK && + write_address(buffers[1].header.data(), &lower_done, 1) == OK && + submit_amm(buffers[0].data.data(), static_cast(offset(buffers[0].header.data())), + 0, &ids[0]) == OK && + submit_amm(buffers[1].data.data(), static_cast(offset(buffers[1].header.data())), + 1, &ids[1]) == OK && wait_amm(ids[1]) == OK, + "AMM lower priority progresses while its high priority waits"); + Check(lower_done == 1 && amm_done == 0, "AMM wait preserves its dependency"); + std::atomic_ref(cpu_fence).store(1, std::memory_order_release); + Check(wait_amm(ids[0]) == OK && amm_done == 1, + "AMM observes an external CPU fence store without a submission notification"); + for (auto &buffer : buffers) { + destroy(buffer.header.data()); + } +} + void CheckAprPaths(const std::filesystem::path &root) { Loader::SymbolDatabase symbols; Libs::LibKernelApr::InitLibKernel_1_Apr(&symbols); @@ -562,6 +686,7 @@ void CheckAprPaths(const std::filesystem::path &root) { Check(resolve(unterminated.data(), paths, 1, ids, sizes, &error_index) == -1 && *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_ENAMETOOLONG, "APR rejects an unterminated prefix"); + CheckAmprOrdering(symbols, expected_id); FileSystem::Umount("/app0"); }